tamakyi
10d23e03cf
- DiffPlaces 的 byCoord/byName 改为多值索引 + take() 逐条认领未用项：真机一个坐标下会有 多条同名同坐标的地点（实测「柳州站」3 条），单槽位时同一坐标最多只配上 2 条，第 3 条 永远判「备份新增」⇒ 每次合并都再插一条重复地点、差异永远归不了零（实测 368→438→439）。 修完：地点新增 70→67、合并后再比对归零、再点合并直接「没有要写入的改动」。 - 到访的「活动」由按名称比改为按身份（ZUID_）比，两边都缺 UUID 才退回名称：16 个活动改名 曾把 355 条到访全变成同因冲突，且逐条选「用实机」只写回 activity_id（映射回同一个活动） 改不掉名字。loadStoredVisits 为此多 JOIN 一次 entity_raw 取 ZACTIVITY.ZUID_。 - 回归用例 4 条（重复地点一一配上、活动改名不算冲突、换活动仍要报、缺 UUID 退回名称）+ 「合并后地点差异归零」断言 3 处；两处改动都做过变异验证。 - 实测（09-28 备份并进 09-14 数据集）：到访冲突 253→6，合并两次地点数 438→439 变为 435→435。
352 lines
11 KiB
Go
352 lines
11 KiB
Go
package web
|
||
|
||
import (
|
||
"context"
|
||
"database/sql"
|
||
"encoding/base64"
|
||
"encoding/hex"
|
||
"fmt"
|
||
"io"
|
||
"log"
|
||
"net/http"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"time"
|
||
|
||
"rond-with-you/internal/ingest"
|
||
"rond-with-you/internal/rond"
|
||
)
|
||
|
||
// rondbackup 的差异预演：把上传的备份与当前展示的数据集按稳定键对齐，只读地摆出
|
||
// 新增 / 仅库里有 / 冲突三类，**不写任何数据**。合并执行是下一步的事。
|
||
|
||
const backupStashExt = ".rondbackup"
|
||
|
||
// diffListLimit 是每类差异在页面上最多列出的条数。867 条全铺出来既慢又没人看，
|
||
// 超过就只给前若干条并注明。
|
||
const diffListLimit = 200
|
||
|
||
func (s *Server) backupStashDir() string {
|
||
return filepath.Join(s.cfg.DataDir, "backup-staging")
|
||
}
|
||
|
||
// adminBackupPreview 收下备份、暂存，跳到差异页。这里不校验备份内容——
|
||
// 差异页反正是要解析一遍的，失败时让它报错即可，免得同一份包解两次。
|
||
func (s *Server) adminBackupPreview(w http.ResponseWriter, r *http.Request) {
|
||
r.Body = http.MaxBytesReader(w, r.Body, maxUploadBytes+1<<20)
|
||
if err := r.ParseMultipartForm(32 << 20); err != nil {
|
||
s.adminFlash(w, r, "err", "读取上传内容失败："+err.Error())
|
||
return
|
||
}
|
||
file, hdr, err := r.FormFile("backup")
|
||
if err != nil {
|
||
s.adminFlash(w, r, "err", "请选择要分析的 .rondbackup 文件")
|
||
return
|
||
}
|
||
defer file.Close()
|
||
name := filepath.Base(hdr.Filename)
|
||
lower := strings.ToLower(name)
|
||
if !strings.HasSuffix(lower, ".rondbackup") && !strings.HasSuffix(lower, ".zip") {
|
||
s.adminFlash(w, r, "err", "文件类型不正确，请上传 rond 导出的 .rondbackup 备份")
|
||
return
|
||
}
|
||
data, err := io.ReadAll(io.LimitReader(file, maxUploadBytes+1))
|
||
if err != nil {
|
||
s.adminFlash(w, r, "err", "读取文件失败："+err.Error())
|
||
return
|
||
}
|
||
if int64(len(data)) > maxUploadBytes {
|
||
s.adminFlash(w, r, "err", "备份文件过大，疑似不是 rond 备份")
|
||
return
|
||
}
|
||
token, err := stashPut(s.backupStashDir(), backupStashExt, data)
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
http.Redirect(w, r, "/admin/preview/"+token, http.StatusSeeOther)
|
||
}
|
||
|
||
// adminBackupPreviewPage 渲染差异页。每次都重新算：库里的数据随时可能被别处改掉。
|
||
func (s *Server) adminBackupPreviewPage(w http.ResponseWriter, r *http.Request) {
|
||
ctx := r.Context()
|
||
p, err := s.pageBase(r, "admin", "备份差异预览")
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
token := r.PathValue("token")
|
||
path, ok := stashPath(s.backupStashDir(), backupStashExt, token)
|
||
if !ok {
|
||
s.adminFlash(w, r, "err", "预览链接无效，请重新上传备份")
|
||
return
|
||
}
|
||
if _, err := os.Stat(path); err != nil {
|
||
s.adminFlash(w, r, "err", "这次预览已过期，请重新上传备份")
|
||
return
|
||
}
|
||
ds, err := s.datasetFor(ctx)
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
if ds == nil {
|
||
s.adminFlash(w, r, "err", "还没有可比较的数据集，请先直接导入一份备份")
|
||
return
|
||
}
|
||
|
||
tmp, err := ingest.TempDir(s.cfg.DataDir, "preview")
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
defer os.RemoveAll(tmp)
|
||
sqlDB, err := rond.Open(path, tmp)
|
||
if err != nil {
|
||
s.adminFlash(w, r, "err", "解析备份失败："+err.Error())
|
||
return
|
||
}
|
||
backup, perr := rond.Parse(sqlDB)
|
||
sqlDB.Close()
|
||
if perr != nil {
|
||
s.adminFlash(w, r, "err", "读取备份数据失败："+perr.Error())
|
||
return
|
||
}
|
||
|
||
storedVisits, err := s.loadStoredVisits(ctx, ds.ID)
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
storedPlaces, err := s.loadStoredPlaces(ctx, ds.ID)
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
storedActs, err := s.loadStoredActivities(ctx, ds.ID)
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
|
||
vd := rond.DiffVisits(storedVisits, backup)
|
||
pd := rond.DiffPlaces(storedPlaces, backup)
|
||
ad := rond.DiffActivities(storedActs, backup)
|
||
|
||
d := BackupDiffData{
|
||
Page: p, Token: token,
|
||
DatasetName: ds.Name,
|
||
VisitCounts: rond.VisitCounts(vd),
|
||
PlaceCounts: rond.PlaceCounts(pd),
|
||
ActCounts: rond.ActivityCounts(ad),
|
||
PlaceAdded: limitSlice(pd.Added, diffListLimit),
|
||
PlaceGone: limitSlice(pd.Removed, diffListLimit),
|
||
PlaceClash: limitSlice(pd.Changed, diffListLimit),
|
||
ActAdded: limitSlice(ad.Added, diffListLimit),
|
||
ActGone: limitSlice(ad.Removed, diffListLimit),
|
||
ActClash: limitSlice(ad.Changed, diffListLimit),
|
||
|
||
BackupVisits: len(backup.Visits), BackupPlaces: len(backup.Locations),
|
||
BackupActs: len(backup.Activities), BackupMovements: len(backup.Movements),
|
||
BackupWeather: len(backup.Weather), BackupRaw: backup.RawVisitNum,
|
||
StoredVisits: len(storedVisits), StoredPlaces: len(storedPlaces),
|
||
StoredActs: len(storedActs),
|
||
}
|
||
d.VisitAdded, d.VisitGone, d.VisitClash = visitRows(vd, backup, diffListLimit)
|
||
d.Truncated = anyTruncated(vd, pd, ad)
|
||
d.StoredMovements, d.StoredWeather, err = s.countOther(ctx, ds.ID)
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
d.Identical = d.VisitCounts.Added+d.VisitCounts.Removed+d.VisitCounts.Changed+
|
||
d.PlaceCounts.Added+d.PlaceCounts.Removed+d.PlaceCounts.Changed+
|
||
d.ActCounts.Added+d.ActCounts.Removed+d.ActCounts.Changed == 0
|
||
s.render(w, "admin/backup-preview", d)
|
||
}
|
||
|
||
// visitRows 把三类到访差异摊平成展示行。备份里的地点/活动只有编号，要借备份自己的
|
||
// 表翻成名称；库里那侧的名称在加载时就一并取出来了。
|
||
func visitRows(vd rond.VisitDiff, b *rond.Backup, limit int) (added, gone, clash []VisitRow) {
|
||
place, act := map[int]string{}, map[int]string{}
|
||
for _, l := range b.Locations {
|
||
place[l.SrcPK] = placeLabel(l.Name, l.Lat, l.Lon)
|
||
}
|
||
for _, a := range b.Activities {
|
||
act[a.SrcPK] = a.Name
|
||
}
|
||
fromBackup := func(v rond.Visit) VisitRow {
|
||
r := VisitRow{Arrival: v.Arrival, Departure: v.Departure, Remark: v.Remark, Emoji: v.Emoji}
|
||
if v.LocationSrcPK != nil {
|
||
r.Place = place[*v.LocationSrcPK]
|
||
}
|
||
if v.ActivitySrcPK != nil {
|
||
r.Activity = act[*v.ActivitySrcPK]
|
||
}
|
||
return r
|
||
}
|
||
for _, v := range limitSlice(vd.Added, limit) {
|
||
added = append(added, fromBackup(v))
|
||
}
|
||
for _, sv := range limitSlice(vd.Removed, limit) {
|
||
gone = append(gone, VisitRow{SrcPK: sv.SrcPK, Arrival: sv.Arrival, Departure: sv.Departure,
|
||
Place: sv.PlaceName, Activity: sv.ActivityName, Remark: sv.Remark, Emoji: sv.Emoji,
|
||
UserAdded: sv.UserAdded})
|
||
}
|
||
for _, c := range limitSlice(vd.Changed, limit) {
|
||
r := fromBackup(c.Backup)
|
||
r.Key = c.DB.UUID
|
||
r.UserAdded = c.DB.UserAdded
|
||
r.Fields = c.Fields
|
||
clash = append(clash, r)
|
||
}
|
||
return
|
||
}
|
||
|
||
func placeLabel(name string, lat, lon float64) string {
|
||
if name != "" {
|
||
return name
|
||
}
|
||
return fmt.Sprintf("%.5f, %.5f", lat, lon)
|
||
}
|
||
|
||
// anyTruncated 报告是否有哪一类差异多到被截断了。
|
||
func anyTruncated(vd rond.VisitDiff, pd rond.PlaceDiff, ad rond.ActivityDiff) bool {
|
||
for _, n := range []int{len(vd.Added), len(vd.Removed), len(vd.Changed),
|
||
len(pd.Added), len(pd.Removed), len(pd.Changed),
|
||
len(ad.Added), len(ad.Removed), len(ad.Changed)} {
|
||
if n > diffListLimit {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func limitSlice[T any](in []T, n int) []T {
|
||
if len(in) <= n {
|
||
return in
|
||
}
|
||
return in[:n]
|
||
}
|
||
|
||
func (s *Server) countOther(ctx context.Context, datasetID int64) (movements, weather int, err error) {
|
||
err = s.db.QueryRowContext(ctx, `SELECT
|
||
(SELECT count(*) FROM movements WHERE dataset_id=$1),
|
||
(SELECT count(*) FROM weather WHERE dataset_id=$1)`, datasetID).Scan(&movements, &weather)
|
||
return
|
||
}
|
||
|
||
// loadStoredVisits 读出该数据集的全部到访，并把原始行留档里的 UUID 解出来当对齐键。
|
||
// 没有留档的行就是站点新建的（真机不会给它 UUID），对齐键留空。
|
||
// 所挂活动的 UUID 也一样：到访层面按活动身份比对，才不会因活动改名而全线报冲突。
|
||
func (s *Server) loadStoredVisits(ctx context.Context, datasetID int64) ([]rond.StoredVisit, error) {
|
||
rows, err := s.db.QueryContext(ctx, `
|
||
SELECT v.src_pk, v.arrival, v.departure, COALESCE(v.remark,''), COALESCE(v.emoji,''),
|
||
v.bookmarked, v.is_user_added, COALESCE(p.src_pk,0),
|
||
COALESCE(p.name,''), COALESCE(p.lat,0), COALESCE(p.lon,0), COALESCE(a.name,''),
|
||
COALESCE(er.raw #>> '{ZIDENTIFIER_,$blob}',''),
|
||
COALESCE(aer.raw #>> '{ZUID_,$blob}','')
|
||
FROM visits v
|
||
LEFT JOIN places p ON p.id = v.place_id
|
||
LEFT JOIN activities a ON a.id = v.activity_id
|
||
LEFT JOIN entity_raw er ON er.dataset_id = v.dataset_id
|
||
AND er.entity = 'ZVISIT' AND er.src_pk = v.src_pk
|
||
LEFT JOIN entity_raw aer ON aer.dataset_id = v.dataset_id
|
||
AND aer.entity = 'ZACTIVITY' AND aer.src_pk = a.src_pk
|
||
WHERE v.dataset_id = $1`, datasetID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var out []rond.StoredVisit
|
||
for rows.Next() {
|
||
var sv rond.StoredVisit
|
||
var arr time.Time
|
||
var dep sql.NullTime
|
||
var placePK int
|
||
var placeName string
|
||
var lat, lon float64
|
||
var b64, actB64 string
|
||
if err := rows.Scan(&sv.SrcPK, &arr, &dep, &sv.Remark, &sv.Emoji, &sv.Bookmarked,
|
||
&sv.UserAdded, &placePK, &placeName, &lat, &lon, &sv.ActivityName, &b64, &actB64); err != nil {
|
||
return nil, err
|
||
}
|
||
sv.Arrival = &arr
|
||
if dep.Valid {
|
||
sv.Departure = &dep.Time
|
||
}
|
||
sv.PlaceName = placeName
|
||
// 没有坐标的地点不能都落到 "0.000000|0.000000" 上，否则两个地点会互相认领
|
||
if placePK > 0 && (lat != 0 || lon != 0) {
|
||
sv.PlaceKey = rond.PlaceCoordKey(lat, lon)
|
||
}
|
||
sv.UUID = uuidFromBase64(b64)
|
||
sv.ActivityUUID = uuidFromBase64(actB64)
|
||
out = append(out, sv)
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
func (s *Server) loadStoredPlaces(ctx context.Context, datasetID int64) ([]rond.StoredPlace, error) {
|
||
rows, err := s.db.QueryContext(ctx, `SELECT src_pk, COALESCE(name,''), COALESCE(lat,0), COALESCE(lon,0),
|
||
COALESCE(province,''), COALESCE(city,''), COALESCE(poi_category,''), visit_count
|
||
FROM places WHERE dataset_id=$1`, datasetID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var out []rond.StoredPlace
|
||
for rows.Next() {
|
||
var sp rond.StoredPlace
|
||
if err := rows.Scan(&sp.SrcPK, &sp.Name, &sp.Lat, &sp.Lon,
|
||
&sp.Province, &sp.City, &sp.POICategory, &sp.VisitCount); err != nil {
|
||
return nil, err
|
||
}
|
||
if sp.Lat != 0 || sp.Lon != 0 {
|
||
sp.Key = rond.PlaceCoordKey(sp.Lat, sp.Lon)
|
||
}
|
||
out = append(out, sp)
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
// loadStoredActivities 读活动类型；UUID 同样从留档里取，用于识别「改了名但其实是同一个」。
|
||
func (s *Server) loadStoredActivities(ctx context.Context, datasetID int64) ([]rond.StoredActivity, error) {
|
||
rows, err := s.db.QueryContext(ctx, `SELECT a.src_pk, a.name,
|
||
COALESCE(er.raw #>> '{ZUID_,$blob}','')
|
||
FROM activities a
|
||
LEFT JOIN entity_raw er ON er.dataset_id = a.dataset_id
|
||
AND er.entity = 'ZACTIVITY' AND er.src_pk = a.src_pk
|
||
WHERE a.dataset_id=$1`, datasetID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var out []rond.StoredActivity
|
||
for rows.Next() {
|
||
var sa rond.StoredActivity
|
||
var b64 string
|
||
if err := rows.Scan(&sa.SrcPK, &sa.Name, &b64); err != nil {
|
||
return nil, err
|
||
}
|
||
sa.UUID = uuidFromBase64(b64)
|
||
out = append(out, sa)
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
// uuidFromBase64 把留档里的 $blob（base64 的 16 字节 UUID）转成小写十六进制，
|
||
// 好跟备份侧 hex(ZIDENTIFIER_) 的写法对上。
|
||
func uuidFromBase64(s string) string {
|
||
if s == "" {
|
||
return ""
|
||
}
|
||
b, err := base64.StdEncoding.DecodeString(s)
|
||
if err != nil || len(b) != 16 {
|
||
log.Printf("留档里的 UUID 不是 16 字节，忽略: %q", s)
|
||
return ""
|
||
}
|
||
return hex.EncodeToString(b)
|
||
}
|