备份与迷雾的差异分析、合并执行，附属数据改挂坐标键

备份（.rondbackup）与迷雾（.fwss）各加一层「先看差异再动手」： - 迷雾：上传先出差异页，说清叠加能补多少、整体替换会丢多少，并在真实底图上用绿/灰/红标出会新增 / 不变 / 会丢的区域 - 备份：与当前数据集按 UUID 逐条比对（rond 每次导出都会重排 Z_PK，编号认不出人），新增合入、冲突逐条选、仅站内有逐条勾选删除，全在一个事务里执行；合入的行写回 entity_raw，保证再传同一份备份时差异归零 - 地点按坐标对齐：站点的「补全地址」会把地点改名，拿名称当键会让一整个地点的到访全变成假冲突 - 附属数据（备注/结论/访客票/图片）从 dataset_id+src_pk 换成坐标键，换数据集不再失联；访客票没有 dataset_id，回填按活跃数据集认领 - 修正 contentHash 拿 Z_PK 当指纹的隐患，以及两处「src_pk 跨快照稳定」的错误注释导出格式（.rondbackup / .fwss）保持原样，往返测试通过。
2026-09-19 16:04:00 +08:00
parent 9e01923244
commit 30c502ce7b
43 changed files with 5968 additions and 170 deletions
@@ -0,0 +1,346 @@
+package web
+
+import (
+	"context"
+	"database/sql"
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"rond-with-you/internal/ingest"
+	"rond-with-you/internal/rond"
+)
+
+// rondbackup 的差异预演：把上传的备份与当前展示的数据集按稳定键对齐，只读地摆出
+// 新增 / 仅库里有 / 冲突三类，**不写任何数据**。合并执行是下一步的事。
+
+const backupStashExt = ".rondbackup"
+
+// diffListLimit 是每类差异在页面上最多列出的条数。867 条全铺出来既慢又没人看，
+// 超过就只给前若干条并注明。
+const diffListLimit = 200
+
+func (s *Server) backupStashDir() string {
+	return filepath.Join(s.cfg.DataDir, "backup-staging")
+}
+
+// adminBackupPreview 收下备份、暂存，跳到差异页。这里不校验备份内容——
+// 差异页反正是要解析一遍的，失败时让它报错即可，免得同一份包解两次。
+func (s *Server) adminBackupPreview(w http.ResponseWriter, r *http.Request) {
+	r.Body = http.MaxBytesReader(w, r.Body, maxUploadBytes+1<<20)
+	if err := r.ParseMultipartForm(32 << 20); err != nil {
+		s.adminFlash(w, r, "err", "读取上传内容失败："+err.Error())
+		return
+	}
+	file, hdr, err := r.FormFile("backup")
+	if err != nil {
+		s.adminFlash(w, r, "err", "请选择要分析的 .rondbackup 文件")
+		return
+	}
+	defer file.Close()
+	name := filepath.Base(hdr.Filename)
+	lower := strings.ToLower(name)
+	if !strings.HasSuffix(lower, ".rondbackup") && !strings.HasSuffix(lower, ".zip") {
+		s.adminFlash(w, r, "err", "文件类型不正确，请上传 rond 导出的 .rondbackup 备份")
+		return
+	}
+	data, err := io.ReadAll(io.LimitReader(file, maxUploadBytes+1))
+	if err != nil {
+		s.adminFlash(w, r, "err", "读取文件失败："+err.Error())
+		return
+	}
+	if int64(len(data)) > maxUploadBytes {
+		s.adminFlash(w, r, "err", "备份文件过大，疑似不是 rond 备份")
+		return
+	}
+	token, err := stashPut(s.backupStashDir(), backupStashExt, data)
+	if err != nil {
+		s.serverError(w, r, err)
+		return
+	}
+	http.Redirect(w, r, "/admin/preview/"+token, http.StatusSeeOther)
+}
+
+// adminBackupPreviewPage 渲染差异页。每次都重新算：库里的数据随时可能被别处改掉。
+func (s *Server) adminBackupPreviewPage(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	p, err := s.pageBase(r, "admin", "备份差异预览")
+	if err != nil {
+		s.serverError(w, r, err)
+		return
+	}
+	token := r.PathValue("token")
+	path, ok := stashPath(s.backupStashDir(), backupStashExt, token)
+	if !ok {
+		s.adminFlash(w, r, "err", "预览链接无效，请重新上传备份")
+		return
+	}
+	if _, err := os.Stat(path); err != nil {
+		s.adminFlash(w, r, "err", "这次预览已过期，请重新上传备份")
+		return
+	}
+	ds, err := s.datasetFor(ctx)
+	if err != nil {
+		s.serverError(w, r, err)
+		return
+	}
+	if ds == nil {
+		s.adminFlash(w, r, "err", "还没有可比较的数据集，请先直接导入一份备份")
+		return
+	}
+
+	tmp, err := ingest.TempDir(s.cfg.DataDir, "preview")
+	if err != nil {
+		s.serverError(w, r, err)
+		return
+	}
+	defer os.RemoveAll(tmp)
+	sqlDB, err := rond.Open(path, tmp)
+	if err != nil {
+		s.adminFlash(w, r, "err", "解析备份失败："+err.Error())
+		return
+	}
+	backup, perr := rond.Parse(sqlDB)
+	sqlDB.Close()
+	if perr != nil {
+		s.adminFlash(w, r, "err", "读取备份数据失败："+perr.Error())
+		return
+	}
+
+	storedVisits, err := s.loadStoredVisits(ctx, ds.ID)
+	if err != nil {
+		s.serverError(w, r, err)
+		return
+	}
+	storedPlaces, err := s.loadStoredPlaces(ctx, ds.ID)
+	if err != nil {
+		s.serverError(w, r, err)
+		return
+	}
+	storedActs, err := s.loadStoredActivities(ctx, ds.ID)
+	if err != nil {
+		s.serverError(w, r, err)
+		return
+	}
+
+	vd := rond.DiffVisits(storedVisits, backup)
+	pd := rond.DiffPlaces(storedPlaces, backup)
+	ad := rond.DiffActivities(storedActs, backup)
+
+	d := BackupDiffData{
+		Page: p, Token: token,
+		DatasetName: ds.Name,
+		VisitCounts: rond.VisitCounts(vd),
+		PlaceCounts: rond.PlaceCounts(pd),
+		ActCounts:   rond.ActivityCounts(ad),
+		PlaceAdded:  limitSlice(pd.Added, diffListLimit),
+		PlaceGone:   limitSlice(pd.Removed, diffListLimit),
+		PlaceClash:  limitSlice(pd.Changed, diffListLimit),
+		ActAdded:    limitSlice(ad.Added, diffListLimit),
+		ActGone:     limitSlice(ad.Removed, diffListLimit),
+		ActClash:    limitSlice(ad.Changed, diffListLimit),
+
+		BackupVisits: len(backup.Visits), BackupPlaces: len(backup.Locations),
+		BackupActs: len(backup.Activities), BackupMovements: len(backup.Movements),
+		BackupWeather: len(backup.Weather), BackupRaw: backup.RawVisitNum,
+		StoredVisits: len(storedVisits), StoredPlaces: len(storedPlaces),
+		StoredActs: len(storedActs),
+	}
+	d.VisitAdded, d.VisitGone, d.VisitClash = visitRows(vd, backup, diffListLimit)
+	d.Truncated = anyTruncated(vd, pd, ad)
+	d.StoredMovements, d.StoredWeather, err = s.countOther(ctx, ds.ID)
+	if err != nil {
+		s.serverError(w, r, err)
+		return
+	}
+	d.Identical = d.VisitCounts.Added+d.VisitCounts.Removed+d.VisitCounts.Changed+
+		d.PlaceCounts.Added+d.PlaceCounts.Removed+d.PlaceCounts.Changed+
+		d.ActCounts.Added+d.ActCounts.Removed+d.ActCounts.Changed == 0
+	s.render(w, "admin/backup-preview", d)
+}
+
+// visitRows 把三类到访差异摊平成展示行。备份里的地点/活动只有编号，要借备份自己的
+// 表翻成名称；库里那侧的名称在加载时就一并取出来了。
+func visitRows(vd rond.VisitDiff, b *rond.Backup, limit int) (added, gone, clash []VisitRow) {
+	place, act := map[int]string{}, map[int]string{}
+	for _, l := range b.Locations {
+		place[l.SrcPK] = placeLabel(l.Name, l.Lat, l.Lon)
+	}
+	for _, a := range b.Activities {
+		act[a.SrcPK] = a.Name
+	}
+	fromBackup := func(v rond.Visit) VisitRow {
+		r := VisitRow{Arrival: v.Arrival, Departure: v.Departure, Remark: v.Remark, Emoji: v.Emoji}
+		if v.LocationSrcPK != nil {
+			r.Place = place[*v.LocationSrcPK]
+		}
+		if v.ActivitySrcPK != nil {
+			r.Activity = act[*v.ActivitySrcPK]
+		}
+		return r
+	}
+	for _, v := range limitSlice(vd.Added, limit) {
+		added = append(added, fromBackup(v))
+	}
+	for _, sv := range limitSlice(vd.Removed, limit) {
+		gone = append(gone, VisitRow{SrcPK: sv.SrcPK, Arrival: sv.Arrival, Departure: sv.Departure,
+			Place: sv.PlaceName, Activity: sv.ActivityName, Remark: sv.Remark, Emoji: sv.Emoji,
+			UserAdded: sv.UserAdded})
+	}
+	for _, c := range limitSlice(vd.Changed, limit) {
+		r := fromBackup(c.Backup)
+		r.Key = c.DB.UUID
+		r.UserAdded = c.DB.UserAdded
+		r.Fields = c.Fields
+		clash = append(clash, r)
+	}
+	return
+}
+
+func placeLabel(name string, lat, lon float64) string {
+	if name != "" {
+		return name
+	}
+	return fmt.Sprintf("%.5f, %.5f", lat, lon)
+}
+
+// anyTruncated 报告是否有哪一类差异多到被截断了。
+func anyTruncated(vd rond.VisitDiff, pd rond.PlaceDiff, ad rond.ActivityDiff) bool {
+	for _, n := range []int{len(vd.Added), len(vd.Removed), len(vd.Changed),
+		len(pd.Added), len(pd.Removed), len(pd.Changed),
+		len(ad.Added), len(ad.Removed), len(ad.Changed)} {
+		if n > diffListLimit {
+			return true
+		}
+	}
+	return false
+}
+
+func limitSlice[T any](in []T, n int) []T {
+	if len(in) <= n {
+		return in
+	}
+	return in[:n]
+}
+
+func (s *Server) countOther(ctx context.Context, datasetID int64) (movements, weather int, err error) {
+	err = s.db.QueryRowContext(ctx, `SELECT
+		(SELECT count(*) FROM movements WHERE dataset_id=$1),
+		(SELECT count(*) FROM weather WHERE dataset_id=$1)`, datasetID).Scan(&movements, &weather)
+	return
+}
+
+// loadStoredVisits 读出该数据集的全部到访，并把原始行留档里的 UUID 解出来当对齐键。
+// 没有留档的行就是站点新建的（真机不会给它 UUID），对齐键留空。
+func (s *Server) loadStoredVisits(ctx context.Context, datasetID int64) ([]rond.StoredVisit, error) {
+	rows, err := s.db.QueryContext(ctx, `
+		SELECT v.src_pk, v.arrival, v.departure, COALESCE(v.remark,''), COALESCE(v.emoji,''),
+		       v.bookmarked, v.is_user_added, COALESCE(p.src_pk,0),
+		       COALESCE(p.name,''), COALESCE(p.lat,0), COALESCE(p.lon,0), COALESCE(a.name,''),
+		       COALESCE(er.raw #>> '{ZIDENTIFIER_,$blob}','')
+		FROM visits v
+		LEFT JOIN places p ON p.id = v.place_id
+		LEFT JOIN activities a ON a.id = v.activity_id
+		LEFT JOIN entity_raw er ON er.dataset_id = v.dataset_id
+			AND er.entity = 'ZVISIT' AND er.src_pk = v.src_pk
+		WHERE v.dataset_id = $1`, datasetID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []rond.StoredVisit
+	for rows.Next() {
+		var sv rond.StoredVisit
+		var arr time.Time
+		var dep sql.NullTime
+		var placePK int
+		var placeName string
+		var lat, lon float64
+		var b64 string
+		if err := rows.Scan(&sv.SrcPK, &arr, &dep, &sv.Remark, &sv.Emoji, &sv.Bookmarked,
+			&sv.UserAdded, &placePK, &placeName, &lat, &lon, &sv.ActivityName, &b64); err != nil {
+			return nil, err
+		}
+		sv.Arrival = &arr
+		if dep.Valid {
+			sv.Departure = &dep.Time
+		}
+		sv.PlaceName = placeName
+		// 没有坐标的地点不能都落到 "0.000000|0.000000" 上，否则两个地点会互相认领
+		if placePK > 0 && (lat != 0 || lon != 0) {
+			sv.PlaceKey = rond.PlaceCoordKey(lat, lon)
+		}
+		sv.UUID = uuidFromBase64(b64)
+		out = append(out, sv)
+	}
+	return out, rows.Err()
+}
+
+func (s *Server) loadStoredPlaces(ctx context.Context, datasetID int64) ([]rond.StoredPlace, error) {
+	rows, err := s.db.QueryContext(ctx, `SELECT src_pk, COALESCE(name,''), COALESCE(lat,0), COALESCE(lon,0),
+		COALESCE(province,''), COALESCE(city,''), COALESCE(poi_category,''), visit_count
+		FROM places WHERE dataset_id=$1`, datasetID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []rond.StoredPlace
+	for rows.Next() {
+		var sp rond.StoredPlace
+		if err := rows.Scan(&sp.SrcPK, &sp.Name, &sp.Lat, &sp.Lon,
+			&sp.Province, &sp.City, &sp.POICategory, &sp.VisitCount); err != nil {
+			return nil, err
+		}
+		if sp.Lat != 0 || sp.Lon != 0 {
+			sp.Key = rond.PlaceCoordKey(sp.Lat, sp.Lon)
+		}
+		out = append(out, sp)
+	}
+	return out, rows.Err()
+}
+
+// loadStoredActivities 读活动类型；UUID 同样从留档里取，用于识别「改了名但其实是同一个」。
+func (s *Server) loadStoredActivities(ctx context.Context, datasetID int64) ([]rond.StoredActivity, error) {
+	rows, err := s.db.QueryContext(ctx, `SELECT a.src_pk, a.name,
+		COALESCE(er.raw #>> '{ZUID_,$blob}','')
+		FROM activities a
+		LEFT JOIN entity_raw er ON er.dataset_id = a.dataset_id
+			AND er.entity = 'ZACTIVITY' AND er.src_pk = a.src_pk
+		WHERE a.dataset_id=$1`, datasetID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []rond.StoredActivity
+	for rows.Next() {
+		var sa rond.StoredActivity
+		var b64 string
+		if err := rows.Scan(&sa.SrcPK, &sa.Name, &b64); err != nil {
+			return nil, err
+		}
+		sa.UUID = uuidFromBase64(b64)
+		out = append(out, sa)
+	}
+	return out, rows.Err()
+}
+
+// uuidFromBase64 把留档里的 $blob（base64 的 16 字节 UUID）转成小写十六进制，
+// 好跟备份侧 hex(ZIDENTIFIER_) 的写法对上。
+func uuidFromBase64(s string) string {
+	if s == "" {
+		return ""
+	}
+	b, err := base64.StdEncoding.DecodeString(s)
+	if err != nil || len(b) != 16 {
+		log.Printf("留档里的 UUID 不是 16 字节，忽略: %q", s)
+		return ""
+	}
+	return hex.EncodeToString(b)
+}