tamakyi
30c502ce7b
备份（.rondbackup）与迷雾（.fwss）各加一层「先看差异再动手」： - 迷雾：上传先出差异页，说清叠加能补多少、整体替换会丢多少，并在真实底图上 用绿/灰/红标出会新增 / 不变 / 会丢的区域 - 备份：与当前数据集按 UUID 逐条比对（rond 每次导出都会重排 Z_PK，编号认不出人）， 新增合入、冲突逐条选、仅站内有逐条勾选删除，全在一个事务里执行； 合入的行写回 entity_raw，保证再传同一份备份时差异归零 - 地点按坐标对齐：站点的「补全地址」会把地点改名，拿名称当键会让一整个地点的 到访全变成假冲突 - 附属数据（备注/结论/访客票/图片）从 dataset_id+src_pk 换成坐标键，换数据集不再失联； 访客票没有 dataset_id，回填按活跃数据集认领 - 修正 contentHash 拿 Z_PK 当指纹的隐患，以及两处「src_pk 跨快照稳定」的错误注释 导出格式（.rondbackup / .fwss）保持原样，往返测试通过。
347 lines
11 KiB
Go
347 lines
11 KiB
Go
package web
|
||
|
||
import (
|
||
"context"
|
||
"database/sql"
|
||
"encoding/base64"
|
||
"encoding/hex"
|
||
"fmt"
|
||
"io"
|
||
"log"
|
||
"net/http"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"time"
|
||
|
||
"rond-with-you/internal/ingest"
|
||
"rond-with-you/internal/rond"
|
||
)
|
||
|
||
// rondbackup 的差异预演：把上传的备份与当前展示的数据集按稳定键对齐，只读地摆出
|
||
// 新增 / 仅库里有 / 冲突三类，**不写任何数据**。合并执行是下一步的事。
|
||
|
||
const backupStashExt = ".rondbackup"
|
||
|
||
// diffListLimit 是每类差异在页面上最多列出的条数。867 条全铺出来既慢又没人看，
|
||
// 超过就只给前若干条并注明。
|
||
const diffListLimit = 200
|
||
|
||
func (s *Server) backupStashDir() string {
|
||
return filepath.Join(s.cfg.DataDir, "backup-staging")
|
||
}
|
||
|
||
// adminBackupPreview 收下备份、暂存，跳到差异页。这里不校验备份内容——
|
||
// 差异页反正是要解析一遍的，失败时让它报错即可，免得同一份包解两次。
|
||
func (s *Server) adminBackupPreview(w http.ResponseWriter, r *http.Request) {
|
||
r.Body = http.MaxBytesReader(w, r.Body, maxUploadBytes+1<<20)
|
||
if err := r.ParseMultipartForm(32 << 20); err != nil {
|
||
s.adminFlash(w, r, "err", "读取上传内容失败："+err.Error())
|
||
return
|
||
}
|
||
file, hdr, err := r.FormFile("backup")
|
||
if err != nil {
|
||
s.adminFlash(w, r, "err", "请选择要分析的 .rondbackup 文件")
|
||
return
|
||
}
|
||
defer file.Close()
|
||
name := filepath.Base(hdr.Filename)
|
||
lower := strings.ToLower(name)
|
||
if !strings.HasSuffix(lower, ".rondbackup") && !strings.HasSuffix(lower, ".zip") {
|
||
s.adminFlash(w, r, "err", "文件类型不正确，请上传 rond 导出的 .rondbackup 备份")
|
||
return
|
||
}
|
||
data, err := io.ReadAll(io.LimitReader(file, maxUploadBytes+1))
|
||
if err != nil {
|
||
s.adminFlash(w, r, "err", "读取文件失败："+err.Error())
|
||
return
|
||
}
|
||
if int64(len(data)) > maxUploadBytes {
|
||
s.adminFlash(w, r, "err", "备份文件过大，疑似不是 rond 备份")
|
||
return
|
||
}
|
||
token, err := stashPut(s.backupStashDir(), backupStashExt, data)
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
http.Redirect(w, r, "/admin/preview/"+token, http.StatusSeeOther)
|
||
}
|
||
|
||
// adminBackupPreviewPage 渲染差异页。每次都重新算：库里的数据随时可能被别处改掉。
|
||
func (s *Server) adminBackupPreviewPage(w http.ResponseWriter, r *http.Request) {
|
||
ctx := r.Context()
|
||
p, err := s.pageBase(r, "admin", "备份差异预览")
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
token := r.PathValue("token")
|
||
path, ok := stashPath(s.backupStashDir(), backupStashExt, token)
|
||
if !ok {
|
||
s.adminFlash(w, r, "err", "预览链接无效，请重新上传备份")
|
||
return
|
||
}
|
||
if _, err := os.Stat(path); err != nil {
|
||
s.adminFlash(w, r, "err", "这次预览已过期，请重新上传备份")
|
||
return
|
||
}
|
||
ds, err := s.datasetFor(ctx)
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
if ds == nil {
|
||
s.adminFlash(w, r, "err", "还没有可比较的数据集，请先直接导入一份备份")
|
||
return
|
||
}
|
||
|
||
tmp, err := ingest.TempDir(s.cfg.DataDir, "preview")
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
defer os.RemoveAll(tmp)
|
||
sqlDB, err := rond.Open(path, tmp)
|
||
if err != nil {
|
||
s.adminFlash(w, r, "err", "解析备份失败："+err.Error())
|
||
return
|
||
}
|
||
backup, perr := rond.Parse(sqlDB)
|
||
sqlDB.Close()
|
||
if perr != nil {
|
||
s.adminFlash(w, r, "err", "读取备份数据失败："+perr.Error())
|
||
return
|
||
}
|
||
|
||
storedVisits, err := s.loadStoredVisits(ctx, ds.ID)
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
storedPlaces, err := s.loadStoredPlaces(ctx, ds.ID)
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
storedActs, err := s.loadStoredActivities(ctx, ds.ID)
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
|
||
vd := rond.DiffVisits(storedVisits, backup)
|
||
pd := rond.DiffPlaces(storedPlaces, backup)
|
||
ad := rond.DiffActivities(storedActs, backup)
|
||
|
||
d := BackupDiffData{
|
||
Page: p, Token: token,
|
||
DatasetName: ds.Name,
|
||
VisitCounts: rond.VisitCounts(vd),
|
||
PlaceCounts: rond.PlaceCounts(pd),
|
||
ActCounts: rond.ActivityCounts(ad),
|
||
PlaceAdded: limitSlice(pd.Added, diffListLimit),
|
||
PlaceGone: limitSlice(pd.Removed, diffListLimit),
|
||
PlaceClash: limitSlice(pd.Changed, diffListLimit),
|
||
ActAdded: limitSlice(ad.Added, diffListLimit),
|
||
ActGone: limitSlice(ad.Removed, diffListLimit),
|
||
ActClash: limitSlice(ad.Changed, diffListLimit),
|
||
|
||
BackupVisits: len(backup.Visits), BackupPlaces: len(backup.Locations),
|
||
BackupActs: len(backup.Activities), BackupMovements: len(backup.Movements),
|
||
BackupWeather: len(backup.Weather), BackupRaw: backup.RawVisitNum,
|
||
StoredVisits: len(storedVisits), StoredPlaces: len(storedPlaces),
|
||
StoredActs: len(storedActs),
|
||
}
|
||
d.VisitAdded, d.VisitGone, d.VisitClash = visitRows(vd, backup, diffListLimit)
|
||
d.Truncated = anyTruncated(vd, pd, ad)
|
||
d.StoredMovements, d.StoredWeather, err = s.countOther(ctx, ds.ID)
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
d.Identical = d.VisitCounts.Added+d.VisitCounts.Removed+d.VisitCounts.Changed+
|
||
d.PlaceCounts.Added+d.PlaceCounts.Removed+d.PlaceCounts.Changed+
|
||
d.ActCounts.Added+d.ActCounts.Removed+d.ActCounts.Changed == 0
|
||
s.render(w, "admin/backup-preview", d)
|
||
}
|
||
|
||
// visitRows 把三类到访差异摊平成展示行。备份里的地点/活动只有编号，要借备份自己的
|
||
// 表翻成名称；库里那侧的名称在加载时就一并取出来了。
|
||
func visitRows(vd rond.VisitDiff, b *rond.Backup, limit int) (added, gone, clash []VisitRow) {
|
||
place, act := map[int]string{}, map[int]string{}
|
||
for _, l := range b.Locations {
|
||
place[l.SrcPK] = placeLabel(l.Name, l.Lat, l.Lon)
|
||
}
|
||
for _, a := range b.Activities {
|
||
act[a.SrcPK] = a.Name
|
||
}
|
||
fromBackup := func(v rond.Visit) VisitRow {
|
||
r := VisitRow{Arrival: v.Arrival, Departure: v.Departure, Remark: v.Remark, Emoji: v.Emoji}
|
||
if v.LocationSrcPK != nil {
|
||
r.Place = place[*v.LocationSrcPK]
|
||
}
|
||
if v.ActivitySrcPK != nil {
|
||
r.Activity = act[*v.ActivitySrcPK]
|
||
}
|
||
return r
|
||
}
|
||
for _, v := range limitSlice(vd.Added, limit) {
|
||
added = append(added, fromBackup(v))
|
||
}
|
||
for _, sv := range limitSlice(vd.Removed, limit) {
|
||
gone = append(gone, VisitRow{SrcPK: sv.SrcPK, Arrival: sv.Arrival, Departure: sv.Departure,
|
||
Place: sv.PlaceName, Activity: sv.ActivityName, Remark: sv.Remark, Emoji: sv.Emoji,
|
||
UserAdded: sv.UserAdded})
|
||
}
|
||
for _, c := range limitSlice(vd.Changed, limit) {
|
||
r := fromBackup(c.Backup)
|
||
r.Key = c.DB.UUID
|
||
r.UserAdded = c.DB.UserAdded
|
||
r.Fields = c.Fields
|
||
clash = append(clash, r)
|
||
}
|
||
return
|
||
}
|
||
|
||
func placeLabel(name string, lat, lon float64) string {
|
||
if name != "" {
|
||
return name
|
||
}
|
||
return fmt.Sprintf("%.5f, %.5f", lat, lon)
|
||
}
|
||
|
||
// anyTruncated 报告是否有哪一类差异多到被截断了。
|
||
func anyTruncated(vd rond.VisitDiff, pd rond.PlaceDiff, ad rond.ActivityDiff) bool {
|
||
for _, n := range []int{len(vd.Added), len(vd.Removed), len(vd.Changed),
|
||
len(pd.Added), len(pd.Removed), len(pd.Changed),
|
||
len(ad.Added), len(ad.Removed), len(ad.Changed)} {
|
||
if n > diffListLimit {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func limitSlice[T any](in []T, n int) []T {
|
||
if len(in) <= n {
|
||
return in
|
||
}
|
||
return in[:n]
|
||
}
|
||
|
||
func (s *Server) countOther(ctx context.Context, datasetID int64) (movements, weather int, err error) {
|
||
err = s.db.QueryRowContext(ctx, `SELECT
|
||
(SELECT count(*) FROM movements WHERE dataset_id=$1),
|
||
(SELECT count(*) FROM weather WHERE dataset_id=$1)`, datasetID).Scan(&movements, &weather)
|
||
return
|
||
}
|
||
|
||
// loadStoredVisits 读出该数据集的全部到访，并把原始行留档里的 UUID 解出来当对齐键。
|
||
// 没有留档的行就是站点新建的（真机不会给它 UUID），对齐键留空。
|
||
func (s *Server) loadStoredVisits(ctx context.Context, datasetID int64) ([]rond.StoredVisit, error) {
|
||
rows, err := s.db.QueryContext(ctx, `
|
||
SELECT v.src_pk, v.arrival, v.departure, COALESCE(v.remark,''), COALESCE(v.emoji,''),
|
||
v.bookmarked, v.is_user_added, COALESCE(p.src_pk,0),
|
||
COALESCE(p.name,''), COALESCE(p.lat,0), COALESCE(p.lon,0), COALESCE(a.name,''),
|
||
COALESCE(er.raw #>> '{ZIDENTIFIER_,$blob}','')
|
||
FROM visits v
|
||
LEFT JOIN places p ON p.id = v.place_id
|
||
LEFT JOIN activities a ON a.id = v.activity_id
|
||
LEFT JOIN entity_raw er ON er.dataset_id = v.dataset_id
|
||
AND er.entity = 'ZVISIT' AND er.src_pk = v.src_pk
|
||
WHERE v.dataset_id = $1`, datasetID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var out []rond.StoredVisit
|
||
for rows.Next() {
|
||
var sv rond.StoredVisit
|
||
var arr time.Time
|
||
var dep sql.NullTime
|
||
var placePK int
|
||
var placeName string
|
||
var lat, lon float64
|
||
var b64 string
|
||
if err := rows.Scan(&sv.SrcPK, &arr, &dep, &sv.Remark, &sv.Emoji, &sv.Bookmarked,
|
||
&sv.UserAdded, &placePK, &placeName, &lat, &lon, &sv.ActivityName, &b64); err != nil {
|
||
return nil, err
|
||
}
|
||
sv.Arrival = &arr
|
||
if dep.Valid {
|
||
sv.Departure = &dep.Time
|
||
}
|
||
sv.PlaceName = placeName
|
||
// 没有坐标的地点不能都落到 "0.000000|0.000000" 上，否则两个地点会互相认领
|
||
if placePK > 0 && (lat != 0 || lon != 0) {
|
||
sv.PlaceKey = rond.PlaceCoordKey(lat, lon)
|
||
}
|
||
sv.UUID = uuidFromBase64(b64)
|
||
out = append(out, sv)
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
func (s *Server) loadStoredPlaces(ctx context.Context, datasetID int64) ([]rond.StoredPlace, error) {
|
||
rows, err := s.db.QueryContext(ctx, `SELECT src_pk, COALESCE(name,''), COALESCE(lat,0), COALESCE(lon,0),
|
||
COALESCE(province,''), COALESCE(city,''), COALESCE(poi_category,''), visit_count
|
||
FROM places WHERE dataset_id=$1`, datasetID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var out []rond.StoredPlace
|
||
for rows.Next() {
|
||
var sp rond.StoredPlace
|
||
if err := rows.Scan(&sp.SrcPK, &sp.Name, &sp.Lat, &sp.Lon,
|
||
&sp.Province, &sp.City, &sp.POICategory, &sp.VisitCount); err != nil {
|
||
return nil, err
|
||
}
|
||
if sp.Lat != 0 || sp.Lon != 0 {
|
||
sp.Key = rond.PlaceCoordKey(sp.Lat, sp.Lon)
|
||
}
|
||
out = append(out, sp)
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
// loadStoredActivities 读活动类型；UUID 同样从留档里取，用于识别「改了名但其实是同一个」。
|
||
func (s *Server) loadStoredActivities(ctx context.Context, datasetID int64) ([]rond.StoredActivity, error) {
|
||
rows, err := s.db.QueryContext(ctx, `SELECT a.src_pk, a.name,
|
||
COALESCE(er.raw #>> '{ZUID_,$blob}','')
|
||
FROM activities a
|
||
LEFT JOIN entity_raw er ON er.dataset_id = a.dataset_id
|
||
AND er.entity = 'ZACTIVITY' AND er.src_pk = a.src_pk
|
||
WHERE a.dataset_id=$1`, datasetID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var out []rond.StoredActivity
|
||
for rows.Next() {
|
||
var sa rond.StoredActivity
|
||
var b64 string
|
||
if err := rows.Scan(&sa.SrcPK, &sa.Name, &b64); err != nil {
|
||
return nil, err
|
||
}
|
||
sa.UUID = uuidFromBase64(b64)
|
||
out = append(out, sa)
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
// uuidFromBase64 把留档里的 $blob（base64 的 16 字节 UUID）转成小写十六进制，
|
||
// 好跟备份侧 hex(ZIDENTIFIER_) 的写法对上。
|
||
func uuidFromBase64(s string) string {
|
||
if s == "" {
|
||
return ""
|
||
}
|
||
b, err := base64.StdEncoding.DecodeString(s)
|
||
if err != nil || len(b) != 16 {
|
||
log.Printf("留档里的 UUID 不是 16 字节，忽略: %q", s)
|
||
return ""
|
||
}
|
||
return hex.EncodeToString(b)
|
||
}
|