tamakyi
8763b850be
- 抽出 rond.MatchPlaces(stored, backup) PlaceMatch：坐标优先、退到「同名且相距 ≤1km」、 多值索引 + 认领即标记已用；DiffPlaces 与 web.mergePlaces 共用它。 - mergePlaces 原来是单槽位 byCoord/byName 自己再配一遍：同坐标/同名的多条备份地点全并到同一行 站内地点，到访被挂到错的分店（实测「茶颜悦色」两家分店相隔 8.3km，到访却指着同一条）。 现在直接用配对结果，再叠加 plan.AddPlaces 的新插入。 - loadStoredPlaces 与 mergePlaces 的地点查询都加 ORDER BY id：候选多于一个时靠顺序定谁配谁， 两侧必须一致，否则配对会飘。 - 删掉只写不读的 PlacePatch.CoordKey、ActivityPatch.Key（容易被误当成查找键）。 - 用例：TestMatchPlacesKeepsDuplicateNameGroupsDistinct（同一坐标/邻近同名都要一条配一条）、 TestMergePlacesMapsSameNameLocationsOneToOne（真库 + 真事务，两条同名备份地点必须映射到两条不同 地点）；两处都做了变异验证（认领不标记已用 / 恢复单槽位映射 → 对应用例变红）。 - 探针库复测（从 dump 恢复、不动本地真库，49 组冲突全「用实机」）：到访挂错地点（>1km）0/953 条、 地点被搬动 >1km 0 个；合并后再比对三类全 0（到访 0/47/0、地点 0/4/0、活动 0/0/0）， 再点合并「按当前的选择没有要写入的改动」。全量验收 293 PASS / 0 FAIL / 0 SKIP。 已知取舍：勾「用实机」但没勾「并进来」时，若该到访在实机上的地点站内没有、且按 1km 规则也配不上， place_id 会被置空（旧行为是挂到一条远方的同名地点上——那正是这次要去掉的错配）。
354 lines
12 KiB
Go
354 lines
12 KiB
Go
package web
|
||
|
||
import (
|
||
"context"
|
||
"database/sql"
|
||
"encoding/base64"
|
||
"encoding/hex"
|
||
"fmt"
|
||
"io"
|
||
"log"
|
||
"net/http"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"time"
|
||
|
||
"rond-with-you/internal/ingest"
|
||
"rond-with-you/internal/rond"
|
||
)
|
||
|
||
// rondbackup 的差异预演：把上传的备份与当前展示的数据集按稳定键对齐，只读地摆出
|
||
// 新增 / 仅库里有 / 冲突三类，**不写任何数据**。合并执行是下一步的事。
|
||
|
||
const backupStashExt = ".rondbackup"
|
||
|
||
// diffListLimit 是每类差异在页面上最多列出的条数。867 条全铺出来既慢又没人看，
|
||
// 超过就只给前若干条并注明。
|
||
const diffListLimit = 200
|
||
|
||
func (s *Server) backupStashDir() string {
|
||
return filepath.Join(s.cfg.DataDir, "backup-staging")
|
||
}
|
||
|
||
// adminBackupPreview 收下备份、暂存，跳到差异页。这里不校验备份内容——
|
||
// 差异页反正是要解析一遍的，失败时让它报错即可，免得同一份包解两次。
|
||
func (s *Server) adminBackupPreview(w http.ResponseWriter, r *http.Request) {
|
||
r.Body = http.MaxBytesReader(w, r.Body, maxUploadBytes+1<<20)
|
||
if err := r.ParseMultipartForm(32 << 20); err != nil {
|
||
s.adminFlash(w, r, "err", "读取上传内容失败："+err.Error())
|
||
return
|
||
}
|
||
file, hdr, err := r.FormFile("backup")
|
||
if err != nil {
|
||
s.adminFlash(w, r, "err", "请选择要分析的 .rondbackup 文件")
|
||
return
|
||
}
|
||
defer file.Close()
|
||
name := filepath.Base(hdr.Filename)
|
||
lower := strings.ToLower(name)
|
||
if !strings.HasSuffix(lower, ".rondbackup") && !strings.HasSuffix(lower, ".zip") {
|
||
s.adminFlash(w, r, "err", "文件类型不正确，请上传 rond 导出的 .rondbackup 备份")
|
||
return
|
||
}
|
||
data, err := io.ReadAll(io.LimitReader(file, maxUploadBytes+1))
|
||
if err != nil {
|
||
s.adminFlash(w, r, "err", "读取文件失败："+err.Error())
|
||
return
|
||
}
|
||
if int64(len(data)) > maxUploadBytes {
|
||
s.adminFlash(w, r, "err", "备份文件过大，疑似不是 rond 备份")
|
||
return
|
||
}
|
||
token, err := stashPut(s.backupStashDir(), backupStashExt, data)
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
http.Redirect(w, r, "/admin/preview/"+token, http.StatusSeeOther)
|
||
}
|
||
|
||
// adminBackupPreviewPage 渲染差异页。每次都重新算：库里的数据随时可能被别处改掉。
|
||
func (s *Server) adminBackupPreviewPage(w http.ResponseWriter, r *http.Request) {
|
||
ctx := r.Context()
|
||
p, err := s.pageBase(r, "admin", "备份差异预览")
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
token := r.PathValue("token")
|
||
path, ok := stashPath(s.backupStashDir(), backupStashExt, token)
|
||
if !ok {
|
||
s.adminFlash(w, r, "err", "预览链接无效，请重新上传备份")
|
||
return
|
||
}
|
||
if _, err := os.Stat(path); err != nil {
|
||
s.adminFlash(w, r, "err", "这次预览已过期，请重新上传备份")
|
||
return
|
||
}
|
||
ds, err := s.datasetFor(ctx)
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
if ds == nil {
|
||
s.adminFlash(w, r, "err", "还没有可比较的数据集，请先直接导入一份备份")
|
||
return
|
||
}
|
||
|
||
tmp, err := ingest.TempDir(s.cfg.DataDir, "preview")
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
defer os.RemoveAll(tmp)
|
||
sqlDB, err := rond.Open(path, tmp)
|
||
if err != nil {
|
||
s.adminFlash(w, r, "err", "解析备份失败："+err.Error())
|
||
return
|
||
}
|
||
backup, perr := rond.Parse(sqlDB)
|
||
sqlDB.Close()
|
||
if perr != nil {
|
||
s.adminFlash(w, r, "err", "读取备份数据失败："+perr.Error())
|
||
return
|
||
}
|
||
|
||
storedVisits, err := s.loadStoredVisits(ctx, ds.ID)
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
storedPlaces, err := s.loadStoredPlaces(ctx, ds.ID)
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
storedActs, err := s.loadStoredActivities(ctx, ds.ID)
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
|
||
vd := rond.DiffVisits(storedVisits, backup)
|
||
pd := rond.DiffPlaces(storedPlaces, backup)
|
||
ad := rond.DiffActivities(storedActs, backup)
|
||
|
||
d := BackupDiffData{
|
||
Page: p, Token: token,
|
||
DatasetName: ds.Name,
|
||
VisitCounts: rond.VisitCounts(vd),
|
||
PlaceCounts: rond.PlaceCounts(pd),
|
||
ActCounts: rond.ActivityCounts(ad),
|
||
PlaceAdded: limitSlice(pd.Added, diffListLimit),
|
||
PlaceGone: limitSlice(pd.Removed, diffListLimit),
|
||
PlaceClash: limitSlice(pd.Changed, diffListLimit),
|
||
ActAdded: limitSlice(ad.Added, diffListLimit),
|
||
ActGone: limitSlice(ad.Removed, diffListLimit),
|
||
ActClash: limitSlice(ad.Changed, diffListLimit),
|
||
|
||
BackupVisits: len(backup.Visits), BackupPlaces: len(backup.Locations),
|
||
BackupActs: len(backup.Activities), BackupMovements: len(backup.Movements),
|
||
BackupWeather: len(backup.Weather), BackupRaw: backup.RawVisitNum,
|
||
StoredVisits: len(storedVisits), StoredPlaces: len(storedPlaces),
|
||
StoredActs: len(storedActs),
|
||
}
|
||
d.VisitAdded, d.VisitGone, d.VisitClash = visitRows(vd, backup, diffListLimit)
|
||
d.Truncated = anyTruncated(vd, pd, ad)
|
||
d.StoredMovements, d.StoredWeather, err = s.countOther(ctx, ds.ID)
|
||
if err != nil {
|
||
s.serverError(w, r, err)
|
||
return
|
||
}
|
||
d.Identical = d.VisitCounts.Added+d.VisitCounts.Removed+d.VisitCounts.Changed+
|
||
d.PlaceCounts.Added+d.PlaceCounts.Removed+d.PlaceCounts.Changed+
|
||
d.ActCounts.Added+d.ActCounts.Removed+d.ActCounts.Changed == 0
|
||
s.render(w, "admin/backup-preview", d)
|
||
}
|
||
|
||
// visitRows 把三类到访差异摊平成展示行。备份里的地点/活动只有编号，要借备份自己的
|
||
// 表翻成名称；库里那侧的名称在加载时就一并取出来了。
|
||
func visitRows(vd rond.VisitDiff, b *rond.Backup, limit int) (added, gone, clash []VisitRow) {
|
||
place, act := map[int]string{}, map[int]string{}
|
||
for _, l := range b.Locations {
|
||
place[l.SrcPK] = placeLabel(l.Name, l.Lat, l.Lon)
|
||
}
|
||
for _, a := range b.Activities {
|
||
act[a.SrcPK] = a.Name
|
||
}
|
||
fromBackup := func(v rond.Visit) VisitRow {
|
||
r := VisitRow{Arrival: v.Arrival, Departure: v.Departure, Remark: v.Remark, Emoji: v.Emoji}
|
||
if v.LocationSrcPK != nil {
|
||
r.Place = place[*v.LocationSrcPK]
|
||
}
|
||
if v.ActivitySrcPK != nil {
|
||
r.Activity = act[*v.ActivitySrcPK]
|
||
}
|
||
return r
|
||
}
|
||
for _, v := range limitSlice(vd.Added, limit) {
|
||
added = append(added, fromBackup(v))
|
||
}
|
||
for _, sv := range limitSlice(vd.Removed, limit) {
|
||
gone = append(gone, VisitRow{SrcPK: sv.SrcPK, Arrival: sv.Arrival, Departure: sv.Departure,
|
||
Place: sv.PlaceName, Activity: sv.ActivityName, Remark: sv.Remark, Emoji: sv.Emoji,
|
||
UserAdded: sv.UserAdded})
|
||
}
|
||
for _, c := range limitSlice(vd.Changed, limit) {
|
||
r := fromBackup(c.Backup)
|
||
r.Key = c.DB.UUID
|
||
r.UserAdded = c.DB.UserAdded
|
||
r.Fields = c.Fields
|
||
clash = append(clash, r)
|
||
}
|
||
return
|
||
}
|
||
|
||
func placeLabel(name string, lat, lon float64) string {
|
||
if name != "" {
|
||
return name
|
||
}
|
||
return fmt.Sprintf("%.5f, %.5f", lat, lon)
|
||
}
|
||
|
||
// anyTruncated 报告是否有哪一类差异多到被截断了。
|
||
func anyTruncated(vd rond.VisitDiff, pd rond.PlaceDiff, ad rond.ActivityDiff) bool {
|
||
for _, n := range []int{len(vd.Added), len(vd.Removed), len(vd.Changed),
|
||
len(pd.Added), len(pd.Removed), len(pd.Changed),
|
||
len(ad.Added), len(ad.Removed), len(ad.Changed)} {
|
||
if n > diffListLimit {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func limitSlice[T any](in []T, n int) []T {
|
||
if len(in) <= n {
|
||
return in
|
||
}
|
||
return in[:n]
|
||
}
|
||
|
||
func (s *Server) countOther(ctx context.Context, datasetID int64) (movements, weather int, err error) {
|
||
err = s.db.QueryRowContext(ctx, `SELECT
|
||
(SELECT count(*) FROM movements WHERE dataset_id=$1),
|
||
(SELECT count(*) FROM weather WHERE dataset_id=$1)`, datasetID).Scan(&movements, &weather)
|
||
return
|
||
}
|
||
|
||
// loadStoredVisits 读出该数据集的全部到访，并把原始行留档里的 UUID 解出来当对齐键。
|
||
// 没有留档的行就是站点新建的（真机不会给它 UUID），对齐键留空。
|
||
// 所挂活动的 UUID 也一样：到访层面按活动身份比对，才不会因活动改名而全线报冲突。
|
||
func (s *Server) loadStoredVisits(ctx context.Context, datasetID int64) ([]rond.StoredVisit, error) {
|
||
rows, err := s.db.QueryContext(ctx, `
|
||
SELECT v.src_pk, v.arrival, v.departure, COALESCE(v.remark,''), COALESCE(v.emoji,''),
|
||
v.bookmarked, v.is_user_added, COALESCE(p.src_pk,0),
|
||
COALESCE(p.name,''), COALESCE(p.lat,0), COALESCE(p.lon,0), COALESCE(a.name,''),
|
||
COALESCE(er.raw #>> '{ZIDENTIFIER_,$blob}',''),
|
||
COALESCE(aer.raw #>> '{ZUID_,$blob}','')
|
||
FROM visits v
|
||
LEFT JOIN places p ON p.id = v.place_id
|
||
LEFT JOIN activities a ON a.id = v.activity_id
|
||
LEFT JOIN entity_raw er ON er.dataset_id = v.dataset_id
|
||
AND er.entity = 'ZVISIT' AND er.src_pk = v.src_pk
|
||
LEFT JOIN entity_raw aer ON aer.dataset_id = v.dataset_id
|
||
AND aer.entity = 'ZACTIVITY' AND aer.src_pk = a.src_pk
|
||
WHERE v.dataset_id = $1`, datasetID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var out []rond.StoredVisit
|
||
for rows.Next() {
|
||
var sv rond.StoredVisit
|
||
var arr time.Time
|
||
var dep sql.NullTime
|
||
var placePK int
|
||
var placeName string
|
||
var lat, lon float64
|
||
var b64, actB64 string
|
||
if err := rows.Scan(&sv.SrcPK, &arr, &dep, &sv.Remark, &sv.Emoji, &sv.Bookmarked,
|
||
&sv.UserAdded, &placePK, &placeName, &lat, &lon, &sv.ActivityName, &b64, &actB64); err != nil {
|
||
return nil, err
|
||
}
|
||
sv.Arrival = &arr
|
||
if dep.Valid {
|
||
sv.Departure = &dep.Time
|
||
}
|
||
sv.PlaceName = placeName
|
||
// 没有坐标的地点不能都落到 "0.000000|0.000000" 上，否则两个地点会互相认领
|
||
if placePK > 0 && (lat != 0 || lon != 0) {
|
||
sv.PlaceKey = rond.PlaceCoordKey(lat, lon)
|
||
}
|
||
sv.UUID = uuidFromBase64(b64)
|
||
sv.ActivityUUID = uuidFromBase64(actB64)
|
||
out = append(out, sv)
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
// loadStoredPlaces 读地点。**必须按 id 排序**：配对的候选多于一个时（同坐标/同名的重复地点）
|
||
// 靠顺序决定谁配谁，合并侧（mergePlaces）用的是同一份顺序与同一套规则。
|
||
func (s *Server) loadStoredPlaces(ctx context.Context, datasetID int64) ([]rond.StoredPlace, error) {
|
||
rows, err := s.db.QueryContext(ctx, `SELECT src_pk, COALESCE(name,''), COALESCE(lat,0), COALESCE(lon,0),
|
||
COALESCE(province,''), COALESCE(city,''), COALESCE(poi_category,''), visit_count
|
||
FROM places WHERE dataset_id=$1 ORDER BY id`, datasetID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var out []rond.StoredPlace
|
||
for rows.Next() {
|
||
var sp rond.StoredPlace
|
||
if err := rows.Scan(&sp.SrcPK, &sp.Name, &sp.Lat, &sp.Lon,
|
||
&sp.Province, &sp.City, &sp.POICategory, &sp.VisitCount); err != nil {
|
||
return nil, err
|
||
}
|
||
if sp.Lat != 0 || sp.Lon != 0 {
|
||
sp.Key = rond.PlaceCoordKey(sp.Lat, sp.Lon)
|
||
}
|
||
out = append(out, sp)
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
// loadStoredActivities 读活动类型；UUID 同样从留档里取，用于识别「改了名但其实是同一个」。
|
||
func (s *Server) loadStoredActivities(ctx context.Context, datasetID int64) ([]rond.StoredActivity, error) {
|
||
rows, err := s.db.QueryContext(ctx, `SELECT a.src_pk, a.name,
|
||
COALESCE(er.raw #>> '{ZUID_,$blob}','')
|
||
FROM activities a
|
||
LEFT JOIN entity_raw er ON er.dataset_id = a.dataset_id
|
||
AND er.entity = 'ZACTIVITY' AND er.src_pk = a.src_pk
|
||
WHERE a.dataset_id=$1`, datasetID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var out []rond.StoredActivity
|
||
for rows.Next() {
|
||
var sa rond.StoredActivity
|
||
var b64 string
|
||
if err := rows.Scan(&sa.SrcPK, &sa.Name, &b64); err != nil {
|
||
return nil, err
|
||
}
|
||
sa.UUID = uuidFromBase64(b64)
|
||
out = append(out, sa)
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
// uuidFromBase64 把留档里的 $blob（base64 的 16 字节 UUID）转成小写十六进制，
|
||
// 好跟备份侧 hex(ZIDENTIFIER_) 的写法对上。
|
||
func uuidFromBase64(s string) string {
|
||
if s == "" {
|
||
return ""
|
||
}
|
||
b, err := base64.StdEncoding.DecodeString(s)
|
||
if err != nil || len(b) != 16 {
|
||
log.Printf("留档里的 UUID 不是 16 字节，忽略: %q", s)
|
||
return ""
|
||
}
|
||
return hex.EncodeToString(b)
|
||
}
|