From 30c502ce7b9d08b757e687d81503d04605a683ee Mon Sep 17 00:00:00 2001 From: tamakyi Date: Sat, 19 Sep 2026 16:04:00 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A4=87=E4=BB=BD=E4=B8=8E=E8=BF=B7=E9=9B=BE?= =?UTF-8?q?=E7=9A=84=E5=B7=AE=E5=BC=82=E5=88=86=E6=9E=90=E3=80=81=E5=90=88?= =?UTF-8?q?=E5=B9=B6=E6=89=A7=E8=A1=8C=EF=BC=8C=E9=99=84=E5=B1=9E=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E6=94=B9=E6=8C=82=E5=9D=90=E6=A0=87=E9=94=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 备份（.rondbackup）与迷雾（.fwss）各加一层「先看差异再动手」： - 迷雾：上传先出差异页，说清叠加能补多少、整体替换会丢多少，并在真实底图上 用绿/灰/红标出会新增 / 不变 / 会丢的区域 - 备份：与当前数据集按 UUID 逐条比对（rond 每次导出都会重排 Z_PK，编号认不出人）， 新增合入、冲突逐条选、仅站内有逐条勾选删除，全在一个事务里执行； 合入的行写回 entity_raw，保证再传同一份备份时差异归零 - 地点按坐标对齐：站点的「补全地址」会把地点改名，拿名称当键会让一整个地点的 到访全变成假冲突 - 附属数据（备注/结论/访客票/图片）从 dataset_id+src_pk 换成坐标键，换数据集不再失联； 访客票没有 dataset_id，回填按活跃数据集认领 - 修正 contentHash 拿 Z_PK 当指纹的隐患，以及两处「src_pk 跨快照稳定」的错误注释 导出格式（.rondbackup / .fwss）保持原样，往返测试通过。 --- internal/db/schema.sql | 163 +++- internal/fog/diff.go | 90 +++ internal/fog/diff_test.go | 88 ++ internal/fog/render.go | 6 + internal/fog/render_diff.go | 200 +++++ internal/fog/render_diff_test.go | 114 +++ internal/ingest/ingest.go | 34 +- internal/ingest/ingest_test.go | 107 +++ internal/notes/notes.go | 99 ++- internal/rond/diff.go | 390 +++++++++ internal/rond/diff_test.go | 293 +++++++ internal/rond/export.go | 9 +- internal/rond/merge.go | 170 ++++ internal/rond/merge_test.go | 76 ++ internal/rond/parse.go | 36 + internal/stats/stats.go | 12 +- internal/votes/votes.go | 112 ++- internal/web/backup_diff.go | 346 ++++++++ internal/web/backup_diff_test.go | 164 ++++ internal/web/backup_merge.go | 753 ++++++++++++++++++ internal/web/backup_merge_test.go | 273 +++++++ internal/web/fog.go | 169 +++- internal/web/fog_diff.go | 94 +++ internal/web/fog_preview_test.go | 422 ++++++++++ internal/web/fog_staging.go | 116 +++ internal/web/handlers_admin.go | 10 +- internal/web/handlers_edit.go | 1 + internal/web/handlers_images.go | 46 +- internal/web/handlers_public.go | 6 +- internal/web/handlers_vote.go | 28 +- internal/web/place_identity_test.go | 149 ++++ internal/web/place_image_test.go | 10 +- internal/web/server.go | 48 +- .../web/templates/admin/backup-preview.html | 310 +++++++ internal/web/templates/admin/fog-preview.html | 174 ++++ internal/web/templates/admin/index.html | 21 +- internal/web/view.go | 71 ++ tools/colprobe/main.go | 89 +++ tools/diffprobe/main.go | 438 ++++++++++ tools/exportprobe/main.go | 40 + tools/fogprobe/main.go | 133 ++++ tools/idprobe/main.go | 171 ++++ tools/sqlprobe/main.go | 57 ++ 43 files changed, 5968 insertions(+), 170 deletions(-) create mode 100644 internal/fog/diff.go create mode 100644 internal/fog/diff_test.go create mode 100644 internal/fog/render_diff.go create mode 100644 internal/fog/render_diff_test.go create mode 100644 internal/ingest/ingest_test.go create mode 100644 internal/rond/diff.go create mode 100644 internal/rond/diff_test.go create mode 100644 internal/rond/merge.go create mode 100644 internal/rond/merge_test.go create mode 100644 internal/web/backup_diff.go create mode 100644 internal/web/backup_diff_test.go create mode 100644 internal/web/backup_merge.go create mode 100644 internal/web/backup_merge_test.go create mode 100644 internal/web/fog_diff.go create mode 100644 internal/web/fog_preview_test.go create mode 100644 internal/web/fog_staging.go create mode 100644 internal/web/place_identity_test.go create mode 100644 internal/web/templates/admin/backup-preview.html create mode 100644 internal/web/templates/admin/fog-preview.html create mode 100644 tools/colprobe/main.go create mode 100644 tools/diffprobe/main.go create mode 100644 tools/exportprobe/main.go create mode 100644 tools/fogprobe/main.go create mode 100644 tools/idprobe/main.go create mode 100644 tools/sqlprobe/main.go diff --git a/internal/db/schema.sql b/internal/db/schema.sql index cf14bb1..3ed116b 100644 --- a/internal/db/schema.sql +++ b/internal/db/schema.sql @@ -171,38 +171,50 @@ CREATE TABLE IF NOT EXISTS fog_blocks ( PRIMARY KEY (user_id, gbx, gby) ); --- 地点备注与别名。按 src_pk（rond 内部地点 ID，跨快照稳定）而不是 places.id 存， --- 这样重新上传备份、数据集换了新 id 之后备注依然挂着。 --- dataset_id 是后加的：src_pk 只在单个数据集内唯一，缺了它切数据集时同编号的 --- 地点会把别人的备注/别名认领过去。 +-- 地点备注与别名。挂**坐标键**（place_key）而不是 src_pk： +-- +-- src_pk（rond 的 Z_PK）只在单个数据集内有效——rond 每次导出/同步都会重排编号 +-- （实测两份备份里 822 条到访的 UUID 完全一致、Z_PK 100% 不同），所以换一份备份、 +-- 换个数据集之后，同编号的完全是另一个地点，备注就再也挂不回去了。 +-- 坐标是地点的地理事实，跨备份稳定：place_key = 经纬度各取 6 位小数拼起来 +-- （约 0.1m 精度），与 Go 侧的 rond.PlaceCoordKey 必须逐位一致。 +-- dataset_id / src_pk 保留下来只作「最后一次写入时挂在哪儿」的排查线索。 CREATE TABLE IF NOT EXISTS place_notes ( user_id BIGINT NOT NULL, + place_key TEXT NOT NULL, dataset_id BIGINT NOT NULL DEFAULT 0, - src_pk INT NOT NULL, + src_pk INT NOT NULL DEFAULT 0, alias TEXT NOT NULL DEFAULT '', note TEXT NOT NULL DEFAULT '', + rond_note TEXT NOT NULL DEFAULT '', updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), - PRIMARY KEY (user_id, dataset_id, src_pk) + PRIMARY KEY (user_id, place_key) ); --- 站长对地点的「官方结论」：推荐(1) / 踩雷(-1)，与自由文本备注分开存，同样按 src_pk。 +-- 站长对地点的「官方结论」：推荐(1) / 踩雷(-1)，与备注一样按坐标键存。 CREATE TABLE IF NOT EXISTS place_votes ( user_id BIGINT NOT NULL, + place_key TEXT NOT NULL, dataset_id BIGINT NOT NULL DEFAULT 0, - src_pk INT NOT NULL, + src_pk INT NOT NULL DEFAULT 0, verdict SMALLINT NOT NULL, updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), - PRIMARY KEY (user_id, dataset_id, src_pk) + PRIMARY KEY (user_id, place_key) ); --- 访客对地点的投票：一人一票、可改可撤。voter_key 由匿名 cookie + IP 派生， --- 与数据集无关（换快照不丢），故不挂 dataset_id。 +-- 访客对地点的投票：一人一票、可改可撤。voter_key 由匿名 cookie + IP 派生。 +-- +-- 这是站点自己的数据（rondbackup 里没有这一项），但同样要认得出「是哪个地点」： +-- 挂 place_key 而不是 src_pk，理由与备注、结论一样。src_pk 只留作排查线索。 +-- 迁移时这张表没有 dataset_id 可依，只能按**活跃数据集**认领那个编号属于哪个地点 +-- （见文件末尾的迁移块）。 CREATE TABLE IF NOT EXISTS place_visitor_votes ( - src_pk INT NOT NULL, + place_key TEXT NOT NULL, voter_key TEXT NOT NULL, vote SMALLINT NOT NULL, + src_pk INT NOT NULL DEFAULT 0, updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), - PRIMARY KEY (src_pk, voter_key) + PRIMARY KEY (place_key, voter_key) ); CREATE INDEX IF NOT EXISTS idx_pvv_src ON place_visitor_votes(src_pk); @@ -309,9 +321,17 @@ UPDATE place_notes SET dataset_id = COALESCE((SELECT id FROM datasets WHERE is_a UPDATE place_votes SET dataset_id = COALESCE((SELECT id FROM datasets WHERE is_active ORDER BY id DESC LIMIT 1), 0) WHERE dataset_id = 0; +-- ⚠️ 这一段只在「还没搬到坐标键」的库上跑：文件末尾那次迁移会把主键换成 +-- (user_id, place_key)，这里要是也跟着改回去，两张表就会在每次启动时来回翻， +-- 永远停在最后一个执行的块上。 DO $$ DECLARE cols text; BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'place_notes' AND column_name = 'place_key') THEN + RETURN; + END IF; + SELECT string_agg(a.attname, ',' ORDER BY array_position(c.conkey, a.attnum)) INTO cols FROM pg_constraint c @@ -395,6 +415,7 @@ ALTER TABLE places ADD COLUMN IF NOT EXISTS src_lon DOUBLE PRECISION; -- SQL 里写不出来。启动时由 ingest.BackfillAllCoordSource 统一处理，那里同时能识别出 -- 「站点导出的备份给新建地点合成的 ZLOCATION 行」，不会把它误当成真机的原始 GPS。 + -- 地点图片：站长给地点配的图。字节由**浏览器**转成 WebP 后再上传（服务端只负责收下与转存）， -- 1 核机器不用扛解码。storage 记这一张实际放在哪、object_key 是本地相对路径或对象存储 key： -- 换了存储后端之后旧图仍按各自那一列去取，不会因为改了设置就全变破图。 @@ -402,6 +423,9 @@ CREATE TABLE IF NOT EXISTS place_images ( id BIGSERIAL PRIMARY KEY, dataset_id BIGINT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE, src_pk INT NOT NULL, + -- place_key 与备注、结论共用同一套坐标键：换数据集后图片还挂得回原地点。 + -- 对象键里仍然带 dataset_id（那只是存储路径，与「认不认得出是同一个地点」无关）。 + place_key TEXT NOT NULL DEFAULT '', storage TEXT NOT NULL DEFAULT 'local', object_key TEXT NOT NULL, width INT NOT NULL DEFAULT 0, @@ -412,3 +436,116 @@ CREATE TABLE IF NOT EXISTS place_images ( UNIQUE (dataset_id, src_pk, object_key) ); CREATE INDEX IF NOT EXISTS idx_place_images_place ON place_images(dataset_id, src_pk, position); + +-- ---------- 附属数据改挂坐标键（一次性迁移）---------- +-- 备注 / 结论 / 图片原本挂 (dataset_id, src_pk)，换个数据集就全看不见了。 +-- 这里把它们搬到坐标键上；只在「还没有 place_key 列」时执行，所以新库直接走上面的 +-- CREATE TABLE，老库走这一段。 +-- +-- 坐标键的算法必须与 Go 侧 rond.PlaceCoordKey 逐位一致： +-- fmt.Sprintf("%.6f|%.6f", lat, lon) ↔ to_char(lat,'FM999990.000000')||'|'||to_char(lon,'FM999990.000000') +-- 已在真实数据上逐条比对过（368/368 相同）。改任何一边都要重新比。 +DO $$ +DECLARE + cols text; +BEGIN + -- 判据是**主键**而不是「列在不在」：半迁移过的库（列补上了、主键还是老的三列） + -- 也得能修好，否则这张表会一直卡在中间状态。 + IF NOT EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'place_notes' AND column_name = 'place_key') THEN + ALTER TABLE place_notes ADD COLUMN place_key TEXT; + END IF; + SELECT string_agg(a.attname, ',' ORDER BY array_position(c.conkey, a.attnum)) + INTO cols FROM pg_constraint c + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = ANY(c.conkey) + WHERE c.conname = 'place_notes_pkey'; + IF cols IS DISTINCT FROM 'user_id,place_key' THEN + UPDATE place_notes n SET place_key = ( + SELECT to_char(p.lat, 'FM999990.000000') || '|' || to_char(p.lon, 'FM999990.000000') + FROM places p + WHERE p.dataset_id = n.dataset_id AND p.src_pk = n.src_pk + AND p.lat IS NOT NULL AND p.lon IS NOT NULL) + WHERE n.place_key IS NULL; + -- 地点已经不在库里的（被删过），留一个不会撞的孤儿键，宁可留着也别丢 + UPDATE place_notes SET place_key = 'orphan:' || dataset_id || ':' || src_pk + WHERE place_key IS NULL; + -- 同一坐标上有多条时只留最新的一条，否则加不上主键 + DELETE FROM place_notes a USING place_notes b + WHERE a.user_id = b.user_id AND a.place_key = b.place_key + AND (a.updated_at, a.ctid) < (b.updated_at, b.ctid); + ALTER TABLE place_notes ALTER COLUMN place_key SET NOT NULL; + ALTER TABLE place_notes DROP CONSTRAINT IF EXISTS place_notes_pkey; + ALTER TABLE place_notes ADD PRIMARY KEY (user_id, place_key); + END IF; + + IF NOT EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'place_votes' AND column_name = 'place_key') THEN + ALTER TABLE place_votes ADD COLUMN place_key TEXT; + END IF; + SELECT string_agg(a.attname, ',' ORDER BY array_position(c.conkey, a.attnum)) + INTO cols FROM pg_constraint c + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = ANY(c.conkey) + WHERE c.conname = 'place_votes_pkey'; + IF cols IS DISTINCT FROM 'user_id,place_key' THEN + UPDATE place_votes n SET place_key = ( + SELECT to_char(p.lat, 'FM999990.000000') || '|' || to_char(p.lon, 'FM999990.000000') + FROM places p + WHERE p.dataset_id = n.dataset_id AND p.src_pk = n.src_pk + AND p.lat IS NOT NULL AND p.lon IS NOT NULL) + WHERE n.place_key IS NULL; + UPDATE place_votes SET place_key = 'orphan:' || dataset_id || ':' || src_pk + WHERE place_key IS NULL; + DELETE FROM place_votes a USING place_votes b + WHERE a.user_id = b.user_id AND a.place_key = b.place_key + AND (a.updated_at, a.ctid) < (b.updated_at, b.ctid); + ALTER TABLE place_votes ALTER COLUMN place_key SET NOT NULL; + ALTER TABLE place_votes DROP CONSTRAINT IF EXISTS place_votes_pkey; + ALTER TABLE place_votes ADD PRIMARY KEY (user_id, place_key); + END IF; + + -- 图片自己有 id 主键，只需补列 + 回填；查询走文件末尾那个索引 + IF NOT EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'place_images' AND column_name = 'place_key') THEN + ALTER TABLE place_images ADD COLUMN place_key TEXT NOT NULL DEFAULT ''; + END IF; + UPDATE place_images i SET place_key = COALESCE(( + SELECT to_char(p.lat, 'FM999990.000000') || '|' || to_char(p.lon, 'FM999990.000000') + FROM places p + WHERE p.dataset_id = i.dataset_id AND p.src_pk = i.src_pk + AND p.lat IS NOT NULL AND p.lon IS NOT NULL), + 'orphan:' || i.dataset_id || ':' || i.src_pk) + WHERE i.place_key = ''; + + -- 访客票：这张表没有 dataset_id，回填时只能按**活跃数据集**认领那个编号属于哪个地点。 + -- 两个数据集里都有同一编号时也只认活跃的那个——这是唯一说得通的猜法， + -- 认不出来的留孤儿键，票不丢。 + IF NOT EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'place_visitor_votes' AND column_name = 'place_key') THEN + ALTER TABLE place_visitor_votes ADD COLUMN place_key TEXT; + END IF; + SELECT string_agg(a.attname, ',' ORDER BY array_position(c.conkey, a.attnum)) + INTO cols FROM pg_constraint c + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = ANY(c.conkey) + WHERE c.conname = 'place_visitor_votes_pkey'; + IF cols IS DISTINCT FROM 'place_key,voter_key' THEN + UPDATE place_visitor_votes vv SET place_key = COALESCE(( + SELECT to_char(p.lat, 'FM999990.000000') || '|' || to_char(p.lon, 'FM999990.000000') + FROM places p JOIN datasets d ON d.id = p.dataset_id + WHERE d.is_active AND p.src_pk = vv.src_pk + AND p.lat IS NOT NULL AND p.lon IS NOT NULL + ORDER BY p.id LIMIT 1), + 'orphan:' || vv.src_pk) + WHERE vv.place_key IS NULL; + -- 同一个访客对同一坐标只剩一条：留最新的 + DELETE FROM place_visitor_votes a USING place_visitor_votes b + WHERE a.voter_key = b.voter_key AND a.place_key = b.place_key + AND (a.updated_at, a.ctid) < (b.updated_at, b.ctid); + ALTER TABLE place_visitor_votes ALTER COLUMN place_key SET NOT NULL; + ALTER TABLE place_visitor_votes DROP CONSTRAINT IF EXISTS place_visitor_votes_pkey; + ALTER TABLE place_visitor_votes ADD PRIMARY KEY (place_key, voter_key); + END IF; +END $$; + +CREATE INDEX IF NOT EXISTS idx_place_images_key ON place_images(place_key, position); + +CREATE INDEX IF NOT EXISTS idx_pvv_key ON place_visitor_votes(place_key); diff --git a/internal/fog/diff.go b/internal/fog/diff.go new file mode 100644 index 0000000..5598659 --- /dev/null +++ b/internal/fog/diff.go @@ -0,0 +1,90 @@ +package fog + +// Diff 是一份待导入快照与当前迷雾之间的差异画像。 +// +// 导入有两种语义完全不同的模式：叠加只做按位或、不可能丢数据；整体替换会先清空， +// 快照里没有的块一律消失。以前两种模式在界面上只是一个下拉框，站长在下手之前 +// 看不到「这次能补多少」和「这次会丢多少」，实测里整体替换静默丢掉过 14% 的已探索区域。 +type Diff struct { + CurBlocks, CurCells int // 库里现有 + IncBlocks, IncCells int // 快照本身 + + // GainBlocks/GainCells 是叠加模式真正新增的部分：快照里当前完全没有的整块， + // 加上重叠块里补亮的格。 + GainBlocks int + GainCells int + + // OverlapBlocks 是与现有块坐标相同的块数，其中 SameBlocks 是一格都不会变的。 + OverlapBlocks int + SameBlocks int + + // LostBlocks/LostCells 是快照里完全没有、只有整体替换才会消失的部分。 + LostBlocks int + LostCells int + + curAreaM2, incAreaM2, gainAreaM2, lostAreaM2 float64 +} + +// DiffBlocks 比对当前块集与一份待导入快照。入参块集不要求有序。 +func DiffBlocks(cur, inc []Block) Diff { + var d Diff + d.CurBlocks, d.IncBlocks = len(cur), len(inc) + + curMap := make(map[[2]int][]byte, len(cur)) + for _, b := range cur { + curMap[[2]int{b.GX, b.GY}] = b.Bitmap + n := popcountSum(b.Bitmap) + d.CurCells += n + d.curAreaM2 += float64(n) * cellAreaM2(b.GX, b.GY) + } + + incSet := make(map[[2]int]struct{}, len(inc)) + for _, b := range inc { + incSet[[2]int{b.GX, b.GY}] = struct{}{} + n := popcountSum(b.Bitmap) + d.IncCells += n + d.incAreaM2 += float64(n) * cellAreaM2(b.GX, b.GY) + + old, ok := curMap[[2]int{b.GX, b.GY}] + if !ok { + d.GainBlocks++ + d.GainCells += n + d.gainAreaM2 += float64(n) * cellAreaM2(b.GX, b.GY) + continue + } + d.OverlapBlocks++ + gain := 0 + for i := range b.Bitmap { + gain += int(popcount[b.Bitmap[i]&^old[i]]) + } + if gain == 0 { + d.SameBlocks++ + continue + } + d.GainCells += gain + d.gainAreaM2 += float64(gain) * cellAreaM2(b.GX, b.GY) + } + + for _, b := range cur { + if _, ok := incSet[[2]int{b.GX, b.GY}]; ok { + continue + } + n := popcountSum(b.Bitmap) + d.LostBlocks++ + d.LostCells += n + d.lostAreaM2 += float64(n) * cellAreaM2(b.GX, b.GY) + } + return d +} + +func (d Diff) CurAreaKm2() float64 { return d.curAreaM2 / 1e6 } +func (d Diff) IncAreaKm2() float64 { return d.incAreaM2 / 1e6 } +func (d Diff) GainAreaKm2() float64 { return d.gainAreaM2 / 1e6 } +func (d Diff) LostAreaKm2() float64 { return d.lostAreaM2 / 1e6 } + +// MergeAreaKm2 是叠加后的总量，CoverAreaKm2 是整体替换后的总量。 +func (d Diff) MergeAreaKm2() float64 { return (d.curAreaM2 + d.gainAreaM2) / 1e6 } +func (d Diff) CoverAreaKm2() float64 { return d.incAreaM2 / 1e6 } + +// Changed 报告叠加是否会让地图有任何变化——为假说明这份快照已经被完全覆盖过。 +func (d Diff) Changed() bool { return d.GainCells > 0 } diff --git a/internal/fog/diff_test.go b/internal/fog/diff_test.go new file mode 100644 index 0000000..3224b00 --- /dev/null +++ b/internal/fog/diff_test.go @@ -0,0 +1,88 @@ +package fog + +import "testing" + +// blk 造一个块，cells 是块内的 (列, 行) 坐标。 +func blk(gx, gy int, cells ...[2]int) Block { + bm := make([]byte, bitmapSize) + for _, c := range cells { + bm[c[1]*8+c[0]/8] |= 0x80 >> (c[0] % 8) + } + return Block{GX: gx, GY: gy, Bitmap: bm} +} + +func TestDiffEmptyBase(t *testing.T) { + d := DiffBlocks(nil, []Block{blk(10, 20, [2]int{0, 0}, [2]int{1, 0})}) + if d.CurBlocks != 0 || d.CurCells != 0 { + t.Fatalf("空库不该有现有数据: %+v", d) + } + if d.GainBlocks != 1 || d.GainCells != 2 || d.IncCells != 2 { + t.Fatalf("空库叠加应为全部新增: %+v", d) + } + if d.LostBlocks != 0 || d.LostCells != 0 || !d.Changed() { + t.Fatalf("空库不该有丢失: %+v", d) + } +} + +func TestDiffIdentical(t *testing.T) { + cur := []Block{blk(10, 20, [2]int{0, 0}, [2]int{1, 0}), blk(11, 20, [2]int{5, 5})} + d := DiffBlocks(cur, cur) + if d.GainCells != 0 || d.Changed() { + t.Fatalf("同一份快照再叠一次不该有任何新增: %+v", d) + } + if d.LostBlocks != 0 || d.LostCells != 0 { + t.Fatalf("同一份快照不该有可丢的部分: %+v", d) + } + if d.OverlapBlocks != 2 || d.SameBlocks != 2 { + t.Fatalf("两个块都应算作「重叠且无变化」: %+v", d) + } +} + +func TestDiffOverlapGain(t *testing.T) { + d := DiffBlocks( + []Block{blk(10, 20, [2]int{0, 0})}, + []Block{blk(10, 20, [2]int{0, 0}, [2]int{3, 4})}) + if d.GainBlocks != 0 || d.GainCells != 1 { + t.Fatalf("重叠块应只补亮多出的那一格: %+v", d) + } + if d.OverlapBlocks != 1 || d.SameBlocks != 0 { + t.Fatalf("重叠但补亮了，不该记为无变化: %+v", d) + } + if d.LostBlocks != 0 { + t.Fatalf("同一个块不该被判为会丢失: %+v", d) + } +} + +func TestDiffDisjoint(t *testing.T) { + d := DiffBlocks( + []Block{blk(10, 20, [2]int{0, 0})}, + []Block{blk(30, 40, [2]int{1, 1})}) + if d.GainBlocks != 1 || d.GainCells != 1 { + t.Fatalf("另一块应是新增: %+v", d) + } + if d.LostBlocks != 1 || d.LostCells != 1 { + t.Fatalf("库里那块在快照里没有，整体替换会丢掉: %+v", d) + } +} + +// 面积按块中心纬度做 cos 修正，越靠近两极越小；总量关系也要自洽。 +func TestDiffAreas(t *testing.T) { + equator := DiffBlocks(nil, []Block{blk(100, WorldCells/BlockCells/2, [2]int{1, 1})}) + high := DiffBlocks(nil, []Block{blk(100, 20000, [2]int{1, 1})}) + if high.GainAreaKm2() <= 0 { + t.Fatalf("高纬单格面积应为正: %v", high.GainAreaKm2()) + } + if !(equator.GainAreaKm2() > high.GainAreaKm2()) { + t.Fatalf("赤道单格应大于高纬: %v vs %v", equator.GainAreaKm2(), high.GainAreaKm2()) + } + + m := DiffBlocks( + []Block{blk(10, 20, [2]int{0, 0})}, + []Block{blk(10, 20, [2]int{0, 0}, [2]int{9, 9})}) + if diff := m.MergeAreaKm2() - (m.CurAreaKm2() + m.GainAreaKm2()); diff > 1e-12 || diff < -1e-12 { + t.Fatalf("叠加后总量应等于现有加新增: %v", diff) + } + if m.CoverAreaKm2() != m.IncAreaKm2() { + t.Fatalf("整体替换后总量应等于快照本身: %v vs %v", m.CoverAreaKm2(), m.IncAreaKm2()) + } +} diff --git a/internal/fog/render.go b/internal/fog/render.go index 96ff4b1..75da91f 100644 --- a/internal/fog/render.go +++ b/internal/fog/render.go @@ -180,6 +180,12 @@ func mercY(lat float64) float64 { return (math.Pi - math.Asinh(math.Tan(rad))) / (2 * math.Pi) * WorldCells } +// BlockLonLat 返回某块中心的 WGS-84 经纬度，供差异地图定位视野。 +func BlockLonLat(gbx, gby int) (lat, lon float64) { + lon, lat = cellLonLat(gbx*BlockCells+BlockCells/2, gby*BlockCells+BlockCells/2) + return +} + func cellLonLat(cx, cy int) (float64, float64) { lon := (float64(cx)+0.5)/WorldCells*360.0 - 180.0 n := 2 * math.Pi * (float64(cy) + 0.5) / WorldCells diff --git a/internal/fog/render_diff.go b/internal/fog/render_diff.go new file mode 100644 index 0000000..85a797e --- /dev/null +++ b/internal/fog/render_diff.go @@ -0,0 +1,200 @@ +package fog + +import ( + "image" + "math" +) + +// 差异图的三种归属。改这里的色值要同步 templates/admin/fog-preview.html 的图例。 +var ( + DiffBothColor = [4]byte{126, 134, 146, 148} // 两边都有：导入不影响 + DiffGainColor = [4]byte{36, 164, 90, 205} // 只有快照有：叠加会新增 + DiffLostColor = [4]byte{214, 56, 56, 210} // 只有库里有：整体替换会丢 +) + +const ( + diffNone uint8 = iota + diffBoth + diffGain + diffLost +) + +// RenderDiffTile 渲染一张 256x256 的差异瓦片，供站长在真实底图上核对 +// 「这份快照能补上哪里」与「整体替换会丢掉哪里」。 +// +// 与 RenderTile 一样按底图坐标系做 WGS84→GCJ02 纠偏；不做描边——三种颜色 +// 已经够区分，再描边在粗缩放级下会糊成一片。 +// +// 逐格判定只在放大到「一格不小于一像素」（z>=14）时做；更粗的级别下一个块还不到 +// 一个像素，按块内占多数的类别整块涂色。绘制顺序 both→gain→lost，让「会新增」 +// 和「会丢」压住底色，重叠处不至于被灰吃掉。 +func RenderDiffTile(cur, inc map[[2]int][]byte, z, x, y int, crs string) *image.NRGBA { + if z < 0 || z > 22 { + return nil + } + step := WorldCells >> z + scale := 256.0 / float64(step) + ox, oy := float64(x*step), float64(y*step) + + layer := make([]uint8, 256*256) + mark := func(px, py, w, h int, kind uint8) { + for r := 0; r < h; r++ { + yy := py + r + if yy < 0 || yy > 255 { + continue + } + row := yy * 256 + for c := 0; c < w; c++ { + xx := px + c + if xx < 0 || xx > 255 { + continue + } + layer[row+xx] = kind + } + } + } + + // 直接遍历块表而不是全球块范围：低缩放级下范围循环会膨胀到数十亿次空查询。 + // GCJ 偏移最多约 700m（74 格），粗裁剪时两侧各放宽两块保证不漏。 + const margin = 2 * BlockCells + lo, hi := float64(x*step-margin), float64((x+1)*step+margin) + rlo, rhi := float64(y*step-margin), float64((y+1)*step+margin) + shift := crs == "gcj02" + + draw := func(pass uint8, gbx, gby int, cb, ib []byte) { + bx, by := float64(gbx*BlockCells), float64(gby*BlockCells) + if bx+BlockCells <= lo || bx >= hi || by+BlockCells <= rlo || by >= rhi { + return + } + ddx, ddy := 0.0, 0.0 + if shift { + ddx, ddy = blockShift(gbx, gby) + } + u0 := (bx + ddx - ox) * scale + v0 := (by + ddy - oy) * scale + + if scale >= 1 { + w := int(math.Ceil(scale)) + for row := 0; row < BlockCells; row++ { + base := row * 8 + for colB := 0; colB < 8; colB++ { + var cbv, ibv byte + if cb != nil { + cbv = cb[base+colB] + } + if ib != nil { + ibv = ib[base+colB] + } + if cbv|ibv == 0 { + continue + } + py := int(math.Floor(v0 + float64(row)*scale)) + for bit := 0; bit < 8; bit++ { + m := byte(0x80 >> bit) + kind := cellKind(cbv&m != 0, ibv&m != 0) + if kind != pass { + continue + } + px := int(math.Floor(u0 + float64(colB*8+bit)*scale)) + mark(px, py, w, w, kind) + } + } + } + return + } + + // 一格不足一像素：整块按多数类涂，三种混在一起的块以数量最多的为准 + if dominantKind(blockDiffCounts(cb, ib)) != pass { + return + } + u1 := (bx + BlockCells + ddx - ox) * scale + v1 := (by + BlockCells + ddy - oy) * scale + px0, py0 := int(math.Floor(u0)), int(math.Floor(v0)) + px1, py1 := int(math.Ceil(u1)), int(math.Ceil(v1)) + if px1-px0 < 1 { + px1 = px0 + 1 + } + if py1-py0 < 1 { + py1 = py0 + 1 + } + mark(px0, py0, px1-px0, py1-py0, pass) + } + + for _, pass := range []uint8{diffBoth, diffGain, diffLost} { + for key, cb := range cur { + draw(pass, key[0], key[1], cb, inc[key]) + } + // 只在快照里的块单独走一遍，两边都有的块不再重复画 + for key, ib := range inc { + if _, dup := cur[key]; dup { + continue + } + draw(pass, key[0], key[1], nil, ib) + } + } + + img := image.NewNRGBA(image.Rect(0, 0, 256, 256)) + for i, kind := range layer { + if kind == diffNone { + continue + } + var c [4]byte + switch kind { + case diffBoth: + c = DiffBothColor + case diffGain: + c = DiffGainColor + default: + c = DiffLostColor + } + o := i * 4 + img.Pix[o], img.Pix[o+1], img.Pix[o+2], img.Pix[o+3] = c[0], c[1], c[2], c[3] + } + return img +} + +func cellKind(inCur, inInc bool) uint8 { + switch { + case inCur && inInc: + return diffBoth + case inInc: + return diffGain + case inCur: + return diffLost + } + return diffNone +} + +// blockDiffCounts 数一个块里三类各占多少格。两个位图都可能缺失（只有一边有这个块）。 +func blockDiffCounts(cb, ib []byte) (both, gain, lost int) { + n := len(cb) + if len(ib) > n { + n = len(ib) + } + for i := 0; i < n; i++ { + var c, in byte + if i < len(cb) { + c = cb[i] + } + if i < len(ib) { + in = ib[i] + } + both += int(popcount[c&in]) + gain += int(popcount[in&^c]) + lost += int(popcount[c&^in]) + } + return +} + +// dominantKind 取三类里占得最多的一类，用来在粗缩放级下决定整块的着色。 +func dominantKind(both, gain, lost int) uint8 { + switch { + case gain > 0 && gain >= both && gain >= lost: + return diffGain + case lost > 0 && lost >= both: + return diffLost + case both > 0: + return diffBoth + } + return diffNone +} diff --git a/internal/fog/render_diff_test.go b/internal/fog/render_diff_test.go new file mode 100644 index 0000000..8ef9d72 --- /dev/null +++ b/internal/fog/render_diff_test.go @@ -0,0 +1,114 @@ +package fog + +import ( + "image" + "math" + "testing" +) + +func bmOf(cells ...[2]int) []byte { + bm := make([]byte, bitmapSize) + for _, c := range cells { + bm[c[1]*8+c[0]/8] |= 0x80 >> (c[0] % 8) + } + return bm +} + +func pixAt(img *image.NRGBA, px, py int) [4]byte { + i := (py*256 + px) * 4 + return [4]byte{img.Pix[i], img.Pix[i+1], img.Pix[i+2], img.Pix[i+3]} +} + +// z=18 时一格正好 16 像素，块 (100,200) 完整落在瓦片 (400,800) 里， +// 格 (col,row) 的中心像素是 (col*16+8, row*16+8)。 +func TestRenderDiffTilePerCell(t *testing.T) { + key := [2]int{100, 200} + cur := map[[2]int][]byte{key: bmOf([2]int{0, 0}, [2]int{4, 0})} + inc := map[[2]int][]byte{key: bmOf([2]int{0, 0}, [2]int{1, 0})} + img := RenderDiffTile(cur, inc, 18, 400, 800, "wgs84") + if img == nil { + t.Fatal("瓦片不该为空") + } + if got := pixAt(img, 8, 8); got != DiffBothColor { + t.Errorf("两边都有的格应为灰色，实际 %v", got) + } + if got := pixAt(img, 24, 8); got != DiffGainColor { + t.Errorf("只有快照有的格应为绿色，实际 %v", got) + } + if got := pixAt(img, 72, 8); got != DiffLostColor { + t.Errorf("只有库里有、整体替换会丢的格应为红色，实际 %v", got) + } + if got := pixAt(img, 200, 200); got[3] != 0 { + t.Errorf("没被探索的格应保持透明，实际 %v", got) + } +} + +func TestRenderDiffTileSingleSided(t *testing.T) { + key := [2]int{100, 200} + onlyCur := RenderDiffTile(map[[2]int][]byte{key: bmOf([2]int{3, 3})}, nil, 18, 400, 800, "wgs84") + if got := pixAt(onlyCur, 56, 56); got != DiffLostColor { + t.Errorf("快照里完全没有的块应整块为红，实际 %v", got) + } + onlyInc := RenderDiffTile(nil, map[[2]int][]byte{key: bmOf([2]int{3, 3})}, 18, 400, 800, "wgs84") + if got := pixAt(onlyInc, 56, 56); got != DiffGainColor { + t.Errorf("库里完全没有的块应整块为绿，实际 %v", got) + } +} + +// z=10 时一格不足一像素，块只占 4x4 像素，按块内多数类涂色。 +func TestRenderDiffTileCoarseZoomUsesDominantKind(t *testing.T) { + key := [2]int{100, 200} + cur := map[[2]int][]byte{key: bmOf([2]int{0, 0})} + inc := map[[2]int][]byte{key: bmOf([2]int{1, 1}, [2]int{2, 2}, [2]int{3, 3})} + img := RenderDiffTile(cur, inc, 10, 1, 3, "wgs84") + + const step, scale = WorldCells >> 10, 256.0 / (WorldCells >> 10) + bx, by := float64(key[0]*BlockCells), float64(key[1]*BlockCells) + px := int(math.Floor((bx - 1*step) * scale)) + py := int(math.Floor((by - 3*step) * scale)) + if got := pixAt(img, px+1, py+1); got != DiffGainColor { + t.Errorf("三格新增、一格不变的块应以绿色为主，实际 %v", got) + } + if w := int(math.Ceil(float64(BlockCells) * scale)); w != 4 { + t.Fatalf("z=10 下块应占 4 像素，实际 %d", w) + } +} + +func TestRenderDiffTileEdgeCases(t *testing.T) { + if RenderDiffTile(nil, nil, 18, 400, 800, "wgs84") == nil { + t.Error("两边都空也应返回一张空瓦片") + } + if RenderDiffTile(nil, nil, 23, 0, 0, "wgs84") != nil { + t.Error("超出缩放范围应返回 nil") + } +} + +func TestDominantKind(t *testing.T) { + cases := []struct { + both, gain, lost int + want uint8 + }{ + {0, 0, 0, diffNone}, + {5, 1, 0, diffBoth}, + {1, 5, 0, diffGain}, + {1, 0, 5, diffLost}, + {2, 2, 0, diffGain}, // 平局让「会新增」胜出 + {3, 1, 1, diffBoth}, + } + for _, c := range cases { + if got := dominantKind(c.both, c.gain, c.lost); got != c.want { + t.Errorf("dominantKind(%d,%d,%d)=%d，期望 %d", c.both, c.gain, c.lost, got, c.want) + } + } +} + +func TestBlockDiffCounts(t *testing.T) { + both, gain, lost := blockDiffCounts(bmOf([2]int{0, 0}, [2]int{1, 0}), bmOf([2]int{0, 0}, [2]int{2, 0})) + if both != 1 || gain != 1 || lost != 1 { + t.Errorf("一块里应各有一格：both=%d gain=%d lost=%d", both, gain, lost) + } + both, gain, lost = blockDiffCounts(nil, bmOf([2]int{5, 5})) + if both != 0 || gain != 1 || lost != 0 { + t.Errorf("只有快照有该块：both=%d gain=%d lost=%d", both, gain, lost) + } +} diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 885e3ef..7c2dfd6 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -67,15 +67,41 @@ func insertEntityRaw(ctx context.Context, tx *sql.Tx, datasetID int64, rows map[ } // contentHash 给备份内容算指纹，用来识别「内容没变、只是重新导出了一次」的上传。 -// rond 每次导出连 WAL 一起打包，字节必然不同，文件摘要挡不住这种重复； -// 但同一批到访的 Z_PK 是稳定的，所以按到访自然键排序后取摘要即可。 +// rond 每次导出连 WAL 一起打包，字节必然不同，文件摘要挡不住这种重复。 +// +// 指纹按**到访/活动/地点的稳定标识**算，绝不能掺 Z_PK：rond 每次导出/同步都会重排 +// 编号（实测两份备份里 822 条到访的 UUID 完全一致、Z_PK 100% 不同），拿编号当指纹会把 +// 「同一批数据重导一次」判成新内容 —— 于是新建数据集，而备注与投票是按 dataset_id +// 隔离的，换了数据集就再也看不见。 +// +// 没有 UUID 的老备份退回「到达时间 + 地点坐标」，仍比编号可靠。 func contentHash(b *rond.Backup) string { - keys := make([]string, 0, len(b.Visits)) + locCoord := make(map[int]string, len(b.Locations)) + keys := make([]string, 0, len(b.Visits)+len(b.Locations)+len(b.Activities)) + for _, l := range b.Locations { + locCoord[l.SrcPK] = fmt.Sprintf("%.6f/%.6f", l.Lat, l.Lon) + keys = append(keys, "L"+l.Name+"/"+locCoord[l.SrcPK]) + } for _, v := range b.Visits { if v.Arrival == nil { + continue // 导入时就被跳过的行，不该影响指纹 + } + if u := b.VisitUUID[v.SrcPK]; u != "" { + keys = append(keys, "V"+u) continue } - keys = append(keys, fmt.Sprintf("%d|%d", v.SrcPK, v.Arrival.Unix())) + loc := "" + if v.LocationSrcPK != nil { + loc = locCoord[*v.LocationSrcPK] + } + keys = append(keys, fmt.Sprintf("T%d/%s", v.Arrival.Unix(), loc)) + } + for _, a := range b.Activities { + if u := b.ActivityUUID[a.SrcPK]; u != "" { + keys = append(keys, "A"+u) + continue + } + keys = append(keys, "A"+a.Name) } sort.Strings(keys) h := sha256.New() diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go new file mode 100644 index 0000000..770624d --- /dev/null +++ b/internal/ingest/ingest_test.go @@ -0,0 +1,107 @@ +package ingest + +import ( + "fmt" + "testing" + "time" + + "rond-with-you/internal/rond" +) + +// sampleBackup 造一份最小的备份：到访 + 地点 + 活动，编号与 UUID 分别给定。 +func sampleBackup(pks []int, uuids []string) *rond.Backup { + b := &rond.Backup{VisitUUID: map[int]string{}, ActivityUUID: map[int]string{}} + for i, pk := range pks { + at := time.Unix(int64(1700000000+i*3600), 0) + b.Visits = append(b.Visits, rond.Visit{SrcPK: pk, Arrival: &at, LocationSrcPK: intp(1)}) + b.VisitUUID[pk] = uuids[i] + } + b.Locations = []rond.Location{{SrcPK: 1, Name: "家", Lat: 24.1, Lon: 109.1}} + b.Activities = []rond.Activity{{SrcPK: 9, Name: "工作"}} + b.ActivityUUID[9] = "cccccccccccccccccccccccccccccccc" + return b +} + +func intp(v int) *int { return &v } + +// 同一批数据重新导出一次：UUID 一个没变、Z_PK 全被打乱（顺序也变了）， +// 指纹必须一样。否则会被判成「新内容」→ 新建数据集 → 按 dataset_id 隔离的 +// 备注与投票全部失联。 +func TestContentHashIgnoresPKReorder(t *testing.T) { + a := sampleBackup([]int{1, 2, 3}, []string{ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "dddddddddddddddddddddddddddddddd", + }) + b := sampleBackup([]int{555, 276, 88}, []string{ + "dddddddddddddddddddddddddddddddd", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + }) + if contentHash(a) != contentHash(b) { + t.Fatal("编号重排不该改变指纹") + } +} + +// 内容真的变了（多一条到访）就得算不同的指纹，否则新数据会被当成重复上传而丢掉。 +func TestContentHashDetectsNewVisit(t *testing.T) { + a := sampleBackup([]int{1, 2}, []string{ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + }) + b := sampleBackup([]int{1, 2, 3}, []string{ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "dddddddddddddddddddddddddddddddd", + }) + if contentHash(a) == contentHash(b) { + t.Fatal("多了一条到访就是不同内容") + } +} + +// 只改了地点名（到访一条没动）也算内容变了：站点是靠地名认地点的。 +func TestContentHashDetectsPlaceRename(t *testing.T) { + a := sampleBackup([]int{1}, []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}) + b := sampleBackup([]int{1}, []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}) + b.Locations[0].Name = "老家" + if contentHash(a) == contentHash(b) { + t.Fatal("地点改名应改变指纹") + } +} + +// 没有 UUID 的老备份退回「到达时间 + 地点坐标」，同一批数据重导仍要认出是同一份。 +func TestContentHashFallsBackWithoutUUID(t *testing.T) { + a := sampleBackup([]int{1, 2}, []string{"", ""}) + b := sampleBackup([]int{700, 900}, []string{"", ""}) + if contentHash(a) != contentHash(b) { + t.Fatal("没有 UUID 时应按到达时间与地点兜底，不该受编号影响") + } + c := sampleBackup([]int{1, 2}, []string{"", ""}) + c.Visits[1].Arrival = ptrTime(time.Unix(1700000000, 0)) // 改掉第二条的时间 + if contentHash(a) == contentHash(c) { + t.Fatal("时间不同应算不同内容") + } +} + +// 没有到达时间的行导入时就被跳过，不该参与指纹。 +func TestContentHashSkipsRowsWithoutArrival(t *testing.T) { + a := sampleBackup([]int{1}, []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}) + b := sampleBackup([]int{1}, []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}) + b.Visits = append(b.Visits, rond.Visit{SrcPK: 42}) + if contentHash(a) != contentHash(b) { + t.Fatal("缺到达时间的行不该影响指纹") + } +} + +// 指纹是稳定的十六进制摘要，不是随手拼的字符串。 +func TestContentHashShape(t *testing.T) { + h := contentHash(sampleBackup([]int{1}, []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"})) + if len(h) != 64 { + t.Fatalf("应是 64 位 sha256 十六进制，实际 %d 位: %s", len(h), h) + } + if _, err := fmt.Sscanf(h, "%x", new(string)); err != nil { + t.Fatalf("不是纯十六进制: %v", err) + } +} + +func ptrTime(t time.Time) *time.Time { return &t } diff --git a/internal/notes/notes.go b/internal/notes/notes.go index 7c92827..403c056 100644 --- a/internal/notes/notes.go +++ b/internal/notes/notes.go @@ -1,11 +1,14 @@ // Package notes 存放站长为地点加的别名与备注。 -// 备注按 rond 的地点 ID（src_pk）而非数据库自增 id 存储， -// 因此重新上传备份、地点换了新 id 之后内容依然挂在原地。 +// +// 备注挂的是**坐标键**（place_key）而不是 rond 的地点编号：src_pk 只在单个数据集内有效 +// （rond 每次导出都会重排 Z_PK，实测两份备份里 822 条到访的 UUID 一致而 Z_PK 100% 不同）， +// 换个数据集同编号就是另一个地点，备注会全部失联。坐标是地理事实，跨数据集稳定。 package notes import ( "context" "database/sql" + "fmt" "strings" "time" ) @@ -20,15 +23,26 @@ type Note struct { type Store struct{ DB *sql.DB } -// All 返回某个数据集的全站备注（站点只有一个站长账号，读的时候不按 user_id 过滤， -// 这样匿名访客也能看到别名；写的时候才需要登录用户的 id）。 -// 后写覆盖先写，保证同一地点只保留一条。必须按 dataset_id 过滤： -// src_pk 只在单个数据集内唯一，跨数据集同编号是完全不同的地点。 +// placeKeyExpr 把某个 places 别名下的坐标换算成 place_key。 +// **必须与 Go 侧的 rond.PlaceCoordKey 逐位一致**： +// +// fmt.Sprintf("%.6f|%.6f") ↔ to_char(x,'FM999990.000000')||'|'||to_char(y,'FM999990.000000') +// +// 已在真实数据上逐条比对（368/368 相同）。改任何一边都要重新比。 +func placeKeyExpr(alias string) string { + return fmt.Sprintf( + "to_char(%[1]s.lat,'FM999990.000000')||'|'||to_char(%[1]s.lon,'FM999990.000000')", alias) +} + +// All 返回某个数据集里「有备注的地点」，键是地点在该数据集内的编号。 +// 站点只有一个站长账号，读的时候不按 user_id 过滤（匿名访客也要看到别名）。 func (s *Store) All(ctx context.Context, datasetID int64) (map[int]Note, error) { out := map[int]Note{} - rows, err := s.DB.QueryContext(ctx, - `SELECT src_pk, alias, note, rond_note, updated_at FROM place_notes - WHERE dataset_id=$1 ORDER BY updated_at DESC`, datasetID) + rows, err := s.DB.QueryContext(ctx, ` + SELECT p.src_pk, n.alias, n.note, n.rond_note, n.updated_at + FROM places p + JOIN place_notes n ON n.place_key = `+placeKeyExpr("p")+` + WHERE p.dataset_id = $1 AND p.lat IS NOT NULL AND p.lon IS NOT NULL`, datasetID) if err != nil { return out, err } @@ -46,9 +60,9 @@ func (s *Store) All(ctx context.Context, datasetID int64) (map[int]Note, error) // SetNote 只改备注、保留别名（批量「清空备注」用）；两者都为空则删除记录。 func (s *Store) SetNote(ctx context.Context, userID, datasetID int64, srcPK int, content string) error { var alias string - err := s.DB.QueryRowContext(ctx, - `SELECT alias FROM place_notes WHERE user_id=$1 AND dataset_id=$2 AND src_pk=$3`, - userID, datasetID, srcPK).Scan(&alias) + err := s.DB.QueryRowContext(ctx, `SELECT n.alias FROM place_notes n + JOIN places p ON n.place_key = `+placeKeyExpr("p")+` + WHERE n.user_id=$1 AND p.dataset_id=$2 AND p.src_pk=$3`, userID, datasetID, srcPK).Scan(&alias) if err != nil && err != sql.ErrNoRows { return err } @@ -57,8 +71,9 @@ func (s *Store) SetNote(ctx context.Context, userID, datasetID int64, srcPK int, // Delete 删除某地点的整条备注与别名。 func (s *Store) Delete(ctx context.Context, userID, datasetID int64, srcPK int) error { - _, err := s.DB.ExecContext(ctx, - `DELETE FROM place_notes WHERE user_id=$1 AND dataset_id=$2 AND src_pk=$3`, userID, datasetID, srcPK) + _, err := s.DB.ExecContext(ctx, `DELETE FROM place_notes n USING places p + WHERE n.user_id=$1 AND p.dataset_id=$2 AND p.src_pk=$3 + AND n.place_key = `+placeKeyExpr("p"), userID, datasetID, srcPK) return err } @@ -68,21 +83,28 @@ func (s *Store) Save(ctx context.Context, userID, datasetID int64, srcPK int, al content = strings.TrimSpace(content) if alias == "" && content == "" { // 没有 rond 备注时直接删行；有则只清空手写部分，别把回填结果一起删掉 - _, err := s.DB.ExecContext(ctx, - `DELETE FROM place_notes WHERE user_id=$1 AND dataset_id=$2 AND src_pk=$3 AND rond_note=''`, - userID, datasetID, srcPK) - if err != nil { + if _, err := s.DB.ExecContext(ctx, `DELETE FROM place_notes n USING places p + WHERE n.user_id=$1 AND p.dataset_id=$2 AND p.src_pk=$3 + AND n.place_key = `+placeKeyExpr("p")+` AND n.rond_note=''`, + userID, datasetID, srcPK); err != nil { return err } - _, err = s.DB.ExecContext(ctx, - `UPDATE place_notes SET alias='', note='', updated_at=now() - WHERE user_id=$1 AND dataset_id=$2 AND src_pk=$3`, userID, datasetID, srcPK) + _, err := s.DB.ExecContext(ctx, `UPDATE place_notes n + SET alias='', note='', updated_at=now() + FROM places p + WHERE n.user_id=$1 AND p.dataset_id=$2 AND p.src_pk=$3 + AND n.place_key = `+placeKeyExpr("p"), userID, datasetID, srcPK) return err } - _, err := s.DB.ExecContext(ctx, `INSERT INTO place_notes (user_id, dataset_id, src_pk, alias, note, updated_at) - VALUES ($1,$2,$3,$4,$5,now()) - ON CONFLICT (user_id, dataset_id, src_pk) - DO UPDATE SET alias=EXCLUDED.alias, note=EXCLUDED.note, updated_at=now()`, + // 先按坐标键算出这条地点该挂在哪，找不到该地点就什么也不写 + _, err := s.DB.ExecContext(ctx, `INSERT INTO place_notes + (user_id, place_key, dataset_id, src_pk, alias, note, updated_at) + SELECT $1, `+placeKeyExpr("p")+`, $2, $3, $4, $5, now() + FROM places p + WHERE p.dataset_id=$2 AND p.src_pk=$3 AND p.lat IS NOT NULL AND p.lon IS NOT NULL + ON CONFLICT (user_id, place_key) DO UPDATE SET + alias=EXCLUDED.alias, note=EXCLUDED.note, + dataset_id=EXCLUDED.dataset_id, src_pk=EXCLUDED.src_pk, updated_at=now()`, userID, datasetID, srcPK, alias, content) return err } @@ -92,20 +114,22 @@ func (s *Store) Save(ctx context.Context, userID, datasetID int64, srcPK int, al // 只重写 rond_note 一列，站长手写的 note 一概不动。返回写了多少行。 func (s *Store) BackfillRond(ctx context.Context, userID, datasetID int64) (int, error) { // DISTINCT 在「日期 + 内容」这一层做，同一天重复写的同一句只留一条 - const agg = ` + agg := ` WITH raw AS ( - SELECT p.src_pk, to_char(v.arrival, 'YYYY-MM-DD') || ' ' || btrim(v.remark) AS line + SELECT ` + placeKeyExpr("p") + ` AS key, + to_char(v.arrival, 'YYYY-MM-DD') || ' ' || btrim(v.remark) AS line FROM visits v JOIN places p ON p.id = v.place_id WHERE v.dataset_id = $1 AND btrim(COALESCE(v.remark, '')) <> '' + AND p.lat IS NOT NULL AND p.lon IS NOT NULL ), uniq AS ( - SELECT DISTINCT src_pk, line, left(line, 10) AS day FROM raw + SELECT DISTINCT key, line, left(line, 10) AS day FROM raw ), agg AS ( - SELECT src_pk, string_agg(line, E'\n' ORDER BY day, line) AS txt FROM uniq GROUP BY src_pk + SELECT key, string_agg(line, E'\n' ORDER BY day, line) AS txt FROM uniq GROUP BY key )` res, err := s.DB.ExecContext(ctx, agg+` - INSERT INTO place_notes (user_id, dataset_id, src_pk, alias, note, rond_note, updated_at) - SELECT $2, $1, src_pk, '', '', txt, now() FROM agg - ON CONFLICT (user_id, dataset_id, src_pk) DO UPDATE SET rond_note = EXCLUDED.rond_note`, + INSERT INTO place_notes (user_id, place_key, dataset_id, src_pk, alias, note, rond_note, updated_at) + SELECT $2, key, $1, 0, '', '', txt, now() FROM agg + ON CONFLICT (user_id, place_key) DO UPDATE SET rond_note = EXCLUDED.rond_note`, datasetID, userID) if err != nil { return 0, err @@ -113,15 +137,16 @@ func (s *Store) BackfillRond(ctx context.Context, userID, datasetID int64) (int, affected, _ := res.RowsAffected() // 备注被删掉的地点要清掉旧值，否则会一直留着过期的 rond 内容。 - // 限定在本数据集的地点里：datasetID 不合法时子查询为空，一行都不会动， - // 不会把别的数据集的回填结果误清掉。 + // 限定在本数据集的地点里：datasetID 不合法时子查询为空，一行都不会动。 if _, err := s.DB.ExecContext(ctx, ` UPDATE place_notes n SET rond_note = '' - WHERE n.user_id = $2 AND n.dataset_id = $1 AND n.rond_note <> '' - AND n.src_pk IN (SELECT src_pk FROM places WHERE dataset_id = $1) + WHERE n.user_id = $2 AND n.rond_note <> '' + AND n.place_key IN (SELECT `+placeKeyExpr("p")+` + FROM places p WHERE p.dataset_id = $1 AND p.lat IS NOT NULL AND p.lon IS NOT NULL) AND NOT EXISTS ( SELECT 1 FROM visits v JOIN places p ON p.id = v.place_id - WHERE v.dataset_id = $1 AND p.src_pk = n.src_pk AND btrim(COALESCE(v.remark, '')) <> '' + WHERE v.dataset_id = $1 AND `+placeKeyExpr("p")+` = n.place_key + AND btrim(COALESCE(v.remark, '')) <> '' )`, datasetID, userID); err != nil { return 0, err } diff --git a/internal/rond/diff.go b/internal/rond/diff.go new file mode 100644 index 0000000..b3096d5 --- /dev/null +++ b/internal/rond/diff.go @@ -0,0 +1,390 @@ +package rond + +import ( + "fmt" + "sort" + "time" +) + +// 两份数据的比对。 +// +// 对齐键的选择是这个文件的全部要点：rond 每次导出/同步都会重排 Z_PK（实测两份备份里 +// 822 条到访的 UUID 完全一致、Z_PK 100% 不同），所以 src_pk 只能在一个数据集内部使用。 +// 跨备份对齐一律走真机写的 UUID；地点/标签/交通方式没有 UUID，退到「名称 + 坐标」。 +// +// 这里只做判定，不碰数据库、也不写任何东西。 + +// StoredVisit 是库里的一条到访，外加从 entity_raw 留档取出的对齐键。 +type StoredVisit struct { + SrcPK int + UUID string // 空 = 站点新建的行（真机不会给它 UUID） + Arrival *time.Time + Departure *time.Time + Remark string + Emoji string + Bookmarked bool + UserAdded bool + PlaceKey string // 地点对齐键，空 = 该到访没有地点 + PlaceName string // 仅用于展示 + ActivityName string +} + +// StoredPlace 是库里的一个地点。 +type StoredPlace struct { + SrcPK int + Key string + Name string + Lat, Lon float64 + Province string + City string + POICategory string + VisitCount int +} + +// StoredActivity 是库里的一个活动类型。 +type StoredActivity struct { + SrcPK int + Name string + UUID string +} + +// FieldChange 是冲突里的一项字段差异。From 是库里当前的值，To 是备份里的值。 +type FieldChange struct { + Label string + From string + To string +} + +// VisitConflict 是同一条到访（UUID 相同）两边字段不一致。 +type VisitConflict struct { + Backup Visit + DB StoredVisit + Fields []FieldChange +} + +// VisitDiff 是到访的差异画像。 +type VisitDiff struct { + Added []Visit // 备份有、库里没有 + Removed []StoredVisit // 库里有、备份里没有 + Changed []VisitConflict + Same int // 对齐上且完全一致 +} + +// PlaceDiff 是地点的差异画像。 +type PlaceDiff struct { + Added []Location + Removed []StoredPlace + Changed []PlaceConflict + Same int +} + +type PlaceConflict struct { + Backup Location + DB StoredPlace + Fields []FieldChange +} + +// ActivityDiff 是活动类型的差异画像（按 UUID 对齐，退不到就按名称）。 +type ActivityDiff struct { + Added []Activity + Removed []StoredActivity + Changed []ActivityConflict + Same int +} + +type ActivityConflict struct { + Backup Activity + DB StoredActivity + Fields []FieldChange +} + +// PlaceCoordKey 是地点的跨备份对齐键：**只认坐标，不认名称**。 +// +// 地点没有 UUID，名称又会被站点自己的「补全缺失地址」改掉（实测：站内把 +// 「体育馆美食街」反查成了「河池市国民体质监测中心」，坐标一字未动，于是 17 条 +// 到访全被误报成「换了地点」）。坐标取 6 位小数（约 0.1m）——足以吸收浮点往返误差， +// 又不会把挨着的两个地点并成一个。 +func PlaceCoordKey(lat, lon float64) string { + return fmt.Sprintf("%.6f|%.6f", lat, lon) +} + +func coordStr(v float64) string { return fmt.Sprintf("%.6f", v) } + +// DiffVisits 按 UUID 对齐两边的到访。 +// +// 真机偶尔有条目没写 UUID（实测 824 行到访里有 2 行），只按 UUID 配对会让同一条既算 +// 「新增」又算「仅站内有」；所以没有 UUID 的行退到「到达时间 + 地点」兜底配对。 +// 库里那些连兜底也配不上的（站点新建的行）一律算「库里有、备份没有」。 +func DiffVisits(stored []StoredVisit, b *Backup) VisitDiff { + placeKey, placeName := map[int]string{}, map[int]string{} + for _, l := range b.Locations { + // 没有坐标的地点不参与坐标对齐，否则它们会共用 "0.000000|0.000000" 互相认领 + if l.Lat != 0 || l.Lon != 0 { + placeKey[l.SrcPK] = PlaceCoordKey(l.Lat, l.Lon) + } + placeName[l.SrcPK] = placeLabel(l.Name, l.Lat, l.Lon) + } + actName := map[int]string{} + for _, a := range b.Activities { + actName[a.SrcPK] = a.Name + } + + used := make([]bool, len(stored)) + byUUID := make(map[string]int, len(stored)) + byTime := map[string][]int{} + for i, sv := range stored { + if sv.UUID != "" { + byUUID[sv.UUID] = i + continue + } + if sv.Arrival != nil { + k := timeKey(*sv.Arrival, sv.PlaceKey) + byTime[k] = append(byTime[k], i) + } + } + + var d VisitDiff + classify := func(i int, v Visit) { + used[i] = true + sv := stored[i] + if fields := visitFieldChanges(sv, v, placeKey, placeName, actName); len(fields) > 0 { + d.Changed = append(d.Changed, VisitConflict{Backup: v, DB: sv, Fields: fields}) + } else { + d.Same++ + } + } + + for _, v := range b.Visits { + if v.Arrival == nil { + continue // 没有到达时间的行导入时就被跳过了，不该被算成「新增」 + } + if uuid := b.VisitUUID[v.SrcPK]; uuid != "" { + if i, ok := byUUID[uuid]; ok { + classify(i, v) + continue + } + d.Added = append(d.Added, v) + continue + } + bpk := "" + if v.LocationSrcPK != nil { + bpk = placeKey[*v.LocationSrcPK] + } + if list := byTime[timeKey(*v.Arrival, bpk)]; len(list) > 0 { + byTime[timeKey(*v.Arrival, bpk)] = list[1:] + classify(list[0], v) + continue + } + d.Added = append(d.Added, v) + } + for i, sv := range stored { + if !used[i] { + d.Removed = append(d.Removed, sv) + } + } + sortVisits(d) + return d +} + +// timeKey 是没有 UUID 时的兜底配对键。只到秒：苹果时间戳转过来是浮点秒， +// 而 PG 存的是微秒，逐位比会永远配不上。 +func timeKey(t time.Time, place string) string { + return fmt.Sprintf("%d|%s", t.Unix(), place) +} + +func visitFieldChanges(sv StoredVisit, v Visit, placeKey, placeName, actName map[int]string) []FieldChange { + var out []FieldChange + add := func(label, from, to string) { + if from != to { + out = append(out, FieldChange{Label: label, From: from, To: to}) + } + } + if !sameTime(sv.Arrival, v.Arrival) { + add("到达时间", fmtTime(sv.Arrival), fmtTime(v.Arrival)) + } + if !sameTime(sv.Departure, v.Departure) { + add("离开时间", fmtTime(sv.Departure), fmtTime(v.Departure)) + } + // 地点按坐标判、显示名称；活动按名称判。都不能比 src_pk——两边编号空间不同。 + bp, bn := "", "" + if v.LocationSrcPK != nil { + bp, bn = placeKey[*v.LocationSrcPK], placeName[*v.LocationSrcPK] + } + // 地点按坐标判、显示名称：名称被「补全地址」反查改过，不算换了地点 + if sv.PlaceKey != bp { + out = append(out, FieldChange{Label: "地点", From: sv.PlaceName, To: bn}) + } + bl := "" + if v.ActivitySrcPK != nil { + bl = actName[*v.ActivitySrcPK] + } + add("活动", sv.ActivityName, bl) + add("备注", sv.Remark, v.Remark) + add("表情", sv.Emoji, v.Emoji) + add("收藏", fmtBool(sv.Bookmarked), fmtBool(v.Bookmarked)) + return out +} + +// placeLabel 给没有名字的地点一个可读的占位。 +func placeLabel(name string, lat, lon float64) string { + if name != "" { + return name + } + return fmt.Sprintf("%.5f, %.5f", lat, lon) +} + +// DiffPlaces 对齐地点。坐标优先（改名不影响配对），配不上再按名称找一遍 +// （覆盖「站点把坐标纠偏过」这种），两样都对不上才算新增。 +func DiffPlaces(stored []StoredPlace, b *Backup) PlaceDiff { + byCoord := make(map[string]StoredPlace, len(stored)) + byName := map[string]StoredPlace{} + for _, sp := range stored { + if sp.Lat != 0 || sp.Lon != 0 { + byCoord[PlaceCoordKey(sp.Lat, sp.Lon)] = sp + } + if _, dup := byName[sp.Name]; !dup && sp.Name != "" { + byName[sp.Name] = sp + } + } + used := map[int]bool{} + var d PlaceDiff + classify := func(sp StoredPlace, l Location) { + used[sp.SrcPK] = true + if fields := placeFieldChanges(sp, l); len(fields) > 0 { + d.Changed = append(d.Changed, PlaceConflict{Backup: l, DB: sp, Fields: fields}) + } else { + d.Same++ + } + } + for _, l := range b.Locations { + if sp, ok := byCoord[PlaceCoordKey(l.Lat, l.Lon)]; ok && !used[sp.SrcPK] { + classify(sp, l) + continue + } + if sp, ok := byName[l.Name]; ok && !used[sp.SrcPK] { + classify(sp, l) + continue + } + d.Added = append(d.Added, l) + } + for _, sp := range stored { + if !used[sp.SrcPK] { + d.Removed = append(d.Removed, sp) + } + } + sort.Slice(d.Added, func(i, j int) bool { return d.Added[i].Name < d.Added[j].Name }) + sort.Slice(d.Removed, func(i, j int) bool { return d.Removed[i].Name < d.Removed[j].Name }) + sort.Slice(d.Changed, func(i, j int) bool { return d.Changed[i].DB.Name < d.Changed[j].DB.Name }) + return d +} + +// placeFieldChanges 比地点上除对齐键之外的字段。名称也算——按坐标配上之后， +// 「名字不一样」正是站长最想看到的那类差异。 +func placeFieldChanges(sp StoredPlace, l Location) []FieldChange { + var out []FieldChange + add := func(label, from, to string) { + if from != to { + out = append(out, FieldChange{Label: label, From: from, To: to}) + } + } + add("名称", sp.Name, l.Name) + add("纬度", coordStr(sp.Lat), coordStr(l.Lat)) + add("经度", coordStr(sp.Lon), coordStr(l.Lon)) + add("省份", sp.Province, l.Province) + add("城市", sp.City, l.City) + add("分类", sp.POICategory, l.POICategory) + return out +} + +// DiffActivities 优先按 UUID 对齐；库里没有 UUID 的（站点新建）退回名称匹配， +// 匹配不上就算「库里有、备份没有」。 +func DiffActivities(stored []StoredActivity, b *Backup) ActivityDiff { + byUUID := map[string]StoredActivity{} + byName := map[string]StoredActivity{} + for _, sa := range stored { + if sa.UUID != "" { + byUUID[sa.UUID] = sa + } + if _, dup := byName[sa.Name]; !dup { + byName[sa.Name] = sa + } + } + var d ActivityDiff + matched := map[int]bool{} + for _, a := range b.Activities { + sa, ok := byUUID[b.ActivityUUID[a.SrcPK]] + if !ok { + if sa, ok = byName[a.Name]; !ok { + d.Added = append(d.Added, a) + continue + } + } + matched[sa.SrcPK] = true + if sa.Name != a.Name { + d.Changed = append(d.Changed, ActivityConflict{Backup: a, DB: sa, + Fields: []FieldChange{{Label: "名称", From: sa.Name, To: a.Name}}}) + } else { + d.Same++ + } + } + for _, sa := range stored { + if !matched[sa.SrcPK] { + d.Removed = append(d.Removed, sa) + } + } + sort.Slice(d.Added, func(i, j int) bool { return d.Added[i].Name < d.Added[j].Name }) + sort.Slice(d.Removed, func(i, j int) bool { return d.Removed[i].Name < d.Removed[j].Name }) + return d +} + +func sortVisits(d VisitDiff) { + tv := func(t *time.Time) int64 { + if t == nil { + return 0 + } + return t.Unix() + } + sort.Slice(d.Added, func(i, j int) bool { return tv(d.Added[i].Arrival) < tv(d.Added[j].Arrival) }) + sort.Slice(d.Removed, func(i, j int) bool { return tv(d.Removed[i].Arrival) < tv(d.Removed[j].Arrival) }) + sort.Slice(d.Changed, func(i, j int) bool { return tv(d.Changed[i].Backup.Arrival) < tv(d.Changed[j].Backup.Arrival) }) +} + +func sameTime(a, b *time.Time) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return a.Equal(*b) +} + +func fmtTime(t *time.Time) string { + if t == nil { + return "—" + } + return t.Local().Format("2006-01-02 15:04") +} + +func fmtBool(v bool) string { + if v { + return "是" + } + return "否" +} + +// Counts 汇总各实体的三类条数，供概览展示。 +type Counts struct { + Added, Removed, Changed, Same int +} + +func (c Counts) Total() int { return c.Added + c.Removed + c.Changed + c.Same } + +func VisitCounts(d VisitDiff) Counts { + return Counts{len(d.Added), len(d.Removed), len(d.Changed), d.Same} +} + +func PlaceCounts(d PlaceDiff) Counts { + return Counts{len(d.Added), len(d.Removed), len(d.Changed), d.Same} +} + +func ActivityCounts(d ActivityDiff) Counts { + return Counts{len(d.Added), len(d.Removed), len(d.Changed), d.Same} +} diff --git a/internal/rond/diff_test.go b/internal/rond/diff_test.go new file mode 100644 index 0000000..20f4242 --- /dev/null +++ b/internal/rond/diff_test.go @@ -0,0 +1,293 @@ +package rond + +import ( + "testing" + "time" +) + +func tp(s string) *time.Time { + t, err := time.Parse("2006-01-02 15:04", s) + if err != nil { + panic(err) + } + return &t +} + +func ip(v int) *int { return &v } + +// 两组编号刻意完全不同：rond 每次导出都会重排 Z_PK，对齐只能靠 UUID。 +const ( + uuidA = "11111111111111111111111111111111" + uuidB = "22222222222222222222222222222222" + uuidC = "33333333333333333333333333333333" +) + +func sampleBackup() *Backup { + return &Backup{ + Visits: []Visit{ + {SrcPK: 555, Arrival: tp("2026-09-01 10:00"), Departure: tp("2026-09-01 11:00"), + LocationSrcPK: ip(3), ActivitySrcPK: ip(7)}, + {SrcPK: 556, Arrival: tp("2026-09-02 10:00"), LocationSrcPK: ip(3)}, + }, + Locations: []Location{ + {SrcPK: 3, Name: "家", Lat: 24.1, Lon: 109.1, City: "柳州市"}, + {SrcPK: 4, Name: "公司", Lat: 24.2, Lon: 109.2, City: "柳州市"}, + }, + Activities: []Activity{ + {SrcPK: 7, Name: "工作"}, + {SrcPK: 8, Name: "散步"}, + }, + VisitUUID: map[int]string{555: uuidA, 556: uuidB}, + ActivityUUID: map[int]string{7: uuidC}, + } +} + +func TestDiffVisitsAlignsByUUIDNotPK(t *testing.T) { + stored := []StoredVisit{ + {SrcPK: 1, UUID: uuidA, Arrival: tp("2026-09-01 10:00"), Departure: tp("2026-09-01 11:00"), + PlaceKey: PlaceCoordKey(24.1, 109.1), ActivityName: "工作"}, + {SrcPK: 2, UUID: uuidB, Arrival: tp("2026-09-02 10:00"), + PlaceKey: PlaceCoordKey(24.1, 109.1)}, + } + d := DiffVisits(stored, sampleBackup()) + if len(d.Added) != 0 || len(d.Removed) != 0 || len(d.Changed) != 0 { + t.Fatalf("两边一致时不该有差异: +%d -%d ~%d", len(d.Added), len(d.Removed), len(d.Changed)) + } + if d.Same != 2 { + t.Fatalf("应配上 2 条，实际 %d", d.Same) + } +} + +func TestDiffVisitsFieldChanges(t *testing.T) { + stored := []StoredVisit{{ + SrcPK: 1, UUID: uuidA, Arrival: tp("2026-09-01 09:00"), Departure: tp("2026-09-01 11:00"), + PlaceKey: PlaceCoordKey(24.1, 109.1), ActivityName: "工作", Remark: "站点备注", + }} + d := DiffVisits(stored, sampleBackup()) + if len(d.Changed) != 1 { + t.Fatalf("到达时间不同应算冲突，实际 %d 条", len(d.Changed)) + } + f := d.Changed[0].Fields + if len(f) != 2 { + t.Fatalf("应报出到达时间与备注两项，实际 %v", f) + } + if f[0].Label != "到达时间" || f[1].Label != "备注" { + t.Fatalf("字段名不对: %v", f) + } + if f[1].From != "站点备注" || f[1].To != "" { + t.Fatalf("方向搞反了（From 应是库里的值）: %+v", f[1]) + } +} + +// 站点新建的到访没有 UUID，而且它的 src_pk 很可能与真机新行的编号撞上—— +// 绝不能因为编号相同就当成同一条。 +func TestDiffVisitsIgnoresPKCollision(t *testing.T) { + stored := []StoredVisit{{ + SrcPK: 555, UUID: "", Arrival: tp("2026-08-01 08:00"), UserAdded: true, + PlaceKey: PlaceCoordKey(30.0, 120.0), + }} + d := DiffVisits(stored, sampleBackup()) + if len(d.Added) != 2 { + t.Fatalf("备份里两条都是新增，实际 %d", len(d.Added)) + } + if len(d.Removed) != 1 { + t.Fatalf("站点补录那条应算「仅库里有」，实际 %d", len(d.Removed)) + } + if !d.Removed[0].UserAdded { + t.Error("站点补录的标记要带上，页面上要区分开") + } +} + +// 真机偶尔有不带 UUID 的到访，只按 UUID 配对会让同一条既算新增又算仅站内有。 +func TestDiffVisitsFallsBackWhenUUIDMissing(t *testing.T) { + b := &Backup{ + Visits: []Visit{ + {SrcPK: 900, Arrival: tp("2026-09-01 10:00"), LocationSrcPK: ip(3)}, + {SrcPK: 901, Arrival: tp("2026-09-02 10:00"), LocationSrcPK: ip(3)}, + }, + Locations: []Location{{SrcPK: 3, Name: "家", Lat: 24.1, Lon: 109.1}}, + VisitUUID: map[int]string{}, // 一条都没有 UUID + ActivityUUID: map[int]string{}, + } + stored := []StoredVisit{ + {SrcPK: 1, Arrival: tp("2026-09-01 10:00"), PlaceKey: PlaceCoordKey(24.1, 109.1)}, + {SrcPK: 2, Arrival: tp("2026-09-02 10:00"), PlaceKey: PlaceCoordKey(24.1, 109.1)}, + } + d := DiffVisits(stored, b) + if len(d.Added) != 0 || len(d.Removed) != 0 { + t.Fatalf("没有 UUID 时应按「到达时间 + 地点」兜底配对: +%d -%d", len(d.Added), len(d.Removed)) + } + if d.Same != 2 { + t.Fatalf("两条都该配上，实际 %d", d.Same) + } +} + +// 兜底配对要认得出「同一条但时间被改过」——时间不同就配不上，此时按新增 + 仅站内有报出， +// 总比静默当成同一条好。 +func TestDiffVisitsFallbackNeedsSameTime(t *testing.T) { + b := &Backup{ + Visits: []Visit{{SrcPK: 900, Arrival: tp("2026-09-01 10:00"), LocationSrcPK: ip(3)}}, + Locations: []Location{{SrcPK: 3, Name: "家", Lat: 24.1, Lon: 109.1}}, + VisitUUID: map[int]string{}, + } + stored := []StoredVisit{{SrcPK: 1, Arrival: tp("2026-09-01 11:00"), PlaceKey: PlaceCoordKey(24.1, 109.1)}} + d := DiffVisits(stored, b) + if len(d.Added) != 1 || len(d.Removed) != 1 { + t.Fatalf("时间对不上时该分别报出: +%d -%d", len(d.Added), len(d.Removed)) + } +} + +func TestDiffVisitsSkipsRowsWithoutArrival(t *testing.T) { + b := sampleBackup() + b.Visits = append(b.Visits, Visit{SrcPK: 557, LocationSrcPK: ip(3)}) + d := DiffVisits(nil, b) + if len(d.Added) != 2 { + t.Fatalf("没有到达时间的行导入时就被跳过，不该算新增，实际 %d", len(d.Added)) + } +} + +func TestDiffPlacesByKey(t *testing.T) { + b := sampleBackup() + stored := []StoredPlace{ + {SrcPK: 10, Key: PlaceCoordKey(24.1, 109.1), Name: "家", Lat: 24.1, Lon: 109.1, City: "柳州市"}, + } + d := DiffPlaces(stored, b) + if len(d.Added) != 1 || d.Added[0].Name != "公司" { + t.Fatalf("应只有「公司」是新增: %+v", d.Added) + } + if len(d.Removed) != 0 || len(d.Changed) != 0 || d.Same != 1 { + t.Fatalf("「家」应配上且无差异: same=%d changed=%d removed=%d", d.Same, len(d.Changed), len(d.Removed)) + } +} + +func TestDiffPlacesReportsFieldChanges(t *testing.T) { + b := sampleBackup() + stored := []StoredPlace{ + {SrcPK: 10, Key: PlaceCoordKey(24.1, 109.1), Name: "家", Lat: 24.1, Lon: 109.1, City: "桂林市"}, + } + d := DiffPlaces(stored, b) + if len(d.Changed) != 1 { + t.Fatalf("城市不同应算冲突，实际 %d", len(d.Changed)) + } + if f := d.Changed[0].Fields; len(f) != 1 || f[0].Label != "城市" || f[0].From != "桂林市" || f[0].To != "柳州市" { + t.Fatalf("字段差异不对: %+v", f) + } +} + +func TestDiffActivities(t *testing.T) { + b := sampleBackup() + stored := []StoredActivity{ + {SrcPK: 20, Name: "工作", UUID: uuidC}, + {SrcPK: 21, Name: "站点自建"}, + } + d := DiffActivities(stored, b) + if len(d.Added) != 1 || d.Added[0].Name != "散步" { + t.Fatalf("「散步」应是新增: %+v", d.Added) + } + if len(d.Removed) != 1 || d.Removed[0].Name != "站点自建" { + t.Fatalf("站点自建的活动应算仅库里有: %+v", d.Removed) + } + if d.Same != 1 { + t.Fatalf("「工作」应配上，实际 same=%d", d.Same) + } +} + +// 活动改了名：UUID 还在，就该认出来是同一个，只报名称变化，而不是「删一个加一个」。 +func TestDiffActivitiesRenameByUUID(t *testing.T) { + b := sampleBackup() + stored := []StoredActivity{{SrcPK: 20, Name: "上班", UUID: uuidC}} + d := DiffActivities(stored, b) + if len(d.Added) != 1 || len(d.Removed) != 0 { + t.Fatalf("只该多出「散步」，「上班」应认出是同一个: +%d -%d", len(d.Added), len(d.Removed)) + } + if len(d.Changed) != 1 || d.Changed[0].Fields[0].From != "上班" || d.Changed[0].Fields[0].To != "工作" { + t.Fatalf("应按 UUID 认出重命名: %+v", d.Changed) + } +} + +func TestPlaceCoordKeyStable(t *testing.T) { + a := PlaceCoordKey(24.1, 109.1) + if a != PlaceCoordKey(24.1, 109.1) { + t.Fatal("同样的输入应得到同样的键") + } + if a == PlaceCoordKey(24.1, 109.2) { + t.Fatal("坐标不同不该算同一个地点") + } +} + +// 站点把地点反查改名（坐标没动）时，到访层面不该报「换了地点」—— +// 实测里这一条会让 17 条到访全变成假冲突。 +func TestDiffVisitsIgnoresPlaceRename(t *testing.T) { + b := &Backup{ + Visits: []Visit{{SrcPK: 5, Arrival: tp("2026-09-01 10:00"), LocationSrcPK: ip(3)}}, + Locations: []Location{{SrcPK: 3, Name: "河池市国民体质监测中心", Lat: 24.691827, Lon: 108.035268}}, + VisitUUID: map[int]string{5: uuidA}, + } + stored := []StoredVisit{{ + SrcPK: 1, UUID: uuidA, Arrival: tp("2026-09-01 10:00"), + PlaceKey: PlaceCoordKey(24.691827, 108.035268), PlaceName: "体育馆美食街", + }} + d := DiffVisits(stored, b) + if len(d.Changed) != 0 { + t.Fatalf("只改了地点名不该算到访冲突: %+v", d.Changed[0].Fields) + } + if d.Same != 1 { + t.Fatalf("应算作一致，实际 same=%d", d.Same) + } +} + +// 反之，地点层面要把改名报出来。 +func TestDiffPlacesReportsRename(t *testing.T) { + b := &Backup{Locations: []Location{ + {SrcPK: 3, Name: "河池市国民体质监测中心", Lat: 24.691827, Lon: 108.035268, City: "河池市"}, + }} + stored := []StoredPlace{{ + SrcPK: 10, Key: PlaceCoordKey(24.691827, 108.035268), Name: "体育馆美食街", + Lat: 24.691827, Lon: 108.035268, City: "河池市", + }} + d := DiffPlaces(stored, b) + if len(d.Added) != 0 || len(d.Removed) != 0 { + t.Fatalf("坐标一样就是同一个地点: +%d -%d", len(d.Added), len(d.Removed)) + } + if len(d.Changed) != 1 { + t.Fatalf("改名应报成一条冲突，实际 %d", len(d.Changed)) + } + f := d.Changed[0].Fields + if len(f) != 1 || f[0].Label != "名称" || f[0].From != "体育馆美食街" || f[0].To != "河池市国民体质监测中心" { + t.Fatalf("字段差异不对: %+v", f) + } +} + +// 没有坐标的地点不能都落到同一个键上，否则两个不同地点会互相认领。 +func TestDiffPlacesWithoutCoordDoesNotCollide(t *testing.T) { + b := &Backup{Locations: []Location{{SrcPK: 3, Name: "甲地"}}} + stored := []StoredPlace{ + {SrcPK: 10, Name: "甲地"}, + {SrcPK: 11, Name: "乙地"}, + } + d := DiffPlaces(stored, b) + if len(d.Added) != 0 || len(d.Removed) != 1 { + t.Fatalf("应只配上同名的「甲地」: +%d -%d", len(d.Added), len(d.Removed)) + } + if d.Removed[0].Name != "乙地" { + t.Fatalf("没配上的应是「乙地」，实际 %q", d.Removed[0].Name) + } + if d.Same != 1 { + t.Fatalf("「甲地」应算一致，实际 %d", d.Same) + } +} + +// 站点把坐标纠偏过（名称没变）也要认得出是同一个地点。 +func TestDiffPlacesMatchesByNameWhenCoordMoved(t *testing.T) { + b := &Backup{Locations: []Location{{SrcPK: 3, Name: "家", Lat: 24.100001, Lon: 109.100001}}} + stored := []StoredPlace{{ + SrcPK: 10, Key: PlaceCoordKey(24.1, 109.1), Name: "家", Lat: 24.1, Lon: 109.1, + }} + d := DiffPlaces(stored, b) + if len(d.Added) != 0 || len(d.Removed) != 0 { + t.Fatalf("名称一样就该配上: +%d -%d", len(d.Added), len(d.Removed)) + } + if len(d.Changed) != 1 || d.Changed[0].Fields[0].Label != "纬度" { + t.Fatalf("应报出坐标差异: %+v", d.Changed) + } +} diff --git a/internal/rond/export.go b/internal/rond/export.go index 2b66d80..5971148 100644 --- a/internal/rond/export.go +++ b/internal/rond/export.go @@ -1332,9 +1332,12 @@ var uuidNamespace = [16]byte{0x6b, 0x1c, 0x9a, 0x2e, 0x42, 0x7d, 0x4f, 0x16, // uuidV5 由命名空间与名字生成 RFC 4122 版本 5 的 UUID（16 字节原始形式）。 // // rond 给每条到访、每个活动都存一个 16 字节 UUID（ZIDENTIFIER_ / ZUID_，真机里全是 v4）， -// 站点这边没有对应字段。这里用确定性算法补：名字只取表名 + src_pk（src_pk 跨快照稳定）， -// 于是同一条记录每次导出拿到的都是同一个 UUID——若改成随机，每次导出的包都会让 rond -// 把同一批记录当成新的。 +// 站点新建的行没有真机 UUID 可继承。这里用确定性算法补：名字取表名 + src_pk +// （src_pk 在**同一个数据集内**稳定，够用了），于是同一条记录每次导出拿到的都是同一个 +// UUID——若改成随机，每次导出的包都会让 rond 把同一批记录当成新的。 +// +// 注意这只保证「同一数据集内幂等」，不是跨备份的稳定标识：rond 那边每次导入都会重排 +// Z_PK，真正的跨备份对齐键是它原生的 ZIDENTIFIER_（见 diff.go）。 func uuidV5(name string) []byte { h := sha1.New() h.Write(uuidNamespace[:]) diff --git a/internal/rond/merge.go b/internal/rond/merge.go new file mode 100644 index 0000000..a0d4ff3 --- /dev/null +++ b/internal/rond/merge.go @@ -0,0 +1,170 @@ +package rond + +// 合并计划：把「差异画像 + 站长在页面上做的选择」翻译成一份可执行的改动清单。 +// 这里只做判定，不碰数据库、也不写任何东西。 + +// 站长对一条冲突的处理方式。 +const ( + PickBackup = "backup" // 用备份（实机）的值覆盖站内 + PickDB = "db" // 保留站内的值 + PickSkip = "skip" // 这条不动 +) + +// Choices 是差异页提交上来的选择。冲突是逐条选的，键分别是到访 UUID、 +// 地点坐标键、活动 UUID。 +type Choices struct { + AddVisits bool // 把备份里多出来的到访并进来 + AddPlaces bool + AddActs bool + + VisitClash map[string]string + PlaceClash map[string]string + ActClash map[string]string + + // 删除：「仅站内有」的记录按库里编号勾选，默认一个都不删。 + // 键是 src_pk 而不是 UUID——站点补录的行压根没有 UUID。 + DelVisits map[int]bool + DelPlaces map[int]bool + DelActs map[int]bool +} + +func (c Choices) visitPick(key string) string { return pickOf(c.VisitClash, key) } +func (c Choices) placePick(key string) string { return pickOf(c.PlaceClash, key) } +func (c Choices) activityPick(key string) string { return pickOf(c.ActClash, key) } + +func pickOf(m map[string]string, key string) string { + switch m[key] { + case PickBackup, PickDB, PickSkip: + return m[key] + } + return PickDB // 没提交或提交了怪值：默认保留站内的，不误伤 +} + +// VisitPatch 表示用备份侧的值覆盖库里的那一条。 +type VisitPatch struct { + UUID string // 对齐键 + V Visit +} + +// PlacePatch 同理，键是坐标键。 +type PlacePatch struct { + CoordKey string + L Location +} + +// ActivityPatch 键是活动 UUID（没有 UUID 时退回名称）。 +type ActivityPatch struct { + Key string + A Activity +} + +// MergePlan 是一次合并要做的全部改动。 +type MergePlan struct { + AddPlaces []Location + UpdPlaces []PlacePatch + DelPlaces []StoredPlace + AddActs []Activity + UpdActs []ActivityPatch + DelActs []StoredActivity + AddVisits []Visit + UpdVisits []VisitPatch + DelVisits []StoredVisit +} + +// Empty 报告这份计划什么事都不用做。 +func (p MergePlan) Empty() bool { + return len(p.AddPlaces)+len(p.UpdPlaces)+len(p.DelPlaces)+ + len(p.AddActs)+len(p.UpdActs)+len(p.DelActs)+ + len(p.AddVisits)+len(p.UpdVisits)+len(p.DelVisits) == 0 +} + +// BuildMergePlan 按选择把差异筛成改动清单。 +func BuildMergePlan(vd VisitDiff, pd PlaceDiff, ad ActivityDiff, c Choices) MergePlan { + var p MergePlan + + if c.AddPlaces { + p.AddPlaces = pd.Added + } + for _, cf := range pd.Changed { + if c.placePick(cf.DB.Key) == PickBackup { + p.UpdPlaces = append(p.UpdPlaces, PlacePatch{CoordKey: cf.DB.Key, L: cf.Backup}) + } + } + for _, sp := range pd.Removed { + if c.DelPlaces[sp.SrcPK] { + p.DelPlaces = append(p.DelPlaces, sp) + } + } + + if c.AddActs { + p.AddActs = ad.Added + } + for _, cf := range ad.Changed { + if c.activityPick(activityKey(cf.DB)) == PickBackup { + p.UpdActs = append(p.UpdActs, ActivityPatch{Key: activityKey(cf.DB), A: cf.Backup}) + } + } + for _, sa := range ad.Removed { + if c.DelActs[sa.SrcPK] { + p.DelActs = append(p.DelActs, sa) + } + } + + if c.AddVisits { + p.AddVisits = vd.Added + // 新到访可能挂在一个「备份里新出现、库里还没有」的地点/活动上： + // 不把它们一起建出来，到访的 place_id / activity_id 就无从映射，只能整条丢掉。 + p.AddPlaces = appendMissingPlaces(p.AddPlaces, pd.Added) + p.AddActs = appendMissingActs(p.AddActs, ad.Added) + } + for _, cf := range vd.Changed { + if c.visitPick(cf.DB.UUID) == PickBackup { + p.UpdVisits = append(p.UpdVisits, VisitPatch{UUID: cf.DB.UUID, V: cf.Backup}) + } + } + for _, sv := range vd.Removed { + if c.DelVisits[sv.SrcPK] { + p.DelVisits = append(p.DelVisits, sv) + } + } + return p +} + +func appendMissingPlaces(have, more []Location) []Location { + seen := make(map[int]bool, len(have)) + for _, l := range have { + seen[l.SrcPK] = true + } + for _, l := range more { + if !seen[l.SrcPK] { + have = append(have, l) + seen[l.SrcPK] = true + } + } + return have +} + +func appendMissingActs(have, more []Activity) []Activity { + seen := make(map[int]bool, len(have)) + for _, a := range have { + seen[a.SrcPK] = true + } + for _, a := range more { + if !seen[a.SrcPK] { + have = append(have, a) + seen[a.SrcPK] = true + } + } + return have +} + +// activityKey 是活动冲突在页面上的标识：优先 UUID（改名也能对上），没有才用名称。 +func activityKey(sa StoredActivity) string { + if sa.UUID != "" { + return sa.UUID + } + return "n:" + sa.Name +} + +// ActivityKey 供页面与处理函数共用同一套键。 +func ActivityKey(sa StoredActivity) string { return activityKey(sa) } diff --git a/internal/rond/merge_test.go b/internal/rond/merge_test.go new file mode 100644 index 0000000..1a5ddc9 --- /dev/null +++ b/internal/rond/merge_test.go @@ -0,0 +1,76 @@ +package rond + +import "testing" + +func TestBuildMergePlanPicks(t *testing.T) { + vd := VisitDiff{ + Added: []Visit{{SrcPK: 900, Arrival: tp("2026-09-09 10:00")}}, + Removed: []StoredVisit{{SrcPK: 5, UUID: uuidA}}, + Changed: []VisitConflict{ + {Backup: Visit{SrcPK: 901, Arrival: tp("2026-09-01 12:00")}, DB: StoredVisit{SrcPK: 1, UUID: uuidA}}, + {Backup: Visit{SrcPK: 902, Arrival: tp("2026-09-02 12:00")}, DB: StoredVisit{SrcPK: 2, UUID: uuidB}}, + }, + } + pd := PlaceDiff{Added: []Location{{SrcPK: 3, Name: "新地点"}}} + ad := ActivityDiff{Added: []Activity{{SrcPK: 8, Name: "散步"}}} + + p := BuildMergePlan(vd, pd, ad, Choices{ + AddVisits: true, + AddPlaces: true, + AddActs: false, + VisitClash: map[string]string{uuidA: PickBackup, uuidB: PickDB}, + }) + if len(p.AddVisits) != 1 || p.AddVisits[0].SrcPK != 900 { + t.Fatalf("新增到访应合入一条: %+v", p.AddVisits) + } + if len(p.UpdVisits) != 1 || p.UpdVisits[0].UUID != uuidA { + t.Fatalf("只有选了「用实机」的那条才该更新: %+v", p.UpdVisits) + } + if len(p.AddPlaces) != 1 { + t.Fatalf("勾了「并进来」就该带上新增地点: %d", len(p.AddPlaces)) + } + // 即使没单独勾活动，合入到访时也要把它可能挂着的「库里还没有的活动」带上， + // 否则那条到访的 activity_id 无从映射，只能整条丢掉。 + if len(p.AddActs) != 1 || p.AddActs[0].Name != "散步" { + t.Fatalf("合入到访应连带它需要的活动: %+v", p.AddActs) + } + // 「仅站内有」永远不进计划：删除是单独的事，绝不顺手做 + if len(p.UpdPlaces)+len(p.UpdActs) != 0 { + t.Fatal("不该有地点/活动更新") + } +} + +// 没提交的键、或提交了怪值，一律当成「保留站内」——不误伤。 +func TestBuildMergePlanDefaultsToKeepDB(t *testing.T) { + vd := VisitDiff{Changed: []VisitConflict{ + {Backup: Visit{SrcPK: 901}, DB: StoredVisit{SrcPK: 1, UUID: uuidA}}, + }} + p := BuildMergePlan(vd, PlaceDiff{}, ActivityDiff{}, Choices{VisitClash: map[string]string{uuidA: "wat"}}) + if len(p.UpdVisits) != 0 { + t.Fatalf("怪值不该被当成「用实机」: %+v", p.UpdVisits) + } + if !p.Empty() { + t.Fatal("什么都没选，计划应当是空的") + } +} + +// 冲突选了「跳过」也不动。 +func TestBuildMergePlanSkip(t *testing.T) { + vd := VisitDiff{Changed: []VisitConflict{ + {Backup: Visit{SrcPK: 901}, DB: StoredVisit{SrcPK: 1, UUID: uuidA}}, + }} + p := BuildMergePlan(vd, PlaceDiff{}, ActivityDiff{}, Choices{VisitClash: map[string]string{uuidA: PickSkip}}) + if !p.Empty() { + t.Fatalf("跳过就不该产生改动: %+v", p) + } +} + +// 活动改了名也要能用 UUID 当键对得上。 +func TestActivityKeyPrefersUUID(t *testing.T) { + if got := ActivityKey(StoredActivity{SrcPK: 1, Name: "上班", UUID: uuidC}); got != uuidC { + t.Fatalf("有 UUID 就用 UUID，实际 %q", got) + } + if got := ActivityKey(StoredActivity{SrcPK: 1, Name: "上班"}); got != "n:上班" { + t.Fatalf("没有 UUID 退回名称，实际 %q", got) + } +} diff --git a/internal/rond/parse.go b/internal/rond/parse.go index 4244175..e51488d 100644 --- a/internal/rond/parse.go +++ b/internal/rond/parse.go @@ -156,6 +156,13 @@ type Backup struct { // RawRows 是各实体在 Core Data 里的完整原始行：「表名 -> Z_PK -> 列名 -> 值」。 // 站点只映射了其中一部分字段，留档让导出能逐列还原，不必依赖基线库补剩下的列。 RawRows map[string]map[int]map[string]any + + // VisitUUID / ActivityUUID 是真机给每条到访、每个活动存的 16 字节 UUID（小写十六进制）。 + // **这是跨备份唯一稳定的对齐键**：rond 每次导出/同步都会重排 Z_PK——实测两份备份里 + // 822 条到访的 UUID 完全一致、Z_PK 100% 不同——所以「这份备份」与「库里的那一份」 + // 之间只能按它对齐。地点/标签/交通方式没有 UUID，只能退到名称与坐标。 + VisitUUID map[int]string + ActivityUUID map[int]string } // Open 解压备份包到 dir，返回可用的 SQLite 连接。 @@ -465,9 +472,38 @@ func Parse(db *sql.DB) (*Backup, error) { } else { log.Printf("抓取原始行留档失败（导出将退回基线补列）: %v", rerr) } + // 对齐键：导出不依赖它，但「两份数据比对」全靠它，缺了就只能放弃逐条对齐 + if b.VisitUUID, err = uuidColumn(db, "ZVISIT", "ZIDENTIFIER_"); err != nil { + return nil, fmt.Errorf("读取到访 UUID: %w", err) + } + if b.ActivityUUID, err = uuidColumn(db, "ZACTIVITY", "ZUID_"); err != nil { + return nil, fmt.Errorf("读取活动 UUID: %w", err) + } return b, nil } +// uuidColumn 读出某表某一列的 UUID，返回 Z_PK -> 小写十六进制。 +// 列不存在（很老的备份）返回空表而不是错误；全零或空的当作没有。 +func uuidColumn(db *sql.DB, table, col string) (map[int]string, error) { + rows, err := db.Query(fmt.Sprintf(`SELECT Z_PK, hex(%s) FROM %s`, col, table)) + if err != nil { + return nil, nil + } + defer rows.Close() + out := map[int]string{} + for rows.Next() { + var pk int + var h sql.NullString + if err := rows.Scan(&pk, &h); err != nil { + return nil, err + } + if h.Valid && strings.Trim(h.String, "0") != "" { + out[pk] = strings.ToLower(h.String) + } + } + return out, rows.Err() +} + // rawTables 是需要整行留档的 Core Data 表（站点管理的实体 + 标签分组）。 var rawTables = []string{"ZLOCATION", "ZVISIT", "ZMOVEMENT", "ZHOURLYWEATHER", "ZACTIVITY", "ZTAG", "ZTAGGROUP", "ZTRANSPORT"} diff --git a/internal/stats/stats.go b/internal/stats/stats.go index d294105..e5c9a7c 100644 --- a/internal/stats/stats.go +++ b/internal/stats/stats.go @@ -666,7 +666,8 @@ const placeCols = `p.id, COALESCE(p.name,''), COALESCE(p.poi_category,''), COALE EXISTS (SELECT 1 FROM visits v3 WHERE v3.place_id=p.id AND v3.is_home), EXISTS (SELECT 1 FROM visits v4 WHERE v4.place_id=p.id AND v4.is_work), p.src_pk, - EXISTS (SELECT 1 FROM place_images pi WHERE pi.dataset_id=p.dataset_id AND pi.src_pk=p.src_pk)` + EXISTS (SELECT 1 FROM place_images pi WHERE pi.place_key = + to_char(p.lat,'FM999990.000000')||'|'||to_char(p.lon,'FM999990.000000'))` // PlaceCols 与 ScanPlace 配对，供包外构造只取单个地点的查询。 const PlaceCols = placeCols @@ -708,11 +709,12 @@ func (d DB) Places(ctx context.Context, f Filter, sortBy string, limit, offset i args := append([]any{f.DatasetID}, inner.args...) base := `FROM places p WHERE p.dataset_id = $1 AND EXISTS (SELECT 1 FROM visits v WHERE ` + inner.expr() + `)` if f.Verdict != 0 { - // 结论存在 place_votes，按 src_pk 关联；占位符接在内层条件之后。 - // dataset_id 也要对上：src_pk 只在单个数据集内唯一。 + // 结论挂坐标键（换数据集也要跟得住），按本数据集的地点圈定范围； + // 占位符接在内层条件之后。 args = append(args, f.Verdict) - base += fmt.Sprintf(` AND EXISTS (SELECT 1 FROM place_votes pv - WHERE pv.src_pk = p.src_pk AND pv.dataset_id = p.dataset_id AND pv.verdict = $%d)`, len(args)) + base += fmt.Sprintf(` AND EXISTS (SELECT 1 FROM place_votes pv WHERE pv.place_key = + to_char(p.lat,'FM999990.000000')||'|'||to_char(p.lon,'FM999990.000000') + AND pv.verdict = $%d)`, len(args)) } q := `SELECT ` + placeCols + ` ` + base + ` ORDER BY ` + order + fmt.Sprintf(` LIMIT %d OFFSET %d`, limit, offset) diff --git a/internal/votes/votes.go b/internal/votes/votes.go index b10d7c2..29c0966 100644 --- a/internal/votes/votes.go +++ b/internal/votes/votes.go @@ -2,14 +2,23 @@ // - place_votes 站长对地点的官方结论（单值） // - place_visitor_votes 访客点赞 / 点踩（一人一票，可改可撤） // -// 两者都按 rond 的地点 ID（src_pk）存储，重新上传备份、地点换了新 id 也不丢。 +// 站长的「官方结论」按**坐标键**（place_key）存储：src_pk 只在单个数据集内有效， +// 换个数据集同编号就是另一个地点，结论会挂错人。 +// 访客票仍按 src_pk 存——那张表没有 dataset_id，无从判断票是投给哪个数据集的， +// 迁移有歧义，留待单独处理。 package votes import ( "context" "database/sql" + "fmt" ) +// placeKeyExpr 把 places p 这一行的坐标换算成 place_key。 +// **必须与 Go 侧的 rond.PlaceCoordKey 以及 notes 包里的同款表达式逐位一致**： +// fmt.Sprintf("%.6f|%.6f") ↔ to_char(x,'FM999990.000000')||'|'||to_char(y,'FM999990.000000') +const placeKeyExpr = `to_char(p.lat,'FM999990.000000')||'|'||to_char(p.lon,'FM999990.000000')` + // VerdictRecommend / VerdictAvoid 是官方结论的取值。 const ( VerdictAvoid = -1 @@ -24,11 +33,13 @@ type Tally struct { Down int `json:"down"` } -// Verdicts 返回某个数据集的站长结论：src_pk -> verdict(1/-1)。 -// 必须按 dataset_id 过滤：src_pk 只在单个数据集内唯一。 +// Verdicts 返回某个数据集的站长结论：该数据集里的 src_pk -> verdict(1/-1)。 func (s *Store) Verdicts(ctx context.Context, datasetID int64) (map[int]int, error) { out := map[int]int{} - rows, err := s.DB.QueryContext(ctx, `SELECT src_pk, verdict FROM place_votes WHERE dataset_id=$1`, datasetID) + rows, err := s.DB.QueryContext(ctx, ` + SELECT p.src_pk, v.verdict FROM places p + JOIN place_votes v ON v.place_key = `+placeKeyExpr+` + WHERE p.dataset_id=$1 AND p.lat IS NOT NULL AND p.lon IS NOT NULL`, datasetID) if err != nil { return out, err } @@ -46,8 +57,9 @@ func (s *Store) Verdicts(ctx context.Context, datasetID int64) (map[int]int, err // Verdict 取单个地点的站长结论；未标注返回 0。 func (s *Store) Verdict(ctx context.Context, datasetID int64, srcPK int) (int, error) { var v int - err := s.DB.QueryRowContext(ctx, - `SELECT verdict FROM place_votes WHERE dataset_id=$1 AND src_pk=$2`, datasetID, srcPK).Scan(&v) + err := s.DB.QueryRowContext(ctx, `SELECT v.verdict FROM place_votes v + JOIN places p ON v.place_key = `+placeKeyExpr+` + WHERE p.dataset_id=$1 AND p.src_pk=$2`, datasetID, srcPK).Scan(&v) if err == sql.ErrNoRows { return 0, nil } @@ -57,24 +69,46 @@ func (s *Store) Verdict(ctx context.Context, datasetID int64, srcPK int) (int, e // SaveVerdict 设置站长结论；verdict 为 0 表示取消结论。 func (s *Store) SaveVerdict(ctx context.Context, userID, datasetID int64, srcPK, verdict int) error { if verdict == 0 { - _, err := s.DB.ExecContext(ctx, - `DELETE FROM place_votes WHERE user_id=$1 AND dataset_id=$2 AND src_pk=$3`, userID, datasetID, srcPK) + _, err := s.DB.ExecContext(ctx, `DELETE FROM place_votes v USING places p + WHERE v.user_id=$1 AND p.dataset_id=$2 AND p.src_pk=$3 + AND v.place_key = `+placeKeyExpr, userID, datasetID, srcPK) return err } - _, err := s.DB.ExecContext(ctx, `INSERT INTO place_votes (user_id, dataset_id, src_pk, verdict, updated_at) - VALUES ($1,$2,$3,$4,now()) - ON CONFLICT (user_id, dataset_id, src_pk) DO UPDATE SET verdict=EXCLUDED.verdict, updated_at=now()`, + // 先按坐标键算出这条地点该挂在哪，找不到该地点就什么也不写 + _, err := s.DB.ExecContext(ctx, `INSERT INTO place_votes + (user_id, place_key, dataset_id, src_pk, verdict, updated_at) + SELECT $1, `+placeKeyExpr+`, $2, $3, $4, now() + FROM places p + WHERE p.dataset_id=$2 AND p.src_pk=$3 AND p.lat IS NOT NULL AND p.lon IS NOT NULL + ON CONFLICT (user_id, place_key) DO UPDATE SET + verdict=EXCLUDED.verdict, dataset_id=EXCLUDED.dataset_id, + src_pk=EXCLUDED.src_pk, updated_at=now()`, userID, datasetID, srcPK, verdict) return err } -// Tallies 返回全部地点的访客票数。票表规模很小（个人站点），一次读全更省事， -// 也避免给 pgx 传数组参数带来的类型歧义。 -func (s *Store) Tallies(ctx context.Context) (map[int]Tally, error) { +// placeKeyOf 把某个数据集里的地点编号换算成坐标键——访客票也是挂在坐标上的， +// 换数据集之后票仍然跟着同一个地点。 +func (s *Store) placeKeyOf(ctx context.Context, datasetID int64, srcPK int) (string, error) { + var key string + err := s.DB.QueryRowContext(ctx, `SELECT `+placeKeyExpr+` FROM places p + WHERE p.dataset_id=$1 AND p.src_pk=$2 AND p.lat IS NOT NULL AND p.lon IS NOT NULL`, + datasetID, srcPK).Scan(&key) + if err == sql.ErrNoRows { + return "", fmt.Errorf("地点不存在或没有坐标") + } + return key, err +} + +// Tallies 返回某个数据集里各地点的访客票数：src_pk -> Tally。 +// 票表规模很小（个人站点），一次读全更省事，也避免给 pgx 传数组参数带来的类型歧义。 +func (s *Store) Tallies(ctx context.Context, datasetID int64) (map[int]Tally, error) { out := map[int]Tally{} - rows, err := s.DB.QueryContext(ctx, `SELECT src_pk, - count(*) FILTER (WHERE vote=1), count(*) FILTER (WHERE vote=-1) - FROM place_visitor_votes GROUP BY src_pk`) + rows, err := s.DB.QueryContext(ctx, `SELECT p.src_pk, + count(*) FILTER (WHERE vv.vote=1), count(*) FILTER (WHERE vv.vote=-1) + FROM places p JOIN place_visitor_votes vv ON vv.place_key = `+placeKeyExpr+` + WHERE p.dataset_id=$1 AND p.lat IS NOT NULL AND p.lon IS NOT NULL + GROUP BY p.src_pk`, datasetID) if err != nil { return out, err } @@ -91,21 +125,24 @@ func (s *Store) Tallies(ctx context.Context) (map[int]Tally, error) { } // Tally 取单个地点的访客票数。 -func (s *Store) Tally(ctx context.Context, srcPK int) (Tally, error) { +func (s *Store) Tally(ctx context.Context, datasetID int64, srcPK int) (Tally, error) { var t Tally - err := s.DB.QueryRowContext(ctx, `SELECT count(*) FILTER (WHERE vote=1), count(*) FILTER (WHERE vote=-1) - FROM place_visitor_votes WHERE src_pk=$1`, srcPK).Scan(&t.Up, &t.Down) + err := s.DB.QueryRowContext(ctx, `SELECT + count(*) FILTER (WHERE vv.vote=1), count(*) FILTER (WHERE vv.vote=-1) + FROM place_visitor_votes vv JOIN places p ON vv.place_key = `+placeKeyExpr+` + WHERE p.dataset_id=$1 AND p.src_pk=$2`, datasetID, srcPK).Scan(&t.Up, &t.Down) return t, err } // VoterAll 返回该访客投过的全部票：src_pk -> vote，供地图一次性标注「我投的」。 -func (s *Store) VoterAll(ctx context.Context, voterKey string) (map[int]int, error) { +func (s *Store) VoterAll(ctx context.Context, datasetID int64, voterKey string) (map[int]int, error) { out := map[int]int{} if voterKey == "" { return out, nil } - rows, err := s.DB.QueryContext(ctx, - `SELECT src_pk, vote FROM place_visitor_votes WHERE voter_key=$1`, voterKey) + rows, err := s.DB.QueryContext(ctx, `SELECT p.src_pk, vv.vote + FROM place_visitor_votes vv JOIN places p ON vv.place_key = `+placeKeyExpr+` + WHERE p.dataset_id=$1 AND vv.voter_key=$2`, datasetID, voterKey) if err != nil { return out, err } @@ -121,13 +158,15 @@ func (s *Store) VoterAll(ctx context.Context, voterKey string) (map[int]int, err } // Voter 返回该访客对某地点的既有投票（1/-1/0）。voterKey 为空表示拿不到身份。 -func (s *Store) Voter(ctx context.Context, srcPK int, voterKey string) int { +func (s *Store) Voter(ctx context.Context, datasetID int64, srcPK int, voterKey string) int { if voterKey == "" { return 0 } var v int - if err := s.DB.QueryRowContext(ctx, - `SELECT vote FROM place_visitor_votes WHERE src_pk=$1 AND voter_key=$2`, srcPK, voterKey).Scan(&v); err != nil { + if err := s.DB.QueryRowContext(ctx, `SELECT vv.vote FROM place_visitor_votes vv + JOIN places p ON vv.place_key = `+placeKeyExpr+` + WHERE p.dataset_id=$1 AND p.src_pk=$2 AND vv.voter_key=$3`, + datasetID, srcPK, voterKey).Scan(&v); err != nil { return 0 } return v @@ -135,30 +174,35 @@ func (s *Store) Voter(ctx context.Context, srcPK int, voterKey string) int { // Cast 记录访客投票：未投过则新增；投同一票则撤销（再点一次取消）；投另一票则改为该票。 // 返回更新后的票数与该访客最终的选择。 -func (s *Store) Cast(ctx context.Context, srcPK int, voterKey string, vote int) (Tally, int, error) { +func (s *Store) Cast(ctx context.Context, datasetID int64, srcPK int, voterKey string, vote int) (Tally, int, error) { + key, err := s.placeKeyOf(ctx, datasetID, srcPK) + if err != nil { + return Tally{}, 0, err + } var cur int - err := s.DB.QueryRowContext(ctx, - `SELECT vote FROM place_visitor_votes WHERE src_pk=$1 AND voter_key=$2`, srcPK, voterKey).Scan(&cur) + err = s.DB.QueryRowContext(ctx, + `SELECT vote FROM place_visitor_votes WHERE place_key=$1 AND voter_key=$2`, key, voterKey).Scan(&cur) switch { case err == sql.ErrNoRows: _, err = s.DB.ExecContext(ctx, - `INSERT INTO place_visitor_votes (src_pk, voter_key, vote) VALUES ($1,$2,$3)`, srcPK, voterKey, vote) + `INSERT INTO place_visitor_votes (place_key, voter_key, vote, src_pk) VALUES ($1,$2,$3,$4)`, + key, voterKey, vote, srcPK) cur = vote case err != nil: return Tally{}, 0, err case cur == vote: _, err = s.DB.ExecContext(ctx, - `DELETE FROM place_visitor_votes WHERE src_pk=$1 AND voter_key=$2`, srcPK, voterKey) + `DELETE FROM place_visitor_votes WHERE place_key=$1 AND voter_key=$2`, key, voterKey) cur = 0 default: _, err = s.DB.ExecContext(ctx, - `UPDATE place_visitor_votes SET vote=$3, updated_at=now() WHERE src_pk=$1 AND voter_key=$2`, - srcPK, voterKey, vote) + `UPDATE place_visitor_votes SET vote=$3, src_pk=$4, updated_at=now() + WHERE place_key=$1 AND voter_key=$2`, key, voterKey, vote, srcPK) cur = vote } if err != nil { return Tally{}, 0, err } - t, err := s.Tally(ctx, srcPK) + t, err := s.Tally(ctx, datasetID, srcPK) return t, cur, err } diff --git a/internal/web/backup_diff.go b/internal/web/backup_diff.go new file mode 100644 index 0000000..a46663a --- /dev/null +++ b/internal/web/backup_diff.go @@ -0,0 +1,346 @@ +package web + +import ( + "context" + "database/sql" + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "rond-with-you/internal/ingest" + "rond-with-you/internal/rond" +) + +// rondbackup 的差异预演：把上传的备份与当前展示的数据集按稳定键对齐，只读地摆出 +// 新增 / 仅库里有 / 冲突三类，**不写任何数据**。合并执行是下一步的事。 + +const backupStashExt = ".rondbackup" + +// diffListLimit 是每类差异在页面上最多列出的条数。867 条全铺出来既慢又没人看， +// 超过就只给前若干条并注明。 +const diffListLimit = 200 + +func (s *Server) backupStashDir() string { + return filepath.Join(s.cfg.DataDir, "backup-staging") +} + +// adminBackupPreview 收下备份、暂存，跳到差异页。这里不校验备份内容—— +// 差异页反正是要解析一遍的，失败时让它报错即可，免得同一份包解两次。 +func (s *Server) adminBackupPreview(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxUploadBytes+1<<20) + if err := r.ParseMultipartForm(32 << 20); err != nil { + s.adminFlash(w, r, "err", "读取上传内容失败："+err.Error()) + return + } + file, hdr, err := r.FormFile("backup") + if err != nil { + s.adminFlash(w, r, "err", "请选择要分析的 .rondbackup 文件") + return + } + defer file.Close() + name := filepath.Base(hdr.Filename) + lower := strings.ToLower(name) + if !strings.HasSuffix(lower, ".rondbackup") && !strings.HasSuffix(lower, ".zip") { + s.adminFlash(w, r, "err", "文件类型不正确，请上传 rond 导出的 .rondbackup 备份") + return + } + data, err := io.ReadAll(io.LimitReader(file, maxUploadBytes+1)) + if err != nil { + s.adminFlash(w, r, "err", "读取文件失败："+err.Error()) + return + } + if int64(len(data)) > maxUploadBytes { + s.adminFlash(w, r, "err", "备份文件过大，疑似不是 rond 备份") + return + } + token, err := stashPut(s.backupStashDir(), backupStashExt, data) + if err != nil { + s.serverError(w, r, err) + return + } + http.Redirect(w, r, "/admin/preview/"+token, http.StatusSeeOther) +} + +// adminBackupPreviewPage 渲染差异页。每次都重新算：库里的数据随时可能被别处改掉。 +func (s *Server) adminBackupPreviewPage(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + p, err := s.pageBase(r, "admin", "备份差异预览") + if err != nil { + s.serverError(w, r, err) + return + } + token := r.PathValue("token") + path, ok := stashPath(s.backupStashDir(), backupStashExt, token) + if !ok { + s.adminFlash(w, r, "err", "预览链接无效，请重新上传备份") + return + } + if _, err := os.Stat(path); err != nil { + s.adminFlash(w, r, "err", "这次预览已过期，请重新上传备份") + return + } + ds, err := s.datasetFor(ctx) + if err != nil { + s.serverError(w, r, err) + return + } + if ds == nil { + s.adminFlash(w, r, "err", "还没有可比较的数据集，请先直接导入一份备份") + return + } + + tmp, err := ingest.TempDir(s.cfg.DataDir, "preview") + if err != nil { + s.serverError(w, r, err) + return + } + defer os.RemoveAll(tmp) + sqlDB, err := rond.Open(path, tmp) + if err != nil { + s.adminFlash(w, r, "err", "解析备份失败："+err.Error()) + return + } + backup, perr := rond.Parse(sqlDB) + sqlDB.Close() + if perr != nil { + s.adminFlash(w, r, "err", "读取备份数据失败："+perr.Error()) + return + } + + storedVisits, err := s.loadStoredVisits(ctx, ds.ID) + if err != nil { + s.serverError(w, r, err) + return + } + storedPlaces, err := s.loadStoredPlaces(ctx, ds.ID) + if err != nil { + s.serverError(w, r, err) + return + } + storedActs, err := s.loadStoredActivities(ctx, ds.ID) + if err != nil { + s.serverError(w, r, err) + return + } + + vd := rond.DiffVisits(storedVisits, backup) + pd := rond.DiffPlaces(storedPlaces, backup) + ad := rond.DiffActivities(storedActs, backup) + + d := BackupDiffData{ + Page: p, Token: token, + DatasetName: ds.Name, + VisitCounts: rond.VisitCounts(vd), + PlaceCounts: rond.PlaceCounts(pd), + ActCounts: rond.ActivityCounts(ad), + PlaceAdded: limitSlice(pd.Added, diffListLimit), + PlaceGone: limitSlice(pd.Removed, diffListLimit), + PlaceClash: limitSlice(pd.Changed, diffListLimit), + ActAdded: limitSlice(ad.Added, diffListLimit), + ActGone: limitSlice(ad.Removed, diffListLimit), + ActClash: limitSlice(ad.Changed, diffListLimit), + + BackupVisits: len(backup.Visits), BackupPlaces: len(backup.Locations), + BackupActs: len(backup.Activities), BackupMovements: len(backup.Movements), + BackupWeather: len(backup.Weather), BackupRaw: backup.RawVisitNum, + StoredVisits: len(storedVisits), StoredPlaces: len(storedPlaces), + StoredActs: len(storedActs), + } + d.VisitAdded, d.VisitGone, d.VisitClash = visitRows(vd, backup, diffListLimit) + d.Truncated = anyTruncated(vd, pd, ad) + d.StoredMovements, d.StoredWeather, err = s.countOther(ctx, ds.ID) + if err != nil { + s.serverError(w, r, err) + return + } + d.Identical = d.VisitCounts.Added+d.VisitCounts.Removed+d.VisitCounts.Changed+ + d.PlaceCounts.Added+d.PlaceCounts.Removed+d.PlaceCounts.Changed+ + d.ActCounts.Added+d.ActCounts.Removed+d.ActCounts.Changed == 0 + s.render(w, "admin/backup-preview", d) +} + +// visitRows 把三类到访差异摊平成展示行。备份里的地点/活动只有编号，要借备份自己的 +// 表翻成名称；库里那侧的名称在加载时就一并取出来了。 +func visitRows(vd rond.VisitDiff, b *rond.Backup, limit int) (added, gone, clash []VisitRow) { + place, act := map[int]string{}, map[int]string{} + for _, l := range b.Locations { + place[l.SrcPK] = placeLabel(l.Name, l.Lat, l.Lon) + } + for _, a := range b.Activities { + act[a.SrcPK] = a.Name + } + fromBackup := func(v rond.Visit) VisitRow { + r := VisitRow{Arrival: v.Arrival, Departure: v.Departure, Remark: v.Remark, Emoji: v.Emoji} + if v.LocationSrcPK != nil { + r.Place = place[*v.LocationSrcPK] + } + if v.ActivitySrcPK != nil { + r.Activity = act[*v.ActivitySrcPK] + } + return r + } + for _, v := range limitSlice(vd.Added, limit) { + added = append(added, fromBackup(v)) + } + for _, sv := range limitSlice(vd.Removed, limit) { + gone = append(gone, VisitRow{SrcPK: sv.SrcPK, Arrival: sv.Arrival, Departure: sv.Departure, + Place: sv.PlaceName, Activity: sv.ActivityName, Remark: sv.Remark, Emoji: sv.Emoji, + UserAdded: sv.UserAdded}) + } + for _, c := range limitSlice(vd.Changed, limit) { + r := fromBackup(c.Backup) + r.Key = c.DB.UUID + r.UserAdded = c.DB.UserAdded + r.Fields = c.Fields + clash = append(clash, r) + } + return +} + +func placeLabel(name string, lat, lon float64) string { + if name != "" { + return name + } + return fmt.Sprintf("%.5f, %.5f", lat, lon) +} + +// anyTruncated 报告是否有哪一类差异多到被截断了。 +func anyTruncated(vd rond.VisitDiff, pd rond.PlaceDiff, ad rond.ActivityDiff) bool { + for _, n := range []int{len(vd.Added), len(vd.Removed), len(vd.Changed), + len(pd.Added), len(pd.Removed), len(pd.Changed), + len(ad.Added), len(ad.Removed), len(ad.Changed)} { + if n > diffListLimit { + return true + } + } + return false +} + +func limitSlice[T any](in []T, n int) []T { + if len(in) <= n { + return in + } + return in[:n] +} + +func (s *Server) countOther(ctx context.Context, datasetID int64) (movements, weather int, err error) { + err = s.db.QueryRowContext(ctx, `SELECT + (SELECT count(*) FROM movements WHERE dataset_id=$1), + (SELECT count(*) FROM weather WHERE dataset_id=$1)`, datasetID).Scan(&movements, &weather) + return +} + +// loadStoredVisits 读出该数据集的全部到访，并把原始行留档里的 UUID 解出来当对齐键。 +// 没有留档的行就是站点新建的（真机不会给它 UUID），对齐键留空。 +func (s *Server) loadStoredVisits(ctx context.Context, datasetID int64) ([]rond.StoredVisit, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT v.src_pk, v.arrival, v.departure, COALESCE(v.remark,''), COALESCE(v.emoji,''), + v.bookmarked, v.is_user_added, COALESCE(p.src_pk,0), + COALESCE(p.name,''), COALESCE(p.lat,0), COALESCE(p.lon,0), COALESCE(a.name,''), + COALESCE(er.raw #>> '{ZIDENTIFIER_,$blob}','') + FROM visits v + LEFT JOIN places p ON p.id = v.place_id + LEFT JOIN activities a ON a.id = v.activity_id + LEFT JOIN entity_raw er ON er.dataset_id = v.dataset_id + AND er.entity = 'ZVISIT' AND er.src_pk = v.src_pk + WHERE v.dataset_id = $1`, datasetID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []rond.StoredVisit + for rows.Next() { + var sv rond.StoredVisit + var arr time.Time + var dep sql.NullTime + var placePK int + var placeName string + var lat, lon float64 + var b64 string + if err := rows.Scan(&sv.SrcPK, &arr, &dep, &sv.Remark, &sv.Emoji, &sv.Bookmarked, + &sv.UserAdded, &placePK, &placeName, &lat, &lon, &sv.ActivityName, &b64); err != nil { + return nil, err + } + sv.Arrival = &arr + if dep.Valid { + sv.Departure = &dep.Time + } + sv.PlaceName = placeName + // 没有坐标的地点不能都落到 "0.000000|0.000000" 上，否则两个地点会互相认领 + if placePK > 0 && (lat != 0 || lon != 0) { + sv.PlaceKey = rond.PlaceCoordKey(lat, lon) + } + sv.UUID = uuidFromBase64(b64) + out = append(out, sv) + } + return out, rows.Err() +} + +func (s *Server) loadStoredPlaces(ctx context.Context, datasetID int64) ([]rond.StoredPlace, error) { + rows, err := s.db.QueryContext(ctx, `SELECT src_pk, COALESCE(name,''), COALESCE(lat,0), COALESCE(lon,0), + COALESCE(province,''), COALESCE(city,''), COALESCE(poi_category,''), visit_count + FROM places WHERE dataset_id=$1`, datasetID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []rond.StoredPlace + for rows.Next() { + var sp rond.StoredPlace + if err := rows.Scan(&sp.SrcPK, &sp.Name, &sp.Lat, &sp.Lon, + &sp.Province, &sp.City, &sp.POICategory, &sp.VisitCount); err != nil { + return nil, err + } + if sp.Lat != 0 || sp.Lon != 0 { + sp.Key = rond.PlaceCoordKey(sp.Lat, sp.Lon) + } + out = append(out, sp) + } + return out, rows.Err() +} + +// loadStoredActivities 读活动类型；UUID 同样从留档里取，用于识别「改了名但其实是同一个」。 +func (s *Server) loadStoredActivities(ctx context.Context, datasetID int64) ([]rond.StoredActivity, error) { + rows, err := s.db.QueryContext(ctx, `SELECT a.src_pk, a.name, + COALESCE(er.raw #>> '{ZUID_,$blob}','') + FROM activities a + LEFT JOIN entity_raw er ON er.dataset_id = a.dataset_id + AND er.entity = 'ZACTIVITY' AND er.src_pk = a.src_pk + WHERE a.dataset_id=$1`, datasetID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []rond.StoredActivity + for rows.Next() { + var sa rond.StoredActivity + var b64 string + if err := rows.Scan(&sa.SrcPK, &sa.Name, &b64); err != nil { + return nil, err + } + sa.UUID = uuidFromBase64(b64) + out = append(out, sa) + } + return out, rows.Err() +} + +// uuidFromBase64 把留档里的 $blob（base64 的 16 字节 UUID）转成小写十六进制， +// 好跟备份侧 hex(ZIDENTIFIER_) 的写法对上。 +func uuidFromBase64(s string) string { + if s == "" { + return "" + } + b, err := base64.StdEncoding.DecodeString(s) + if err != nil || len(b) != 16 { + log.Printf("留档里的 UUID 不是 16 字节，忽略: %q", s) + return "" + } + return hex.EncodeToString(b) +} diff --git a/internal/web/backup_diff_test.go b/internal/web/backup_diff_test.go new file mode 100644 index 0000000..b0fece0 --- /dev/null +++ b/internal/web/backup_diff_test.go @@ -0,0 +1,164 @@ +package web + +import ( + "bytes" + "mime/multipart" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" +) + +// previewBackup 走真实的上传处理函数：POST → 303 → GET 差异页，返回渲染结果。 +func previewBackup(t *testing.T, f *movementFixture, path string) string { + t.Helper() + _, page := previewBackupToken(t, f, path) + return page +} + +// previewBackupToken 同上，但把令牌也交出来，供后续的合并步骤使用。 +func previewBackupToken(t *testing.T, f *movementFixture, path string) (string, string) { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("读取 %s 失败: %v", path, err) + } + var buf bytes.Buffer + mw := multipart.NewWriter(&buf) + fw, err := mw.CreateFormFile("backup", filepath.Base(path)) + if err != nil { + t.Fatalf("构造上传表单失败: %v", err) + } + if _, err := fw.Write(data); err != nil { + t.Fatalf("写入上传内容失败: %v", err) + } + mw.Close() + + rec := httptest.NewRecorder() + req := httptest.NewRequest("POST", "/admin/preview", &buf) + req.Header.Set("Content-Type", mw.FormDataContentType()) + f.srv.adminBackupPreview(rec, req.WithContext(f.ctx)) + if rec.Code != http.StatusSeeOther { + t.Fatalf("上传应跳转到差异页: code=%d body=%s", rec.Code, rec.Body.String()) + } + loc := rec.Header().Get("Location") + token := strings.TrimPrefix(loc, "/admin/preview/") + if token == loc || len(token) != stashTokenLen { + t.Fatalf("差异页地址不对: %q", loc) + } + + rec = httptest.NewRecorder() + req = httptest.NewRequest("GET", loc, nil).WithContext(f.ctx) + req.SetPathValue("token", token) + f.srv.adminBackupPreviewPage(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("差异页渲染失败: code=%d body=%s", rec.Code, rec.Body.String()) + } + return token, rec.Body.String() +} + +// visitCounts 从概览表里取出「到访」那一行的 (备份新增, 仅站内有, 冲突)。 +// 不能全页 Contains 数字——地点行也有同样的 data-label，会张冠李戴。 +func visitCounts(t *testing.T, page string) (added, gone, clash int) { + t.Helper() + row := regexp.MustCompile(`(?s)到访(.*?)`).FindStringSubmatch(page) + if row == nil { + t.Fatal("概览表里找不到「到访」行") + } + nums := regexp.MustCompile(`data-label="(备份新增|仅站内有|冲突)">(\d+)<`).FindAllStringSubmatch(row[1], -1) + if len(nums) != 3 { + t.Fatalf("到访行应有三个计数，实际 %d", len(nums)) + } + for _, n := range nums { + v, _ := strconv.Atoi(n[2]) + switch n[1] { + case "备份新增": + added = v + case "仅站内有": + gone = v + case "冲突": + clash = v + } + } + return +} + +// 同一份备份再传一次：UUID 对得上、字段也没变，到访应当一条不差。 +// +// 这条也顺带钉住「按 UUID 而不是编号对齐」：库内那些行的编号与备份里完全不同 +// （rond 重排过），只按编号比会让 800 多条全变冲突。 +// +// 只断言到访，不断言「完全一致」整句：导入后站点会反查补全地点（改名称/地址）， +// 地点层面因此真的存在差异，那是预期行为，不该由这个用例来管。 +func TestBackupDiffIdentical(t *testing.T) { + f := newMovementFixture(t) + page := previewBackup(t, f, f.src) + if a, g, c := visitCounts(t, page); a != 0 || g != 0 || c != 0 { + t.Fatalf("同一份备份的到访不该有差异: 新增 %d 仅站内有 %d 冲突 %d", a, g, c) + } +} + +// 站内改过备注、又补录了一条到访，差异页要把这两件事分别认出来。 +// +// 这里的关键是「按 UUID 对齐」：库里那些行的 Z_PK 与备份里完全不同 +// （rond 重排过），只按编号比会把 800 多条全判成冲突。 +func TestBackupDiffDetectsSiteEdits(t *testing.T) { + f := newMovementFixture(t) + + var pk int + if err := f.pg.QueryRowContext(f.ctx, `UPDATE visits SET remark='站点改过的备注' + WHERE id = (SELECT id FROM visits WHERE dataset_id=$1 ORDER BY src_pk LIMIT 1) + RETURNING src_pk`, f.dsID).Scan(&pk); err != nil { + t.Fatalf("改备注失败: %v", err) + } + if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO visits + (dataset_id, src_pk, arrival, departure, is_user_added, remark) + VALUES ($1, 999999, now(), now(), true, '站点补录的一条')`, f.dsID); err != nil { + t.Fatalf("插入补录到访失败: %v", err) + } + + page := previewBackup(t, f, f.src) + // 「站点改过的备注」只可能从冲突行的字段差异里渲染出来；「站内补录」是那条补录行的来源标记 + for _, want := range []string{"备份差异预览", "站点改过的备注", "站点补录的一条", ">站内补录<"} { + if !strings.Contains(page, want) { + t.Errorf("差异页缺少 %q", want) + } + } + a, g, c := visitCounts(t, page) + if c != 1 { + t.Errorf("改过一条备注，到访冲突数应为 1，实际 %d", c) + } + if g != 1 { + t.Errorf("补录了一条，到访「仅站内有」应为 1，实际 %d", g) + } + if a != 0 { + t.Errorf("备份里没有多出来的到访，实际 %d", a) + } +} + +// 差异页是只读的：渲染前后库里的行数必须一模一样。 +func TestBackupDiffPreviewWritesNothing(t *testing.T) { + f := newMovementFixture(t) + + count := func() (visits, places, raw int) { + t.Helper() + if err := f.pg.QueryRowContext(f.ctx, `SELECT + (SELECT count(*) FROM visits WHERE dataset_id=$1), + (SELECT count(*) FROM places WHERE dataset_id=$1), + (SELECT count(*) FROM entity_raw WHERE dataset_id=$1)`, f.dsID). + Scan(&visits, &places, &raw); err != nil { + t.Fatalf("统计失败: %v", err) + } + return + } + v0, p0, r0 := count() + previewBackup(t, f, f.src) + v1, p1, r1 := count() + if v0 != v1 || p0 != p1 || r0 != r1 { + t.Fatalf("差异页不该写库: 到访 %d→%d 地点 %d→%d 留档 %d→%d", v0, v1, p0, p1, r0, r1) + } +} diff --git a/internal/web/backup_merge.go b/internal/web/backup_merge.go new file mode 100644 index 0000000..c7b0cf6 --- /dev/null +++ b/internal/web/backup_merge.go @@ -0,0 +1,753 @@ +package web + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "log" + "net/http" + "os" + "strconv" + "strings" + + "rond-with-you/internal/geo" + "rond-with-you/internal/ingest" + "rond-with-you/internal/rond" +) + +// 差异页确认后的合并执行。 +// +// 全部改动在一个事务里完成：半合并的数据集比没合并更难收拾（比如地点建了、 +// 到访没建，地图上就多一个永远没人去的点）。删除不在这里——删地点会级联删掉 +// 它的全部到访，那件事单独再议。 +// +// 编号是这里最容易出错的地方：**库里那侧的编号空间才是唯一有效的**。 +// 新插入的行拿 nextSrcPK 发的新号，于是位移的 from_visit_src、天气的 visit_src +// 都必须写「库里的新号」，而不是备份里的原号。 + +type mergeStats struct { + Places, Activities, Tags int + Visits, Movements, Weather int + UpdPlaces, UpdActs, UpdVisits int + DelPlaces, DelActs, DelVisits int +} + +func (s *Server) adminBackupMerge(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + ds, err := s.datasetFor(ctx) + if err != nil { + s.serverError(w, r, err) + return + } + if ds == nil { + s.adminFlash(w, r, "err", "没有可合并的数据集") + return + } + token := r.PathValue("token") + path, ok := stashPath(s.backupStashDir(), backupStashExt, token) + if !ok { + s.adminFlash(w, r, "err", "合并链接无效，请重新上传备份") + return + } + if _, err := os.Stat(path); err != nil { + s.adminFlash(w, r, "err", "这次预览已过期，请重新上传备份") + return + } + if err := r.ParseForm(); err != nil { + s.serverError(w, r, err) + return + } + backup, err := s.parseBackupFile(path) + if err != nil { + s.adminFlash(w, r, "err", err.Error()) + return + } + visits, places, acts, err := s.loadStoredAll(ctx, ds.ID) + if err != nil { + s.serverError(w, r, err) + return + } + + // 重新算一遍差异再执行：从渲染差异页到点下按钮之间，库可能已经被别处改过 + vd := rond.DiffVisits(visits, backup) + pd := rond.DiffPlaces(places, backup) + ad := rond.DiffActivities(acts, backup) + addNew := r.PostFormValue("add_new") == "1" + plan := rond.BuildMergePlan(vd, pd, ad, rond.Choices{ + AddVisits: addNew, AddPlaces: addNew, AddActs: addNew, + VisitClash: pickForm(r, "vc_"), + PlaceClash: pickForm(r, "pc_"), + ActClash: pickForm(r, "ac_"), + DelVisits: pickIntForm(r, "dv_"), + DelPlaces: pickIntForm(r, "dp_"), + DelActs: pickIntForm(r, "da_"), + }) + if plan.Empty() { + s.adminFlash(w, r, "ok", "按当前的选择没有要写入的改动，数据未变") + return + } + st, err := s.applyMergePlan(ctx, ds, backup, plan, visits) + if err != nil { + log.Printf("合并失败: %v", err) + s.adminFlash(w, r, "err", "合并失败，已整体回滚、数据未变："+err.Error()) + return + } + stashDrop(s.backupStashDir(), backupStashExt, token) + + msg := fmt.Sprintf("合并完成：新增 %d 个地点 / %d 个活动 / %d 条到访；更新 %d 个地点 / %d 个活动 / %d 条到访", + st.Places, st.Activities, st.Visits, st.UpdPlaces, st.UpdActs, st.UpdVisits) + if st.DelVisits+st.DelActs+st.DelPlaces > 0 { + msg += fmt.Sprintf("；删除 %d 条到访 / %d 个活动 / %d 个地点", st.DelVisits, st.DelActs, st.DelPlaces) + } + if st.Tags > 0 { + msg += fmt.Sprintf("；顺带建了 %d 个标签", st.Tags) + } + if st.Movements+st.Weather > 0 { + msg += fmt.Sprintf("；随到访带入 %d 条位移、%d 条天气", st.Movements, st.Weather) + } + s.adminFlash(w, r, "ok", msg) +} + +// pickIntForm 收集「前缀 + 整数」形式的勾选，比如 dv_825=1（要删除的到访）。 +func pickIntForm(r *http.Request, prefix string) map[int]bool { + out := map[int]bool{} + for k, vs := range r.PostForm { + if len(vs) == 0 || vs[0] == "" || !strings.HasPrefix(k, prefix) { + continue + } + if n, err := strconv.Atoi(strings.TrimPrefix(k, prefix)); err == nil { + out[n] = true + } + } + return out +} + +// pickForm 收集「前缀 + 键」形式的逐条选择，比如 vc_=backup。 +func pickForm(r *http.Request, prefix string) map[string]string { + out := map[string]string{} + for k, vs := range r.PostForm { + if len(vs) == 0 || !strings.HasPrefix(k, prefix) { + continue + } + out[strings.TrimPrefix(k, prefix)] = vs[0] + } + return out +} + +func (s *Server) loadStoredAll(ctx context.Context, dsID int64) ([]rond.StoredVisit, []rond.StoredPlace, []rond.StoredActivity, error) { + visits, err := s.loadStoredVisits(ctx, dsID) + if err != nil { + return nil, nil, nil, err + } + places, err := s.loadStoredPlaces(ctx, dsID) + if err != nil { + return nil, nil, nil, err + } + acts, err := s.loadStoredActivities(ctx, dsID) + if err != nil { + return nil, nil, nil, err + } + return visits, places, acts, nil +} + +func (s *Server) parseBackupFile(path string) (*rond.Backup, error) { + tmp, err := ingest.TempDir(s.cfg.DataDir, "merge") + if err != nil { + return nil, err + } + defer os.RemoveAll(tmp) + db, err := rond.Open(path, tmp) + if err != nil { + return nil, fmt.Errorf("解析备份失败：%w", err) + } + defer db.Close() + b, err := rond.Parse(db) + if err != nil { + return nil, fmt.Errorf("读取备份数据失败：%w", err) + } + return b, nil +} + +func (s *Server) applyMergePlan(ctx context.Context, ds *Dataset, b *rond.Backup, plan rond.MergePlan, + storedVisits []rond.StoredVisit) (*mergeStats, error) { + st := &mergeStats{} + err := s.withTx(ctx, func(ex execer) error { + placeID, err := s.mergePlaces(ctx, ex, ds.ID, b, plan, st) + if err != nil { + return err + } + actID, err := s.mergeActivities(ctx, ex, ds.ID, b, plan, st) + if err != nil { + return err + } + tagID, err := s.mergeTags(ctx, ex, ds.ID, b, st) + if err != nil { + return err + } + if err := s.mergeVisits(ctx, ex, ds.ID, b, plan, placeID, actID, tagID, storedVisits, st); err != nil { + return err + } + // 删除放在最后：先让新增/更新落定，再按最终状态删 + if err := s.applyDeletions(ctx, ex, ds.ID, plan, st); err != nil { + return err + } + return s.recountPlaces(ctx, ex, ds.ID) + }) + if err != nil { + return nil, err + } + return st, nil +} + +// mergePlaces 先把备份地点映射到库里的 places.id（已存在的直接映射）， +// 再处理冲突更新与新增，返回「备份地点编号 -> 库 id」。 +func (s *Server) mergePlaces(ctx context.Context, ex execer, dsID int64, b *rond.Backup, + plan rond.MergePlan, st *mergeStats) (map[int]int64, error) { + byCoord, byName := map[string]int64{}, map[string]int64{} + rows, err := ex.QueryContext(ctx, `SELECT id, COALESCE(name,''), COALESCE(lat,0), COALESCE(lon,0) + FROM places WHERE dataset_id=$1`, dsID) + if err != nil { + return nil, err + } + for rows.Next() { + var id int64 + var name string + var lat, lon float64 + if err := rows.Scan(&id, &name, &lat, &lon); err != nil { + rows.Close() + return nil, err + } + if lat != 0 || lon != 0 { + byCoord[rond.PlaceCoordKey(lat, lon)] = id + } + if _, dup := byName[name]; !dup && name != "" { + byName[name] = id + } + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + + placeID := map[int]int64{} + for _, l := range b.Locations { + if l.Lat != 0 || l.Lon != 0 { + if id, ok := byCoord[rond.PlaceCoordKey(l.Lat, l.Lon)]; ok { + placeID[l.SrcPK] = id + continue + } + } + if l.Name != "" { + if id, ok := byName[l.Name]; ok { + placeID[l.SrcPK] = id + } + } + } + + for _, up := range plan.UpdPlaces { + id := placeID[up.L.SrcPK] + if id == 0 { + id = byCoord[up.CoordKey] + } + if id == 0 { + id = byName[up.L.Name] + } + if id == 0 { + return nil, fmt.Errorf("找不到要更新的地点（%s）", up.L.Name) + } + if _, err := ex.ExecContext(ctx, `UPDATE places SET name=$1, poi_category=$2, lat=$3, lon=$4, + country_code=$5, province=$6, city=$7, district=$8, sublocality=$9, thoroughfare=$10, timezone=$11 + WHERE id=$12`, + nullStr(up.L.Name), nullStr(up.L.POICategory), up.L.Lat, up.L.Lon, + nullStr(up.L.CountryCode), nullStr(up.L.Province), nullStr(up.L.City), nullStr(up.L.District), + nullStr(up.L.Sublocality), nullStr(up.L.Thoroughfare), nullStr(up.L.Timezone), id); err != nil { + return nil, fmt.Errorf("更新地点: %w", err) + } + placeID[up.L.SrcPK] = id + st.UpdPlaces++ + } + + for _, l := range plan.AddPlaces { + pk, err := s.nextSrcPK(ctx, ex, dsID, placeSeq) + if err != nil { + return nil, err + } + var id int64 + if err := ex.QueryRowContext(ctx, `INSERT INTO places + (dataset_id, src_pk, name, poi_category, lat, lon, country_code, province, city, + district, sublocality, thoroughfare, timezone) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13) RETURNING id`, + dsID, pk, nullStr(l.Name), nullStr(l.POICategory), l.Lat, l.Lon, nullStr(l.CountryCode), + nullStr(l.Province), nullStr(l.City), nullStr(l.District), nullStr(l.Sublocality), + nullStr(l.Thoroughfare), nullStr(l.Timezone)).Scan(&id); err != nil { + return nil, fmt.Errorf("新增地点: %w", err) + } + placeID[l.SrcPK] = id + st.Places++ + if err := writeRawRow(ctx, ex, dsID, "ZLOCATION", pk, b.RawRows["ZLOCATION"][l.SrcPK]); err != nil { + return nil, err + } + } + return placeID, nil +} + +func (s *Server) mergeActivities(ctx context.Context, ex execer, dsID int64, b *rond.Backup, + plan rond.MergePlan, st *mergeStats) (map[int]int64, error) { + byUUID, byName := map[string]int64{}, map[string]int64{} + rows, err := ex.QueryContext(ctx, `SELECT a.id, a.name, COALESCE(er.raw #>> '{ZUID_,$blob}','') + FROM activities a + LEFT JOIN entity_raw er ON er.dataset_id = a.dataset_id + AND er.entity = 'ZACTIVITY' AND er.src_pk = a.src_pk + WHERE a.dataset_id=$1`, dsID) + if err != nil { + return nil, err + } + for rows.Next() { + var id int64 + var name, b64 string + if err := rows.Scan(&id, &name, &b64); err != nil { + rows.Close() + return nil, err + } + if u := uuidFromBase64(b64); u != "" { + byUUID[u] = id + } + if _, dup := byName[name]; !dup && name != "" { + byName[name] = id + } + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + + actID := map[int]int64{} + for _, a := range b.Activities { + if u := b.ActivityUUID[a.SrcPK]; u != "" { + if id, ok := byUUID[u]; ok { + actID[a.SrcPK] = id + continue + } + } + if id, ok := byName[a.Name]; ok { + actID[a.SrcPK] = id + } + } + + for _, up := range plan.UpdActs { + id := actID[up.A.SrcPK] + if id == 0 { + id = byName[up.A.Name] + } + if id == 0 { + return nil, fmt.Errorf("找不到要更新的活动（%s）", up.A.Name) + } + if _, err := ex.ExecContext(ctx, `UPDATE activities SET name=$1, color=$2, icon=$3, + is_home=$4, is_work=$5, is_excluded=$6, is_archived=$7 WHERE id=$8`, + up.A.Name, nullStr(up.A.Color), nullStr(up.A.Icon), up.A.IsHome, up.A.IsWork, + up.A.IsExcluded, up.A.IsArchived, id); err != nil { + return nil, fmt.Errorf("更新活动: %w", err) + } + st.UpdActs++ + } + + for _, a := range plan.AddActs { + pk, err := s.nextSrcPK(ctx, ex, dsID, activitySeq) + if err != nil { + return nil, err + } + var id int64 + if err := ex.QueryRowContext(ctx, `INSERT INTO activities + (dataset_id, src_pk, name, color, icon, is_home, is_work, is_excluded, is_archived) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) RETURNING id`, + dsID, pk, a.Name, nullStr(a.Color), nullStr(a.Icon), + a.IsHome, a.IsWork, a.IsExcluded, a.IsArchived).Scan(&id); err != nil { + return nil, fmt.Errorf("新增活动: %w", err) + } + actID[a.SrcPK] = id + st.Activities++ + if err := writeRawRow(ctx, ex, dsID, "ZACTIVITY", pk, b.RawRows["ZACTIVITY"][a.SrcPK]); err != nil { + return nil, err + } + } + return actID, nil +} + +// mergeTags 只做「库里有同名的就复用、没有就建」——标签是纯名称实体，没有别的标识。 +func (s *Server) mergeTags(ctx context.Context, ex execer, dsID int64, b *rond.Backup, st *mergeStats) (map[int]int64, error) { + byName := map[string]int64{} + rows, err := ex.QueryContext(ctx, `SELECT id, name FROM tags WHERE dataset_id=$1`, dsID) + if err != nil { + return nil, err + } + for rows.Next() { + var id int64 + var name string + if err := rows.Scan(&id, &name); err != nil { + rows.Close() + return nil, err + } + byName[name] = id + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + + tagID := map[int]int64{} + for _, t := range b.Tags { + if id, ok := byName[t.Name]; ok { + tagID[t.SrcPK] = id + continue + } + pk, err := s.nextSrcPK(ctx, ex, dsID, tagSeq) + if err != nil { + return nil, err + } + var id int64 + if err := ex.QueryRowContext(ctx, `INSERT INTO tags (dataset_id, src_pk, name, color, group_name) + VALUES ($1,$2,$3,$4,NULLIF($5,'')) RETURNING id`, + dsID, pk, t.Name, nullStr(t.Color), t.GroupName).Scan(&id); err != nil { + return nil, fmt.Errorf("新增标签: %w", err) + } + byName[t.Name] = id + tagID[t.SrcPK] = id + st.Tags++ + if err := writeRawRow(ctx, ex, dsID, "ZTAG", pk, b.RawRows["ZTAG"][t.SrcPK]); err != nil { + return nil, err + } + } + return tagID, nil +} + +func (s *Server) mergeVisits(ctx context.Context, ex execer, dsID int64, b *rond.Backup, plan rond.MergePlan, + placeID, actID, tagID map[int]int64, storedVisits []rond.StoredVisit, st *mergeStats) error { + // 备份到访编号 -> 库里的 visits.src_pk：位移的 from/to_visit_src 与天气的 + // visit_src 写的都是**库里的编号**，端点在备份里的原号在这里换成库里的号。 + visSrc := map[int]int{} + storedByUUID := map[string]rond.StoredVisit{} + for _, sv := range storedVisits { + if sv.UUID != "" { + storedByUUID[sv.UUID] = sv + } + } + for _, v := range b.Visits { + if u := b.VisitUUID[v.SrcPK]; u != "" { + if sv, ok := storedByUUID[u]; ok { + visSrc[v.SrcPK] = sv.SrcPK + } + } + } + + activityByPK := map[int]rond.Activity{} + for _, a := range b.Activities { + activityByPK[a.SrcPK] = a + } + visitPlace := map[int]int{} + for _, v := range b.Visits { + if v.LocationSrcPK != nil { + visitPlace[v.SrcPK] = *v.LocationSrcPK + } + } + + resolve := func(v rond.Visit) (pid, aid any, isHome, isWork bool) { + if v.LocationSrcPK != nil { + if x, ok := placeID[*v.LocationSrcPK]; ok { + pid = x + } + } + if v.ActivitySrcPK != nil { + if x, ok := actID[*v.ActivitySrcPK]; ok { + aid = x + } + if a, ok := activityByPK[*v.ActivitySrcPK]; ok { + isHome, isWork = a.IsHome, a.IsWork + } + } + return + } + + for _, up := range plan.UpdVisits { + sv, ok := storedByUUID[up.UUID] + if !ok { + return fmt.Errorf("找不到要更新的到访（%s）", up.UUID) + } + pid, aid, isHome, isWork := resolve(up.V) + if _, err := ex.ExecContext(ctx, `UPDATE visits SET place_id=$1, activity_id=$2, arrival=$3, departure=$4, + duration_min=$5, is_home=$6, is_work=$7, bookmarked=$8, remark=NULLIF($9,''), emoji=NULLIF($10,'') + WHERE dataset_id=$11 AND src_pk=$12`, + pid, aid, up.V.Arrival, up.V.Departure, visitDuration(up.V), isHome, isWork, + up.V.Bookmarked, up.V.Remark, up.V.Emoji, dsID, sv.SrcPK); err != nil { + return fmt.Errorf("更新到访: %w", err) + } + st.UpdVisits++ + } + + addedVisit := map[int]bool{} + for _, v := range plan.AddVisits { + pid, aid, isHome, isWork := resolve(v) + pk, err := s.nextSrcPK(ctx, ex, dsID, visitSeq) + if err != nil { + return err + } + var id int64 + if err := ex.QueryRowContext(ctx, `INSERT INTO visits + (dataset_id, src_pk, place_id, activity_id, arrival, departure, duration_min, + is_home, is_work, bookmarked, is_user_added, remark, emoji, weather_symbol) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,NULLIF($12,''),NULLIF($13,''),NULLIF($14,'')) + ON CONFLICT DO NOTHING RETURNING id`, + dsID, pk, pid, aid, v.Arrival, v.Departure, visitDuration(v), isHome, isWork, + v.Bookmarked, v.UserAdded, v.Remark, v.Emoji, v.WeatherSymbol).Scan(&id); err != nil { + if err == sql.ErrNoRows { + continue + } + return fmt.Errorf("新增到访: %w", err) + } + + visSrc[v.SrcPK] = pk + addedVisit[v.SrcPK] = true + st.Visits++ + // 留档必须写：下次再比对时库侧就是靠这里的 UUID 认出这一条， + // 少了它只能退到「到达时间 + 坐标」兜底。 + if err := writeRawRow(ctx, ex, dsID, "ZVISIT", pk, b.RawRows["ZVISIT"][v.SrcPK]); err != nil { + return err + } + for _, tpk := range v.TagSrcPKs { + if tid, ok := tagID[tpk]; ok { + if _, err := ex.ExecContext(ctx, `INSERT INTO visit_tags (visit_id, tag_id) + VALUES ($1,$2) ON CONFLICT DO NOTHING`, id, tid); err != nil { + return fmt.Errorf("写入到访标签: %w", err) + } + } + } + } + + transName, transColor, transIcon := map[int]string{}, map[int]string{}, map[int]string{} + for _, t := range b.Transports { + transName[t.SrcPK], transColor[t.SrcPK], transIcon[t.SrcPK] = t.Name, t.Color, t.Icon + } + coordOf := map[int][2]float64{} + for _, l := range b.Locations { + coordOf[l.SrcPK] = [2]float64{l.Lat, l.Lon} + } + // 位移与天气跟着**新插入的到访**走：只插这一轮真正补进来的那些， + // 否则库里已有的位移会被整份复制一遍。另一端可以是老到访（库里已经有）。 + for _, m := range b.Movements { + if m.FromVisitSrc == nil || m.ToVisitSrc == nil { + continue + } + if !addedVisit[*m.FromVisitSrc] && !addedVisit[*m.ToVisitSrc] { + continue + } + fsrc, ok1 := visSrc[*m.FromVisitSrc] + tsrc, ok2 := visSrc[*m.ToVisitSrc] + if !ok1 || !ok2 { + continue + } + var fromPID, toPID any + if fp, ok := visitPlace[*m.FromVisitSrc]; ok { + if x, ok := placeID[fp]; ok { + fromPID = x + } + } + if tp, ok := visitPlace[*m.ToVisitSrc]; ok { + if x, ok := placeID[tp]; ok { + toPID = x + } + } + var dist any + if fp, ok1 := visitPlace[*m.FromVisitSrc]; ok1 { + if tp, ok2 := visitPlace[*m.ToVisitSrc]; ok2 { + c1, c2 := coordOf[fp], coordOf[tp] + if c1[0] != 0 && c2[0] != 0 { + dist = geo.Haversine(c1[0], c1[1], c2[0], c2[1]) + } + } + } + var tname, tcolor, ticon any + if m.TransportSrc != nil { + tname = nullStr(transName[*m.TransportSrc]) + tcolor = nullStr(transColor[*m.TransportSrc]) + ticon = nullStr(transIcon[*m.TransportSrc]) + } + var dur any + if m.Start != nil && m.End != nil { + if d := int(m.End.Sub(*m.Start).Minutes()); d >= 0 { + dur = d + } + } + pk, err := s.nextSrcPK(ctx, ex, dsID, movementSeq) + if err != nil { + return err + } + if _, err := ex.ExecContext(ctx, `INSERT INTO movements + (dataset_id, src_pk, transport_src, transport_name, transport_color, transport_icon, + from_visit_src, to_visit_src, from_place_id, to_place_id, started_at, ended_at, duration_min, distance_km) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT DO NOTHING`, + dsID, pk, m.TransportSrc, tname, tcolor, ticon, fsrc, tsrc, + fromPID, toPID, m.Start, m.End, dur, dist); err != nil { + return fmt.Errorf("新增位移: %w", err) + } + st.Movements++ + if err := writeRawRow(ctx, ex, dsID, "ZMOVEMENT", pk, b.RawRows["ZMOVEMENT"][m.SrcPK]); err != nil { + return err + } + } + for _, w := range b.Weather { + if w.VisitSrc == nil || w.At == nil { + continue + } + if !addedVisit[*w.VisitSrc] { + continue + } + vsrc, ok := visSrc[*w.VisitSrc] + if !ok { + continue + } + // 站内删到访时会把天气的归属置空而保留记录（见 editVisitDelete）。 + // 这类「同一时刻、没有归属」的天气先认领回来，否则同一条到访找回来之后 + // 会多出一份重复的天气。 + var claimed int64 + if err := ex.QueryRowContext(ctx, `UPDATE weather SET visit_src=$1 + WHERE dataset_id=$2 AND at=$3 AND visit_src IS NULL RETURNING id`, + vsrc, dsID, w.At).Scan(&claimed); err == nil { + continue + } + pk, err := s.nextSrcPK(ctx, ex, dsID, weatherSeq) + if err != nil { + return err + } + if _, err := ex.ExecContext(ctx, `INSERT INTO weather + (dataset_id, src_pk, visit_src, at, is_daylight, temperature_c, apparent_c, humidity, + precipitation_amount, precipitation_chance, wind_speed, visibility, uv_index, + condition, symbol, uv_category, wind_direction) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,NULLIF($14,''),NULLIF($15,''),NULLIF($16,''),NULLIF($17,'')) + ON CONFLICT DO NOTHING`, + dsID, pk, vsrc, w.At, w.IsDaylight, w.TempC, w.ApparentC, w.Humidity, + w.PrecipAmt, w.PrecipPct, w.WindSpeed, w.Visibility, w.UVIndex, + w.Condition, w.Symbol, w.UVCategory, w.WindDir); err != nil { + return fmt.Errorf("新增天气: %w", err) + } + st.Weather++ + if err := writeRawRow(ctx, ex, dsID, "ZHOURLYWEATHER", pk, b.RawRows["ZHOURLYWEATHER"][w.SrcPK]); err != nil { + return err + } + } + + return nil +} + +// applyDeletions 处理「仅站内有」里被勾选删除的记录。顺序是到访 → 活动 → 地点： +// 删地点会级联删掉它名下的全部到访（外键 ON DELETE CASCADE），所以反过来会误删； +// 而且**有到访引用时直接拒绝**，让站长先处理到访，而不是悄悄带走一批。 +func (s *Server) applyDeletions(ctx context.Context, ex execer, dsID int64, plan rond.MergePlan, st *mergeStats) error { + for _, sv := range plan.DelVisits { + if _, err := ex.ExecContext(ctx, `DELETE FROM movements WHERE dataset_id=$1 + AND (from_visit_src=$2 OR to_visit_src=$2)`, dsID, sv.SrcPK); err != nil { + return fmt.Errorf("删除到访的位移: %w", err) + } + // 天气保留、只解除归属——与站点 editVisitDelete 的处理一致 + if _, err := ex.ExecContext(ctx, `UPDATE weather SET visit_src=NULL + WHERE dataset_id=$1 AND visit_src=$2`, dsID, sv.SrcPK); err != nil { + return fmt.Errorf("解除天气归属: %w", err) + } + if _, err := ex.ExecContext(ctx, `DELETE FROM visits WHERE dataset_id=$1 AND src_pk=$2`, + dsID, sv.SrcPK); err != nil { + return fmt.Errorf("删除到访: %w", err) + } + if _, err := ex.ExecContext(ctx, `DELETE FROM entity_raw WHERE dataset_id=$1 + AND entity='ZVISIT' AND src_pk=$2`, dsID, sv.SrcPK); err != nil { + return err + } + st.DelVisits++ + } + for _, sa := range plan.DelActs { + // 到访的 activity_id 是 ON DELETE SET NULL，删活动不会带走到访 + if _, err := ex.ExecContext(ctx, `DELETE FROM activities WHERE dataset_id=$1 AND src_pk=$2`, + dsID, sa.SrcPK); err != nil { + return fmt.Errorf("删除活动: %w", err) + } + if _, err := ex.ExecContext(ctx, `DELETE FROM entity_raw WHERE dataset_id=$1 + AND entity='ZACTIVITY' AND src_pk=$2`, dsID, sa.SrcPK); err != nil { + return err + } + st.DelActs++ + } + for _, sp := range plan.DelPlaces { + var n int + if err := ex.QueryRowContext(ctx, `SELECT count(*) FROM visits v + JOIN places p ON p.id = v.place_id + WHERE p.dataset_id=$1 AND p.src_pk=$2`, dsID, sp.SrcPK).Scan(&n); err != nil { + return err + } + if n > 0 { + return fmt.Errorf("地点「%s」名下还有 %d 条到访，删它会连带删掉那些到访——请先处理这些到访", + sp.Name, n) + } + if _, err := ex.ExecContext(ctx, `DELETE FROM places WHERE dataset_id=$1 AND src_pk=$2`, + dsID, sp.SrcPK); err != nil { + return fmt.Errorf("删除地点: %w", err) + } + if _, err := ex.ExecContext(ctx, `DELETE FROM entity_raw WHERE dataset_id=$1 + AND entity='ZLOCATION' AND src_pk=$2`, dsID, sp.SrcPK); err != nil { + return err + } + st.DelPlaces++ + } + return nil +} + +// recountPlaces 重算地点聚合与数据集汇总。全量重算而不是只动受影响的行： +// 367 个地点两条 SQL 就够，省得漏掉任何一条派生列。 +func (s *Server) recountPlaces(ctx context.Context, ex execer, dsID int64) error { + if _, err := ex.ExecContext(ctx, `UPDATE places SET visit_count=0, dwell_minutes=0, + first_visit_at=NULL, last_visit_at=NULL WHERE dataset_id=$1`, dsID); err != nil { + return err + } + if _, err := ex.ExecContext(ctx, `UPDATE places p SET visit_count=sub.c, dwell_minutes=sub.d, + first_visit_at=sub.f, last_visit_at=sub.l + FROM (SELECT place_id, count(*) c, + COALESCE(sum(LEAST(COALESCE(duration_min,0),10080)),0) d, + min(arrival) f, max(arrival) l + FROM visits WHERE dataset_id=$1 AND place_id IS NOT NULL GROUP BY place_id) sub + WHERE p.id = sub.place_id`, dsID); err != nil { + return err + } + _, err := ex.ExecContext(ctx, `UPDATE datasets SET + visit_count=(SELECT count(*) FROM visits WHERE dataset_id=$1), + place_count=(SELECT count(*) FROM places WHERE dataset_id=$1), + first_visit_at=(SELECT min(arrival) FROM visits WHERE dataset_id=$1), + last_visit_at=(SELECT max(arrival) FROM visits WHERE dataset_id=$1) + WHERE id=$1`, dsID) + return err +} + +// writeRawRow 给新插入的行留一份原始行：导出时要靠它逐列还原， +// 再次比对时要靠它认出 UUID（否则只能退到「时间 + 坐标」兜底）。 +func writeRawRow(ctx context.Context, ex execer, dsID int64, entity string, pk int, raw map[string]any) error { + if len(raw) == 0 { + return nil + } + blob, err := json.Marshal(raw) + if err != nil { + log.Printf("留档 %s#%d 序列化失败: %v", entity, pk, err) + return nil + } + _, err = ex.ExecContext(ctx, `INSERT INTO entity_raw (dataset_id, entity, src_pk, raw, skipped) + VALUES ($1,$2,$3,$4,false) + ON CONFLICT (dataset_id, entity, src_pk) DO UPDATE SET raw=EXCLUDED.raw`, + dsID, entity, pk, string(blob)) + return err +} + +func visitDuration(v rond.Visit) any { + if v.Arrival == nil || v.Departure == nil { + return nil + } + m := int(v.Departure.Sub(*v.Arrival).Minutes()) + if m < 0 { + m = 0 + } + return m +} diff --git a/internal/web/backup_merge_test.go b/internal/web/backup_merge_test.go new file mode 100644 index 0000000..58031f7 --- /dev/null +++ b/internal/web/backup_merge_test.go @@ -0,0 +1,273 @@ +package web + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// mergeBackup 走真实的合并处理函数；失败（重定向里带 err=）直接判错。 +func mergeBackup(t *testing.T, f *movementFixture, token, body string) { + t.Helper() + if loc := mergeBackupAllowErr(t, f, token, body); strings.Contains(loc, "err=") { + t.Fatalf("合并失败: %s", loc) + } +} + +// mergeBackupAllowErr 同上，但把落点交出来，供「预期会失败」的用例断言。 +func mergeBackupAllowErr(t *testing.T, f *movementFixture, token, body string) string { + t.Helper() + rec := httptest.NewRecorder() + req := httptest.NewRequest("POST", "/admin/preview/"+token+"/merge", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.SetPathValue("token", token) + f.srv.adminBackupMerge(rec, req.WithContext(f.ctx)) + if rec.Code != http.StatusSeeOther { + t.Fatalf("合并应跳回后台: code=%d body=%s", rec.Code, rec.Body.String()) + } + return rec.Header().Get("Location") +} + +// addExtraVisit 造一条「仅站内有」的到访：没有留档，于是没有 UUID，比对时必然落在仅站内有里。 +func (f *movementFixture) addExtraVisit(t *testing.T, pk int, remark string) { + t.Helper() + if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO visits + (dataset_id, src_pk, arrival, departure, is_user_added, remark) + VALUES ($1,$2,now(),now(),true,$3)`, f.dsID, pk, remark); err != nil { + t.Fatalf("插入补录到访失败: %v", err) + } +} + +func (f *movementFixture) scalar(t *testing.T, q string, args ...any) int { + t.Helper() + var n int + if err := f.pg.QueryRowContext(f.ctx, q, args...).Scan(&n); err != nil { + t.Fatalf("查询失败: %v", err) + } + return n +} + +// 合并把备份里多出来的记录写进去，而且写完要「复原」——同一份备份再比一次必须一条不差。 +// +// 这条测试同时钉住三件事： +// - 新增行用 nextSrcPK 发新号，且**写进 entity_raw**（否则下次比对认不出 UUID）； +// - 位移/天气的外键指向库里的新编号（写错就成孤儿）； +// - 合并是幂等的：再传同一份备份，差异归零。 +func TestBackupMergeRestoresMissingVisits(t *testing.T) { + f := newMovementFixture(t) + + visitCount := func() int { + return f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1`, f.dsID) + } + all := visitCount() + movCount := f.scalar(t, `SELECT count(*) FROM movements WHERE dataset_id=$1`, f.dsID) + weatherCount := f.scalar(t, `SELECT count(*) FROM weather WHERE dataset_id=$1`, f.dsID) + // 原始数据里就有指向「因缺到达时间而没入库」的到访的位移，先记下基线再比 + orphanQ := `SELECT count(*) FROM movements m WHERE m.dataset_id=$1 + AND (m.from_visit_src IS NULL OR NOT EXISTS ( + SELECT 1 FROM visits v WHERE v.dataset_id=$1 AND v.src_pk=m.from_visit_src))` + orphans0 := f.scalar(t, orphanQ, f.dsID) + + // 制造差异：按**站点删到访的方式**删掉最早的两条（见 editVisitDelete）—— + // 位移删掉、天气的归属置空、到访删掉。直接 DELETE visits 会留下孤儿天气， + // 那是测试手法造成的，不是产品行为。 + var pk1, pk2 int + if err := f.pg.QueryRowContext(f.ctx, `SELECT + (SELECT src_pk FROM visits WHERE dataset_id=$1 ORDER BY arrival LIMIT 1), + (SELECT src_pk FROM visits WHERE dataset_id=$1 ORDER BY arrival OFFSET 1 LIMIT 1)`, + f.dsID).Scan(&pk1, &pk2); err != nil { + t.Fatalf("取待删到访失败: %v", err) + } + for _, q := range []string{ + `DELETE FROM movements WHERE dataset_id=$1 AND (from_visit_src IN ($2,$3) OR to_visit_src IN ($2,$3))`, + `UPDATE weather SET visit_src=NULL WHERE dataset_id=$1 AND visit_src IN ($2,$3)`, + `DELETE FROM visits WHERE dataset_id=$1 AND src_pk IN ($2,$3)`, + } { + if _, err := f.pg.ExecContext(f.ctx, q, f.dsID, pk1, pk2); err != nil { + t.Fatalf("预置删除失败: %v", err) + } + } + if got := visitCount(); got != all-2 { + t.Fatalf("预置失败：应有 %d 条，实际 %d", all-2, got) + } + + token, page := previewBackupToken(t, f, f.src) + if a, _, _ := visitCounts(t, page); a != 2 { + t.Fatalf("差异页应报 2 条新增，实际 %d", a) + } + + mergeBackup(t, f, token, "add_new=1") + + if got := visitCount(); got != all { + t.Fatalf("合并后到访数应回到 %d，实际 %d", all, got) + } + // 新补进来的那两条必须各有一份留档（下次比对就靠它认 UUID）。 + // 被删掉的行会留下孤儿留档——那是老数据，不影响，所以只查新编号。 + if got := f.scalar(t, `SELECT count(*) FROM entity_raw er + WHERE er.dataset_id=$1 AND er.entity='ZVISIT' AND EXISTS ( + SELECT 1 FROM visits v WHERE v.dataset_id=$1 AND v.src_pk=er.src_pk)`, f.dsID); got != all { + t.Errorf("每条在库的到访都该有原始行留档，实际 %d / %d", got, all) + } + if got := f.scalar(t, `SELECT count(*) FROM movements WHERE dataset_id=$1`, f.dsID); got != movCount { + t.Errorf("位移应恢复到 %d，实际 %d", movCount, got) + } + if got := f.scalar(t, `SELECT count(*) FROM weather WHERE dataset_id=$1`, f.dsID); got != weatherCount { + t.Errorf("天气应恢复到 %d，实际 %d", weatherCount, got) + } + // 合并不该新增孤儿位移（原始数据里本来就有一些，见 orphanQ 的注释） + if orphans := f.scalar(t, orphanQ, f.dsID); orphans > orphans0 { + t.Errorf("孤儿位移变多了：%d → %d", orphans0, orphans) + } + + // 幂等：再比一次应当完全一致 + page = previewBackup(t, f, f.src) + if a, g, c := visitCounts(t, page); a != 0 || g != 0 || c != 0 { + t.Fatalf("合并后再比对应当没有差异，实际 新增 %d / 仅站内有 %d / 冲突 %d", a, g, c) + } +} + +// 冲突逐条选「用实机」就把备份的值写进去，选「用站内」就原样不动。 +func TestBackupMergeConflictChoice(t *testing.T) { + f := newMovementFixture(t) + + var pk int + if err := f.pg.QueryRowContext(f.ctx, `UPDATE visits SET remark='站内改过的备注' + WHERE id = (SELECT id FROM visits WHERE dataset_id=$1 ORDER BY src_pk LIMIT 1) + RETURNING src_pk`, f.dsID).Scan(&pk); err != nil { + t.Fatalf("改备注失败: %v", err) + } + // 这条的 UUID 就是页面上的冲突键 + var b64 string + if err := f.pg.QueryRowContext(f.ctx, `SELECT COALESCE(raw #>> '{ZIDENTIFIER_,$blob}','') + FROM entity_raw WHERE dataset_id=$1 AND entity='ZVISIT' AND src_pk=$2`, f.dsID, pk).Scan(&b64); err != nil { + t.Fatalf("取 UUID 失败: %v", err) + } + uuid := uuidFromBase64(b64) + if uuid == "" { + t.Fatal("这条到访应当有 UUID") + } + + remark := func() string { + var s string + if err := f.pg.QueryRowContext(f.ctx, `SELECT COALESCE(remark,'') FROM visits + WHERE dataset_id=$1 AND src_pk=$2`, f.dsID, pk).Scan(&s); err != nil { + t.Fatalf("读备注失败: %v", err) + } + return s + } + + // 用站内：备注保持站里那份 + token, _ := previewBackupToken(t, f, f.src) + mergeBackup(t, f, token, "vc_"+uuid+"=db") + if got := remark(); got != "站内改过的备注" { + t.Fatalf("选「用站内」不该改动，实际 %q", got) + } + + // 用实机：备注被备份里的值（空）覆盖回去 + token, _ = previewBackupToken(t, f, f.src) + mergeBackup(t, f, token, "vc_"+uuid+"=backup") + if got := remark(); got != "" { + t.Fatalf("选「用实机」应覆盖成备份里的值，实际 %q", got) + } +} + +// 什么都不勾、冲突全跳过时，合并应当什么都不写。 +func TestBackupMergeNoopKeepsData(t *testing.T) { + f := newMovementFixture(t) + before := f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1`, f.dsID) + + token, _ := previewBackupToken(t, f, f.src) + mergeBackup(t, f, token, "add_new=0") + + if got := f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1`, f.dsID); got != before { + t.Fatalf("没勾任何改动却动了数据: %d → %d", before, got) + } +} + +// 删除必须「勾了才删」：没勾的一个都不动，勾了的那条（连带它的位移）才消失。 +func TestBackupMergeDeletesOnlyChecked(t *testing.T) { + f := newMovementFixture(t) + visitCount := func() int { + return f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1`, f.dsID) + } + before := visitCount() + + f.addExtraVisit(t, 900001, "稍后删掉") + f.addExtraVisit(t, 900002, "留着不动") + if got := visitCount(); got != before+2 { + t.Fatalf("预置失败: %d", got) + } + + // 只勾一条 + token, page := previewBackupToken(t, f, f.src) + if _, g, _ := visitCounts(t, page); g != 2 { + t.Fatalf("差异页应报 2 条仅站内有，实际 %d", g) + } + mergeBackup(t, f, token, "dv_900001=1") + + if got := visitCount(); got != before+1 { + t.Fatalf("应只删掉勾选的那条，实际剩 %d（原 %d）", got, before) + } + if n := f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1 AND src_pk=900002`, f.dsID); n != 1 { + t.Error("没勾的那条不该被动") + } + // 删掉的到访不该留下留档 + if n := f.scalar(t, `SELECT count(*) FROM entity_raw WHERE dataset_id=$1 + AND entity='ZVISIT' AND src_pk=900001`, f.dsID); n != 0 { + t.Error("删除到访时要连它的原始行留档一起清掉") + } + + // 再合并一次、一条都不勾：剩下那条必须还在 + token, _ = previewBackupToken(t, f, f.src) + mergeBackup(t, f, token, "add_new=0") + if got := visitCount(); got != before+1 { + t.Fatalf("不勾就不该删，实际 %d", got) + } +} + +// 名下有到访的地点必须拒绝删除——那个外键是 ON DELETE CASCADE， +// 真删下去会把到访一起带走。拒绝后还要整体回滚，别留下半截。 +func TestBackupMergeRefusesDeletingPlaceInUse(t *testing.T) { + f := newMovementFixture(t) + visitCount := func() int { + return f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1`, f.dsID) + } + beforeVisits := visitCount() + + // 一个没有到访的空地点：允许删 + if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO places (dataset_id, src_pk, name, lat, lon) + VALUES ($1, 910001, '临时空地点', 30.0, 120.0)`, f.dsID); err != nil { + t.Fatalf("插入地点失败: %v", err) + } + // 一个有到访的地点：必须拒绝 + if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO places (dataset_id, src_pk, name, lat, lon) + VALUES ($1, 910002, '有到访的地点', 31.0, 121.0)`, f.dsID); err != nil { + t.Fatalf("插入地点失败: %v", err) + } + if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO visits (dataset_id, src_pk, place_id, arrival, departure) + VALUES ($1, 910003, (SELECT id FROM places WHERE dataset_id=$1 AND src_pk=910002), now(), now())`, + f.dsID); err != nil { + t.Fatalf("插入到访失败: %v", err) + } + + // 先删空地点：应当成功 + token, _ := previewBackupToken(t, f, f.src) + mergeBackup(t, f, token, "dp_910001=1") + if n := f.scalar(t, `SELECT count(*) FROM places WHERE dataset_id=$1 AND src_pk=910001`, f.dsID); n != 0 { + t.Fatal("空地点应当被删掉") + } + + // 再删有到访的地点：应当被拒绝 + token, _ = previewBackupToken(t, f, f.src) + loc := mergeBackupAllowErr(t, f, token, "dp_910002=1") + if !strings.Contains(loc, "err=") { + t.Fatal("名下有到访的地点应当拒绝删除") + } + if n := f.scalar(t, `SELECT count(*) FROM places WHERE dataset_id=$1 AND src_pk=910002`, f.dsID); n != 1 { + t.Error("被拒绝后地点不该消失") + } + if got := visitCount(); got != beforeVisits+1 { + t.Errorf("被拒绝后到访数不该变，实际 %d（期望 %d）", got, beforeVisits+1) + } +} diff --git a/internal/web/fog.go b/internal/web/fog.go index 88586fa..690c8d7 100644 --- a/internal/web/fog.go +++ b/internal/web/fog.go @@ -3,11 +3,13 @@ package web import ( "bytes" "context" + "errors" "fmt" "image" "image/png" "io" "log" + "math" "net/http" "path/filepath" "sort" @@ -111,41 +113,57 @@ func fogTileCacheControl(isAdmin bool) string { return "public, max-age=3600" } -// adminFogUpload 导入 .fwss：mode=overlay 按位或合并进现有迷雾， -// mode=cover 清空后整体替换。 -func (s *Server) adminFogUpload(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - u := userFrom(ctx) +// readFogSnapshot 从 multipart 表单里读出 .fwss 的字节并做基本校验。 +func readFogSnapshot(w http.ResponseWriter, r *http.Request) ([]byte, error) { r.Body = http.MaxBytesReader(w, r.Body, maxFogUploadBytes+1<<20) if err := r.ParseMultipartForm(32 << 20); err != nil { - s.adminFlash(w, r, "err", "读取上传内容失败："+err.Error()) - return + return nil, fmt.Errorf("读取上传内容失败：%w", err) } file, hdr, err := r.FormFile("fwss") if err != nil { - s.adminFlash(w, r, "err", "请选择要上传的 .fwss 迷雾快照") - return + return nil, errors.New("请选择要上传的 .fwss 迷雾快照") } defer file.Close() name := strings.ToLower(filepath.Base(hdr.Filename)) if !strings.HasSuffix(name, ".fwss") && !strings.HasSuffix(name, ".zip") { - s.adminFlash(w, r, "err", "文件类型不正确，请上传 Fog of World 导出的 .fwss 快照") - return + return nil, errors.New("文件类型不正确，请上传 Fog of World 导出的 .fwss 快照") } + data, err := io.ReadAll(io.LimitReader(file, maxFogUploadBytes+1)) + if err != nil { + return nil, fmt.Errorf("读取文件失败：%w", err) + } + if int64(len(data)) > maxFogUploadBytes { + return nil, errors.New("迷雾快照超过 64MB，疑似文件有误") + } + return data, nil +} + +// adminFogUpload 导入 .fwss：mode=overlay 按位或合并进现有迷雾， +// mode=cover 清空后整体替换。 +// +// 带 stash 令牌的请求来自差异页的确认按钮，快照已经在暂存目录里， +// 不用让浏览器把几十 MB 再传一遍。 +func (s *Server) adminFogUpload(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + u := userFrom(ctx) mode := r.PostFormValue("mode") if mode != "overlay" && mode != "cover" { mode = "overlay" } - data, err := io.ReadAll(io.LimitReader(file, maxFogUploadBytes+1)) - if err != nil { - s.adminFlash(w, r, "err", "读取文件失败："+err.Error()) - return - } - if int64(len(data)) > maxFogUploadBytes { - s.adminFlash(w, r, "err", "迷雾快照超过 64MB，疑似文件有误") + token := r.URL.Query().Get("stash") + var data []byte + var err error + if token != "" { + if data, err = s.loadStashedFog(token); err != nil { + s.adminFlash(w, r, "err", err.Error()) + return + } + } else if data, err = readFogSnapshot(w, r); err != nil { + s.adminFlash(w, r, "err", err.Error()) return } + blocks, cells, err := fog.Decode(data) if err != nil { s.adminFlash(w, r, "err", "解析迷雾快照失败："+err.Error()) @@ -181,10 +199,125 @@ func (s *Server) adminFogUpload(w http.ResponseWriter, r *http.Request) { if mode == "cover" { verb = "整体替换" } + // 导入成功才丢掉暂存件；失败时留着，站长可以回差异页重试 + if token != "" { + s.dropStashedFog(token) + } s.adminFlash(w, r, "ok", fmt.Sprintf( "迷雾导入成功（%s）：本次写入 %d 块 / %d 格，当前共 %d 块 / %d 格", verb, applied, cells, b, c)) } +// adminFogPreview 先把快照收下暂存，不写库，跳到差异页—— +// 叠加能补多少、整体替换会丢多少，都得让站长在下手之前看见。 +func (s *Server) adminFogPreview(w http.ResponseWriter, r *http.Request) { + data, err := readFogSnapshot(w, r) + if err != nil { + s.adminFlash(w, r, "err", err.Error()) + return + } + // 先解一遍确认是真快照，免得把坏文件暂存下来、到差异页才发现 + if _, _, err := fog.Decode(data); err != nil { + s.adminFlash(w, r, "err", "解析迷雾快照失败："+err.Error()) + return + } + token, err := s.stashFog(data) + if err != nil { + s.serverError(w, r, err) + return + } + http.Redirect(w, r, "/admin/fog/preview/"+token, http.StatusSeeOther) +} + +// adminFogPreviewPage 渲染差异页。每次都重新算一遍：库里可能刚被别的操作改过。 +func (s *Server) adminFogPreviewPage(w http.ResponseWriter, r *http.Request) { + p, err := s.pageBase(r, "admin", "迷雾导入预览") + if err != nil { + s.serverError(w, r, err) + return + } + token := r.PathValue("token") + data, err := s.loadStashedFog(token) + if err != nil { + s.adminFlash(w, r, "err", err.Error()) + return + } + blocks, _, err := fog.Decode(data) + if err != nil { + s.adminFlash(w, r, "err", "解析迷雾快照失败："+err.Error()) + return + } + if len(blocks) == 0 { + s.adminFlash(w, r, "err", "快照里没有找到迷雾瓦片数据") + return + } + layers, err := fog.RawLayers(data) + if err != nil { + log.Printf("解析快照的其它图层失败: %v", err) + } + ix, err := s.fogIndex() + if err != nil { + s.serverError(w, r, err) + return + } + var cur []fog.Block + ix.View(func(m map[[2]int][]byte) { + cur = make([]fog.Block, 0, len(m)) + for k, bm := range m { + cur = append(cur, fog.Block{GX: k[0], GY: k[1], Bitmap: bm}) + } + }) + tilde, hash, other := fogLayerCounts(layers) + d := FogPreviewData{ + Page: p, Token: token, Diff: fog.DiffBlocks(cur, blocks), + TildeLayers: tilde, HashLayers: hash, OtherLayers: other, + } + d.MinLat, d.MinLon, d.MaxLat, d.MaxLon, d.HasBounds = diffBounds(cur, blocks, s.cfg.MapCRS) + // 布局按这个标记引入 Leaflet，漏了地图整块不出现 + d.Page.HasMap = d.HasBounds + s.render(w, "admin/fog-preview", d) +} + +// diffBounds 返回两边块集覆盖范围的包围盒，供差异地图定位。迷雾网格是 WGS-84， +// 高德这类底图按 GCJ-02 摆放，所以坐标要跟着底图走，否则视野会偏出几百米。 +func diffBounds(cur, inc []fog.Block, crs string) (minLat, minLon, maxLat, maxLon float64, ok bool) { + visit := func(gbx, gby int) { + lat, lon := fog.BlockLonLat(gbx, gby) + if crs == "gcj02" { + lat, lon = geo.WGS84ToGCJ02(lat, lon) + } + if !ok { + minLat, maxLat, minLon, maxLon, ok = lat, lat, lon, lon, true + return + } + minLat, maxLat = math.Min(minLat, lat), math.Max(maxLat, lat) + minLon, maxLon = math.Min(minLon, lon), math.Max(maxLon, lon) + } + for _, b := range cur { + visit(b.GX, b.GY) + } + for _, b := range inc { + visit(b.GX, b.GY) + } + return +} + +// fogLayerCounts 按前缀归类快照里「已探索层之外」的条目。 +// Model/~ 与 Model/# 两层站点不解析、只原样留存，但整体替换会把它们一并换掉， +// 差异页要把这件事说清楚。 +func fogLayerCounts(layers map[string][]byte) (tilde, hash, other int) { + for p := range layers { + switch { + case strings.HasPrefix(p, "Model/~/") || strings.HasPrefix(p, "Model~/"): + tilde++ + case strings.HasPrefix(p, "Model/#/") || strings.HasPrefix(p, "Model#/"): + hash++ + default: + other++ + } + } + return +} + // ---------- 迷雾覆盖率 ---------- // FogCity 是单个城市（或省份）的迷雾点亮统计：地点里有多少个落在已探索格内。 diff --git a/internal/web/fog_diff.go b/internal/web/fog_diff.go new file mode 100644 index 0000000..ceeac48 --- /dev/null +++ b/internal/web/fog_diff.go @@ -0,0 +1,94 @@ +package web + +import ( + "bytes" + "image/png" + "net/http" + "strconv" + "sync" + "time" + + "rond-with-you/internal/fog" +) + +// 差异瓦片要反复读同一份快照、同一份库内迷雾，单槽缓存住两边的块集即可—— +// 站长一次只会看一份预览。缓存里的 map 加载完就只读，渲染时并发遍历是安全的。 +var ( + diffSetsMu sync.Mutex + diffSetsToken string + diffSetsAt time.Time + diffSetsCur map[[2]int][]byte + diffSetsInc map[[2]int][]byte +) + +// diffSetsTTL 短于暂存的 2 小时：库内迷雾随时可能被别的操作改掉， +// 缓存太久会让差异图与页面上那组数字对不上。 +const diffSetsTTL = 10 * time.Minute + +func (s *Server) fogDiffSets(token string) (cur, inc map[[2]int][]byte, err error) { + diffSetsMu.Lock() + defer diffSetsMu.Unlock() + if token == diffSetsToken && time.Since(diffSetsAt) < diffSetsTTL { + return diffSetsCur, diffSetsInc, nil + } + data, err := s.loadStashedFog(token) + if err != nil { + return nil, nil, err + } + blocks, _, err := fog.Decode(data) + if err != nil { + return nil, nil, err + } + ix, err := s.fogIndex() + if err != nil { + return nil, nil, err + } + cur = map[[2]int][]byte{} + ix.View(func(m map[[2]int][]byte) { + for k, bm := range m { + cur[k] = bm + } + }) + inc = make(map[[2]int][]byte, len(blocks)) + for _, b := range blocks { + inc[[2]int{b.GX, b.GY}] = b.Bitmap + } + diffSetsToken, diffSetsCur, diffSetsInc, diffSetsAt = token, cur, inc, time.Now() + return cur, inc, nil +} + +// adminFogDiffTile 渲染一张差异瓦片：绿=叠加会新增、灰=两边都有、红=整体替换会丢。 +// 只给站长看，不涉及遮罩。 +func (s *Server) adminFogDiffTile(w http.ResponseWriter, r *http.Request) { + z, err1 := strconv.Atoi(r.PathValue("z")) + x, err2 := strconv.Atoi(r.PathValue("x")) + y, err3 := strconv.Atoi(r.PathValue("y")) + if err1 != nil || err2 != nil || err3 != nil || z < 0 || z > 22 || + x < 0 || y < 0 || x >= 1<= 1< c[0] && c[1] > c[2]: + return "gain" + case c[0] > c[1] && c[0] > c[2]: + return "lost" + } + return "both" +} + +func pixAtImg(img image.Image, px, py int) [4]byte { + r, g, b, a := img.At(px, py).RGBA() + return [4]byte{byte(r >> 8), byte(g >> 8), byte(b >> 8), byte(a >> 8)} +} + +// 令牌来自 URL，必须挡死路径穿越。 +func TestFogStashTokenRejectsTraversal(t *testing.T) { + s := &Server{cfg: &config.Config{DataDir: t.TempDir()}} + for _, bad := range []string{"", "..", "../../etc/passwd", strings.Repeat("a", 32) + "/../x", strings.ToUpper(strings.Repeat("a", 32))} { + if p, ok := s.fogStashPath(bad); ok { + t.Errorf("令牌 %q 不该通过校验，得到 %s", bad, p) + } + } + if _, ok := s.fogStashPath(strings.Repeat("0", 32)); !ok { + t.Error("32 位十六进制令牌应当通过") + } +} + +func newFogBlock(gx, gy int, cells ...[2]int) fog.Block { + bm := make([]byte, 512) + for _, c := range cells { + bm[c[1]*8+c[0]/8] |= 0x80 >> (c[0] % 8) + } + return fog.Block{GX: gx, GY: gy, Bitmap: bm} +} + +func fogUploadRequest(t *testing.T, data []byte) *http.Request { + t.Helper() + var buf bytes.Buffer + mw := multipart.NewWriter(&buf) + fw, err := mw.CreateFormFile("fwss", "Snapshot-test.fwss") + if err != nil { + t.Fatalf("构造上传表单失败: %v", err) + } + if _, err := fw.Write(data); err != nil { + t.Fatalf("写入上传内容失败: %v", err) + } + mw.Close() + req := httptest.NewRequest("POST", "/admin/fog/preview", &buf) + req.Header.Set("Content-Type", mw.FormDataContentType()) + return req +} diff --git a/internal/web/fog_staging.go b/internal/web/fog_staging.go new file mode 100644 index 0000000..ab513d0 --- /dev/null +++ b/internal/web/fog_staging.go @@ -0,0 +1,116 @@ +package web + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +// 待确认的上传（迷雾快照、rond 备份）都先落到磁盘暂存，让站长看完差异再决定怎么导入； +// 看完直接走人的靠 TTL 回收，不至于把几十 MB 的文件一直留在盘上。 +// +// 令牌只认固定长度的十六进制串，这样即便来自 URL 也拼不出 ../ 之类的路径。 + +const ( + stashTokenLen = 32 // 16 字节的十六进制 + stashTTL = 2 * time.Hour +) + +func stashPut(dir, ext string, data []byte) (string, error) { + if err := os.MkdirAll(dir, 0o700); err != nil { + return "", err + } + var raw [stashTokenLen / 2]byte + if _, err := rand.Read(raw[:]); err != nil { + return "", err + } + token := hex.EncodeToString(raw[:]) + if err := os.WriteFile(filepath.Join(dir, token+ext), data, 0o600); err != nil { + return "", err + } + stashSweep(dir, ext) + return token, nil +} + +func stashPath(dir, ext, token string) (string, bool) { + if len(token) != stashTokenLen { + return "", false + } + for i := 0; i < len(token); i++ { + c := token[i] + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return "", false + } + } + return filepath.Join(dir, token+ext), true +} + +func stashRead(dir, ext, token, what string) ([]byte, error) { + p, ok := stashPath(dir, ext, token) + if !ok { + return nil, fmt.Errorf("预览链接无效，请重新上传%s", what) + } + data, err := os.ReadFile(p) + if err != nil { + return nil, fmt.Errorf("这次预览已过期，请重新上传%s", what) + } + return data, nil +} + +func stashDrop(dir, ext, token string) { + if p, ok := stashPath(dir, ext, token); ok { + os.Remove(p) + } +} + +func stashSweep(dir, ext string) { + entries, err := os.ReadDir(dir) + if err != nil { + return + } + cutoff := time.Now().Add(-stashTTL) + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ext) { + continue + } + if info, err := e.Info(); err == nil && info.ModTime().After(cutoff) { + continue + } + os.Remove(filepath.Join(dir, e.Name())) + } +} + +// ---------- 迷雾快照 ---------- + +// fogStashTokenLen 是迷雾预览令牌的长度，测试与调用方按它校验。 +const fogStashTokenLen = stashTokenLen + +const fogStashExt = ".fwss" + +func (s *Server) fogStashDir() string { + return filepath.Join(s.cfg.DataDir, "fog-staging") +} + +func (s *Server) stashFog(data []byte) (string, error) { + return stashPut(s.fogStashDir(), fogStashExt, data) +} + +func (s *Server) fogStashPath(token string) (string, bool) { + return stashPath(s.fogStashDir(), fogStashExt, token) +} + +func (s *Server) loadStashedFog(token string) ([]byte, error) { + return stashRead(s.fogStashDir(), fogStashExt, token, "快照") +} + +func (s *Server) dropStashedFog(token string) { + stashDrop(s.fogStashDir(), fogStashExt, token) +} + +func (s *Server) sweepFogStash() { + stashSweep(s.fogStashDir(), fogStashExt) +} diff --git a/internal/web/handlers_admin.go b/internal/web/handlers_admin.go index d8d26ca..9a7530a 100644 --- a/internal/web/handlers_admin.go +++ b/internal/web/handlers_admin.go @@ -769,12 +769,12 @@ func (s *Server) notesPage(w http.ResponseWriter, r *http.Request) { case "note": order = "n.updated_at DESC NULLS LAST, p.name" case "talk": - order = "(COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.src_pk = p.src_pk),0)) DESC, p.visit_count DESC" + order = "(COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.place_key = " + placeKeyExpr + "),0)) DESC, p.visit_count DESC" } base := `FROM places p - LEFT JOIN place_notes n ON n.src_pk = p.src_pk AND n.dataset_id = p.dataset_id - LEFT JOIN place_votes v ON v.src_pk = p.src_pk AND v.dataset_id = p.dataset_id + LEFT JOIN place_notes n ON n.place_key = ` + placeKeyExpr + ` + LEFT JOIN place_votes v ON v.place_key = ` + placeKeyExpr + ` WHERE ` + strings.Join(where, " AND ") var total int @@ -790,8 +790,8 @@ func (s *Server) notesPage(w http.ResponseWriter, r *http.Request) { limitArgs := append(append([]any{}, args...), notesPerPage, (d.PageNo-1)*notesPerPage) q2 := `SELECT p.src_pk, COALESCE(n.alias,''), COALESCE(n.note,''), COALESCE(n.rond_note,''), COALESCE(v.verdict,0), - COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.src_pk=p.src_pk AND vv.vote=1),0), - COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.src_pk=p.src_pk AND vv.vote=-1),0), + COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.place_key = ` + placeKeyExpr + ` AND vv.vote=1),0), + COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.place_key = ` + placeKeyExpr + ` AND vv.vote=-1),0), COALESCE(n.updated_at, to_timestamp(0)), p.id, COALESCE(p.name,''), COALESCE(p.city,''), COALESCE(p.visit_count,0) ` + base + ` ORDER BY ` + order + fmt.Sprintf(` LIMIT $%d OFFSET $%d`, len(args)+1, len(args)+2) diff --git a/internal/web/handlers_edit.go b/internal/web/handlers_edit.go index 12a52e7..1b6d7b5 100644 --- a/internal/web/handlers_edit.go +++ b/internal/web/handlers_edit.go @@ -1742,6 +1742,7 @@ var ( activitySeq = srcPKSeq{"activities", "src_pk", "Activity", "ZACTIVITY"} weatherSeq = srcPKSeq{"weather", "src_pk", "HourlyWeather", "ZHOURLYWEATHER"} movementSeq = srcPKSeq{"movements", "src_pk", "Movement", "ZMOVEMENT"} + tagSeq = srcPKSeq{"tags", "src_pk", "Tag", "ZTAG"} transportSeq = srcPKSeq{"movements", "transport_src", "Transport", "ZTRANSPORT"} ) diff --git a/internal/web/handlers_images.go b/internal/web/handlers_images.go index fc28c76..79ced8a 100644 --- a/internal/web/handlers_images.go +++ b/internal/web/handlers_images.go @@ -3,6 +3,7 @@ package web import ( "context" "crypto/sha256" + "database/sql" "encoding/hex" "fmt" "io" @@ -26,6 +27,11 @@ const ( imageUploadMem = 8 << 20 // multipart 解析用的内存上限 ) +// placeKeyExpr 把 places p 那一行的坐标换算成 place_key，图片、备注、结论共用这个键。 +// **必须与 rond.PlaceCoordKey 以及 notes/votes 包里的同款表达式逐位一致**： +// fmt.Sprintf("%.6f|%.6f") ↔ to_char(x,'FM999990.000000')||'|'||to_char(y,'FM999990.000000') +const placeKeyExpr = `to_char(p.lat,'FM999990.000000')||'|'||to_char(p.lon,'FM999990.000000')` + // PlaceImage 是模板里用的地点图片。 type PlaceImage struct { ID int64 @@ -57,14 +63,18 @@ func (s *Server) savePlaceImage(ctx context.Context, datasetID int64, srcPK int, if err := s.media.Save(ctx, backend, key, data, "image/webp"); err != nil { return 0, "", err } + pkey, err := s.placeKeyOf(ctx, datasetID, srcPK) + if err != nil { + return 0, "", err + } var id int64 if err := s.db.QueryRowContext(ctx, `INSERT INTO place_images - (dataset_id, src_pk, storage, object_key, width, height, bytes, position) - VALUES ($1,$2,$3,$4,$5,$6,$7, - COALESCE((SELECT max(position)+1 FROM place_images WHERE dataset_id=$1 AND src_pk=$2), 0)) + (dataset_id, src_pk, place_key, storage, object_key, width, height, bytes, position) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8, + COALESCE((SELECT max(position)+1 FROM place_images WHERE place_key=$3), 0)) ON CONFLICT (dataset_id, src_pk, object_key) DO UPDATE SET bytes=EXCLUDED.bytes RETURNING id`, - datasetID, srcPK, backend, key, w, h, int64(len(data))).Scan(&id); err != nil { + datasetID, srcPK, pkey, backend, key, w, h, int64(len(data))).Scan(&id); err != nil { return 0, "", err } return id, s.media.PublicURL(backend, key), nil @@ -95,9 +105,15 @@ func (s *Server) adminPlaceImageUpload(w http.ResponseWriter, r *http.Request) { return } + pkey, err := s.placeKeyOf(ctx, ds.ID, src) + if err != nil { + s.serverError(w, r, err) + return + } var count int + // 按坐标键数：换数据集后图片仍跟着同一个地点，配额也要跟着 if err := s.db.QueryRowContext(ctx, `SELECT count(*) FROM place_images - WHERE dataset_id=$1 AND src_pk=$2`, ds.ID, src).Scan(&count); err != nil { + WHERE place_key=$1`, pkey).Scan(&count); err != nil { s.serverError(w, r, err) return } @@ -196,10 +212,28 @@ func (s *Server) imageBackend(ctx context.Context) (string, error) { return setting.MediaBackendLocal, nil } +// placeKeyOf 把某个数据集里的地点编号换算成坐标键。 +// 换数据集之后仍靠它认出「还是同一个地点」，所以图片、备注、结论都挂在这上面。 +func (s *Server) placeKeyOf(ctx context.Context, datasetID int64, srcPK int) (string, error) { + var key string + err := s.db.QueryRowContext(ctx, `SELECT `+placeKeyExpr+` FROM places p + WHERE p.dataset_id=$1 AND p.src_pk=$2 AND p.lat IS NOT NULL AND p.lon IS NOT NULL`, + datasetID, srcPK).Scan(&key) + if err == sql.ErrNoRows { + return "", fmt.Errorf("地点不存在或没有坐标") + } + return key, err +} + // placeImagesOf 读某个地点的图片，按 position 排序。 +// 按坐标键查而不是 (dataset_id, src_pk)：换一份备份之后图片还跟着同一个地点。 func (s *Server) placeImagesOf(ctx context.Context, datasetID int64, srcPK int) ([]PlaceImage, error) { + pkey, err := s.placeKeyOf(ctx, datasetID, srcPK) + if err != nil { + return nil, nil // 地点没有坐标就当作没有图 + } rows, err := s.db.QueryContext(ctx, `SELECT id, storage, object_key, width, height, bytes - FROM place_images WHERE dataset_id=$1 AND src_pk=$2 ORDER BY position, id`, datasetID, srcPK) + FROM place_images WHERE place_key=$1 ORDER BY position, id`, pkey) if err != nil { return nil, err } diff --git a/internal/web/handlers_public.go b/internal/web/handlers_public.go index 585dc81..2e98e91 100644 --- a/internal/web/handlers_public.go +++ b/internal/web/handlers_public.go @@ -183,7 +183,7 @@ func (s *Server) attachVotes(ctx context.Context, datasetID int64, places []stat if err != nil { return } - tallies, err := s.votes.Tallies(ctx) + tallies, err := s.votes.Tallies(ctx, datasetID) if err != nil { return } @@ -539,8 +539,8 @@ func (s *Server) placePage(w http.ResponseWriter, r *http.Request) { } } d.Verdict, _ = s.votes.Verdict(r.Context(), p.Dataset.ID, place.SrcPK) - d.Votes, _ = s.votes.Tally(r.Context(), place.SrcPK) - d.MyVote = s.votes.Voter(r.Context(), place.SrcPK, s.voterKeyFromRequest(r)) + d.Votes, _ = s.votes.Tally(r.Context(), p.Dataset.ID, place.SrcPK) + d.MyVote = s.votes.Voter(r.Context(), p.Dataset.ID, place.SrcPK, s.voterKeyFromRequest(r)) // 图片：访客也要看得到，所以不分管理员 if imgs, err := s.placeImagesOf(r.Context(), p.Dataset.ID, place.SrcPK); err == nil { d.Images = imgs diff --git a/internal/web/handlers_vote.go b/internal/web/handlers_vote.go index cf86cad..26827c8 100644 --- a/internal/web/handlers_vote.go +++ b/internal/web/handlers_vote.go @@ -102,12 +102,12 @@ func (s *Server) apiVotes(w http.ResponseWriter, r *http.Request) { s.json(w, map[string]string{"error": err.Error()}) return } - tallies, err := s.votes.Tallies(ctx) + tallies, err := s.votes.Tallies(ctx, votesDatasetID) if err != nil { s.json(w, map[string]string{"error": err.Error()}) return } - mine, err := s.votes.VoterAll(ctx, s.voterKeyFromRequest(r)) + mine, err := s.votes.VoterAll(ctx, votesDatasetID, s.voterKeyFromRequest(r)) if err != nil { s.json(w, map[string]string{"error": err.Error()}) return @@ -145,10 +145,16 @@ func (s *Server) apiVerdict(w http.ResponseWriter, r *http.Request) { // apiVote 记录访客对某地点的点赞 / 点踩，返回最新票数与该访客的选择。 func (s *Server) apiVote(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() if err := r.ParseForm(); err != nil { s.json(w, map[string]string{"error": "请求解析失败"}) return } + ds, err := s.datasetFor(ctx) + if err != nil || ds == nil { + s.json(w, map[string]string{"error": "还没有可投票的数据集"}) + return + } srcPK, err := strconv.Atoi(r.PostFormValue("src_pk")) vote, _ := strconv.Atoi(r.PostFormValue("vote")) if err != nil || (vote != 1 && vote != -1) { @@ -160,7 +166,7 @@ func (s *Server) apiVote(w http.ResponseWriter, r *http.Request) { s.json(w, map[string]string{"error": "无法识别访客"}) return } - t, mine, err := s.votes.Cast(r.Context(), srcPK, key, vote) + t, mine, err := s.votes.Cast(ctx, ds.ID, srcPK, key, vote) if err != nil { s.json(w, map[string]string{"error": err.Error()}) return @@ -170,11 +176,17 @@ func (s *Server) apiVote(w http.ResponseWriter, r *http.Request) { // voteTotals 汇总站长结论数量与访客票数，供统计页展示。 func (s *Server) voteTotals(ctx context.Context, datasetID int64) (rec, avoid, up, down int, err error) { + // 两张票表都挂坐标键，统计要按「本数据集的地点」圈定范围， + // 不能再按它们自己的 dataset_id 过滤——那个只是最后写入时留下的线索。 err = s.db.QueryRowContext(ctx, `SELECT - (SELECT count(*) FROM place_votes WHERE dataset_id=$1 AND verdict=1), - (SELECT count(*) FROM place_votes WHERE dataset_id=$1 AND verdict=-1), - (SELECT count(*) FROM place_visitor_votes WHERE vote=1), - (SELECT count(*) FROM place_visitor_votes WHERE vote=-1)`, datasetID). + (SELECT count(*) FROM place_votes v JOIN places p ON v.place_key = `+placeKeyExpr+` + WHERE p.dataset_id=$1 AND v.verdict=1), + (SELECT count(*) FROM place_votes v JOIN places p ON v.place_key = `+placeKeyExpr+` + WHERE p.dataset_id=$1 AND v.verdict=-1), + (SELECT count(*) FROM place_visitor_votes vv JOIN places p ON vv.place_key = `+placeKeyExpr+` + WHERE p.dataset_id=$1 AND vv.vote=1), + (SELECT count(*) FROM place_visitor_votes vv JOIN places p ON vv.place_key = `+placeKeyExpr+` + WHERE p.dataset_id=$1 AND vv.vote=-1)`, datasetID). Scan(&rec, &avoid, &up, &down) return } @@ -183,7 +195,7 @@ func (s *Server) voteTotals(ctx context.Context, datasetID int64) (rec, avoid, u func (s *Server) verdictPlaces(ctx context.Context, datasetID int64, verdict, limit int) ([]stats.Place, error) { q := `SELECT ` + stats.PlaceCols + ` FROM places p WHERE p.dataset_id=$1 AND EXISTS (SELECT 1 FROM place_votes pv - WHERE pv.src_pk=p.src_pk AND pv.dataset_id=p.dataset_id AND pv.verdict=$2) + WHERE pv.place_key = ` + placeKeyExpr + ` AND pv.verdict=$2) ORDER BY p.visit_count DESC LIMIT ` + strconv.Itoa(limit) rows, err := s.db.QueryContext(ctx, q, datasetID, verdict) if err != nil { diff --git a/internal/web/place_identity_test.go b/internal/web/place_identity_test.go new file mode 100644 index 0000000..0cc9f3d --- /dev/null +++ b/internal/web/place_identity_test.go @@ -0,0 +1,149 @@ +package web + +import ( + "fmt" + "testing" + "time" + + "rond-with-you/internal/votes" +) + +// testPlace 在两个数据集里各建一个**同坐标**的地点，返回它们的编号与第二个数据集。 +// +// 坐标是随手挑的专属值：备注按坐标共享，而读取按设计不按 user_id 过滤（单站长站点）， +// 借用真实数据里的坐标会把别的测试残留的记录串进来。 +func testPlace(t *testing.T, f *movementFixture, uid int64, lat, lon float64) (pkA, pkB int, dsB int64) { + t.Helper() + const pkAVal, pkBVal = 660001, 660002 + if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO places (dataset_id, src_pk, name, lat, lon) + VALUES ($1,$2,'甲侧地点',$3,$4)`, f.dsID, pkAVal, lat, lon); err != nil { + t.Fatalf("建地点失败: %v", err) + } + sha := fmt.Sprintf("%064d", time.Now().UnixNano()) + if err := f.pg.QueryRowContext(f.ctx, `INSERT INTO datasets (user_id, original_name, sha256, status, is_active) + VALUES ($1,'另一个数据集',$2,'done',false) RETURNING id`, uid, sha).Scan(&dsB); err != nil { + t.Fatalf("建数据集失败: %v", err) + } + if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO places (dataset_id, src_pk, name, lat, lon) + VALUES ($1,$2,'乙侧地点',$3,$4)`, dsB, pkBVal, lat, lon); err != nil { + t.Fatalf("建地点失败: %v", err) + } + key := fmt.Sprintf("%.6f|%.6f", lat, lon) + t.Cleanup(func() { + f.pg.ExecContext(f.ctx, `DELETE FROM place_notes WHERE place_key=$1`, key) + f.pg.ExecContext(f.ctx, `DELETE FROM place_votes WHERE place_key=$1`, key) + f.pg.ExecContext(f.ctx, `DELETE FROM place_visitor_votes WHERE place_key=$1`, key) + f.pg.ExecContext(f.ctx, `DELETE FROM datasets WHERE id=$1`, dsB) + }) + return pkAVal, pkBVal, dsB +} + +// 访客票也一样：它是站点自己的数据（rondbackup 里没有这一项）， +// 但同样要认得出「是哪个地点」，所以也挂坐标键。 +func TestVisitorVotesFollowPlaceAcrossDatasets(t *testing.T) { + f := newMovementFixture(t) + uid := userFrom(f.ctx).ID + pkA, pkB, dsB := testPlace(t, f, uid, 77.777777, 88.888888) + + if _, mine, err := f.srv.votes.Cast(f.ctx, f.dsID, pkA, "visitor-a", 1); err != nil || mine != 1 { + t.Fatalf("投票失败: mine=%d err=%v", mine, err) + } + + tally, err := f.srv.votes.Tally(f.ctx, dsB, pkB) + if err != nil { + t.Fatalf("读票数失败: %v", err) + } + if tally.Up != 1 || tally.Down != 0 { + t.Fatalf("换数据集后票数应照旧，实际 %+v", tally) + } + if v := f.srv.votes.Voter(f.ctx, dsB, pkB, "visitor-a"); v != 1 { + t.Fatalf("「我投的」也该认得出来，实际 %d", v) + } + // 再点一次是撤销：从哪一侧点都该撤掉同一张票 + if _, mine, err := f.srv.votes.Cast(f.ctx, dsB, pkB, "visitor-a", 1); err != nil || mine != 0 { + t.Fatalf("再点一次应撤销: mine=%d err=%v", mine, err) + } + if tally, err := f.srv.votes.Tally(f.ctx, f.dsID, pkA); err != nil || tally.Up != 0 { + t.Fatalf("撤销后另一边也该归零: %+v err=%v", tally, err) + } +} + +// 备注、结论、图片挂的是坐标键，所以**换数据集之后仍然跟着同一个地点**—— +// 这正是这次迁移要解决的问题。老结构挂 (dataset_id, src_pk)， +// 换一份备份同编号就是另一个地点，内容全部失联。 +func TestNotesAndVerdictsFollowPlaceAcrossDatasets(t *testing.T) { + f := newMovementFixture(t) + uid := userFrom(f.ctx).ID + pkA, pkB, dsB := testPlace(t, f, uid, 11.111111, 22.222222) + + if err := f.srv.notes.Save(f.ctx, uid, f.dsID, pkA, "别名甲", "备注甲"); err != nil { + t.Fatalf("写备注失败: %v", err) + } + if err := f.srv.votes.SaveVerdict(f.ctx, uid, f.dsID, pkA, votes.VerdictRecommend); err != nil { + t.Fatalf("写结论失败: %v", err) + } + + // 从「另一个数据集」读：编号完全不同，但坐标一致，应当照样读得到 + all, err := f.srv.notes.All(f.ctx, dsB) + if err != nil { + t.Fatalf("读备注失败: %v", err) + } + got, ok := all[pkB] + if !ok { + t.Fatal("换数据集后应当仍能读到这个地点的备注") + } + if got.Alias != "别名甲" || got.Content != "备注甲" { + t.Fatalf("备注应原样跟过来，实际 alias=%q note=%q", got.Alias, got.Content) + } + + v, err := f.srv.votes.Verdict(f.ctx, dsB, pkB) + if err != nil { + t.Fatalf("读结论失败: %v", err) + } + if v != votes.VerdictRecommend { + t.Fatalf("结论应跟过来，实际 %d", v) + } +} + +// 两个数据集里的编号完全不同，但备注与结论是同一份——改一处两边都变。 +func TestNotesUpdateVisibleFromOtherDataset(t *testing.T) { + f := newMovementFixture(t) + uid := userFrom(f.ctx).ID + pkA, pkB, dsB := testPlace(t, f, uid, 33.333333, 44.444444) + + if err := f.srv.notes.Save(f.ctx, uid, dsB, pkB, "别名乙", "备注乙"); err != nil { + t.Fatalf("在新数据集写备注失败: %v", err) + } + back, err := f.srv.notes.All(f.ctx, f.dsID) + if err != nil { + t.Fatalf("回读失败: %v", err) + } + if back[pkA].Content != "备注乙" { + t.Fatalf("备注是同一份数据，改一处两边都该变，实际 %q", back[pkA].Content) + } +} + +// 坐标不同的地点绝不能互相认领——键是坐标，不是「大概位置」。 +func TestNotesDoNotLeakBetweenDifferentPlaces(t *testing.T) { + f := newMovementFixture(t) + uid := userFrom(f.ctx).ID + pkA, _, dsB := testPlace(t, f, uid, 55.555555, 66.666666) + + if err := f.srv.notes.Save(f.ctx, uid, f.dsID, pkA, "别名甲", "备注甲"); err != nil { + t.Fatalf("写备注失败: %v", err) + } + // 在乙数据集里放一个坐标差 0.001 度（约 100m）的地点，是完全不同的地方 + const otherPK = 660003 + if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO places (dataset_id, src_pk, name, lat, lon) + VALUES ($1,$2,'隔壁那家', 55.556555, 66.666666)`, dsB, otherPK); err != nil { + t.Fatalf("建地点失败: %v", err) + } + + all, err := f.srv.notes.All(f.ctx, dsB) + if err != nil { + t.Fatalf("读备注失败: %v", err) + } + if _, ok := all[otherPK]; ok { + t.Fatal("坐标不同的地点不该认领到别人的备注") + } +} diff --git a/internal/web/place_image_test.go b/internal/web/place_image_test.go index fdda16f..a701a07 100644 --- a/internal/web/place_image_test.go +++ b/internal/web/place_image_test.go @@ -317,14 +317,16 @@ func TestVisitorPlacePageCarriesImageURLs(t *testing.T) { var placeID int64 var srcPK int if err := f.pg.QueryRowContext(f.ctx, `SELECT id, src_pk FROM places - WHERE dataset_id=$1 ORDER BY id LIMIT 1`, f.dsID).Scan(&placeID, &srcPK); err != nil { + WHERE dataset_id=$1 AND lat IS NOT NULL ORDER BY id LIMIT 1`, f.dsID).Scan(&placeID, &srcPK); err != nil { t.Fatalf("找不到地点: %v", err) } const key = "9/visitor-page-test.webp" - // 这里只验发下去的地址，不验能否取回字节，所以不用真的写文件 + // 这里只验发下去的地址，不验能否取回字节，所以不用真的写文件。 + // place_key 得照真实写入路径一并算好——读取是认坐标键的，不认 (dataset_id, src_pk)。 if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO place_images - (dataset_id, src_pk, storage, object_key, width, height, bytes, position) - VALUES ($1,$2,'local',$3,1200,900,1024,0) + (dataset_id, src_pk, place_key, storage, object_key, width, height, bytes, position) + SELECT $1, $2, `+placeKeyExpr+`, 'local', $3, 1200, 900, 1024, 0 + FROM places p WHERE p.dataset_id=$1 AND p.src_pk=$2 ON CONFLICT (dataset_id, src_pk, object_key) DO NOTHING`, f.dsID, srcPK, key); err != nil { t.Fatalf("插入图片记录失败: %v", err) diff --git a/internal/web/server.go b/internal/web/server.go index 92076a7..5ba9a92 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -40,26 +40,28 @@ var pageFiles = map[string]struct { files []string entry string }{ - "home": {[]string{"templates/layout.html", "templates/home.html"}, "layout"}, - "map": {[]string{"templates/layout.html", "templates/map.html"}, "layout"}, - "places": {[]string{"templates/layout.html", "templates/places.html"}, "layout"}, - "place": {[]string{"templates/layout.html", "templates/place.html"}, "layout"}, - "timeline": {[]string{"templates/layout.html", "templates/timeline.html"}, "layout"}, - "stats": {[]string{"templates/layout.html", "templates/stats.html"}, "layout"}, - "report": {[]string{"templates/layout.html", "templates/report.html"}, "layout"}, - "admin/index": {[]string{"templates/layout.html", "templates/admin/index.html"}, "layout"}, - "admin/settings": {[]string{"templates/layout.html", "templates/admin/settings.html"}, "layout"}, - "admin/password": {[]string{"templates/layout.html", "templates/admin/password.html"}, "layout"}, - "admin/login": {[]string{"templates/admin/login.html"}, "login"}, - "admin/twofa-login": {[]string{"templates/admin/twofa-login.html"}, "login2"}, - "admin/twofa": {[]string{"templates/layout.html", "templates/admin/twofa.html"}, "layout"}, - "admin/fog-mask": {[]string{"templates/layout.html", "templates/admin/fog-mask.html"}, "layout"}, - "admin/notes": {[]string{"templates/layout.html", "templates/admin/notes.html"}, "layout"}, - "admin/edit": {[]string{"templates/layout.html", "templates/admin/edit.html"}, "layout"}, - "admin/topics": {[]string{"templates/layout.html", "templates/admin/topics.html"}, "layout"}, - "admin/photos": {[]string{"templates/layout.html", "templates/admin/photos.html"}, "layout"}, - "topics": {[]string{"templates/layout.html", "templates/topics.html"}, "layout"}, - "topic": {[]string{"templates/layout.html", "templates/topic.html"}, "layout"}, + "home": {[]string{"templates/layout.html", "templates/home.html"}, "layout"}, + "map": {[]string{"templates/layout.html", "templates/map.html"}, "layout"}, + "places": {[]string{"templates/layout.html", "templates/places.html"}, "layout"}, + "place": {[]string{"templates/layout.html", "templates/place.html"}, "layout"}, + "timeline": {[]string{"templates/layout.html", "templates/timeline.html"}, "layout"}, + "stats": {[]string{"templates/layout.html", "templates/stats.html"}, "layout"}, + "report": {[]string{"templates/layout.html", "templates/report.html"}, "layout"}, + "admin/index": {[]string{"templates/layout.html", "templates/admin/index.html"}, "layout"}, + "admin/settings": {[]string{"templates/layout.html", "templates/admin/settings.html"}, "layout"}, + "admin/password": {[]string{"templates/layout.html", "templates/admin/password.html"}, "layout"}, + "admin/login": {[]string{"templates/admin/login.html"}, "login"}, + "admin/twofa-login": {[]string{"templates/admin/twofa-login.html"}, "login2"}, + "admin/twofa": {[]string{"templates/layout.html", "templates/admin/twofa.html"}, "layout"}, + "admin/fog-mask": {[]string{"templates/layout.html", "templates/admin/fog-mask.html"}, "layout"}, + "admin/fog-preview": {[]string{"templates/layout.html", "templates/admin/fog-preview.html"}, "layout"}, + "admin/backup-preview": {[]string{"templates/layout.html", "templates/admin/backup-preview.html"}, "layout"}, + "admin/notes": {[]string{"templates/layout.html", "templates/admin/notes.html"}, "layout"}, + "admin/edit": {[]string{"templates/layout.html", "templates/admin/edit.html"}, "layout"}, + "admin/topics": {[]string{"templates/layout.html", "templates/admin/topics.html"}, "layout"}, + "admin/photos": {[]string{"templates/layout.html", "templates/admin/photos.html"}, "layout"}, + "topics": {[]string{"templates/layout.html", "templates/topics.html"}, "layout"}, + "topic": {[]string{"templates/layout.html", "templates/topic.html"}, "layout"}, } type pageSet struct { @@ -229,6 +231,9 @@ func (s *Server) routes() { s.mux.HandleFunc("POST /admin/twofa/disable", s.requireAdmin(s.twofaDisable)) s.mux.HandleFunc("GET /admin", s.requireAdmin(s.adminHome)) s.mux.HandleFunc("POST /admin/upload", s.requireAdmin(s.adminUpload)) + s.mux.HandleFunc("POST /admin/preview", s.requireAdmin(s.adminBackupPreview)) + s.mux.HandleFunc("GET /admin/preview/{token}", s.requireAdmin(s.adminBackupPreviewPage)) + s.mux.HandleFunc("POST /admin/preview/{token}/merge", s.requireAdmin(s.adminBackupMerge)) s.mux.HandleFunc("POST /admin/datasets/{id}/activate", s.requireAdmin(s.adminActivate)) s.mux.HandleFunc("POST /admin/datasets/{id}/delete", s.requireAdmin(s.adminDelete)) s.mux.HandleFunc("POST /admin/place/note", s.requireAdmin(s.adminPlaceNote)) @@ -261,6 +266,9 @@ func (s *Server) routes() { s.mux.HandleFunc("POST /admin/edit/weather", s.requireAdmin(s.editWeatherCreate)) s.mux.HandleFunc("POST /admin/gpx", s.requireAdmin(s.adminGpxUpload)) s.mux.HandleFunc("POST /admin/gpx/delete", s.requireAdmin(s.adminGpxDelete)) + s.mux.HandleFunc("POST /admin/fog/preview", s.requireAdmin(s.adminFogPreview)) + s.mux.HandleFunc("GET /admin/fog/preview/{token}", s.requireAdmin(s.adminFogPreviewPage)) + s.mux.HandleFunc("GET /admin/fog/preview/{token}/{z}/{x}/{y}", s.requireAdmin(s.adminFogDiffTile)) s.mux.HandleFunc("POST /admin/fog", s.requireAdmin(s.adminFogUpload)) s.mux.HandleFunc("POST /admin/fog/clear", s.requireAdmin(s.adminFogClear)) s.mux.HandleFunc("GET /admin/fog/export", s.requireAdmin(s.adminFogExport)) diff --git a/internal/web/templates/admin/backup-preview.html b/internal/web/templates/admin/backup-preview.html new file mode 100644 index 0000000..5cd263e --- /dev/null +++ b/internal/web/templates/admin/backup-preview.html @@ -0,0 +1,310 @@ +{{define "content"}} +
+
+
+

备份差异预览

+
+ 这份备份与当前展示的「{{.DatasetName}}」逐条比对的结果。还没有导入，站内数据一个字节都没动。 +
+
+
← 返回后台
+
+ + {{if .Identical}} +
+
这份备份与当前数据集完全一致：没有新增、没有被删、也没有冲突。
+
+ {{end}} + + {{if .Truncated}} +
差异较多，下面每类只列出前 200 条，确切数量看概览表。
+ {{end}} + +
+

概览

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
实体备份站内备份新增仅站内有冲突一致
到访{{num .BackupVisits}}{{num .StoredVisits}}{{num .VisitCounts.Added}}{{num .VisitCounts.Removed}}{{num .VisitCounts.Changed}}{{num .VisitCounts.Same}}
地点{{num .BackupPlaces}}{{num .StoredPlaces}}{{num .PlaceCounts.Added}}{{num .PlaceCounts.Removed}}{{num .PlaceCounts.Changed}}{{num .PlaceCounts.Same}}
活动类型{{num .BackupActs}}{{num .StoredActs}}{{num .ActCounts.Added}}{{num .ActCounts.Removed}}{{num .ActCounts.Changed}}{{num .ActCounts.Same}}
+
+
+ 对齐按每条记录的 UUID 做：rond 每次导出都会重排记录编号（同样的 822 条到访，两次导出的编号 100% 不同，UUID 却逐条一模一样）， + 所以只有 UUID 认得出来是同一条；地点没有 UUID，退到「名称 + 坐标」。 + 站内补录的记录不会出现在真机备份里，一律归到「仅站内有」。 +
+
+ +
+

到访

+ +
备份里多出来的（{{num .VisitCounts.Added}}） —— 目前站内没有这些记录
+ {{if .VisitAdded}} +
+ + + + {{range .VisitAdded}} + + + + + + + + {{end}} + +
到达离开地点活动备注
{{dtt .Arrival}}{{dtt .Departure}}{{if .Place}}{{.Place}}{{else}}—{{end}}{{if .Activity}}{{.Activity}}{{else}}—{{end}}{{if .Remark}}{{.Remark}}{{else}}—{{end}}
+
+ {{else}}
没有
{{end}} + +
+ 只在站内有（{{num .VisitCounts.Removed}}） —— 备份里找不到；可能是你在这里补录的，也可能是实机上删掉的。 + 默认一条都不删；确实想清掉的，勾上左列的「合并时删除」。 +
+ {{if .VisitGone}} +
+ + + + {{range .VisitGone}} + + + + + + + + + + {{end}} + +
到达离开地点活动备注来源合并时删除
{{dtt .Arrival}}{{dtt .Departure}}{{if .Place}}{{.Place}}{{else}}—{{end}}{{if .Activity}}{{.Activity}}{{else}}—{{end}}{{if .Remark}}{{.Remark}}{{else}}—{{end}}{{if .UserAdded}}站内补录{{else}}备份里已无{{end}} + +
+
+ {{else}}
没有
{{end}} + +
两边都有但不一样（{{num .VisitCounts.Changed}}） —— 左边是站内当前的值，右边是备份里的值
+ {{if .VisitClash}} +
+ + + + {{range .VisitClash}} + + + + + + + + {{end}} + +
到达地点活动差异合并时
{{dtt .Arrival}}{{if .Place}}{{.Place}}{{else}}—{{end}}{{if .Activity}}{{.Activity}}{{else}}—{{end}} + {{range .Fields}} +
+ {{.Label}} + {{if .From}}{{.From}}{{else}}（空）{{end}} + → + {{if .To}}{{.To}}{{else}}（空）{{end}} +
+ {{end}} + 		+ + + +
+
+ {{else}}
没有
{{end}} +
+ +
+

地点

+
备份里多出来的（{{num .PlaceCounts.Added}}）
+ {{if .PlaceAdded}} +
+ + + + {{range .PlaceAdded}} + + + + + + + {{end}} + +
名称省 / 市分类坐标
{{if .Name}}{{.Name}}{{else}}（未命名）{{end}}{{.Province}} {{.City}}{{if .POICategory}}{{.POICategory}}{{else}}—{{end}}{{printf "%.5f" .Lat}}, {{printf "%.5f" .Lon}}
+
+ {{else}}
没有
{{end}} + +
+ 只在站内有（{{num .PlaceCounts.Removed}}） + —— 名下还有到访的地点删不掉（会连带删掉那些到访），会直接报错让你先处理。 +
+ {{if .PlaceGone}} +
+ + + + {{range .PlaceGone}} + + + + + + + + {{end}} + +
名称省 / 市到访坐标合并时删除
{{if .Name}}{{.Name}}{{else}}（未命名）{{end}}{{.Province}} {{.City}}{{num .VisitCount}}{{printf "%.5f" .Lat}}, {{printf "%.5f" .Lon}} + {{if gt .VisitCount 0}} + 还有到访，不能删 + {{else}} + + {{end}} +
+
+ {{else}}
没有
{{end}} + +
两边都有但地址不一样（{{num .PlaceCounts.Changed}}）
+ {{if .PlaceClash}} +
+ + + + {{range .PlaceClash}} + + + + + + {{end}} + +
名称差异合并时
{{if .DB.Name}}{{.DB.Name}}{{else}}（未命名）{{end}} + {{range .Fields}} +
+ {{.Label}} + {{if .From}}{{.From}}{{else}}（空）{{end}} + → + {{if .To}}{{.To}}{{else}}（空）{{end}} +
+ {{end}} + 		+ + + +
+
+ {{else}}
没有
{{end}} +
+ +
+

活动类型

+
+ 新增 {{num .ActCounts.Added}} · 仅站内有 {{num .ActCounts.Removed}} · 冲突 {{num .ActCounts.Changed}} · 一致 {{num .ActCounts.Same}} +
+
+
+
备份新增
+ {{range .ActAdded}}
{{.Name}}
{{else}}
没有
{{end}} +
+
+
仅站内有（勾上＝合并时删除）
+ {{range .ActGone}} + + {{else}}
没有
{{end}} +
+
+
名称不同
+ {{range .ActClash}} +
+ {{range .Fields}}
{{.From}} → {{.To}}
{{end}} +
+ + + +
+
+ {{else}}
没有
{{end}} +
+
+
+ +
+

其它实体（只列数量）

+
+ + + + + + + +
实体备份站内
位移{{num .BackupMovements}}{{num .StoredMovements}}
小时天气{{num .BackupWeather}}{{num .StoredWeather}}
原始定位点{{num .BackupRaw}}不单独落库
+
+
+ 位移跟着它的起止到访走、天气跟着到访走，所以这两项按到访对齐结果一并处理，不单独做逐条比对。 +
+
+ +
+

执行合并

+
+ 把选中的改动写进当前数据集「{{.DatasetName}}」。整个过程跑在一个事务里，任何一步失败都会整体回滚，数据不会半改。 +
「仅站内有」的记录默认一条都不删；确实要清掉的，在上面各表里逐条勾「合并时删除」。 + 名下有到访的地点会被拒绝删除（否则会连带删掉那些到访），先处理到访再来。 +
+
+ + +
+ 合并后再传一次同一份备份，差异应当变成「完全一致」——那说明对齐键（UUID 与坐标）都写进去了。 +
+
+
+{{end}} diff --git a/internal/web/templates/admin/fog-preview.html b/internal/web/templates/admin/fog-preview.html new file mode 100644 index 0000000..cbd9038 --- /dev/null +++ b/internal/web/templates/admin/fog-preview.html @@ -0,0 +1,174 @@ +{{define "content"}} +
+
+
+

迷雾导入预览

+
这份快照还没有写入。先看清两种导入方式各自的后果，再决定用哪一种。
+
+
+ ← 返回后台 +
+
+ +
+

数量对比

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
状态块格面积
当前迷雾{{num .Diff.CurBlocks}}{{num .Diff.CurCells}}{{printf "%.1f" .Diff.CurAreaKm2}} km²
这份快照{{num .Diff.IncBlocks}}{{num .Diff.IncCells}}{{printf "%.1f" .Diff.IncAreaKm2}} km²
叠加导入后{{num .Diff.CurBlocks}} +{{num .Diff.GainBlocks}}{{num .Diff.CurCells}} +{{num .Diff.GainCells}}{{printf "%.1f" .Diff.MergeAreaKm2}} km²
整体替换后{{num .Diff.IncBlocks}}{{num .Diff.IncCells}}{{printf "%.1f" .Diff.CoverAreaKm2}} km²
+
+
+ 两个块坐标相同就只按位合并，所以「叠加后」的块数不会等于两边相加——快照里有 {{num .Diff.OverlapBlocks}} 个块与当前迷雾落在同一块， + 其中 {{num .Diff.SameBlocks}} 个一格都不会变。 +
+
+ + {{if .HasBounds}} +
+

差异地图

+
在真实底图上核对差异落在哪些区域，可以缩放平移。
+
+ + 叠加会新增 + + + 两边都有 + + + 整体替换会丢掉 + +
+
+
+ {{end}} + +
+

叠加导入

+ {{if .Diff.Changed}} +
+ 会补上当前迷雾没有的 {{num .Diff.GainBlocks}} 块、{{num .Diff.GainCells}} 格 + （约 {{printf "%.1f" .Diff.GainAreaKm2}} km²）。叠加只做按位合并，不会丢失任何已有数据。 +
+ {{else}} +
+ 这份快照的点亮区域已经被当前迷雾完全覆盖——叠加导入不会带来任何变化， + 多半是同一批数据又导出了一次。 +
+ {{end}} +
+ + +
+
+ +
+

整体替换

+ {{if gt .Diff.LostCells 0}} +
+ ⚠️ 快照里没有当前迷雾的 {{num .Diff.LostBlocks}} 块、{{num .Diff.LostCells}} 格 + （约 {{printf "%.1f" .Diff.LostAreaKm2}} km²）。整体替换后，这些区域会从点亮变回未探索，且无法恢复。 +
+ {{else}} +
+ 这份快照完整包含了当前迷雾的全部点亮区域，替换不会丢数据。 +
+ {{end}} + {{if and (eq .TildeLayers 0) (eq .HashLayers 0)}} +
+ ⚠️ 快照里没有 Model/~ 与 Model/# 两层数据，替换会把现有的这两层一并清掉。 +
+ {{else}} +
+ 另外，站点不解析但会留存的 Model/~（{{.TildeLayers}} 项）与 + Model/#（{{.HashLayers}} 项）也会一并换成这份快照里的{{if .OtherLayers}}，另有 {{.OtherLayers}} 项其它条目{{end}}。 +
+ {{end}} +
+ + +
+
+
+ + +{{end}} diff --git a/internal/web/templates/admin/index.html b/internal/web/templates/admin/index.html index f164c4b..018485c 100644 --- a/internal/web/templates/admin/index.html +++ b/internal/web/templates/admin/index.html @@ -29,6 +29,7 @@ 上传是以备份为准重建数据：站点里删掉的地点、改过的记录，只要下次传一份 内容不同的备份，就会被备份里的原样覆盖回来（重复传同一份文件不会，会提示「完全一致」）。 在站点上新加的地点属于当时那个数据集，传新备份后要看它们，可在下面的数据集列表里切回。 +
拿不准就先用「分析差异」：它会逐条比对这份备份与站内当前数据，告诉你哪些是新增、哪些只在站内有、哪些两边不一样，看完再决定要不要导入。
- +
+ + +
@@ -53,21 +57,18 @@ 尚未导入迷雾——把 Fog of World 的 .fwss（在 App 内导出，通常在 zip 里）上传即可点亮地图。 {{end}} -
+ - - +
+
+ 上传后先不写入：会先给出这份快照能补多少新区域、以及「整体替换」会丢掉多少已有区域， + 确认之后才真正导入。 +
{{if gt .FogBlocks 0}}
导出当前迷雾（.fwss） diff --git a/internal/web/view.go b/internal/web/view.go index 17646f7..6c1f898 100644 --- a/internal/web/view.go +++ b/internal/web/view.go @@ -14,6 +14,7 @@ import ( "rond-with-you/internal/auth" "rond-with-you/internal/fog" + "rond-with-you/internal/rond" "rond-with-you/internal/setting" "rond-with-you/internal/stats" "rond-with-you/internal/votes" @@ -637,6 +638,74 @@ type AdminData struct { DefaultPW bool } +// VisitRow 是一条到访摊平后的展示行。地点与活动写名称而不是编号—— +// 两边的编号空间完全不同，编号对人没有任何意义。 +type VisitRow struct { + // Key 是这条冲突的对齐键（到访 UUID），合并表单拿它当字段名 + Key string + // SrcPK 只有「仅站内有」的行有意义：要删哪条就勾哪条 + SrcPK int + Arrival *time.Time + Departure *time.Time + Place string + Activity string + Remark string + Emoji string + UserAdded bool // 站点补录的行 + Fields []rond.FieldChange +} + +// BackupDiffData 是「备份差异预览」页的数据。 +// +// 上限说明：列表按 diffListLimit 截断，页面上只提示「还有更多」，不做分页—— +// 预演是给人下判断用的，看前 200 条足够判断要不要继续；真正要逐条取舍的是 +// 后面那个合并流程（那里才有逐条操作）。 +type BackupDiffData struct { + Page + Token string + DatasetName string + + VisitCounts rond.Counts + PlaceCounts rond.Counts + ActCounts rond.Counts + + VisitAdded, VisitGone, VisitClash []VisitRow + PlaceAdded []rond.Location + PlaceGone []rond.StoredPlace + PlaceClash []rond.PlaceConflict + ActAdded []rond.Activity + ActGone []rond.StoredActivity + ActClash []rond.ActivityConflict + + // 两侧总量，用于「有没有少数据」的直觉判断 + BackupVisits, StoredVisits int + BackupPlaces, StoredPlaces int + BackupActs, StoredActs int + BackupMovements, StoredMovements int + BackupWeather, StoredWeather int + BackupRaw int + + Truncated bool + Identical bool +} + +// FogPreviewData 是「迷雾导入预览」页的数据：先把差异摆出来，再决定叠加还是整体替换。 +type FogPreviewData struct { + Page + Token string + Diff fog.Diff + // 快照里不参与渲染、但整体替换会一并换掉的图层条目数 + TildeLayers int + HashLayers int + OtherLayers int + + // 差异地图的初始视野：两边块集覆盖范围的包围盒（已按底图坐标系换算）。 + // 没有任何块时 HasBounds 为假，模板不画地图。 + HasBounds bool + MinLat, MinLon float64 + MaxLat, MaxLon float64 +} + type NotesData struct { Page Rows []NoteRow @@ -1247,6 +1316,8 @@ func templateFuncs() template.FuncMap { "km": func(f float64) string { return fmt.Sprintf("%.1f", f) }, "f1": func(f float64) string { return fmt.Sprintf("%.1f", f) }, "dt": func(t time.Time) string { return t.Local().Format("2006-01-02 15:04") }, + // actKey 给活动冲突生成表单字段名，与处理函数共用同一套键 + "actKey": rond.ActivityKey, // dtt 接受 *time.Time，nil 返回空串，便于可选时间字段直接渲染 "dtt": func(t *time.Time) string { if t == nil { diff --git a/tools/colprobe/main.go b/tools/colprobe/main.go new file mode 100644 index 0000000..4345b14 --- /dev/null +++ b/tools/colprobe/main.go @@ -0,0 +1,89 @@ +// 一次性探针：验证 ZLOCATION.ZPLACEID / ZVISIT.ZEVENTIDENTIFIER 是否跨备份稳定。 +package main + +import ( + "fmt" + "os" + + "rond-with-you/internal/rond" +) + +type snap struct { + placeID map[string]int + eventID map[string]int + nulls int + total int + evTotal int +} + +func load(p string) *snap { + dir, err := os.MkdirTemp("", "colprobe") + if err != nil { + panic(err) + } + defer os.RemoveAll(dir) + db, err := rond.Open(p, dir) + if err != nil { + panic(err) + } + defer db.Close() + s := &snap{placeID: map[string]int{}, eventID: map[string]int{}} + + rows, err := db.Query(`SELECT Z_PK, COALESCE(ZPLACEID,'') FROM ZLOCATION`) + if err != nil { + panic(err) + } + for rows.Next() { + var pk int + var id string + rows.Scan(&pk, &id) + s.total++ + if id == "" { + s.nulls++ + } else { + s.placeID[id] = pk + } + } + rows.Close() + + rows, err = db.Query(`SELECT COALESCE(ZEVENTIDENTIFIER,'') FROM ZVISIT`) + if err != nil { + panic(err) + } + for rows.Next() { + var id string + rows.Scan(&id) + if id != "" { + s.eventID[id] = 1 + s.evTotal++ + } + } + rows.Close() + fmt.Printf("%s: ZLOCATION %d 个（ZPLACEID 空 %d），ZEVENTIDENTIFIER 非空 %d\n", + p, s.total, s.nulls, s.evTotal) + return s +} + +func main() { + a := load(os.Args[1]) + b := load(os.Args[2]) + inter, samePK := 0, 0 + for k, pa := range a.placeID { + if pb, ok := b.placeID[k]; ok { + inter++ + if pa == pb { + samePK++ + } + } + } + fmt.Printf("\nZPLACEID 交集 %d（A 独有 %d，B 独有 %d），其中 Z_PK 相同 %d\n", + inter, len(a.placeID)-inter, len(b.placeID)-inter, samePK) + + ei := 0 + for k := range a.eventID { + if _, ok := b.eventID[k]; ok { + ei++ + } + } + fmt.Printf("ZEVENTIDENTIFIER 交集 %d（A %d / B %d）\n", ei, len(a.eventID), len(b.eventID)) +} diff --git a/tools/diffprobe/main.go b/tools/diffprobe/main.go new file mode 100644 index 0000000..a6edb49 --- /dev/null +++ b/tools/diffprobe/main.go @@ -0,0 +1,438 @@ +// 一次性探针：把一份 .rondbackup 与 PG 里某个数据集的记录逐条对齐，输出差异画像。 +// 用完即可删，不属于站点功能。 +package main + +import ( + "context" + "database/sql" + "fmt" + "os" + "sort" + "time" + + _ "github.com/jackc/pgx/v5/stdlib" + + "rond-with-you/internal/rond" +) + +func main() { + if len(os.Args) < 4 { + fmt.Println("用法: diffprobe ") + os.Exit(2) + } + archive, dsn := os.Args[1], os.Args[2] + var datasetID int64 + fmt.Sscanf(os.Args[3], "%d", &datasetID) + + tmp, err := os.MkdirTemp("", "diffprobe") + must(err) + defer os.RemoveAll(tmp) + + sqlDB, err := rond.Open(archive, tmp) + must(err) + b, err := rond.Parse(sqlDB) + sqlDB.Close() + must(err) + + pg, err := sql.Open("pgx", dsn) + must(err) + defer pg.Close() + ctx := context.Background() + must(pg.PingContext(ctx)) + + fmt.Printf("=== 备份 %s ===\n", archive) + fmt.Printf("备份: 到访 %d / 地点 %d / 位移 %d / 天气 %d / 活动 %d / 标签 %d / 原始定位 %d\n", + len(b.Visits), len(b.Locations), len(b.Movements), len(b.Weather), + len(b.Activities), len(b.Tags), b.RawVisitNum) + var bFirst, bLast sql.NullTime + for _, v := range b.Visits { + if v.Arrival == nil { + continue + } + if !bFirst.Valid || v.Arrival.Before(bFirst.Time) { + bFirst = sql.NullTime{Time: *v.Arrival, Valid: true} + } + if !bLast.Valid || v.Arrival.After(bLast.Time) { + bLast = sql.NullTime{Time: *v.Arrival, Valid: true} + } + } + fmt.Printf("备份时间跨度: %s ~ %s\n", ts(bFirst), ts(bLast)) + + fmt.Printf("\n=== 库内 dataset #%d ===\n", datasetID) + var name string + var cnt, pcnt, raw int + var gFirst, gLast sql.NullTime + must(pg.QueryRowContext(ctx, `SELECT original_name, visit_count, place_count, raw_visit_count, + first_visit_at, last_visit_at FROM datasets WHERE id=$1`, datasetID). + Scan(&name, &cnt, &pcnt, &raw, &gFirst, &gLast)) + fmt.Printf("%s: 到访 %d / 地点 %d / 原始定位 %d，跨度 %s ~ %s\n", name, cnt, pcnt, raw, ts(gFirst), ts(gLast)) + + diffVisits(ctx, pg, datasetID, b) + diffPlaces(ctx, pg, datasetID, b) + diffMovements(ctx, pg, datasetID, b) + diffWeather(ctx, pg, datasetID, b) + diffActivities(ctx, pg, datasetID, b) + diffTags(ctx, pg, datasetID, b) +} + +func ts(t sql.NullTime) string { + if !t.Valid { + return "-" + } + return t.Time.Local().Format("2006-01-02 15:04") +} + +type visitRow struct { + pk, placeSrc, actSrc int + arrival, departure *time.Time + remark, emoji string + bookmarked, userAdd bool +} + +func diffVisits(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) { + fmt.Printf("\n---------- 到访 ----------\n") + rows, err := pg.QueryContext(ctx, `SELECT v.src_pk, COALESCE(p.src_pk,0), COALESCE(a.src_pk,0), + v.arrival, v.departure, COALESCE(v.remark,''), COALESCE(v.emoji,''), v.bookmarked, v.is_user_added + FROM visits v LEFT JOIN places p ON p.id=v.place_id LEFT JOIN activities a ON a.id=v.activity_id + WHERE v.dataset_id=$1`, dsID) + must(err) + db := map[int]visitRow{} + for rows.Next() { + var r visitRow + var arr time.Time + var dep sql.NullTime + must(rows.Scan(&r.pk, &r.placeSrc, &r.actSrc, &arr, &dep, &r.remark, &r.emoji, &r.bookmarked, &r.userAdd)) + r.arrival = &arr + if dep.Valid { + r.departure = &dep.Time + } + db[r.pk] = r + } + rows.Close() + + bk := map[int]rond.Visit{} + for _, v := range b.Visits { + bk[v.SrcPK] = v + } + + var onlyBackup, onlyDB, bothDiff, same []int + for pk, v := range bk { + r, ok := db[pk] + if !ok { + onlyBackup = append(onlyBackup, pk) + continue + } + if vdiff(v, r) != "" { + bothDiff = append(bothDiff, pk) + continue + } + same = append(same, pk) + } + for pk := range db { + if _, ok := bk[pk]; !ok { + onlyDB = append(onlyDB, pk) + } + } + sort.Ints(onlyBackup) + sort.Ints(onlyDB) + sort.Ints(bothDiff) + + fmt.Printf("同号且一致: %d\n", len(same)) + fmt.Printf("仅备份有（实机新增，库里没有）: %d\n", len(onlyBackup)) + fmt.Printf("仅库里有（站点侧新增/实机已删除）: %d\n", len(onlyDB)) + fmt.Printf("同号但内容不同（冲突）: %d\n", len(bothDiff)) + + // 按到达时间做多重集匹配：判断「同号冲突」到底是同批数据换了主键，还是两批无关数据 + byTime := map[string][]int{} + for pk, r := range db { + if r.arrival != nil { + byTime[r.arrival.UTC().Format(time.RFC3339)] = append(byTime[r.arrival.UTC().Format(time.RFC3339)], pk) + } + } + matched, miss := 0, 0 + pair := [][2]int{} + for pk, v := range bk { + if v.Arrival == nil { + continue + } + k := v.Arrival.UTC().Format(time.RFC3339) + if list := byTime[k]; len(list) > 0 { + matched++ + pair = append(pair, [2]int{pk, list[0]}) + byTime[k] = list[1:] + } else { + miss++ + } + } + fmt.Printf("\n[时间对齐] 备份到访按「到达时刻」能在库内找到同刻记录: %d 条，找不到: %d 条\n", matched, miss) + sort.Slice(pair, func(i, j int) bool { return pair[i][0] < pair[j][0] }) + fmt.Printf("[时间对齐] 同刻对的 (备份pk→库pk) 前 12 组: ") + for i, p := range pair { + if i >= 12 { + break + } + fmt.Printf("%d→%d ", p[0], p[1]) + } + fmt.Println() + nDiff := 0 + for _, p := range pair { + if p[0] != p[1] { + nDiff++ + } + } + fmt.Printf("[时间对齐] 同刻但主键不同: %d / %d\n", nDiff, len(pair)) + + show := func(label string, pks []int, f func(pk int) string) { + if len(pks) == 0 { + return + } + fmt.Printf("\n[%s] 共 %d 条，前 8 条：\n", label, len(pks)) + for i, pk := range pks { + if i >= 8 { + break + } + fmt.Printf(" #%d %s\n", pk, f(pk)) + } + } + show("仅备份有", onlyBackup, func(pk int) string { + v := bk[pk] + return fmt.Sprintf("到访 %s 离开 %s 地点src=%s", ft(v.Arrival), ft(v.Departure), ps(v.LocationSrcPK)) + }) + show("仅库里有", onlyDB, func(pk int) string { + r := db[pk] + return fmt.Sprintf("到访 %s 离开 %s 地点src=%d 站点新增=%v 备注=%q", + ft(r.arrival), ftp(r.departure), r.placeSrc, r.userAdd, r.remark) + }) + show("冲突", bothDiff, func(pk int) string { + return vdiff(bk[pk], db[pk]) + }) +} + +func vdiff(v rond.Visit, r visitRow) string { + var d string + if !eq(v.Arrival, r.arrival) { + d += fmt.Sprintf("到达 %s→%s ", ft(v.Arrival), ft(r.arrival)) + } + if !eq(v.Departure, r.departure) { + d += fmt.Sprintf("离开 %s→%s ", ft(v.Departure), ft(r.departure)) + } + if pv(v.LocationSrcPK) != r.placeSrc { + d += fmt.Sprintf("地点 %s→%d ", ps(v.LocationSrcPK), r.placeSrc) + } + if av(v.ActivitySrcPK) != r.actSrc { + d += fmt.Sprintf("活动 %s→%d ", ps(v.ActivitySrcPK), r.actSrc) + } + if v.Remark != r.remark { + d += fmt.Sprintf("备注 %q→%q ", v.Remark, r.remark) + } + if v.Emoji != r.emoji { + d += fmt.Sprintf("表情 %q→%q ", v.Emoji, r.emoji) + } + if v.Bookmarked != r.bookmarked { + d += fmt.Sprintf("收藏 %v→%v ", v.Bookmarked, r.bookmarked) + } + return d +} + +func diffPlaces(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) { + fmt.Printf("\n---------- 地点 ----------\n") + type prow struct { + name, poi, cc, prov, city, dist, sub, thor, tz string + lat, lon float64 + } + rows, err := pg.QueryContext(ctx, `SELECT src_pk, COALESCE(name,''), COALESCE(poi_category,''), + COALESCE(country_code,''), COALESCE(province,''), COALESCE(city,''), COALESCE(district,''), + COALESCE(sublocality,''), COALESCE(thoroughfare,''), COALESCE(timezone,''), + COALESCE(lat,0), COALESCE(lon,0) FROM places WHERE dataset_id=$1`, dsID) + must(err) + db := map[int]prow{} + for rows.Next() { + var pk int + var r prow + must(rows.Scan(&pk, &r.name, &r.poi, &r.cc, &r.prov, &r.city, &r.dist, &r.sub, &r.thor, &r.tz, &r.lat, &r.lon)) + db[pk] = r + } + rows.Close() + + bk := map[int]rond.Location{} + for _, l := range b.Locations { + bk[l.SrcPK] = l + } + var onlyB, onlyD, diff []int + for pk, l := range bk { + r, ok := db[pk] + if !ok { + onlyB = append(onlyB, pk) + continue + } + if l.Name != r.name || l.POICategory != r.poi || l.Province != r.prov || l.City != r.city || + l.District != r.dist || l.Sublocality != r.sub || l.Thoroughfare != r.thor || l.Timezone != r.tz || + l.CountryCode != r.cc || !near(l.Lat, r.lat) || !near(l.Lon, r.lon) { + diff = append(diff, pk) + } + } + for pk := range db { + if _, ok := bk[pk]; !ok { + onlyD = append(onlyD, pk) + } + } + sort.Ints(onlyB) + sort.Ints(onlyD) + sort.Ints(diff) + fmt.Printf("仅备份有: %d，仅库里有: %d，同号内容不同: %d\n", len(onlyB), len(onlyD), len(diff)) + for i, pk := range onlyB { + if i >= 6 { + break + } + fmt.Printf(" 仅备份 #%d %s (%s %s %s)\n", pk, bk[pk].Name, bk[pk].Province, bk[pk].City, bk[pk].POICategory) + } + for i, pk := range onlyD { + if i >= 6 { + break + } + fmt.Printf(" 仅库内 #%d %s (%s %s)\n", pk, db[pk].name, db[pk].prov, db[pk].city) + } + for i, pk := range diff { + if i >= 6 { + break + } + l, r := bk[pk], db[pk] + fmt.Printf(" 冲突 #%d 备份[name=%q city=%q %.6f,%.6f] 库[name=%q city=%q %.6f,%.6f]\n", + pk, l.Name, l.City, l.Lat, l.Lon, r.name, r.city, r.lat, r.lon) + } +} + +func diffMovements(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) { + fmt.Printf("\n---------- 位移 ----------\n") + rows, err := pg.QueryContext(ctx, `SELECT src_pk, COALESCE(transport_src,0), + COALESCE(from_visit_src,0), COALESCE(to_visit_src,0), started_at, ended_at + FROM movements WHERE dataset_id=$1`, dsID) + must(err) + type mrow struct { + t, f, to int + s, e *time.Time + } + db := map[int]mrow{} + for rows.Next() { + var pk int + var r mrow + var s, e sql.NullTime + must(rows.Scan(&pk, &r.t, &r.f, &r.to, &s, &e)) + if s.Valid { + r.s = &s.Time + } + if e.Valid { + r.e = &e.Time + } + db[pk] = r + } + rows.Close() + var onlyB, onlyD int + for _, m := range b.Movements { + if _, ok := db[m.SrcPK]; !ok { + onlyB++ + } + } + for pk := range db { + found := false + for _, m := range b.Movements { + if m.SrcPK == pk { + found = true + break + } + } + if !found { + onlyD++ + } + } + fmt.Printf("备份 %d 条 / 库内 %d 条；仅备份有 %d，仅库里有 %d\n", len(b.Movements), len(db), onlyB, onlyD) +} + +func diffWeather(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) { + var n int + must(pg.QueryRowContext(ctx, `SELECT count(*) FROM weather WHERE dataset_id=$1`, dsID).Scan(&n)) + fmt.Printf("\n---------- 天气 ----------\n备份 %d 条 / 库内 %d 条\n", len(b.Weather), n) +} + +func diffActivities(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) { + fmt.Printf("\n---------- 活动类型 ----------\n") + rows, err := pg.QueryContext(ctx, `SELECT src_pk, name FROM activities WHERE dataset_id=$1`, dsID) + must(err) + db := map[int]string{} + for rows.Next() { + var pk int + var n string + must(rows.Scan(&pk, &n)) + db[pk] = n + } + rows.Close() + for _, a := range b.Activities { + n, ok := db[a.SrcPK] + if !ok { + fmt.Printf(" 仅备份 #%d %q\n", a.SrcPK, a.Name) + } else if n != a.Name { + fmt.Printf(" 冲突 #%d 备份=%q 库=%q\n", a.SrcPK, a.Name, n) + } + } + for pk, n := range db { + found := false + for _, a := range b.Activities { + if a.SrcPK == pk { + found = true + } + } + if !found { + fmt.Printf(" 仅库内 #%d %q\n", pk, n) + } + } +} + +func diffTags(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) { + var n int + must(pg.QueryRowContext(ctx, `SELECT count(*) FROM tags WHERE dataset_id=$1`, dsID).Scan(&n)) + fmt.Printf("\n---------- 标签 ----------\n备份 %d 条 / 库内 %d 条\n", len(b.Tags), n) +} + +func eq(a, b *time.Time) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return a.Equal(*b) +} + +func pv(p *int) int { + if p == nil { + return 0 + } + return *p +} + +func av(p *int) int { return pv(p) } + +func ps(p *int) string { + if p == nil { + return "-" + } + return fmt.Sprint(*p) +} + +func ft(t *time.Time) string { + if t == nil { + return "-" + } + return t.Local().Format("01-02 15:04") +} + +func ftp(t *time.Time) string { return ft(t) } + +func near(a, b float64) bool { + d := a - b + return d < 1e-9 && d > -1e-9 +} + +func must(err error) { + if err != nil { + fmt.Fprintln(os.Stderr, "错误:", err) + os.Exit(1) + } +} diff --git a/tools/exportprobe/main.go b/tools/exportprobe/main.go new file mode 100644 index 0000000..5c5f116 --- /dev/null +++ b/tools/exportprobe/main.go @@ -0,0 +1,40 @@ +// 一次性探针：调用站点的导出逻辑产出一份 .rondbackup，用于核对 +// 「站点导出 → 导入 rond → rond 再导出」这条链路上 UUID 与 Z_PK 是否还连贯。 +package main + +import ( + "context" + "database/sql" + "fmt" + "os" + "strconv" + + _ "github.com/jackc/pgx/v5/stdlib" + + "rond-with-you/internal/rond" +) + +func main() { + if len(os.Args) < 5 { + fmt.Println("用法: exportprobe ") + os.Exit(2) + } + dsID, err := strconv.ParseInt(os.Args[2], 10, 64) + must(err) + db, err := sql.Open("pgx", os.Args[1]) + must(err) + defer db.Close() + + f, err := os.Create(os.Args[4]) + must(err) + defer f.Close() + must(rond.ExportRondbackup(context.Background(), db, dsID, os.Args[3], f)) + fmt.Printf("已导出到 %s\n", os.Args[4]) +} + +func must(err error) { + if err != nil { + fmt.Fprintln(os.Stderr, "错误:", err) + os.Exit(1) + } +} diff --git a/tools/fogprobe/main.go b/tools/fogprobe/main.go new file mode 100644 index 0000000..496956b --- /dev/null +++ b/tools/fogprobe/main.go @@ -0,0 +1,133 @@ +// 一次性探针：把一份 .fwss 与 PG 里的 fog_blocks 逐块比对，量化「叠加 / 替换」两种模式的影响。 +package main + +import ( + "database/sql" + "encoding/hex" + "fmt" + "os" + "strconv" + "strings" + + _ "github.com/jackc/pgx/v5/stdlib" + + "rond-with-you/internal/fog" +) + +func main() { + if len(os.Args) < 3 { + fmt.Println("用法: fogprobe [userID]") + os.Exit(2) + } + uid := int64(1) + if len(os.Args) > 3 { + uid, _ = strconv.ParseInt(os.Args[3], 10, 64) + } + data, err := os.ReadFile(os.Args[1]) + must(err) + inc, incCells, err := fog.Decode(data) + must(err) + fmt.Printf("快照 %s: %d 块 / %d 格\n", os.Args[1], len(inc), incCells) + + pg, err := sql.Open("pgx", os.Args[2]) + must(err) + defer pg.Close() + rows, err := pg.Query(`SELECT gbx, gby, bitmap FROM fog_blocks WHERE user_id=$1`, uid) + must(err) + local := map[[2]int][]byte{} + localCells := 0 + for rows.Next() { + var gx, gy int + var bm []byte + must(rows.Scan(&gx, &gy, &bm)) + local[[2]int{gx, gy}] = bm + localCells += popcount(bm) + } + rows.Close() + fmt.Printf("库内: %d 块 / %d 格\n", len(local), localCells) + + var newBlocks, newCells int + var novBlock, overlap int + var overlapGain int + for _, b := range inc { + k := [2]int{b.GX, b.GY} + old, ok := local[k] + if !ok { + newBlocks++ + newCells += popcount(b.Bitmap) + continue + } + novBlock++ + gain := 0 + for i := range b.Bitmap { + gain += poptab[b.Bitmap[i]&^old[i]] + } + if gain > 0 { + overlap++ + overlapGain += gain + } + } + var lostBlocks, lostCells, lostFull int + for k, bm := range local { + found := false + for _, b := range inc { + if b.GX == k[0] && b.GY == k[1] { + found = true + break + } + } + if !found { + lostBlocks++ + lostCells += popcount(bm) + lostFull++ + } + } + fmt.Printf("\n【叠加合并 overlay】\n") + fmt.Printf(" 新增整块: %d 块（+%d 格）\n", newBlocks, newCells) + fmt.Printf(" 与已有块重叠: %d 块，其中 %d 块含新格（+%d 格）\n", novBlock, overlap, overlapGain) + fmt.Printf(" 合计新增格: %d，合并后总量: %d 块 / %d 格\n", + newCells+overlapGain, len(local)+newBlocks, localCells+newCells+overlapGain) + + fmt.Printf("\n【整体替换 cover】\n") + fmt.Printf(" 库里现有 %d 块会被丢弃（快照里完全没有这些块），损失 %d 格\n", lostFull, lostCells) + fmt.Printf(" 替换后总量: %d 块 / %d 格\n", len(inc), incCells) + fmt.Printf(" 净变化: %+d 块 / %+d 格\n", len(inc)-len(local), incCells-localCells) + + fmt.Printf("\n【其它图层】\n") + layers, err := fog.RawLayers(data) + if err == nil { + for k, v := range layers { + fmt.Printf(" %s (%d 字节, %s)\n", k, len(v), hex.EncodeToString(v[:min(8, len(v))])) + } + } +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +var poptab = func() [256]int { + var t [256]int + for i := range t { + t[i] = strings.Count(strconv.FormatUint(uint64(i), 2), "1") + } + return t +}() + +func popcount(b []byte) int { + n := 0 + for _, v := range b { + n += poptab[v] + } + return n +} + +func must(err error) { + if err != nil { + fmt.Fprintln(os.Stderr, "错误:", err) + os.Exit(1) + } +} diff --git a/tools/idprobe/main.go b/tools/idprobe/main.go new file mode 100644 index 0000000..72eee4d --- /dev/null +++ b/tools/idprobe/main.go @@ -0,0 +1,171 @@ +// 一次性探针：检查 rond 备份里是否存在「跨备份稳定」的对齐键。 +// 用完即可删。 +package main + +import ( + "database/sql" + "fmt" + "os" + "strings" + + "rond-with-you/internal/rond" +) + +func main() { + if len(os.Args) < 3 { + fmt.Println("用法: idprobe ") + os.Exit(2) + } + a := load(os.Args[1]) + b := load(os.Args[2]) + fmt.Printf("\n=== A=%s ===\n=== B=%s ===\n", os.Args[1], os.Args[2]) + + fmt.Printf("\n---------- ZVISIT.ZIDENTIFIER_ 稳定性 ----------\n") + fmt.Printf("A 有 UUID %d / %d，B 有 UUID %d / %d\n", len(a.visitUUID), len(a.visitPK), len(b.visitUUID), len(b.visitPK)) + inter := 0 + for u := range a.visitUUID { + if _, ok := b.visitUUID[u]; ok { + inter++ + } + } + fmt.Printf("两次备份 UUID 交集: %d（A 独有 %d，B 独有 %d）\n", + inter, len(a.visitUUID)-inter, len(b.visitUUID)-inter) + // UUID 相同则对比 pk 是否相同 + samePK, diffPK := 0, 0 + for u, pa := range a.visitUUID { + if pb, ok := b.visitUUID[u]; ok { + if pa == pb { + samePK++ + } else { + diffPK++ + } + } + } + fmt.Printf("UUID 相同者中：Z_PK 也相同 %d，Z_PK 已变 %d\n", samePK, diffPK) + + fmt.Printf("\n---------- ZACTIVITY.ZUID_ ----------\n") + fmt.Printf("A UUID %d / %d，B UUID %d / %d\n", len(a.actUUID), len(a.actPK), len(b.actUUID), len(b.actPK)) + same, dif := 0, 0 + for u, pa := range a.actUUID { + if pb, ok := b.actUUID[u]; ok { + if pa == pb { + same++ + } else { + dif++ + } + } + } + fmt.Printf("交集配对：Z_PK 相同 %d，Z_PK 已变 %d\n", same, dif) + + fmt.Printf("\n---------- ZLOCATION 可用稳定列 ----------\n") + fmt.Printf("A 列: %s\n", strings.Join(a.locCols, ", ")) + fmt.Printf("A 地点 %d，B 地点 %d\n", len(a.locPK), len(b.locPK)) + // 名称+粗坐标是否可作键 + type lk struct { + name string + lat float64 + lon float64 + } + keyA := map[string][]int{} + for pk, l := range a.locPK { + k := fmt.Sprintf("%s|%.5f|%.5f", l.name, l.lat, l.lon) + keyA[k] = append(keyA[k], pk) + } + hit, miss := 0, 0 + for pk, l := range b.locPK { + k := fmt.Sprintf("%s|%.5f|%.5f", l.name, l.lat, l.lon) + if _, ok := keyA[k]; ok { + hit++ + } else { + miss++ + if miss <= 5 { + fmt.Printf(" B 独有地点 #%d %q %.6f,%.6f\n", pk, l.name, l.lat, l.lon) + } + } + } + fmt.Printf("按「名称+坐标(1e-5)」键：B 命中 A %d，未命中 %d\n", hit, miss) +} + +type loc struct { + name string + lat, lon float64 +} + +type snap struct { + visitUUID map[string]int + visitPK map[int]bool + actUUID map[string]int + actPK map[int]bool + locPK map[int]loc + locCols []string +} + +func load(path string) *snap { + dir, err := os.MkdirTemp("", "idprobe") + must(err) + defer os.RemoveAll(dir) + db, err := rond.Open(path, dir) + must(err) + defer db.Close() + + s := &snap{visitUUID: map[string]int{}, visitPK: map[int]bool{}, + actUUID: map[string]int{}, actPK: map[int]bool{}, locPK: map[int]loc{}} + + rows, err := db.Query(`SELECT Z_PK, hex(ZIDENTIFIER_) FROM ZVISIT`) + must(err) + for rows.Next() { + var pk int + var u sql.NullString + must(rows.Scan(&pk, &u)) + s.visitPK[pk] = true + if u.Valid && u.String != "" && strings.Trim(u.String, "0") != "" { + s.visitUUID[u.String] = pk + } + } + rows.Close() + + rows, err = db.Query(`SELECT Z_PK, hex(ZUID_), COALESCE(ZNAME_,'') FROM ZACTIVITY`) + must(err) + for rows.Next() { + var pk int + var u sql.NullString + var n string + must(rows.Scan(&pk, &u, &n)) + s.actPK[pk] = true + if u.Valid && u.String != "" && strings.Trim(u.String, "0") != "" { + s.actUUID[u.String] = pk + } + } + rows.Close() + + rows, err = db.Query(`PRAGMA table_info(ZLOCATION)`) + must(err) + for rows.Next() { + var cid int + var name, typ string + var notnull int + var dflt sql.NullString + var pk int + must(rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk)) + s.locCols = append(s.locCols, name) + } + rows.Close() + + rows, err = db.Query(`SELECT Z_PK, COALESCE(ZNAME_,''), COALESCE(ZLATITUDE,0), COALESCE(ZLONGITUDE,0) FROM ZLOCATION`) + must(err) + for rows.Next() { + var pk int + var l loc + must(rows.Scan(&pk, &l.name, &l.lat, &l.lon)) + s.locPK[pk] = l + } + rows.Close() + return s +} + +func must(err error) { + if err != nil { + fmt.Fprintln(os.Stderr, "错误:", err) + os.Exit(1) + } +} diff --git a/tools/sqlprobe/main.go b/tools/sqlprobe/main.go new file mode 100644 index 0000000..88466bc --- /dev/null +++ b/tools/sqlprobe/main.go @@ -0,0 +1,57 @@ +// 一次性探针：在 rond 备份的 SQLite 上跑一条查询并打印结果（BLOB 打十六进制）。 +package main + +import ( + "fmt" + "os" + "strings" + + "rond-with-you/internal/rond" +) + +func main() { + if len(os.Args) < 3 { + fmt.Println(`用法: sqlprobe "SELECT ..."`) + os.Exit(2) + } + dir, err := os.MkdirTemp("", "sqlprobe") + if err != nil { + panic(err) + } + defer os.RemoveAll(dir) + db, err := rond.Open(os.Args[1], dir) + if err != nil { + panic(err) + } + defer db.Close() + + rows, err := db.Query(os.Args[2]) + if err != nil { + panic(err) + } + defer rows.Close() + cols, _ := rows.Columns() + fmt.Println(strings.Join(cols, " | ")) + for rows.Next() { + vals := make([]any, len(cols)) + ptrs := make([]any, len(cols)) + for i := range vals { + ptrs[i] = &vals[i] + } + if err := rows.Scan(ptrs...); err != nil { + panic(err) + } + parts := make([]string, len(cols)) + for i, v := range vals { + switch t := v.(type) { + case nil: + parts[i] = "-" + case []byte: + parts[i] = fmt.Sprintf("%x", t) + default: + parts[i] = fmt.Sprint(t) + } + } + fmt.Println(strings.Join(parts, " | ")) + } +}