备份与迷雾的差异分析、合并执行,附属数据改挂坐标键

备份(.rondbackup)与迷雾(.fwss)各加一层「先看差异再动手」:

- 迷雾:上传先出差异页,说清叠加能补多少、整体替换会丢多少,并在真实底图上
  用绿/灰/红标出会新增 / 不变 / 会丢的区域
- 备份:与当前数据集按 UUID 逐条比对(rond 每次导出都会重排 Z_PK,编号认不出人),
  新增合入、冲突逐条选、仅站内有逐条勾选删除,全在一个事务里执行;
  合入的行写回 entity_raw,保证再传同一份备份时差异归零
- 地点按坐标对齐:站点的「补全地址」会把地点改名,拿名称当键会让一整个地点的
  到访全变成假冲突
- 附属数据(备注/结论/访客票/图片)从 dataset_id+src_pk 换成坐标键,换数据集不再失联;
  访客票没有 dataset_id,回填按活跃数据集认领
- 修正 contentHash 拿 Z_PK 当指纹的隐患,以及两处「src_pk 跨快照稳定」的错误注释

导出格式(.rondbackup / .fwss)保持原样,往返测试通过。
This commit is contained in:
tamakyi
2026-09-19 16:04:00 +08:00
parent 9e01923244
commit 30c502ce7b
43 changed files with 5968 additions and 170 deletions
+150 -13
View File
@@ -171,38 +171,50 @@ CREATE TABLE IF NOT EXISTS fog_blocks (
PRIMARY KEY (user_id, gbx, gby) PRIMARY KEY (user_id, gbx, gby)
); );
-- 地点备注与别名。按 src_pk(rond 内部地点 ID,跨快照稳定)而不是 places.id 存, -- 地点备注与别名。挂**坐标键**(place_key)而不是 src_pk:
-- 这样重新上传备份、数据集换了新 id 之后备注依然挂着。 --
-- dataset_id 是后加的:src_pk 只在单个数据集内唯一,缺了它切数据集时同编号的 -- src_pk(rond 的 Z_PK)只在单个数据集内有效——rond 每次导出/同步都会重排编号
-- 地点会把别人的备注/别名认领过去。 -- (实测两份备份里 822 条到访的 UUID 完全一致、Z_PK 100% 不同),所以换一份备份、
-- 换个数据集之后,同编号的完全是另一个地点,备注就再也挂不回去了。
-- 坐标是地点的地理事实,跨备份稳定:place_key = 经纬度各取 6 位小数拼起来
-- (约 0.1m 精度),与 Go 侧的 rond.PlaceCoordKey 必须逐位一致。
-- dataset_id / src_pk 保留下来只作「最后一次写入时挂在哪儿」的排查线索。
CREATE TABLE IF NOT EXISTS place_notes ( CREATE TABLE IF NOT EXISTS place_notes (
user_id BIGINT NOT NULL, user_id BIGINT NOT NULL,
place_key TEXT NOT NULL,
dataset_id BIGINT NOT NULL DEFAULT 0, dataset_id BIGINT NOT NULL DEFAULT 0,
src_pk INT NOT NULL, src_pk INT NOT NULL DEFAULT 0,
alias TEXT NOT NULL DEFAULT '', alias TEXT NOT NULL DEFAULT '',
note TEXT NOT NULL DEFAULT '', note TEXT NOT NULL DEFAULT '',
rond_note TEXT NOT NULL DEFAULT '',
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, dataset_id, src_pk) PRIMARY KEY (user_id, place_key)
); );
-- 站长对地点的「官方结论」:推荐(1) / 踩雷(-1),与自由文本备注分开存,同样按 src_pk。 -- 站长对地点的「官方结论」:推荐(1) / 踩雷(-1),与备注一样按坐标键存。
CREATE TABLE IF NOT EXISTS place_votes ( CREATE TABLE IF NOT EXISTS place_votes (
user_id BIGINT NOT NULL, user_id BIGINT NOT NULL,
place_key TEXT NOT NULL,
dataset_id BIGINT NOT NULL DEFAULT 0, dataset_id BIGINT NOT NULL DEFAULT 0,
src_pk INT NOT NULL, src_pk INT NOT NULL DEFAULT 0,
verdict SMALLINT NOT NULL, verdict SMALLINT NOT NULL,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, dataset_id, src_pk) PRIMARY KEY (user_id, place_key)
); );
-- 访客对地点的投票:一人一票、可改可撤。voter_key 由匿名 cookie + IP 派生, -- 访客对地点的投票:一人一票、可改可撤。voter_key 由匿名 cookie + IP 派生。
-- 与数据集无关(换快照不丢),故不挂 dataset_id。 --
-- 这是站点自己的数据(rondbackup 里没有这一项),但同样要认得出「是哪个地点」:
-- 挂 place_key 而不是 src_pk,理由与备注、结论一样。src_pk 只留作排查线索。
-- 迁移时这张表没有 dataset_id 可依,只能按**活跃数据集**认领那个编号属于哪个地点
-- (见文件末尾的迁移块)。
CREATE TABLE IF NOT EXISTS place_visitor_votes ( CREATE TABLE IF NOT EXISTS place_visitor_votes (
src_pk INT NOT NULL, place_key TEXT NOT NULL,
voter_key TEXT NOT NULL, voter_key TEXT NOT NULL,
vote SMALLINT NOT NULL, vote SMALLINT NOT NULL,
src_pk INT NOT NULL DEFAULT 0,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (src_pk, voter_key) PRIMARY KEY (place_key, voter_key)
); );
CREATE INDEX IF NOT EXISTS idx_pvv_src ON place_visitor_votes(src_pk); CREATE INDEX IF NOT EXISTS idx_pvv_src ON place_visitor_votes(src_pk);
@@ -309,9 +321,17 @@ UPDATE place_notes SET dataset_id = COALESCE((SELECT id FROM datasets WHERE is_a
UPDATE place_votes SET dataset_id = COALESCE((SELECT id FROM datasets WHERE is_active ORDER BY id DESC LIMIT 1), 0) UPDATE place_votes SET dataset_id = COALESCE((SELECT id FROM datasets WHERE is_active ORDER BY id DESC LIMIT 1), 0)
WHERE dataset_id = 0; WHERE dataset_id = 0;
-- ⚠️ 这一段只在「还没搬到坐标键」的库上跑:文件末尾那次迁移会把主键换成
-- (user_id, place_key),这里要是也跟着改回去,两张表就会在每次启动时来回翻,
-- 永远停在最后一个执行的块上。
DO $$ DO $$
DECLARE cols text; DECLARE cols text;
BEGIN BEGIN
IF EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_name = 'place_notes' AND column_name = 'place_key') THEN
RETURN;
END IF;
SELECT string_agg(a.attname, ',' ORDER BY array_position(c.conkey, a.attnum)) SELECT string_agg(a.attname, ',' ORDER BY array_position(c.conkey, a.attnum))
INTO cols INTO cols
FROM pg_constraint c FROM pg_constraint c
@@ -395,6 +415,7 @@ ALTER TABLE places ADD COLUMN IF NOT EXISTS src_lon DOUBLE PRECISION;
-- SQL 里写不出来。启动时由 ingest.BackfillAllCoordSource 统一处理,那里同时能识别出 -- SQL 里写不出来。启动时由 ingest.BackfillAllCoordSource 统一处理,那里同时能识别出
-- 「站点导出的备份给新建地点合成的 ZLOCATION 行」,不会把它误当成真机的原始 GPS。 -- 「站点导出的备份给新建地点合成的 ZLOCATION 行」,不会把它误当成真机的原始 GPS。
-- 地点图片:站长给地点配的图。字节由**浏览器**转成 WebP 后再上传(服务端只负责收下与转存), -- 地点图片:站长给地点配的图。字节由**浏览器**转成 WebP 后再上传(服务端只负责收下与转存),
-- 1 核机器不用扛解码。storage 记这一张实际放在哪、object_key 是本地相对路径或对象存储 key: -- 1 核机器不用扛解码。storage 记这一张实际放在哪、object_key 是本地相对路径或对象存储 key:
-- 换了存储后端之后旧图仍按各自那一列去取,不会因为改了设置就全变破图。 -- 换了存储后端之后旧图仍按各自那一列去取,不会因为改了设置就全变破图。
@@ -402,6 +423,9 @@ CREATE TABLE IF NOT EXISTS place_images (
id BIGSERIAL PRIMARY KEY, id BIGSERIAL PRIMARY KEY,
dataset_id BIGINT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE, dataset_id BIGINT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE,
src_pk INT NOT NULL, src_pk INT NOT NULL,
-- place_key 与备注、结论共用同一套坐标键:换数据集后图片还挂得回原地点。
-- 对象键里仍然带 dataset_id(那只是存储路径,与「认不认得出是同一个地点」无关)。
place_key TEXT NOT NULL DEFAULT '',
storage TEXT NOT NULL DEFAULT 'local', storage TEXT NOT NULL DEFAULT 'local',
object_key TEXT NOT NULL, object_key TEXT NOT NULL,
width INT NOT NULL DEFAULT 0, width INT NOT NULL DEFAULT 0,
@@ -412,3 +436,116 @@ CREATE TABLE IF NOT EXISTS place_images (
UNIQUE (dataset_id, src_pk, object_key) UNIQUE (dataset_id, src_pk, object_key)
); );
CREATE INDEX IF NOT EXISTS idx_place_images_place ON place_images(dataset_id, src_pk, position); CREATE INDEX IF NOT EXISTS idx_place_images_place ON place_images(dataset_id, src_pk, position);
-- ---------- 附属数据改挂坐标键(一次性迁移)----------
-- 备注 / 结论 / 图片原本挂 (dataset_id, src_pk),换个数据集就全看不见了。
-- 这里把它们搬到坐标键上;只在「还没有 place_key 列」时执行,所以新库直接走上面的
-- CREATE TABLE,老库走这一段。
--
-- 坐标键的算法必须与 Go 侧 rond.PlaceCoordKey 逐位一致:
-- fmt.Sprintf("%.6f|%.6f", lat, lon) ↔ to_char(lat,'FM999990.000000')||'|'||to_char(lon,'FM999990.000000')
-- 已在真实数据上逐条比对过(368/368 相同)。改任何一边都要重新比。
DO $$
DECLARE
cols text;
BEGIN
-- 判据是**主键**而不是「列在不在」:半迁移过的库(列补上了、主键还是老的三列)
-- 也得能修好,否则这张表会一直卡在中间状态。
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_name = 'place_notes' AND column_name = 'place_key') THEN
ALTER TABLE place_notes ADD COLUMN place_key TEXT;
END IF;
SELECT string_agg(a.attname, ',' ORDER BY array_position(c.conkey, a.attnum))
INTO cols FROM pg_constraint c
JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = ANY(c.conkey)
WHERE c.conname = 'place_notes_pkey';
IF cols IS DISTINCT FROM 'user_id,place_key' THEN
UPDATE place_notes n SET place_key = (
SELECT to_char(p.lat, 'FM999990.000000') || '|' || to_char(p.lon, 'FM999990.000000')
FROM places p
WHERE p.dataset_id = n.dataset_id AND p.src_pk = n.src_pk
AND p.lat IS NOT NULL AND p.lon IS NOT NULL)
WHERE n.place_key IS NULL;
-- 地点已经不在库里的(被删过),留一个不会撞的孤儿键,宁可留着也别丢
UPDATE place_notes SET place_key = 'orphan:' || dataset_id || ':' || src_pk
WHERE place_key IS NULL;
-- 同一坐标上有多条时只留最新的一条,否则加不上主键
DELETE FROM place_notes a USING place_notes b
WHERE a.user_id = b.user_id AND a.place_key = b.place_key
AND (a.updated_at, a.ctid) < (b.updated_at, b.ctid);
ALTER TABLE place_notes ALTER COLUMN place_key SET NOT NULL;
ALTER TABLE place_notes DROP CONSTRAINT IF EXISTS place_notes_pkey;
ALTER TABLE place_notes ADD PRIMARY KEY (user_id, place_key);
END IF;
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_name = 'place_votes' AND column_name = 'place_key') THEN
ALTER TABLE place_votes ADD COLUMN place_key TEXT;
END IF;
SELECT string_agg(a.attname, ',' ORDER BY array_position(c.conkey, a.attnum))
INTO cols FROM pg_constraint c
JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = ANY(c.conkey)
WHERE c.conname = 'place_votes_pkey';
IF cols IS DISTINCT FROM 'user_id,place_key' THEN
UPDATE place_votes n SET place_key = (
SELECT to_char(p.lat, 'FM999990.000000') || '|' || to_char(p.lon, 'FM999990.000000')
FROM places p
WHERE p.dataset_id = n.dataset_id AND p.src_pk = n.src_pk
AND p.lat IS NOT NULL AND p.lon IS NOT NULL)
WHERE n.place_key IS NULL;
UPDATE place_votes SET place_key = 'orphan:' || dataset_id || ':' || src_pk
WHERE place_key IS NULL;
DELETE FROM place_votes a USING place_votes b
WHERE a.user_id = b.user_id AND a.place_key = b.place_key
AND (a.updated_at, a.ctid) < (b.updated_at, b.ctid);
ALTER TABLE place_votes ALTER COLUMN place_key SET NOT NULL;
ALTER TABLE place_votes DROP CONSTRAINT IF EXISTS place_votes_pkey;
ALTER TABLE place_votes ADD PRIMARY KEY (user_id, place_key);
END IF;
-- 图片自己有 id 主键,只需补列 + 回填;查询走文件末尾那个索引
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_name = 'place_images' AND column_name = 'place_key') THEN
ALTER TABLE place_images ADD COLUMN place_key TEXT NOT NULL DEFAULT '';
END IF;
UPDATE place_images i SET place_key = COALESCE((
SELECT to_char(p.lat, 'FM999990.000000') || '|' || to_char(p.lon, 'FM999990.000000')
FROM places p
WHERE p.dataset_id = i.dataset_id AND p.src_pk = i.src_pk
AND p.lat IS NOT NULL AND p.lon IS NOT NULL),
'orphan:' || i.dataset_id || ':' || i.src_pk)
WHERE i.place_key = '';
-- 访客票:这张表没有 dataset_id,回填时只能按**活跃数据集**认领那个编号属于哪个地点。
-- 两个数据集里都有同一编号时也只认活跃的那个——这是唯一说得通的猜法,
-- 认不出来的留孤儿键,票不丢。
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_name = 'place_visitor_votes' AND column_name = 'place_key') THEN
ALTER TABLE place_visitor_votes ADD COLUMN place_key TEXT;
END IF;
SELECT string_agg(a.attname, ',' ORDER BY array_position(c.conkey, a.attnum))
INTO cols FROM pg_constraint c
JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = ANY(c.conkey)
WHERE c.conname = 'place_visitor_votes_pkey';
IF cols IS DISTINCT FROM 'place_key,voter_key' THEN
UPDATE place_visitor_votes vv SET place_key = COALESCE((
SELECT to_char(p.lat, 'FM999990.000000') || '|' || to_char(p.lon, 'FM999990.000000')
FROM places p JOIN datasets d ON d.id = p.dataset_id
WHERE d.is_active AND p.src_pk = vv.src_pk
AND p.lat IS NOT NULL AND p.lon IS NOT NULL
ORDER BY p.id LIMIT 1),
'orphan:' || vv.src_pk)
WHERE vv.place_key IS NULL;
-- 同一个访客对同一坐标只剩一条:留最新的
DELETE FROM place_visitor_votes a USING place_visitor_votes b
WHERE a.voter_key = b.voter_key AND a.place_key = b.place_key
AND (a.updated_at, a.ctid) < (b.updated_at, b.ctid);
ALTER TABLE place_visitor_votes ALTER COLUMN place_key SET NOT NULL;
ALTER TABLE place_visitor_votes DROP CONSTRAINT IF EXISTS place_visitor_votes_pkey;
ALTER TABLE place_visitor_votes ADD PRIMARY KEY (place_key, voter_key);
END IF;
END $$;
CREATE INDEX IF NOT EXISTS idx_place_images_key ON place_images(place_key, position);
CREATE INDEX IF NOT EXISTS idx_pvv_key ON place_visitor_votes(place_key);
+90
View File
@@ -0,0 +1,90 @@
package fog
// Diff 是一份待导入快照与当前迷雾之间的差异画像。
//
// 导入有两种语义完全不同的模式:叠加只做按位或、不可能丢数据;整体替换会先清空,
// 快照里没有的块一律消失。以前两种模式在界面上只是一个下拉框,站长在下手之前
// 看不到「这次能补多少」和「这次会丢多少」,实测里整体替换静默丢掉过 14% 的已探索区域。
type Diff struct {
CurBlocks, CurCells int // 库里现有
IncBlocks, IncCells int // 快照本身
// GainBlocks/GainCells 是叠加模式真正新增的部分:快照里当前完全没有的整块,
// 加上重叠块里补亮的格。
GainBlocks int
GainCells int
// OverlapBlocks 是与现有块坐标相同的块数,其中 SameBlocks 是一格都不会变的。
OverlapBlocks int
SameBlocks int
// LostBlocks/LostCells 是快照里完全没有、只有整体替换才会消失的部分。
LostBlocks int
LostCells int
curAreaM2, incAreaM2, gainAreaM2, lostAreaM2 float64
}
// DiffBlocks 比对当前块集与一份待导入快照。入参块集不要求有序。
func DiffBlocks(cur, inc []Block) Diff {
var d Diff
d.CurBlocks, d.IncBlocks = len(cur), len(inc)
curMap := make(map[[2]int][]byte, len(cur))
for _, b := range cur {
curMap[[2]int{b.GX, b.GY}] = b.Bitmap
n := popcountSum(b.Bitmap)
d.CurCells += n
d.curAreaM2 += float64(n) * cellAreaM2(b.GX, b.GY)
}
incSet := make(map[[2]int]struct{}, len(inc))
for _, b := range inc {
incSet[[2]int{b.GX, b.GY}] = struct{}{}
n := popcountSum(b.Bitmap)
d.IncCells += n
d.incAreaM2 += float64(n) * cellAreaM2(b.GX, b.GY)
old, ok := curMap[[2]int{b.GX, b.GY}]
if !ok {
d.GainBlocks++
d.GainCells += n
d.gainAreaM2 += float64(n) * cellAreaM2(b.GX, b.GY)
continue
}
d.OverlapBlocks++
gain := 0
for i := range b.Bitmap {
gain += int(popcount[b.Bitmap[i]&^old[i]])
}
if gain == 0 {
d.SameBlocks++
continue
}
d.GainCells += gain
d.gainAreaM2 += float64(gain) * cellAreaM2(b.GX, b.GY)
}
for _, b := range cur {
if _, ok := incSet[[2]int{b.GX, b.GY}]; ok {
continue
}
n := popcountSum(b.Bitmap)
d.LostBlocks++
d.LostCells += n
d.lostAreaM2 += float64(n) * cellAreaM2(b.GX, b.GY)
}
return d
}
func (d Diff) CurAreaKm2() float64 { return d.curAreaM2 / 1e6 }
func (d Diff) IncAreaKm2() float64 { return d.incAreaM2 / 1e6 }
func (d Diff) GainAreaKm2() float64 { return d.gainAreaM2 / 1e6 }
func (d Diff) LostAreaKm2() float64 { return d.lostAreaM2 / 1e6 }
// MergeAreaKm2 是叠加后的总量,CoverAreaKm2 是整体替换后的总量。
func (d Diff) MergeAreaKm2() float64 { return (d.curAreaM2 + d.gainAreaM2) / 1e6 }
func (d Diff) CoverAreaKm2() float64 { return d.incAreaM2 / 1e6 }
// Changed 报告叠加是否会让地图有任何变化——为假说明这份快照已经被完全覆盖过。
func (d Diff) Changed() bool { return d.GainCells > 0 }
+88
View File
@@ -0,0 +1,88 @@
package fog
import "testing"
// blk 造一个块,cells 是块内的 (列, 行) 坐标。
func blk(gx, gy int, cells ...[2]int) Block {
bm := make([]byte, bitmapSize)
for _, c := range cells {
bm[c[1]*8+c[0]/8] |= 0x80 >> (c[0] % 8)
}
return Block{GX: gx, GY: gy, Bitmap: bm}
}
func TestDiffEmptyBase(t *testing.T) {
d := DiffBlocks(nil, []Block{blk(10, 20, [2]int{0, 0}, [2]int{1, 0})})
if d.CurBlocks != 0 || d.CurCells != 0 {
t.Fatalf("空库不该有现有数据: %+v", d)
}
if d.GainBlocks != 1 || d.GainCells != 2 || d.IncCells != 2 {
t.Fatalf("空库叠加应为全部新增: %+v", d)
}
if d.LostBlocks != 0 || d.LostCells != 0 || !d.Changed() {
t.Fatalf("空库不该有丢失: %+v", d)
}
}
func TestDiffIdentical(t *testing.T) {
cur := []Block{blk(10, 20, [2]int{0, 0}, [2]int{1, 0}), blk(11, 20, [2]int{5, 5})}
d := DiffBlocks(cur, cur)
if d.GainCells != 0 || d.Changed() {
t.Fatalf("同一份快照再叠一次不该有任何新增: %+v", d)
}
if d.LostBlocks != 0 || d.LostCells != 0 {
t.Fatalf("同一份快照不该有可丢的部分: %+v", d)
}
if d.OverlapBlocks != 2 || d.SameBlocks != 2 {
t.Fatalf("两个块都应算作「重叠且无变化」: %+v", d)
}
}
func TestDiffOverlapGain(t *testing.T) {
d := DiffBlocks(
[]Block{blk(10, 20, [2]int{0, 0})},
[]Block{blk(10, 20, [2]int{0, 0}, [2]int{3, 4})})
if d.GainBlocks != 0 || d.GainCells != 1 {
t.Fatalf("重叠块应只补亮多出的那一格: %+v", d)
}
if d.OverlapBlocks != 1 || d.SameBlocks != 0 {
t.Fatalf("重叠但补亮了,不该记为无变化: %+v", d)
}
if d.LostBlocks != 0 {
t.Fatalf("同一个块不该被判为会丢失: %+v", d)
}
}
func TestDiffDisjoint(t *testing.T) {
d := DiffBlocks(
[]Block{blk(10, 20, [2]int{0, 0})},
[]Block{blk(30, 40, [2]int{1, 1})})
if d.GainBlocks != 1 || d.GainCells != 1 {
t.Fatalf("另一块应是新增: %+v", d)
}
if d.LostBlocks != 1 || d.LostCells != 1 {
t.Fatalf("库里那块在快照里没有,整体替换会丢掉: %+v", d)
}
}
// 面积按块中心纬度做 cos 修正,越靠近两极越小;总量关系也要自洽。
func TestDiffAreas(t *testing.T) {
equator := DiffBlocks(nil, []Block{blk(100, WorldCells/BlockCells/2, [2]int{1, 1})})
high := DiffBlocks(nil, []Block{blk(100, 20000, [2]int{1, 1})})
if high.GainAreaKm2() <= 0 {
t.Fatalf("高纬单格面积应为正: %v", high.GainAreaKm2())
}
if !(equator.GainAreaKm2() > high.GainAreaKm2()) {
t.Fatalf("赤道单格应大于高纬: %v vs %v", equator.GainAreaKm2(), high.GainAreaKm2())
}
m := DiffBlocks(
[]Block{blk(10, 20, [2]int{0, 0})},
[]Block{blk(10, 20, [2]int{0, 0}, [2]int{9, 9})})
if diff := m.MergeAreaKm2() - (m.CurAreaKm2() + m.GainAreaKm2()); diff > 1e-12 || diff < -1e-12 {
t.Fatalf("叠加后总量应等于现有加新增: %v", diff)
}
if m.CoverAreaKm2() != m.IncAreaKm2() {
t.Fatalf("整体替换后总量应等于快照本身: %v vs %v", m.CoverAreaKm2(), m.IncAreaKm2())
}
}
+6
View File
@@ -180,6 +180,12 @@ func mercY(lat float64) float64 {
return (math.Pi - math.Asinh(math.Tan(rad))) / (2 * math.Pi) * WorldCells return (math.Pi - math.Asinh(math.Tan(rad))) / (2 * math.Pi) * WorldCells
} }
// BlockLonLat 返回某块中心的 WGS-84 经纬度,供差异地图定位视野。
func BlockLonLat(gbx, gby int) (lat, lon float64) {
lon, lat = cellLonLat(gbx*BlockCells+BlockCells/2, gby*BlockCells+BlockCells/2)
return
}
func cellLonLat(cx, cy int) (float64, float64) { func cellLonLat(cx, cy int) (float64, float64) {
lon := (float64(cx)+0.5)/WorldCells*360.0 - 180.0 lon := (float64(cx)+0.5)/WorldCells*360.0 - 180.0
n := 2 * math.Pi * (float64(cy) + 0.5) / WorldCells n := 2 * math.Pi * (float64(cy) + 0.5) / WorldCells
+200
View File
@@ -0,0 +1,200 @@
package fog
import (
"image"
"math"
)
// 差异图的三种归属。改这里的色值要同步 templates/admin/fog-preview.html 的图例。
var (
DiffBothColor = [4]byte{126, 134, 146, 148} // 两边都有:导入不影响
DiffGainColor = [4]byte{36, 164, 90, 205} // 只有快照有:叠加会新增
DiffLostColor = [4]byte{214, 56, 56, 210} // 只有库里有:整体替换会丢
)
const (
diffNone uint8 = iota
diffBoth
diffGain
diffLost
)
// RenderDiffTile 渲染一张 256x256 的差异瓦片,供站长在真实底图上核对
// 「这份快照能补上哪里」与「整体替换会丢掉哪里」。
//
// 与 RenderTile 一样按底图坐标系做 WGS84→GCJ02 纠偏;不做描边——三种颜色
// 已经够区分,再描边在粗缩放级下会糊成一片。
//
// 逐格判定只在放大到「一格不小于一像素」(z>=14)时做;更粗的级别下一个块还不到
// 一个像素,按块内占多数的类别整块涂色。绘制顺序 both→gain→lost,让「会新增」
// 和「会丢」压住底色,重叠处不至于被灰吃掉。
func RenderDiffTile(cur, inc map[[2]int][]byte, z, x, y int, crs string) *image.NRGBA {
if z < 0 || z > 22 {
return nil
}
step := WorldCells >> z
scale := 256.0 / float64(step)
ox, oy := float64(x*step), float64(y*step)
layer := make([]uint8, 256*256)
mark := func(px, py, w, h int, kind uint8) {
for r := 0; r < h; r++ {
yy := py + r
if yy < 0 || yy > 255 {
continue
}
row := yy * 256
for c := 0; c < w; c++ {
xx := px + c
if xx < 0 || xx > 255 {
continue
}
layer[row+xx] = kind
}
}
}
// 直接遍历块表而不是全球块范围:低缩放级下范围循环会膨胀到数十亿次空查询。
// GCJ 偏移最多约 700m(74 格),粗裁剪时两侧各放宽两块保证不漏。
const margin = 2 * BlockCells
lo, hi := float64(x*step-margin), float64((x+1)*step+margin)
rlo, rhi := float64(y*step-margin), float64((y+1)*step+margin)
shift := crs == "gcj02"
draw := func(pass uint8, gbx, gby int, cb, ib []byte) {
bx, by := float64(gbx*BlockCells), float64(gby*BlockCells)
if bx+BlockCells <= lo || bx >= hi || by+BlockCells <= rlo || by >= rhi {
return
}
ddx, ddy := 0.0, 0.0
if shift {
ddx, ddy = blockShift(gbx, gby)
}
u0 := (bx + ddx - ox) * scale
v0 := (by + ddy - oy) * scale
if scale >= 1 {
w := int(math.Ceil(scale))
for row := 0; row < BlockCells; row++ {
base := row * 8
for colB := 0; colB < 8; colB++ {
var cbv, ibv byte
if cb != nil {
cbv = cb[base+colB]
}
if ib != nil {
ibv = ib[base+colB]
}
if cbv|ibv == 0 {
continue
}
py := int(math.Floor(v0 + float64(row)*scale))
for bit := 0; bit < 8; bit++ {
m := byte(0x80 >> bit)
kind := cellKind(cbv&m != 0, ibv&m != 0)
if kind != pass {
continue
}
px := int(math.Floor(u0 + float64(colB*8+bit)*scale))
mark(px, py, w, w, kind)
}
}
}
return
}
// 一格不足一像素:整块按多数类涂,三种混在一起的块以数量最多的为准
if dominantKind(blockDiffCounts(cb, ib)) != pass {
return
}
u1 := (bx + BlockCells + ddx - ox) * scale
v1 := (by + BlockCells + ddy - oy) * scale
px0, py0 := int(math.Floor(u0)), int(math.Floor(v0))
px1, py1 := int(math.Ceil(u1)), int(math.Ceil(v1))
if px1-px0 < 1 {
px1 = px0 + 1
}
if py1-py0 < 1 {
py1 = py0 + 1
}
mark(px0, py0, px1-px0, py1-py0, pass)
}
for _, pass := range []uint8{diffBoth, diffGain, diffLost} {
for key, cb := range cur {
draw(pass, key[0], key[1], cb, inc[key])
}
// 只在快照里的块单独走一遍,两边都有的块不再重复画
for key, ib := range inc {
if _, dup := cur[key]; dup {
continue
}
draw(pass, key[0], key[1], nil, ib)
}
}
img := image.NewNRGBA(image.Rect(0, 0, 256, 256))
for i, kind := range layer {
if kind == diffNone {
continue
}
var c [4]byte
switch kind {
case diffBoth:
c = DiffBothColor
case diffGain:
c = DiffGainColor
default:
c = DiffLostColor
}
o := i * 4
img.Pix[o], img.Pix[o+1], img.Pix[o+2], img.Pix[o+3] = c[0], c[1], c[2], c[3]
}
return img
}
func cellKind(inCur, inInc bool) uint8 {
switch {
case inCur && inInc:
return diffBoth
case inInc:
return diffGain
case inCur:
return diffLost
}
return diffNone
}
// blockDiffCounts 数一个块里三类各占多少格。两个位图都可能缺失(只有一边有这个块)。
func blockDiffCounts(cb, ib []byte) (both, gain, lost int) {
n := len(cb)
if len(ib) > n {
n = len(ib)
}
for i := 0; i < n; i++ {
var c, in byte
if i < len(cb) {
c = cb[i]
}
if i < len(ib) {
in = ib[i]
}
both += int(popcount[c&in])
gain += int(popcount[in&^c])
lost += int(popcount[c&^in])
}
return
}
// dominantKind 取三类里占得最多的一类,用来在粗缩放级下决定整块的着色。
func dominantKind(both, gain, lost int) uint8 {
switch {
case gain > 0 && gain >= both && gain >= lost:
return diffGain
case lost > 0 && lost >= both:
return diffLost
case both > 0:
return diffBoth
}
return diffNone
}
+114
View File
@@ -0,0 +1,114 @@
package fog
import (
"image"
"math"
"testing"
)
func bmOf(cells ...[2]int) []byte {
bm := make([]byte, bitmapSize)
for _, c := range cells {
bm[c[1]*8+c[0]/8] |= 0x80 >> (c[0] % 8)
}
return bm
}
func pixAt(img *image.NRGBA, px, py int) [4]byte {
i := (py*256 + px) * 4
return [4]byte{img.Pix[i], img.Pix[i+1], img.Pix[i+2], img.Pix[i+3]}
}
// z=18 时一格正好 16 像素,块 (100,200) 完整落在瓦片 (400,800) 里,
// 格 (col,row) 的中心像素是 (col*16+8, row*16+8)。
func TestRenderDiffTilePerCell(t *testing.T) {
key := [2]int{100, 200}
cur := map[[2]int][]byte{key: bmOf([2]int{0, 0}, [2]int{4, 0})}
inc := map[[2]int][]byte{key: bmOf([2]int{0, 0}, [2]int{1, 0})}
img := RenderDiffTile(cur, inc, 18, 400, 800, "wgs84")
if img == nil {
t.Fatal("瓦片不该为空")
}
if got := pixAt(img, 8, 8); got != DiffBothColor {
t.Errorf("两边都有的格应为灰色,实际 %v", got)
}
if got := pixAt(img, 24, 8); got != DiffGainColor {
t.Errorf("只有快照有的格应为绿色,实际 %v", got)
}
if got := pixAt(img, 72, 8); got != DiffLostColor {
t.Errorf("只有库里有、整体替换会丢的格应为红色,实际 %v", got)
}
if got := pixAt(img, 200, 200); got[3] != 0 {
t.Errorf("没被探索的格应保持透明,实际 %v", got)
}
}
func TestRenderDiffTileSingleSided(t *testing.T) {
key := [2]int{100, 200}
onlyCur := RenderDiffTile(map[[2]int][]byte{key: bmOf([2]int{3, 3})}, nil, 18, 400, 800, "wgs84")
if got := pixAt(onlyCur, 56, 56); got != DiffLostColor {
t.Errorf("快照里完全没有的块应整块为红,实际 %v", got)
}
onlyInc := RenderDiffTile(nil, map[[2]int][]byte{key: bmOf([2]int{3, 3})}, 18, 400, 800, "wgs84")
if got := pixAt(onlyInc, 56, 56); got != DiffGainColor {
t.Errorf("库里完全没有的块应整块为绿,实际 %v", got)
}
}
// z=10 时一格不足一像素,块只占 4x4 像素,按块内多数类涂色。
func TestRenderDiffTileCoarseZoomUsesDominantKind(t *testing.T) {
key := [2]int{100, 200}
cur := map[[2]int][]byte{key: bmOf([2]int{0, 0})}
inc := map[[2]int][]byte{key: bmOf([2]int{1, 1}, [2]int{2, 2}, [2]int{3, 3})}
img := RenderDiffTile(cur, inc, 10, 1, 3, "wgs84")
const step, scale = WorldCells >> 10, 256.0 / (WorldCells >> 10)
bx, by := float64(key[0]*BlockCells), float64(key[1]*BlockCells)
px := int(math.Floor((bx - 1*step) * scale))
py := int(math.Floor((by - 3*step) * scale))
if got := pixAt(img, px+1, py+1); got != DiffGainColor {
t.Errorf("三格新增、一格不变的块应以绿色为主,实际 %v", got)
}
if w := int(math.Ceil(float64(BlockCells) * scale)); w != 4 {
t.Fatalf("z=10 下块应占 4 像素,实际 %d", w)
}
}
func TestRenderDiffTileEdgeCases(t *testing.T) {
if RenderDiffTile(nil, nil, 18, 400, 800, "wgs84") == nil {
t.Error("两边都空也应返回一张空瓦片")
}
if RenderDiffTile(nil, nil, 23, 0, 0, "wgs84") != nil {
t.Error("超出缩放范围应返回 nil")
}
}
func TestDominantKind(t *testing.T) {
cases := []struct {
both, gain, lost int
want uint8
}{
{0, 0, 0, diffNone},
{5, 1, 0, diffBoth},
{1, 5, 0, diffGain},
{1, 0, 5, diffLost},
{2, 2, 0, diffGain}, // 平局让「会新增」胜出
{3, 1, 1, diffBoth},
}
for _, c := range cases {
if got := dominantKind(c.both, c.gain, c.lost); got != c.want {
t.Errorf("dominantKind(%d,%d,%d)=%d,期望 %d", c.both, c.gain, c.lost, got, c.want)
}
}
}
func TestBlockDiffCounts(t *testing.T) {
both, gain, lost := blockDiffCounts(bmOf([2]int{0, 0}, [2]int{1, 0}), bmOf([2]int{0, 0}, [2]int{2, 0}))
if both != 1 || gain != 1 || lost != 1 {
t.Errorf("一块里应各有一格:both=%d gain=%d lost=%d", both, gain, lost)
}
both, gain, lost = blockDiffCounts(nil, bmOf([2]int{5, 5}))
if both != 0 || gain != 1 || lost != 0 {
t.Errorf("只有快照有该块:both=%d gain=%d lost=%d", both, gain, lost)
}
}
+30 -4
View File
@@ -67,15 +67,41 @@ func insertEntityRaw(ctx context.Context, tx *sql.Tx, datasetID int64, rows map[
} }
// contentHash 给备份内容算指纹,用来识别「内容没变、只是重新导出了一次」的上传。 // contentHash 给备份内容算指纹,用来识别「内容没变、只是重新导出了一次」的上传。
// rond 每次导出连 WAL 一起打包,字节必然不同,文件摘要挡不住这种重复; // rond 每次导出连 WAL 一起打包,字节必然不同,文件摘要挡不住这种重复。
// 但同一批到访的 Z_PK 是稳定的,所以按到访自然键排序后取摘要即可。 //
// 指纹按**到访/活动/地点的稳定标识**算,绝不能掺 Z_PK:rond 每次导出/同步都会重排
// 编号(实测两份备份里 822 条到访的 UUID 完全一致、Z_PK 100% 不同),拿编号当指纹会把
// 「同一批数据重导一次」判成新内容 —— 于是新建数据集,而备注与投票是按 dataset_id
// 隔离的,换了数据集就再也看不见。
//
// 没有 UUID 的老备份退回「到达时间 + 地点坐标」,仍比编号可靠。
func contentHash(b *rond.Backup) string { func contentHash(b *rond.Backup) string {
keys := make([]string, 0, len(b.Visits)) locCoord := make(map[int]string, len(b.Locations))
keys := make([]string, 0, len(b.Visits)+len(b.Locations)+len(b.Activities))
for _, l := range b.Locations {
locCoord[l.SrcPK] = fmt.Sprintf("%.6f/%.6f", l.Lat, l.Lon)
keys = append(keys, "L"+l.Name+"/"+locCoord[l.SrcPK])
}
for _, v := range b.Visits { for _, v := range b.Visits {
if v.Arrival == nil { if v.Arrival == nil {
continue // 导入时就被跳过的行,不该影响指纹
}
if u := b.VisitUUID[v.SrcPK]; u != "" {
keys = append(keys, "V"+u)
continue continue
} }
keys = append(keys, fmt.Sprintf("%d|%d", v.SrcPK, v.Arrival.Unix())) loc := ""
if v.LocationSrcPK != nil {
loc = locCoord[*v.LocationSrcPK]
}
keys = append(keys, fmt.Sprintf("T%d/%s", v.Arrival.Unix(), loc))
}
for _, a := range b.Activities {
if u := b.ActivityUUID[a.SrcPK]; u != "" {
keys = append(keys, "A"+u)
continue
}
keys = append(keys, "A"+a.Name)
} }
sort.Strings(keys) sort.Strings(keys)
h := sha256.New() h := sha256.New()
+107
View File
@@ -0,0 +1,107 @@
package ingest
import (
"fmt"
"testing"
"time"
"rond-with-you/internal/rond"
)
// sampleBackup 造一份最小的备份:到访 + 地点 + 活动,编号与 UUID 分别给定。
func sampleBackup(pks []int, uuids []string) *rond.Backup {
b := &rond.Backup{VisitUUID: map[int]string{}, ActivityUUID: map[int]string{}}
for i, pk := range pks {
at := time.Unix(int64(1700000000+i*3600), 0)
b.Visits = append(b.Visits, rond.Visit{SrcPK: pk, Arrival: &at, LocationSrcPK: intp(1)})
b.VisitUUID[pk] = uuids[i]
}
b.Locations = []rond.Location{{SrcPK: 1, Name: "家", Lat: 24.1, Lon: 109.1}}
b.Activities = []rond.Activity{{SrcPK: 9, Name: "工作"}}
b.ActivityUUID[9] = "cccccccccccccccccccccccccccccccc"
return b
}
func intp(v int) *int { return &v }
// 同一批数据重新导出一次:UUID 一个没变、Z_PK 全被打乱(顺序也变了),
// 指纹必须一样。否则会被判成「新内容」→ 新建数据集 → 按 dataset_id 隔离的
// 备注与投票全部失联。
func TestContentHashIgnoresPKReorder(t *testing.T) {
a := sampleBackup([]int{1, 2, 3}, []string{
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
"dddddddddddddddddddddddddddddddd",
})
b := sampleBackup([]int{555, 276, 88}, []string{
"dddddddddddddddddddddddddddddddd",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
})
if contentHash(a) != contentHash(b) {
t.Fatal("编号重排不该改变指纹")
}
}
// 内容真的变了(多一条到访)就得算不同的指纹,否则新数据会被当成重复上传而丢掉。
func TestContentHashDetectsNewVisit(t *testing.T) {
a := sampleBackup([]int{1, 2}, []string{
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
})
b := sampleBackup([]int{1, 2, 3}, []string{
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
"dddddddddddddddddddddddddddddddd",
})
if contentHash(a) == contentHash(b) {
t.Fatal("多了一条到访就是不同内容")
}
}
// 只改了地点名(到访一条没动)也算内容变了:站点是靠地名认地点的。
func TestContentHashDetectsPlaceRename(t *testing.T) {
a := sampleBackup([]int{1}, []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"})
b := sampleBackup([]int{1}, []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"})
b.Locations[0].Name = "老家"
if contentHash(a) == contentHash(b) {
t.Fatal("地点改名应改变指纹")
}
}
// 没有 UUID 的老备份退回「到达时间 + 地点坐标」,同一批数据重导仍要认出是同一份。
func TestContentHashFallsBackWithoutUUID(t *testing.T) {
a := sampleBackup([]int{1, 2}, []string{"", ""})
b := sampleBackup([]int{700, 900}, []string{"", ""})
if contentHash(a) != contentHash(b) {
t.Fatal("没有 UUID 时应按到达时间与地点兜底,不该受编号影响")
}
c := sampleBackup([]int{1, 2}, []string{"", ""})
c.Visits[1].Arrival = ptrTime(time.Unix(1700000000, 0)) // 改掉第二条的时间
if contentHash(a) == contentHash(c) {
t.Fatal("时间不同应算不同内容")
}
}
// 没有到达时间的行导入时就被跳过,不该参与指纹。
func TestContentHashSkipsRowsWithoutArrival(t *testing.T) {
a := sampleBackup([]int{1}, []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"})
b := sampleBackup([]int{1}, []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"})
b.Visits = append(b.Visits, rond.Visit{SrcPK: 42})
if contentHash(a) != contentHash(b) {
t.Fatal("缺到达时间的行不该影响指纹")
}
}
// 指纹是稳定的十六进制摘要,不是随手拼的字符串。
func TestContentHashShape(t *testing.T) {
h := contentHash(sampleBackup([]int{1}, []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}))
if len(h) != 64 {
t.Fatalf("应是 64 位 sha256 十六进制,实际 %d 位: %s", len(h), h)
}
if _, err := fmt.Sscanf(h, "%x", new(string)); err != nil {
t.Fatalf("不是纯十六进制: %v", err)
}
}
func ptrTime(t time.Time) *time.Time { return &t }
+62 -37
View File
@@ -1,11 +1,14 @@
// Package notes 存放站长为地点加的别名与备注。 // Package notes 存放站长为地点加的别名与备注。
// 备注按 rond 的地点 ID(src_pk)而非数据库自增 id 存储, //
// 因此重新上传备份、地点换了新 id 之后内容依然挂在原地。 // 备注挂的是**坐标键**(place_key)而不是 rond 的地点编号:src_pk 只在单个数据集内有效
// (rond 每次导出都会重排 Z_PK,实测两份备份里 822 条到访的 UUID 一致而 Z_PK 100% 不同),
// 换个数据集同编号就是另一个地点,备注会全部失联。坐标是地理事实,跨数据集稳定。
package notes package notes
import ( import (
"context" "context"
"database/sql" "database/sql"
"fmt"
"strings" "strings"
"time" "time"
) )
@@ -20,15 +23,26 @@ type Note struct {
type Store struct{ DB *sql.DB } type Store struct{ DB *sql.DB }
// All 返回某个数据集的全站备注(站点只有一个站长账号,读的时候不按 user_id 过滤, // placeKeyExpr 把某个 places 别名下的坐标换算成 place_key。
// 这样匿名访客也能看到别名;写的时候才需要登录用户的 id)。 // **必须与 Go 侧的 rond.PlaceCoordKey 逐位一致**:
// 后写覆盖先写,保证同一地点只保留一条。必须按 dataset_id 过滤: //
// src_pk 只在单个数据集内唯一,跨数据集同编号是完全不同的地点。 // fmt.Sprintf("%.6f|%.6f") ↔ to_char(x,'FM999990.000000')||'|'||to_char(y,'FM999990.000000')
//
// 已在真实数据上逐条比对(368/368 相同)。改任何一边都要重新比。
func placeKeyExpr(alias string) string {
return fmt.Sprintf(
"to_char(%[1]s.lat,'FM999990.000000')||'|'||to_char(%[1]s.lon,'FM999990.000000')", alias)
}
// All 返回某个数据集里「有备注的地点」,键是地点在该数据集内的编号。
// 站点只有一个站长账号,读的时候不按 user_id 过滤(匿名访客也要看到别名)。
func (s *Store) All(ctx context.Context, datasetID int64) (map[int]Note, error) { func (s *Store) All(ctx context.Context, datasetID int64) (map[int]Note, error) {
out := map[int]Note{} out := map[int]Note{}
rows, err := s.DB.QueryContext(ctx, rows, err := s.DB.QueryContext(ctx, `
`SELECT src_pk, alias, note, rond_note, updated_at FROM place_notes SELECT p.src_pk, n.alias, n.note, n.rond_note, n.updated_at
WHERE dataset_id=$1 ORDER BY updated_at DESC`, datasetID) FROM places p
JOIN place_notes n ON n.place_key = `+placeKeyExpr("p")+`
WHERE p.dataset_id = $1 AND p.lat IS NOT NULL AND p.lon IS NOT NULL`, datasetID)
if err != nil { if err != nil {
return out, err return out, err
} }
@@ -46,9 +60,9 @@ func (s *Store) All(ctx context.Context, datasetID int64) (map[int]Note, error)
// SetNote 只改备注、保留别名(批量「清空备注」用);两者都为空则删除记录。 // SetNote 只改备注、保留别名(批量「清空备注」用);两者都为空则删除记录。
func (s *Store) SetNote(ctx context.Context, userID, datasetID int64, srcPK int, content string) error { func (s *Store) SetNote(ctx context.Context, userID, datasetID int64, srcPK int, content string) error {
var alias string var alias string
err := s.DB.QueryRowContext(ctx, err := s.DB.QueryRowContext(ctx, `SELECT n.alias FROM place_notes n
`SELECT alias FROM place_notes WHERE user_id=$1 AND dataset_id=$2 AND src_pk=$3`, JOIN places p ON n.place_key = `+placeKeyExpr("p")+`
userID, datasetID, srcPK).Scan(&alias) WHERE n.user_id=$1 AND p.dataset_id=$2 AND p.src_pk=$3`, userID, datasetID, srcPK).Scan(&alias)
if err != nil && err != sql.ErrNoRows { if err != nil && err != sql.ErrNoRows {
return err return err
} }
@@ -57,8 +71,9 @@ func (s *Store) SetNote(ctx context.Context, userID, datasetID int64, srcPK int,
// Delete 删除某地点的整条备注与别名。 // Delete 删除某地点的整条备注与别名。
func (s *Store) Delete(ctx context.Context, userID, datasetID int64, srcPK int) error { func (s *Store) Delete(ctx context.Context, userID, datasetID int64, srcPK int) error {
_, err := s.DB.ExecContext(ctx, _, err := s.DB.ExecContext(ctx, `DELETE FROM place_notes n USING places p
`DELETE FROM place_notes WHERE user_id=$1 AND dataset_id=$2 AND src_pk=$3`, userID, datasetID, srcPK) WHERE n.user_id=$1 AND p.dataset_id=$2 AND p.src_pk=$3
AND n.place_key = `+placeKeyExpr("p"), userID, datasetID, srcPK)
return err return err
} }
@@ -68,21 +83,28 @@ func (s *Store) Save(ctx context.Context, userID, datasetID int64, srcPK int, al
content = strings.TrimSpace(content) content = strings.TrimSpace(content)
if alias == "" && content == "" { if alias == "" && content == "" {
// 没有 rond 备注时直接删行;有则只清空手写部分,别把回填结果一起删掉 // 没有 rond 备注时直接删行;有则只清空手写部分,别把回填结果一起删掉
_, err := s.DB.ExecContext(ctx, if _, err := s.DB.ExecContext(ctx, `DELETE FROM place_notes n USING places p
`DELETE FROM place_notes WHERE user_id=$1 AND dataset_id=$2 AND src_pk=$3 AND rond_note=''`, WHERE n.user_id=$1 AND p.dataset_id=$2 AND p.src_pk=$3
userID, datasetID, srcPK) AND n.place_key = `+placeKeyExpr("p")+` AND n.rond_note=''`,
if err != nil { userID, datasetID, srcPK); err != nil {
return err return err
} }
_, err = s.DB.ExecContext(ctx, _, err := s.DB.ExecContext(ctx, `UPDATE place_notes n
`UPDATE place_notes SET alias='', note='', updated_at=now() SET alias='', note='', updated_at=now()
WHERE user_id=$1 AND dataset_id=$2 AND src_pk=$3`, userID, datasetID, srcPK) FROM places p
WHERE n.user_id=$1 AND p.dataset_id=$2 AND p.src_pk=$3
AND n.place_key = `+placeKeyExpr("p"), userID, datasetID, srcPK)
return err return err
} }
_, err := s.DB.ExecContext(ctx, `INSERT INTO place_notes (user_id, dataset_id, src_pk, alias, note, updated_at) // 先按坐标键算出这条地点该挂在哪,找不到该地点就什么也不写
VALUES ($1,$2,$3,$4,$5,now()) _, err := s.DB.ExecContext(ctx, `INSERT INTO place_notes
ON CONFLICT (user_id, dataset_id, src_pk) (user_id, place_key, dataset_id, src_pk, alias, note, updated_at)
DO UPDATE SET alias=EXCLUDED.alias, note=EXCLUDED.note, updated_at=now()`, SELECT $1, `+placeKeyExpr("p")+`, $2, $3, $4, $5, now()
FROM places p
WHERE p.dataset_id=$2 AND p.src_pk=$3 AND p.lat IS NOT NULL AND p.lon IS NOT NULL
ON CONFLICT (user_id, place_key) DO UPDATE SET
alias=EXCLUDED.alias, note=EXCLUDED.note,
dataset_id=EXCLUDED.dataset_id, src_pk=EXCLUDED.src_pk, updated_at=now()`,
userID, datasetID, srcPK, alias, content) userID, datasetID, srcPK, alias, content)
return err return err
} }
@@ -92,20 +114,22 @@ func (s *Store) Save(ctx context.Context, userID, datasetID int64, srcPK int, al
// 只重写 rond_note 一列,站长手写的 note 一概不动。返回写了多少行。 // 只重写 rond_note 一列,站长手写的 note 一概不动。返回写了多少行。
func (s *Store) BackfillRond(ctx context.Context, userID, datasetID int64) (int, error) { func (s *Store) BackfillRond(ctx context.Context, userID, datasetID int64) (int, error) {
// DISTINCT 在「日期 + 内容」这一层做,同一天重复写的同一句只留一条 // DISTINCT 在「日期 + 内容」这一层做,同一天重复写的同一句只留一条
const agg = ` agg := `
WITH raw AS ( WITH raw AS (
SELECT p.src_pk, to_char(v.arrival, 'YYYY-MM-DD') || ' ' || btrim(v.remark) AS line SELECT ` + placeKeyExpr("p") + ` AS key,
to_char(v.arrival, 'YYYY-MM-DD') || ' ' || btrim(v.remark) AS line
FROM visits v JOIN places p ON p.id = v.place_id FROM visits v JOIN places p ON p.id = v.place_id
WHERE v.dataset_id = $1 AND btrim(COALESCE(v.remark, '')) <> '' WHERE v.dataset_id = $1 AND btrim(COALESCE(v.remark, '')) <> ''
AND p.lat IS NOT NULL AND p.lon IS NOT NULL
), uniq AS ( ), uniq AS (
SELECT DISTINCT src_pk, line, left(line, 10) AS day FROM raw SELECT DISTINCT key, line, left(line, 10) AS day FROM raw
), agg AS ( ), agg AS (
SELECT src_pk, string_agg(line, E'\n' ORDER BY day, line) AS txt FROM uniq GROUP BY src_pk SELECT key, string_agg(line, E'\n' ORDER BY day, line) AS txt FROM uniq GROUP BY key
)` )`
res, err := s.DB.ExecContext(ctx, agg+` res, err := s.DB.ExecContext(ctx, agg+`
INSERT INTO place_notes (user_id, dataset_id, src_pk, alias, note, rond_note, updated_at) INSERT INTO place_notes (user_id, place_key, dataset_id, src_pk, alias, note, rond_note, updated_at)
SELECT $2, $1, src_pk, '', '', txt, now() FROM agg SELECT $2, key, $1, 0, '', '', txt, now() FROM agg
ON CONFLICT (user_id, dataset_id, src_pk) DO UPDATE SET rond_note = EXCLUDED.rond_note`, ON CONFLICT (user_id, place_key) DO UPDATE SET rond_note = EXCLUDED.rond_note`,
datasetID, userID) datasetID, userID)
if err != nil { if err != nil {
return 0, err return 0, err
@@ -113,15 +137,16 @@ func (s *Store) BackfillRond(ctx context.Context, userID, datasetID int64) (int,
affected, _ := res.RowsAffected() affected, _ := res.RowsAffected()
// 备注被删掉的地点要清掉旧值,否则会一直留着过期的 rond 内容。 // 备注被删掉的地点要清掉旧值,否则会一直留着过期的 rond 内容。
// 限定在本数据集的地点里:datasetID 不合法时子查询为空,一行都不会动, // 限定在本数据集的地点里:datasetID 不合法时子查询为空,一行都不会动。
// 不会把别的数据集的回填结果误清掉。
if _, err := s.DB.ExecContext(ctx, ` if _, err := s.DB.ExecContext(ctx, `
UPDATE place_notes n SET rond_note = '' UPDATE place_notes n SET rond_note = ''
WHERE n.user_id = $2 AND n.dataset_id = $1 AND n.rond_note <> '' WHERE n.user_id = $2 AND n.rond_note <> ''
AND n.src_pk IN (SELECT src_pk FROM places WHERE dataset_id = $1) AND n.place_key IN (SELECT `+placeKeyExpr("p")+`
FROM places p WHERE p.dataset_id = $1 AND p.lat IS NOT NULL AND p.lon IS NOT NULL)
AND NOT EXISTS ( AND NOT EXISTS (
SELECT 1 FROM visits v JOIN places p ON p.id = v.place_id SELECT 1 FROM visits v JOIN places p ON p.id = v.place_id
WHERE v.dataset_id = $1 AND p.src_pk = n.src_pk AND btrim(COALESCE(v.remark, '')) <> '' WHERE v.dataset_id = $1 AND `+placeKeyExpr("p")+` = n.place_key
AND btrim(COALESCE(v.remark, '')) <> ''
)`, datasetID, userID); err != nil { )`, datasetID, userID); err != nil {
return 0, err return 0, err
} }
+390
View File
@@ -0,0 +1,390 @@
package rond
import (
"fmt"
"sort"
"time"
)
// 两份数据的比对。
//
// 对齐键的选择是这个文件的全部要点:rond 每次导出/同步都会重排 Z_PK(实测两份备份里
// 822 条到访的 UUID 完全一致、Z_PK 100% 不同),所以 src_pk 只能在一个数据集内部使用。
// 跨备份对齐一律走真机写的 UUID;地点/标签/交通方式没有 UUID,退到「名称 + 坐标」。
//
// 这里只做判定,不碰数据库、也不写任何东西。
// StoredVisit 是库里的一条到访,外加从 entity_raw 留档取出的对齐键。
type StoredVisit struct {
SrcPK int
UUID string // 空 = 站点新建的行(真机不会给它 UUID)
Arrival *time.Time
Departure *time.Time
Remark string
Emoji string
Bookmarked bool
UserAdded bool
PlaceKey string // 地点对齐键,空 = 该到访没有地点
PlaceName string // 仅用于展示
ActivityName string
}
// StoredPlace 是库里的一个地点。
type StoredPlace struct {
SrcPK int
Key string
Name string
Lat, Lon float64
Province string
City string
POICategory string
VisitCount int
}
// StoredActivity 是库里的一个活动类型。
type StoredActivity struct {
SrcPK int
Name string
UUID string
}
// FieldChange 是冲突里的一项字段差异。From 是库里当前的值,To 是备份里的值。
type FieldChange struct {
Label string
From string
To string
}
// VisitConflict 是同一条到访(UUID 相同)两边字段不一致。
type VisitConflict struct {
Backup Visit
DB StoredVisit
Fields []FieldChange
}
// VisitDiff 是到访的差异画像。
type VisitDiff struct {
Added []Visit // 备份有、库里没有
Removed []StoredVisit // 库里有、备份里没有
Changed []VisitConflict
Same int // 对齐上且完全一致
}
// PlaceDiff 是地点的差异画像。
type PlaceDiff struct {
Added []Location
Removed []StoredPlace
Changed []PlaceConflict
Same int
}
type PlaceConflict struct {
Backup Location
DB StoredPlace
Fields []FieldChange
}
// ActivityDiff 是活动类型的差异画像(按 UUID 对齐,退不到就按名称)。
type ActivityDiff struct {
Added []Activity
Removed []StoredActivity
Changed []ActivityConflict
Same int
}
type ActivityConflict struct {
Backup Activity
DB StoredActivity
Fields []FieldChange
}
// PlaceCoordKey 是地点的跨备份对齐键:**只认坐标,不认名称**。
//
// 地点没有 UUID,名称又会被站点自己的「补全缺失地址」改掉(实测:站内把
// 「体育馆美食街」反查成了「河池市国民体质监测中心」,坐标一字未动,于是 17 条
// 到访全被误报成「换了地点」)。坐标取 6 位小数(约 0.1m)——足以吸收浮点往返误差,
// 又不会把挨着的两个地点并成一个。
func PlaceCoordKey(lat, lon float64) string {
return fmt.Sprintf("%.6f|%.6f", lat, lon)
}
func coordStr(v float64) string { return fmt.Sprintf("%.6f", v) }
// DiffVisits 按 UUID 对齐两边的到访。
//
// 真机偶尔有条目没写 UUID(实测 824 行到访里有 2 行),只按 UUID 配对会让同一条既算
// 「新增」又算「仅站内有」;所以没有 UUID 的行退到「到达时间 + 地点」兜底配对。
// 库里那些连兜底也配不上的(站点新建的行)一律算「库里有、备份没有」。
func DiffVisits(stored []StoredVisit, b *Backup) VisitDiff {
placeKey, placeName := map[int]string{}, map[int]string{}
for _, l := range b.Locations {
// 没有坐标的地点不参与坐标对齐,否则它们会共用 "0.000000|0.000000" 互相认领
if l.Lat != 0 || l.Lon != 0 {
placeKey[l.SrcPK] = PlaceCoordKey(l.Lat, l.Lon)
}
placeName[l.SrcPK] = placeLabel(l.Name, l.Lat, l.Lon)
}
actName := map[int]string{}
for _, a := range b.Activities {
actName[a.SrcPK] = a.Name
}
used := make([]bool, len(stored))
byUUID := make(map[string]int, len(stored))
byTime := map[string][]int{}
for i, sv := range stored {
if sv.UUID != "" {
byUUID[sv.UUID] = i
continue
}
if sv.Arrival != nil {
k := timeKey(*sv.Arrival, sv.PlaceKey)
byTime[k] = append(byTime[k], i)
}
}
var d VisitDiff
classify := func(i int, v Visit) {
used[i] = true
sv := stored[i]
if fields := visitFieldChanges(sv, v, placeKey, placeName, actName); len(fields) > 0 {
d.Changed = append(d.Changed, VisitConflict{Backup: v, DB: sv, Fields: fields})
} else {
d.Same++
}
}
for _, v := range b.Visits {
if v.Arrival == nil {
continue // 没有到达时间的行导入时就被跳过了,不该被算成「新增」
}
if uuid := b.VisitUUID[v.SrcPK]; uuid != "" {
if i, ok := byUUID[uuid]; ok {
classify(i, v)
continue
}
d.Added = append(d.Added, v)
continue
}
bpk := ""
if v.LocationSrcPK != nil {
bpk = placeKey[*v.LocationSrcPK]
}
if list := byTime[timeKey(*v.Arrival, bpk)]; len(list) > 0 {
byTime[timeKey(*v.Arrival, bpk)] = list[1:]
classify(list[0], v)
continue
}
d.Added = append(d.Added, v)
}
for i, sv := range stored {
if !used[i] {
d.Removed = append(d.Removed, sv)
}
}
sortVisits(d)
return d
}
// timeKey 是没有 UUID 时的兜底配对键。只到秒:苹果时间戳转过来是浮点秒,
// 而 PG 存的是微秒,逐位比会永远配不上。
func timeKey(t time.Time, place string) string {
return fmt.Sprintf("%d|%s", t.Unix(), place)
}
func visitFieldChanges(sv StoredVisit, v Visit, placeKey, placeName, actName map[int]string) []FieldChange {
var out []FieldChange
add := func(label, from, to string) {
if from != to {
out = append(out, FieldChange{Label: label, From: from, To: to})
}
}
if !sameTime(sv.Arrival, v.Arrival) {
add("到达时间", fmtTime(sv.Arrival), fmtTime(v.Arrival))
}
if !sameTime(sv.Departure, v.Departure) {
add("离开时间", fmtTime(sv.Departure), fmtTime(v.Departure))
}
// 地点按坐标判、显示名称;活动按名称判。都不能比 src_pk——两边编号空间不同。
bp, bn := "", ""
if v.LocationSrcPK != nil {
bp, bn = placeKey[*v.LocationSrcPK], placeName[*v.LocationSrcPK]
}
// 地点按坐标判、显示名称:名称被「补全地址」反查改过,不算换了地点
if sv.PlaceKey != bp {
out = append(out, FieldChange{Label: "地点", From: sv.PlaceName, To: bn})
}
bl := ""
if v.ActivitySrcPK != nil {
bl = actName[*v.ActivitySrcPK]
}
add("活动", sv.ActivityName, bl)
add("备注", sv.Remark, v.Remark)
add("表情", sv.Emoji, v.Emoji)
add("收藏", fmtBool(sv.Bookmarked), fmtBool(v.Bookmarked))
return out
}
// placeLabel 给没有名字的地点一个可读的占位。
func placeLabel(name string, lat, lon float64) string {
if name != "" {
return name
}
return fmt.Sprintf("%.5f, %.5f", lat, lon)
}
// DiffPlaces 对齐地点。坐标优先(改名不影响配对),配不上再按名称找一遍
// (覆盖「站点把坐标纠偏过」这种),两样都对不上才算新增。
func DiffPlaces(stored []StoredPlace, b *Backup) PlaceDiff {
byCoord := make(map[string]StoredPlace, len(stored))
byName := map[string]StoredPlace{}
for _, sp := range stored {
if sp.Lat != 0 || sp.Lon != 0 {
byCoord[PlaceCoordKey(sp.Lat, sp.Lon)] = sp
}
if _, dup := byName[sp.Name]; !dup && sp.Name != "" {
byName[sp.Name] = sp
}
}
used := map[int]bool{}
var d PlaceDiff
classify := func(sp StoredPlace, l Location) {
used[sp.SrcPK] = true
if fields := placeFieldChanges(sp, l); len(fields) > 0 {
d.Changed = append(d.Changed, PlaceConflict{Backup: l, DB: sp, Fields: fields})
} else {
d.Same++
}
}
for _, l := range b.Locations {
if sp, ok := byCoord[PlaceCoordKey(l.Lat, l.Lon)]; ok && !used[sp.SrcPK] {
classify(sp, l)
continue
}
if sp, ok := byName[l.Name]; ok && !used[sp.SrcPK] {
classify(sp, l)
continue
}
d.Added = append(d.Added, l)
}
for _, sp := range stored {
if !used[sp.SrcPK] {
d.Removed = append(d.Removed, sp)
}
}
sort.Slice(d.Added, func(i, j int) bool { return d.Added[i].Name < d.Added[j].Name })
sort.Slice(d.Removed, func(i, j int) bool { return d.Removed[i].Name < d.Removed[j].Name })
sort.Slice(d.Changed, func(i, j int) bool { return d.Changed[i].DB.Name < d.Changed[j].DB.Name })
return d
}
// placeFieldChanges 比地点上除对齐键之外的字段。名称也算——按坐标配上之后,
// 「名字不一样」正是站长最想看到的那类差异。
func placeFieldChanges(sp StoredPlace, l Location) []FieldChange {
var out []FieldChange
add := func(label, from, to string) {
if from != to {
out = append(out, FieldChange{Label: label, From: from, To: to})
}
}
add("名称", sp.Name, l.Name)
add("纬度", coordStr(sp.Lat), coordStr(l.Lat))
add("经度", coordStr(sp.Lon), coordStr(l.Lon))
add("省份", sp.Province, l.Province)
add("城市", sp.City, l.City)
add("分类", sp.POICategory, l.POICategory)
return out
}
// DiffActivities 优先按 UUID 对齐;库里没有 UUID 的(站点新建)退回名称匹配,
// 匹配不上就算「库里有、备份没有」。
func DiffActivities(stored []StoredActivity, b *Backup) ActivityDiff {
byUUID := map[string]StoredActivity{}
byName := map[string]StoredActivity{}
for _, sa := range stored {
if sa.UUID != "" {
byUUID[sa.UUID] = sa
}
if _, dup := byName[sa.Name]; !dup {
byName[sa.Name] = sa
}
}
var d ActivityDiff
matched := map[int]bool{}
for _, a := range b.Activities {
sa, ok := byUUID[b.ActivityUUID[a.SrcPK]]
if !ok {
if sa, ok = byName[a.Name]; !ok {
d.Added = append(d.Added, a)
continue
}
}
matched[sa.SrcPK] = true
if sa.Name != a.Name {
d.Changed = append(d.Changed, ActivityConflict{Backup: a, DB: sa,
Fields: []FieldChange{{Label: "名称", From: sa.Name, To: a.Name}}})
} else {
d.Same++
}
}
for _, sa := range stored {
if !matched[sa.SrcPK] {
d.Removed = append(d.Removed, sa)
}
}
sort.Slice(d.Added, func(i, j int) bool { return d.Added[i].Name < d.Added[j].Name })
sort.Slice(d.Removed, func(i, j int) bool { return d.Removed[i].Name < d.Removed[j].Name })
return d
}
func sortVisits(d VisitDiff) {
tv := func(t *time.Time) int64 {
if t == nil {
return 0
}
return t.Unix()
}
sort.Slice(d.Added, func(i, j int) bool { return tv(d.Added[i].Arrival) < tv(d.Added[j].Arrival) })
sort.Slice(d.Removed, func(i, j int) bool { return tv(d.Removed[i].Arrival) < tv(d.Removed[j].Arrival) })
sort.Slice(d.Changed, func(i, j int) bool { return tv(d.Changed[i].Backup.Arrival) < tv(d.Changed[j].Backup.Arrival) })
}
func sameTime(a, b *time.Time) bool {
if a == nil || b == nil {
return a == nil && b == nil
}
return a.Equal(*b)
}
func fmtTime(t *time.Time) string {
if t == nil {
return "—"
}
return t.Local().Format("2006-01-02 15:04")
}
func fmtBool(v bool) string {
if v {
return "是"
}
return "否"
}
// Counts 汇总各实体的三类条数,供概览展示。
type Counts struct {
Added, Removed, Changed, Same int
}
func (c Counts) Total() int { return c.Added + c.Removed + c.Changed + c.Same }
func VisitCounts(d VisitDiff) Counts {
return Counts{len(d.Added), len(d.Removed), len(d.Changed), d.Same}
}
func PlaceCounts(d PlaceDiff) Counts {
return Counts{len(d.Added), len(d.Removed), len(d.Changed), d.Same}
}
func ActivityCounts(d ActivityDiff) Counts {
return Counts{len(d.Added), len(d.Removed), len(d.Changed), d.Same}
}
+293
View File
@@ -0,0 +1,293 @@
package rond
import (
"testing"
"time"
)
func tp(s string) *time.Time {
t, err := time.Parse("2006-01-02 15:04", s)
if err != nil {
panic(err)
}
return &t
}
func ip(v int) *int { return &v }
// 两组编号刻意完全不同:rond 每次导出都会重排 Z_PK,对齐只能靠 UUID。
const (
uuidA = "11111111111111111111111111111111"
uuidB = "22222222222222222222222222222222"
uuidC = "33333333333333333333333333333333"
)
func sampleBackup() *Backup {
return &Backup{
Visits: []Visit{
{SrcPK: 555, Arrival: tp("2026-09-01 10:00"), Departure: tp("2026-09-01 11:00"),
LocationSrcPK: ip(3), ActivitySrcPK: ip(7)},
{SrcPK: 556, Arrival: tp("2026-09-02 10:00"), LocationSrcPK: ip(3)},
},
Locations: []Location{
{SrcPK: 3, Name: "家", Lat: 24.1, Lon: 109.1, City: "柳州市"},
{SrcPK: 4, Name: "公司", Lat: 24.2, Lon: 109.2, City: "柳州市"},
},
Activities: []Activity{
{SrcPK: 7, Name: "工作"},
{SrcPK: 8, Name: "散步"},
},
VisitUUID: map[int]string{555: uuidA, 556: uuidB},
ActivityUUID: map[int]string{7: uuidC},
}
}
func TestDiffVisitsAlignsByUUIDNotPK(t *testing.T) {
stored := []StoredVisit{
{SrcPK: 1, UUID: uuidA, Arrival: tp("2026-09-01 10:00"), Departure: tp("2026-09-01 11:00"),
PlaceKey: PlaceCoordKey(24.1, 109.1), ActivityName: "工作"},
{SrcPK: 2, UUID: uuidB, Arrival: tp("2026-09-02 10:00"),
PlaceKey: PlaceCoordKey(24.1, 109.1)},
}
d := DiffVisits(stored, sampleBackup())
if len(d.Added) != 0 || len(d.Removed) != 0 || len(d.Changed) != 0 {
t.Fatalf("两边一致时不该有差异: +%d -%d ~%d", len(d.Added), len(d.Removed), len(d.Changed))
}
if d.Same != 2 {
t.Fatalf("应配上 2 条,实际 %d", d.Same)
}
}
func TestDiffVisitsFieldChanges(t *testing.T) {
stored := []StoredVisit{{
SrcPK: 1, UUID: uuidA, Arrival: tp("2026-09-01 09:00"), Departure: tp("2026-09-01 11:00"),
PlaceKey: PlaceCoordKey(24.1, 109.1), ActivityName: "工作", Remark: "站点备注",
}}
d := DiffVisits(stored, sampleBackup())
if len(d.Changed) != 1 {
t.Fatalf("到达时间不同应算冲突,实际 %d 条", len(d.Changed))
}
f := d.Changed[0].Fields
if len(f) != 2 {
t.Fatalf("应报出到达时间与备注两项,实际 %v", f)
}
if f[0].Label != "到达时间" || f[1].Label != "备注" {
t.Fatalf("字段名不对: %v", f)
}
if f[1].From != "站点备注" || f[1].To != "" {
t.Fatalf("方向搞反了(From 应是库里的值): %+v", f[1])
}
}
// 站点新建的到访没有 UUID,而且它的 src_pk 很可能与真机新行的编号撞上——
// 绝不能因为编号相同就当成同一条。
func TestDiffVisitsIgnoresPKCollision(t *testing.T) {
stored := []StoredVisit{{
SrcPK: 555, UUID: "", Arrival: tp("2026-08-01 08:00"), UserAdded: true,
PlaceKey: PlaceCoordKey(30.0, 120.0),
}}
d := DiffVisits(stored, sampleBackup())
if len(d.Added) != 2 {
t.Fatalf("备份里两条都是新增,实际 %d", len(d.Added))
}
if len(d.Removed) != 1 {
t.Fatalf("站点补录那条应算「仅库里有」,实际 %d", len(d.Removed))
}
if !d.Removed[0].UserAdded {
t.Error("站点补录的标记要带上,页面上要区分开")
}
}
// 真机偶尔有不带 UUID 的到访,只按 UUID 配对会让同一条既算新增又算仅站内有。
func TestDiffVisitsFallsBackWhenUUIDMissing(t *testing.T) {
b := &Backup{
Visits: []Visit{
{SrcPK: 900, Arrival: tp("2026-09-01 10:00"), LocationSrcPK: ip(3)},
{SrcPK: 901, Arrival: tp("2026-09-02 10:00"), LocationSrcPK: ip(3)},
},
Locations: []Location{{SrcPK: 3, Name: "家", Lat: 24.1, Lon: 109.1}},
VisitUUID: map[int]string{}, // 一条都没有 UUID
ActivityUUID: map[int]string{},
}
stored := []StoredVisit{
{SrcPK: 1, Arrival: tp("2026-09-01 10:00"), PlaceKey: PlaceCoordKey(24.1, 109.1)},
{SrcPK: 2, Arrival: tp("2026-09-02 10:00"), PlaceKey: PlaceCoordKey(24.1, 109.1)},
}
d := DiffVisits(stored, b)
if len(d.Added) != 0 || len(d.Removed) != 0 {
t.Fatalf("没有 UUID 时应按「到达时间 + 地点」兜底配对: +%d -%d", len(d.Added), len(d.Removed))
}
if d.Same != 2 {
t.Fatalf("两条都该配上,实际 %d", d.Same)
}
}
// 兜底配对要认得出「同一条但时间被改过」——时间不同就配不上,此时按新增 + 仅站内有报出,
// 总比静默当成同一条好。
func TestDiffVisitsFallbackNeedsSameTime(t *testing.T) {
b := &Backup{
Visits: []Visit{{SrcPK: 900, Arrival: tp("2026-09-01 10:00"), LocationSrcPK: ip(3)}},
Locations: []Location{{SrcPK: 3, Name: "家", Lat: 24.1, Lon: 109.1}},
VisitUUID: map[int]string{},
}
stored := []StoredVisit{{SrcPK: 1, Arrival: tp("2026-09-01 11:00"), PlaceKey: PlaceCoordKey(24.1, 109.1)}}
d := DiffVisits(stored, b)
if len(d.Added) != 1 || len(d.Removed) != 1 {
t.Fatalf("时间对不上时该分别报出: +%d -%d", len(d.Added), len(d.Removed))
}
}
func TestDiffVisitsSkipsRowsWithoutArrival(t *testing.T) {
b := sampleBackup()
b.Visits = append(b.Visits, Visit{SrcPK: 557, LocationSrcPK: ip(3)})
d := DiffVisits(nil, b)
if len(d.Added) != 2 {
t.Fatalf("没有到达时间的行导入时就被跳过,不该算新增,实际 %d", len(d.Added))
}
}
func TestDiffPlacesByKey(t *testing.T) {
b := sampleBackup()
stored := []StoredPlace{
{SrcPK: 10, Key: PlaceCoordKey(24.1, 109.1), Name: "家", Lat: 24.1, Lon: 109.1, City: "柳州市"},
}
d := DiffPlaces(stored, b)
if len(d.Added) != 1 || d.Added[0].Name != "公司" {
t.Fatalf("应只有「公司」是新增: %+v", d.Added)
}
if len(d.Removed) != 0 || len(d.Changed) != 0 || d.Same != 1 {
t.Fatalf("「家」应配上且无差异: same=%d changed=%d removed=%d", d.Same, len(d.Changed), len(d.Removed))
}
}
func TestDiffPlacesReportsFieldChanges(t *testing.T) {
b := sampleBackup()
stored := []StoredPlace{
{SrcPK: 10, Key: PlaceCoordKey(24.1, 109.1), Name: "家", Lat: 24.1, Lon: 109.1, City: "桂林市"},
}
d := DiffPlaces(stored, b)
if len(d.Changed) != 1 {
t.Fatalf("城市不同应算冲突,实际 %d", len(d.Changed))
}
if f := d.Changed[0].Fields; len(f) != 1 || f[0].Label != "城市" || f[0].From != "桂林市" || f[0].To != "柳州市" {
t.Fatalf("字段差异不对: %+v", f)
}
}
func TestDiffActivities(t *testing.T) {
b := sampleBackup()
stored := []StoredActivity{
{SrcPK: 20, Name: "工作", UUID: uuidC},
{SrcPK: 21, Name: "站点自建"},
}
d := DiffActivities(stored, b)
if len(d.Added) != 1 || d.Added[0].Name != "散步" {
t.Fatalf("「散步」应是新增: %+v", d.Added)
}
if len(d.Removed) != 1 || d.Removed[0].Name != "站点自建" {
t.Fatalf("站点自建的活动应算仅库里有: %+v", d.Removed)
}
if d.Same != 1 {
t.Fatalf("「工作」应配上,实际 same=%d", d.Same)
}
}
// 活动改了名:UUID 还在,就该认出来是同一个,只报名称变化,而不是「删一个加一个」。
func TestDiffActivitiesRenameByUUID(t *testing.T) {
b := sampleBackup()
stored := []StoredActivity{{SrcPK: 20, Name: "上班", UUID: uuidC}}
d := DiffActivities(stored, b)
if len(d.Added) != 1 || len(d.Removed) != 0 {
t.Fatalf("只该多出「散步」,「上班」应认出是同一个: +%d -%d", len(d.Added), len(d.Removed))
}
if len(d.Changed) != 1 || d.Changed[0].Fields[0].From != "上班" || d.Changed[0].Fields[0].To != "工作" {
t.Fatalf("应按 UUID 认出重命名: %+v", d.Changed)
}
}
func TestPlaceCoordKeyStable(t *testing.T) {
a := PlaceCoordKey(24.1, 109.1)
if a != PlaceCoordKey(24.1, 109.1) {
t.Fatal("同样的输入应得到同样的键")
}
if a == PlaceCoordKey(24.1, 109.2) {
t.Fatal("坐标不同不该算同一个地点")
}
}
// 站点把地点反查改名(坐标没动)时,到访层面不该报「换了地点」——
// 实测里这一条会让 17 条到访全变成假冲突。
func TestDiffVisitsIgnoresPlaceRename(t *testing.T) {
b := &Backup{
Visits: []Visit{{SrcPK: 5, Arrival: tp("2026-09-01 10:00"), LocationSrcPK: ip(3)}},
Locations: []Location{{SrcPK: 3, Name: "河池市国民体质监测中心", Lat: 24.691827, Lon: 108.035268}},
VisitUUID: map[int]string{5: uuidA},
}
stored := []StoredVisit{{
SrcPK: 1, UUID: uuidA, Arrival: tp("2026-09-01 10:00"),
PlaceKey: PlaceCoordKey(24.691827, 108.035268), PlaceName: "体育馆美食街",
}}
d := DiffVisits(stored, b)
if len(d.Changed) != 0 {
t.Fatalf("只改了地点名不该算到访冲突: %+v", d.Changed[0].Fields)
}
if d.Same != 1 {
t.Fatalf("应算作一致,实际 same=%d", d.Same)
}
}
// 反之,地点层面要把改名报出来。
func TestDiffPlacesReportsRename(t *testing.T) {
b := &Backup{Locations: []Location{
{SrcPK: 3, Name: "河池市国民体质监测中心", Lat: 24.691827, Lon: 108.035268, City: "河池市"},
}}
stored := []StoredPlace{{
SrcPK: 10, Key: PlaceCoordKey(24.691827, 108.035268), Name: "体育馆美食街",
Lat: 24.691827, Lon: 108.035268, City: "河池市",
}}
d := DiffPlaces(stored, b)
if len(d.Added) != 0 || len(d.Removed) != 0 {
t.Fatalf("坐标一样就是同一个地点: +%d -%d", len(d.Added), len(d.Removed))
}
if len(d.Changed) != 1 {
t.Fatalf("改名应报成一条冲突,实际 %d", len(d.Changed))
}
f := d.Changed[0].Fields
if len(f) != 1 || f[0].Label != "名称" || f[0].From != "体育馆美食街" || f[0].To != "河池市国民体质监测中心" {
t.Fatalf("字段差异不对: %+v", f)
}
}
// 没有坐标的地点不能都落到同一个键上,否则两个不同地点会互相认领。
func TestDiffPlacesWithoutCoordDoesNotCollide(t *testing.T) {
b := &Backup{Locations: []Location{{SrcPK: 3, Name: "甲地"}}}
stored := []StoredPlace{
{SrcPK: 10, Name: "甲地"},
{SrcPK: 11, Name: "乙地"},
}
d := DiffPlaces(stored, b)
if len(d.Added) != 0 || len(d.Removed) != 1 {
t.Fatalf("应只配上同名的「甲地」: +%d -%d", len(d.Added), len(d.Removed))
}
if d.Removed[0].Name != "乙地" {
t.Fatalf("没配上的应是「乙地」,实际 %q", d.Removed[0].Name)
}
if d.Same != 1 {
t.Fatalf("「甲地」应算一致,实际 %d", d.Same)
}
}
// 站点把坐标纠偏过(名称没变)也要认得出是同一个地点。
func TestDiffPlacesMatchesByNameWhenCoordMoved(t *testing.T) {
b := &Backup{Locations: []Location{{SrcPK: 3, Name: "家", Lat: 24.100001, Lon: 109.100001}}}
stored := []StoredPlace{{
SrcPK: 10, Key: PlaceCoordKey(24.1, 109.1), Name: "家", Lat: 24.1, Lon: 109.1,
}}
d := DiffPlaces(stored, b)
if len(d.Added) != 0 || len(d.Removed) != 0 {
t.Fatalf("名称一样就该配上: +%d -%d", len(d.Added), len(d.Removed))
}
if len(d.Changed) != 1 || d.Changed[0].Fields[0].Label != "纬度" {
t.Fatalf("应报出坐标差异: %+v", d.Changed)
}
}
+6 -3
View File
@@ -1332,9 +1332,12 @@ var uuidNamespace = [16]byte{0x6b, 0x1c, 0x9a, 0x2e, 0x42, 0x7d, 0x4f, 0x16,
// uuidV5 由命名空间与名字生成 RFC 4122 版本 5 的 UUID(16 字节原始形式)。 // uuidV5 由命名空间与名字生成 RFC 4122 版本 5 的 UUID(16 字节原始形式)。
// //
// rond 给每条到访、每个活动都存一个 16 字节 UUID(ZIDENTIFIER_ / ZUID_,真机里全是 v4), // rond 给每条到访、每个活动都存一个 16 字节 UUID(ZIDENTIFIER_ / ZUID_,真机里全是 v4),
// 站点这边没有对应字段。这里用确定性算法补:名字只取表名 + src_pk(src_pk 跨快照稳定), // 站点新建的行没有真机 UUID 可继承。这里用确定性算法补:名字取表名 + src_pk
// 于是同一条记录每次导出拿到的都是同一个 UUID——若改成随机,每次导出的包都会让 rond // (src_pk 在**同一个数据集内**稳定,够用了),于是同一条记录每次导出拿到的都是同一个
// 把同一批记录当成新的。 // UUID——若改成随机,每次导出的包都会让 rond 把同一批记录当成新的。
//
// 注意这只保证「同一数据集内幂等」,不是跨备份的稳定标识:rond 那边每次导入都会重排
// Z_PK,真正的跨备份对齐键是它原生的 ZIDENTIFIER_(见 diff.go)。
func uuidV5(name string) []byte { func uuidV5(name string) []byte {
h := sha1.New() h := sha1.New()
h.Write(uuidNamespace[:]) h.Write(uuidNamespace[:])
+170
View File
@@ -0,0 +1,170 @@
package rond
// 合并计划:把「差异画像 + 站长在页面上做的选择」翻译成一份可执行的改动清单。
// 这里只做判定,不碰数据库、也不写任何东西。
// 站长对一条冲突的处理方式。
const (
PickBackup = "backup" // 用备份(实机)的值覆盖站内
PickDB = "db" // 保留站内的值
PickSkip = "skip" // 这条不动
)
// Choices 是差异页提交上来的选择。冲突是逐条选的,键分别是到访 UUID、
// 地点坐标键、活动 UUID。
type Choices struct {
AddVisits bool // 把备份里多出来的到访并进来
AddPlaces bool
AddActs bool
VisitClash map[string]string
PlaceClash map[string]string
ActClash map[string]string
// 删除:「仅站内有」的记录按库里编号勾选,默认一个都不删。
// 键是 src_pk 而不是 UUID——站点补录的行压根没有 UUID。
DelVisits map[int]bool
DelPlaces map[int]bool
DelActs map[int]bool
}
func (c Choices) visitPick(key string) string { return pickOf(c.VisitClash, key) }
func (c Choices) placePick(key string) string { return pickOf(c.PlaceClash, key) }
func (c Choices) activityPick(key string) string { return pickOf(c.ActClash, key) }
func pickOf(m map[string]string, key string) string {
switch m[key] {
case PickBackup, PickDB, PickSkip:
return m[key]
}
return PickDB // 没提交或提交了怪值:默认保留站内的,不误伤
}
// VisitPatch 表示用备份侧的值覆盖库里的那一条。
type VisitPatch struct {
UUID string // 对齐键
V Visit
}
// PlacePatch 同理,键是坐标键。
type PlacePatch struct {
CoordKey string
L Location
}
// ActivityPatch 键是活动 UUID(没有 UUID 时退回名称)。
type ActivityPatch struct {
Key string
A Activity
}
// MergePlan 是一次合并要做的全部改动。
type MergePlan struct {
AddPlaces []Location
UpdPlaces []PlacePatch
DelPlaces []StoredPlace
AddActs []Activity
UpdActs []ActivityPatch
DelActs []StoredActivity
AddVisits []Visit
UpdVisits []VisitPatch
DelVisits []StoredVisit
}
// Empty 报告这份计划什么事都不用做。
func (p MergePlan) Empty() bool {
return len(p.AddPlaces)+len(p.UpdPlaces)+len(p.DelPlaces)+
len(p.AddActs)+len(p.UpdActs)+len(p.DelActs)+
len(p.AddVisits)+len(p.UpdVisits)+len(p.DelVisits) == 0
}
// BuildMergePlan 按选择把差异筛成改动清单。
func BuildMergePlan(vd VisitDiff, pd PlaceDiff, ad ActivityDiff, c Choices) MergePlan {
var p MergePlan
if c.AddPlaces {
p.AddPlaces = pd.Added
}
for _, cf := range pd.Changed {
if c.placePick(cf.DB.Key) == PickBackup {
p.UpdPlaces = append(p.UpdPlaces, PlacePatch{CoordKey: cf.DB.Key, L: cf.Backup})
}
}
for _, sp := range pd.Removed {
if c.DelPlaces[sp.SrcPK] {
p.DelPlaces = append(p.DelPlaces, sp)
}
}
if c.AddActs {
p.AddActs = ad.Added
}
for _, cf := range ad.Changed {
if c.activityPick(activityKey(cf.DB)) == PickBackup {
p.UpdActs = append(p.UpdActs, ActivityPatch{Key: activityKey(cf.DB), A: cf.Backup})
}
}
for _, sa := range ad.Removed {
if c.DelActs[sa.SrcPK] {
p.DelActs = append(p.DelActs, sa)
}
}
if c.AddVisits {
p.AddVisits = vd.Added
// 新到访可能挂在一个「备份里新出现、库里还没有」的地点/活动上:
// 不把它们一起建出来,到访的 place_id / activity_id 就无从映射,只能整条丢掉。
p.AddPlaces = appendMissingPlaces(p.AddPlaces, pd.Added)
p.AddActs = appendMissingActs(p.AddActs, ad.Added)
}
for _, cf := range vd.Changed {
if c.visitPick(cf.DB.UUID) == PickBackup {
p.UpdVisits = append(p.UpdVisits, VisitPatch{UUID: cf.DB.UUID, V: cf.Backup})
}
}
for _, sv := range vd.Removed {
if c.DelVisits[sv.SrcPK] {
p.DelVisits = append(p.DelVisits, sv)
}
}
return p
}
func appendMissingPlaces(have, more []Location) []Location {
seen := make(map[int]bool, len(have))
for _, l := range have {
seen[l.SrcPK] = true
}
for _, l := range more {
if !seen[l.SrcPK] {
have = append(have, l)
seen[l.SrcPK] = true
}
}
return have
}
func appendMissingActs(have, more []Activity) []Activity {
seen := make(map[int]bool, len(have))
for _, a := range have {
seen[a.SrcPK] = true
}
for _, a := range more {
if !seen[a.SrcPK] {
have = append(have, a)
seen[a.SrcPK] = true
}
}
return have
}
// activityKey 是活动冲突在页面上的标识:优先 UUID(改名也能对上),没有才用名称。
func activityKey(sa StoredActivity) string {
if sa.UUID != "" {
return sa.UUID
}
return "n:" + sa.Name
}
// ActivityKey 供页面与处理函数共用同一套键。
func ActivityKey(sa StoredActivity) string { return activityKey(sa) }
+76
View File
@@ -0,0 +1,76 @@
package rond
import "testing"
func TestBuildMergePlanPicks(t *testing.T) {
vd := VisitDiff{
Added: []Visit{{SrcPK: 900, Arrival: tp("2026-09-09 10:00")}},
Removed: []StoredVisit{{SrcPK: 5, UUID: uuidA}},
Changed: []VisitConflict{
{Backup: Visit{SrcPK: 901, Arrival: tp("2026-09-01 12:00")}, DB: StoredVisit{SrcPK: 1, UUID: uuidA}},
{Backup: Visit{SrcPK: 902, Arrival: tp("2026-09-02 12:00")}, DB: StoredVisit{SrcPK: 2, UUID: uuidB}},
},
}
pd := PlaceDiff{Added: []Location{{SrcPK: 3, Name: "新地点"}}}
ad := ActivityDiff{Added: []Activity{{SrcPK: 8, Name: "散步"}}}
p := BuildMergePlan(vd, pd, ad, Choices{
AddVisits: true,
AddPlaces: true,
AddActs: false,
VisitClash: map[string]string{uuidA: PickBackup, uuidB: PickDB},
})
if len(p.AddVisits) != 1 || p.AddVisits[0].SrcPK != 900 {
t.Fatalf("新增到访应合入一条: %+v", p.AddVisits)
}
if len(p.UpdVisits) != 1 || p.UpdVisits[0].UUID != uuidA {
t.Fatalf("只有选了「用实机」的那条才该更新: %+v", p.UpdVisits)
}
if len(p.AddPlaces) != 1 {
t.Fatalf("勾了「并进来」就该带上新增地点: %d", len(p.AddPlaces))
}
// 即使没单独勾活动,合入到访时也要把它可能挂着的「库里还没有的活动」带上,
// 否则那条到访的 activity_id 无从映射,只能整条丢掉。
if len(p.AddActs) != 1 || p.AddActs[0].Name != "散步" {
t.Fatalf("合入到访应连带它需要的活动: %+v", p.AddActs)
}
// 「仅站内有」永远不进计划:删除是单独的事,绝不顺手做
if len(p.UpdPlaces)+len(p.UpdActs) != 0 {
t.Fatal("不该有地点/活动更新")
}
}
// 没提交的键、或提交了怪值,一律当成「保留站内」——不误伤。
func TestBuildMergePlanDefaultsToKeepDB(t *testing.T) {
vd := VisitDiff{Changed: []VisitConflict{
{Backup: Visit{SrcPK: 901}, DB: StoredVisit{SrcPK: 1, UUID: uuidA}},
}}
p := BuildMergePlan(vd, PlaceDiff{}, ActivityDiff{}, Choices{VisitClash: map[string]string{uuidA: "wat"}})
if len(p.UpdVisits) != 0 {
t.Fatalf("怪值不该被当成「用实机」: %+v", p.UpdVisits)
}
if !p.Empty() {
t.Fatal("什么都没选,计划应当是空的")
}
}
// 冲突选了「跳过」也不动。
func TestBuildMergePlanSkip(t *testing.T) {
vd := VisitDiff{Changed: []VisitConflict{
{Backup: Visit{SrcPK: 901}, DB: StoredVisit{SrcPK: 1, UUID: uuidA}},
}}
p := BuildMergePlan(vd, PlaceDiff{}, ActivityDiff{}, Choices{VisitClash: map[string]string{uuidA: PickSkip}})
if !p.Empty() {
t.Fatalf("跳过就不该产生改动: %+v", p)
}
}
// 活动改了名也要能用 UUID 当键对得上。
func TestActivityKeyPrefersUUID(t *testing.T) {
if got := ActivityKey(StoredActivity{SrcPK: 1, Name: "上班", UUID: uuidC}); got != uuidC {
t.Fatalf("有 UUID 就用 UUID,实际 %q", got)
}
if got := ActivityKey(StoredActivity{SrcPK: 1, Name: "上班"}); got != "n:上班" {
t.Fatalf("没有 UUID 退回名称,实际 %q", got)
}
}
+36
View File
@@ -156,6 +156,13 @@ type Backup struct {
// RawRows 是各实体在 Core Data 里的完整原始行:「表名 -> Z_PK -> 列名 -> 值」。 // RawRows 是各实体在 Core Data 里的完整原始行:「表名 -> Z_PK -> 列名 -> 值」。
// 站点只映射了其中一部分字段,留档让导出能逐列还原,不必依赖基线库补剩下的列。 // 站点只映射了其中一部分字段,留档让导出能逐列还原,不必依赖基线库补剩下的列。
RawRows map[string]map[int]map[string]any RawRows map[string]map[int]map[string]any
// VisitUUID / ActivityUUID 是真机给每条到访、每个活动存的 16 字节 UUID(小写十六进制)。
// **这是跨备份唯一稳定的对齐键**:rond 每次导出/同步都会重排 Z_PK——实测两份备份里
// 822 条到访的 UUID 完全一致、Z_PK 100% 不同——所以「这份备份」与「库里的那一份」
// 之间只能按它对齐。地点/标签/交通方式没有 UUID,只能退到名称与坐标。
VisitUUID map[int]string
ActivityUUID map[int]string
} }
// Open 解压备份包到 dir,返回可用的 SQLite 连接。 // Open 解压备份包到 dir,返回可用的 SQLite 连接。
@@ -465,9 +472,38 @@ func Parse(db *sql.DB) (*Backup, error) {
} else { } else {
log.Printf("抓取原始行留档失败(导出将退回基线补列): %v", rerr) log.Printf("抓取原始行留档失败(导出将退回基线补列): %v", rerr)
} }
// 对齐键:导出不依赖它,但「两份数据比对」全靠它,缺了就只能放弃逐条对齐
if b.VisitUUID, err = uuidColumn(db, "ZVISIT", "ZIDENTIFIER_"); err != nil {
return nil, fmt.Errorf("读取到访 UUID: %w", err)
}
if b.ActivityUUID, err = uuidColumn(db, "ZACTIVITY", "ZUID_"); err != nil {
return nil, fmt.Errorf("读取活动 UUID: %w", err)
}
return b, nil return b, nil
} }
// uuidColumn 读出某表某一列的 UUID,返回 Z_PK -> 小写十六进制。
// 列不存在(很老的备份)返回空表而不是错误;全零或空的当作没有。
func uuidColumn(db *sql.DB, table, col string) (map[int]string, error) {
rows, err := db.Query(fmt.Sprintf(`SELECT Z_PK, hex(%s) FROM %s`, col, table))
if err != nil {
return nil, nil
}
defer rows.Close()
out := map[int]string{}
for rows.Next() {
var pk int
var h sql.NullString
if err := rows.Scan(&pk, &h); err != nil {
return nil, err
}
if h.Valid && strings.Trim(h.String, "0") != "" {
out[pk] = strings.ToLower(h.String)
}
}
return out, rows.Err()
}
// rawTables 是需要整行留档的 Core Data 表(站点管理的实体 + 标签分组)。 // rawTables 是需要整行留档的 Core Data 表(站点管理的实体 + 标签分组)。
var rawTables = []string{"ZLOCATION", "ZVISIT", "ZMOVEMENT", "ZHOURLYWEATHER", "ZACTIVITY", "ZTAG", "ZTAGGROUP", "ZTRANSPORT"} var rawTables = []string{"ZLOCATION", "ZVISIT", "ZMOVEMENT", "ZHOURLYWEATHER", "ZACTIVITY", "ZTAG", "ZTAGGROUP", "ZTRANSPORT"}
+7 -5
View File
@@ -666,7 +666,8 @@ const placeCols = `p.id, COALESCE(p.name,''), COALESCE(p.poi_category,''), COALE
EXISTS (SELECT 1 FROM visits v3 WHERE v3.place_id=p.id AND v3.is_home), EXISTS (SELECT 1 FROM visits v3 WHERE v3.place_id=p.id AND v3.is_home),
EXISTS (SELECT 1 FROM visits v4 WHERE v4.place_id=p.id AND v4.is_work), EXISTS (SELECT 1 FROM visits v4 WHERE v4.place_id=p.id AND v4.is_work),
p.src_pk, p.src_pk,
EXISTS (SELECT 1 FROM place_images pi WHERE pi.dataset_id=p.dataset_id AND pi.src_pk=p.src_pk)` EXISTS (SELECT 1 FROM place_images pi WHERE pi.place_key =
to_char(p.lat,'FM999990.000000')||'|'||to_char(p.lon,'FM999990.000000'))`
// PlaceCols 与 ScanPlace 配对,供包外构造只取单个地点的查询。 // PlaceCols 与 ScanPlace 配对,供包外构造只取单个地点的查询。
const PlaceCols = placeCols const PlaceCols = placeCols
@@ -708,11 +709,12 @@ func (d DB) Places(ctx context.Context, f Filter, sortBy string, limit, offset i
args := append([]any{f.DatasetID}, inner.args...) args := append([]any{f.DatasetID}, inner.args...)
base := `FROM places p WHERE p.dataset_id = $1 AND EXISTS (SELECT 1 FROM visits v WHERE ` + inner.expr() + `)` base := `FROM places p WHERE p.dataset_id = $1 AND EXISTS (SELECT 1 FROM visits v WHERE ` + inner.expr() + `)`
if f.Verdict != 0 { if f.Verdict != 0 {
// 结论存在 place_votes,按 src_pk 关联;占位符接在内层条件之后。 // 结论挂坐标键(换数据集也要跟得住),按本数据集的地点圈定范围;
// dataset_id 也要对上:src_pk 只在单个数据集内唯一。 // 占位符接在内层条件之后。
args = append(args, f.Verdict) args = append(args, f.Verdict)
base += fmt.Sprintf(` AND EXISTS (SELECT 1 FROM place_votes pv base += fmt.Sprintf(` AND EXISTS (SELECT 1 FROM place_votes pv WHERE pv.place_key =
WHERE pv.src_pk = p.src_pk AND pv.dataset_id = p.dataset_id AND pv.verdict = $%d)`, len(args)) to_char(p.lat,'FM999990.000000')||'|'||to_char(p.lon,'FM999990.000000')
AND pv.verdict = $%d)`, len(args))
} }
q := `SELECT ` + placeCols + ` ` + base + ` ORDER BY ` + order + fmt.Sprintf(` LIMIT %d OFFSET %d`, limit, offset) q := `SELECT ` + placeCols + ` ` + base + ` ORDER BY ` + order + fmt.Sprintf(` LIMIT %d OFFSET %d`, limit, offset)
+78 -34
View File
@@ -2,14 +2,23 @@
// - place_votes 站长对地点的官方结论(单值) // - place_votes 站长对地点的官方结论(单值)
// - place_visitor_votes 访客点赞 / 点踩(一人一票,可改可撤) // - place_visitor_votes 访客点赞 / 点踩(一人一票,可改可撤)
// //
// 两者都按 rond 的地点 ID(src_pk)存储,重新上传备份、地点换了新 id 也不丢。 // 站长的「官方结论」按**坐标键**(place_key)存储:src_pk 只在单个数据集内有效,
// 换个数据集同编号就是另一个地点,结论会挂错人。
// 访客票仍按 src_pk 存——那张表没有 dataset_id,无从判断票是投给哪个数据集的,
// 迁移有歧义,留待单独处理。
package votes package votes
import ( import (
"context" "context"
"database/sql" "database/sql"
"fmt"
) )
// placeKeyExpr 把 places p 这一行的坐标换算成 place_key。
// **必须与 Go 侧的 rond.PlaceCoordKey 以及 notes 包里的同款表达式逐位一致**:
// fmt.Sprintf("%.6f|%.6f") ↔ to_char(x,'FM999990.000000')||'|'||to_char(y,'FM999990.000000')
const placeKeyExpr = `to_char(p.lat,'FM999990.000000')||'|'||to_char(p.lon,'FM999990.000000')`
// VerdictRecommend / VerdictAvoid 是官方结论的取值。 // VerdictRecommend / VerdictAvoid 是官方结论的取值。
const ( const (
VerdictAvoid = -1 VerdictAvoid = -1
@@ -24,11 +33,13 @@ type Tally struct {
Down int `json:"down"` Down int `json:"down"`
} }
// Verdicts 返回某个数据集的站长结论:src_pk -> verdict(1/-1)。 // Verdicts 返回某个数据集的站长结论:该数据集里的 src_pk -> verdict(1/-1)。
// 必须按 dataset_id 过滤:src_pk 只在单个数据集内唯一。
func (s *Store) Verdicts(ctx context.Context, datasetID int64) (map[int]int, error) { func (s *Store) Verdicts(ctx context.Context, datasetID int64) (map[int]int, error) {
out := map[int]int{} out := map[int]int{}
rows, err := s.DB.QueryContext(ctx, `SELECT src_pk, verdict FROM place_votes WHERE dataset_id=$1`, datasetID) rows, err := s.DB.QueryContext(ctx, `
SELECT p.src_pk, v.verdict FROM places p
JOIN place_votes v ON v.place_key = `+placeKeyExpr+`
WHERE p.dataset_id=$1 AND p.lat IS NOT NULL AND p.lon IS NOT NULL`, datasetID)
if err != nil { if err != nil {
return out, err return out, err
} }
@@ -46,8 +57,9 @@ func (s *Store) Verdicts(ctx context.Context, datasetID int64) (map[int]int, err
// Verdict 取单个地点的站长结论;未标注返回 0。 // Verdict 取单个地点的站长结论;未标注返回 0。
func (s *Store) Verdict(ctx context.Context, datasetID int64, srcPK int) (int, error) { func (s *Store) Verdict(ctx context.Context, datasetID int64, srcPK int) (int, error) {
var v int var v int
err := s.DB.QueryRowContext(ctx, err := s.DB.QueryRowContext(ctx, `SELECT v.verdict FROM place_votes v
`SELECT verdict FROM place_votes WHERE dataset_id=$1 AND src_pk=$2`, datasetID, srcPK).Scan(&v) JOIN places p ON v.place_key = `+placeKeyExpr+`
WHERE p.dataset_id=$1 AND p.src_pk=$2`, datasetID, srcPK).Scan(&v)
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
return 0, nil return 0, nil
} }
@@ -57,24 +69,46 @@ func (s *Store) Verdict(ctx context.Context, datasetID int64, srcPK int) (int, e
// SaveVerdict 设置站长结论;verdict 为 0 表示取消结论。 // SaveVerdict 设置站长结论;verdict 为 0 表示取消结论。
func (s *Store) SaveVerdict(ctx context.Context, userID, datasetID int64, srcPK, verdict int) error { func (s *Store) SaveVerdict(ctx context.Context, userID, datasetID int64, srcPK, verdict int) error {
if verdict == 0 { if verdict == 0 {
_, err := s.DB.ExecContext(ctx, _, err := s.DB.ExecContext(ctx, `DELETE FROM place_votes v USING places p
`DELETE FROM place_votes WHERE user_id=$1 AND dataset_id=$2 AND src_pk=$3`, userID, datasetID, srcPK) WHERE v.user_id=$1 AND p.dataset_id=$2 AND p.src_pk=$3
AND v.place_key = `+placeKeyExpr, userID, datasetID, srcPK)
return err return err
} }
_, err := s.DB.ExecContext(ctx, `INSERT INTO place_votes (user_id, dataset_id, src_pk, verdict, updated_at) // 先按坐标键算出这条地点该挂在哪,找不到该地点就什么也不写
VALUES ($1,$2,$3,$4,now()) _, err := s.DB.ExecContext(ctx, `INSERT INTO place_votes
ON CONFLICT (user_id, dataset_id, src_pk) DO UPDATE SET verdict=EXCLUDED.verdict, updated_at=now()`, (user_id, place_key, dataset_id, src_pk, verdict, updated_at)
SELECT $1, `+placeKeyExpr+`, $2, $3, $4, now()
FROM places p
WHERE p.dataset_id=$2 AND p.src_pk=$3 AND p.lat IS NOT NULL AND p.lon IS NOT NULL
ON CONFLICT (user_id, place_key) DO UPDATE SET
verdict=EXCLUDED.verdict, dataset_id=EXCLUDED.dataset_id,
src_pk=EXCLUDED.src_pk, updated_at=now()`,
userID, datasetID, srcPK, verdict) userID, datasetID, srcPK, verdict)
return err return err
} }
// Tallies 返回全部地点的访客票数。票表规模很小(个人站点),一次读全更省事, // placeKeyOf 把某个数据集里的地点编号换算成坐标键——访客票也是挂在坐标上的,
// 也避免给 pgx 传数组参数带来的类型歧义。 // 换数据集之后票仍然跟着同一个地点。
func (s *Store) Tallies(ctx context.Context) (map[int]Tally, error) { func (s *Store) placeKeyOf(ctx context.Context, datasetID int64, srcPK int) (string, error) {
var key string
err := s.DB.QueryRowContext(ctx, `SELECT `+placeKeyExpr+` FROM places p
WHERE p.dataset_id=$1 AND p.src_pk=$2 AND p.lat IS NOT NULL AND p.lon IS NOT NULL`,
datasetID, srcPK).Scan(&key)
if err == sql.ErrNoRows {
return "", fmt.Errorf("地点不存在或没有坐标")
}
return key, err
}
// Tallies 返回某个数据集里各地点的访客票数:src_pk -> Tally。
// 票表规模很小(个人站点),一次读全更省事,也避免给 pgx 传数组参数带来的类型歧义。
func (s *Store) Tallies(ctx context.Context, datasetID int64) (map[int]Tally, error) {
out := map[int]Tally{} out := map[int]Tally{}
rows, err := s.DB.QueryContext(ctx, `SELECT src_pk, rows, err := s.DB.QueryContext(ctx, `SELECT p.src_pk,
count(*) FILTER (WHERE vote=1), count(*) FILTER (WHERE vote=-1) count(*) FILTER (WHERE vv.vote=1), count(*) FILTER (WHERE vv.vote=-1)
FROM place_visitor_votes GROUP BY src_pk`) FROM places p JOIN place_visitor_votes vv ON vv.place_key = `+placeKeyExpr+`
WHERE p.dataset_id=$1 AND p.lat IS NOT NULL AND p.lon IS NOT NULL
GROUP BY p.src_pk`, datasetID)
if err != nil { if err != nil {
return out, err return out, err
} }
@@ -91,21 +125,24 @@ func (s *Store) Tallies(ctx context.Context) (map[int]Tally, error) {
} }
// Tally 取单个地点的访客票数。 // Tally 取单个地点的访客票数。
func (s *Store) Tally(ctx context.Context, srcPK int) (Tally, error) { func (s *Store) Tally(ctx context.Context, datasetID int64, srcPK int) (Tally, error) {
var t Tally var t Tally
err := s.DB.QueryRowContext(ctx, `SELECT count(*) FILTER (WHERE vote=1), count(*) FILTER (WHERE vote=-1) err := s.DB.QueryRowContext(ctx, `SELECT
FROM place_visitor_votes WHERE src_pk=$1`, srcPK).Scan(&t.Up, &t.Down) count(*) FILTER (WHERE vv.vote=1), count(*) FILTER (WHERE vv.vote=-1)
FROM place_visitor_votes vv JOIN places p ON vv.place_key = `+placeKeyExpr+`
WHERE p.dataset_id=$1 AND p.src_pk=$2`, datasetID, srcPK).Scan(&t.Up, &t.Down)
return t, err return t, err
} }
// VoterAll 返回该访客投过的全部票:src_pk -> vote,供地图一次性标注「我投的」。 // VoterAll 返回该访客投过的全部票:src_pk -> vote,供地图一次性标注「我投的」。
func (s *Store) VoterAll(ctx context.Context, voterKey string) (map[int]int, error) { func (s *Store) VoterAll(ctx context.Context, datasetID int64, voterKey string) (map[int]int, error) {
out := map[int]int{} out := map[int]int{}
if voterKey == "" { if voterKey == "" {
return out, nil return out, nil
} }
rows, err := s.DB.QueryContext(ctx, rows, err := s.DB.QueryContext(ctx, `SELECT p.src_pk, vv.vote
`SELECT src_pk, vote FROM place_visitor_votes WHERE voter_key=$1`, voterKey) FROM place_visitor_votes vv JOIN places p ON vv.place_key = `+placeKeyExpr+`
WHERE p.dataset_id=$1 AND vv.voter_key=$2`, datasetID, voterKey)
if err != nil { if err != nil {
return out, err return out, err
} }
@@ -121,13 +158,15 @@ func (s *Store) VoterAll(ctx context.Context, voterKey string) (map[int]int, err
} }
// Voter 返回该访客对某地点的既有投票(1/-1/0)。voterKey 为空表示拿不到身份。 // Voter 返回该访客对某地点的既有投票(1/-1/0)。voterKey 为空表示拿不到身份。
func (s *Store) Voter(ctx context.Context, srcPK int, voterKey string) int { func (s *Store) Voter(ctx context.Context, datasetID int64, srcPK int, voterKey string) int {
if voterKey == "" { if voterKey == "" {
return 0 return 0
} }
var v int var v int
if err := s.DB.QueryRowContext(ctx, if err := s.DB.QueryRowContext(ctx, `SELECT vv.vote FROM place_visitor_votes vv
`SELECT vote FROM place_visitor_votes WHERE src_pk=$1 AND voter_key=$2`, srcPK, voterKey).Scan(&v); err != nil { JOIN places p ON vv.place_key = `+placeKeyExpr+`
WHERE p.dataset_id=$1 AND p.src_pk=$2 AND vv.voter_key=$3`,
datasetID, srcPK, voterKey).Scan(&v); err != nil {
return 0 return 0
} }
return v return v
@@ -135,30 +174,35 @@ func (s *Store) Voter(ctx context.Context, srcPK int, voterKey string) int {
// Cast 记录访客投票:未投过则新增;投同一票则撤销(再点一次取消);投另一票则改为该票。 // Cast 记录访客投票:未投过则新增;投同一票则撤销(再点一次取消);投另一票则改为该票。
// 返回更新后的票数与该访客最终的选择。 // 返回更新后的票数与该访客最终的选择。
func (s *Store) Cast(ctx context.Context, srcPK int, voterKey string, vote int) (Tally, int, error) { func (s *Store) Cast(ctx context.Context, datasetID int64, srcPK int, voterKey string, vote int) (Tally, int, error) {
key, err := s.placeKeyOf(ctx, datasetID, srcPK)
if err != nil {
return Tally{}, 0, err
}
var cur int var cur int
err := s.DB.QueryRowContext(ctx, err = s.DB.QueryRowContext(ctx,
`SELECT vote FROM place_visitor_votes WHERE src_pk=$1 AND voter_key=$2`, srcPK, voterKey).Scan(&cur) `SELECT vote FROM place_visitor_votes WHERE place_key=$1 AND voter_key=$2`, key, voterKey).Scan(&cur)
switch { switch {
case err == sql.ErrNoRows: case err == sql.ErrNoRows:
_, err = s.DB.ExecContext(ctx, _, err = s.DB.ExecContext(ctx,
`INSERT INTO place_visitor_votes (src_pk, voter_key, vote) VALUES ($1,$2,$3)`, srcPK, voterKey, vote) `INSERT INTO place_visitor_votes (place_key, voter_key, vote, src_pk) VALUES ($1,$2,$3,$4)`,
key, voterKey, vote, srcPK)
cur = vote cur = vote
case err != nil: case err != nil:
return Tally{}, 0, err return Tally{}, 0, err
case cur == vote: case cur == vote:
_, err = s.DB.ExecContext(ctx, _, err = s.DB.ExecContext(ctx,
`DELETE FROM place_visitor_votes WHERE src_pk=$1 AND voter_key=$2`, srcPK, voterKey) `DELETE FROM place_visitor_votes WHERE place_key=$1 AND voter_key=$2`, key, voterKey)
cur = 0 cur = 0
default: default:
_, err = s.DB.ExecContext(ctx, _, err = s.DB.ExecContext(ctx,
`UPDATE place_visitor_votes SET vote=$3, updated_at=now() WHERE src_pk=$1 AND voter_key=$2`, `UPDATE place_visitor_votes SET vote=$3, src_pk=$4, updated_at=now()
srcPK, voterKey, vote) WHERE place_key=$1 AND voter_key=$2`, key, voterKey, vote, srcPK)
cur = vote cur = vote
} }
if err != nil { if err != nil {
return Tally{}, 0, err return Tally{}, 0, err
} }
t, err := s.Tally(ctx, srcPK) t, err := s.Tally(ctx, datasetID, srcPK)
return t, cur, err return t, cur, err
} }
+346
View File
@@ -0,0 +1,346 @@
package web
import (
"context"
"database/sql"
"encoding/base64"
"encoding/hex"
"fmt"
"io"
"log"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"rond-with-you/internal/ingest"
"rond-with-you/internal/rond"
)
// rondbackup 的差异预演:把上传的备份与当前展示的数据集按稳定键对齐,只读地摆出
// 新增 / 仅库里有 / 冲突三类,**不写任何数据**。合并执行是下一步的事。
const backupStashExt = ".rondbackup"
// diffListLimit 是每类差异在页面上最多列出的条数。867 条全铺出来既慢又没人看,
// 超过就只给前若干条并注明。
const diffListLimit = 200
func (s *Server) backupStashDir() string {
return filepath.Join(s.cfg.DataDir, "backup-staging")
}
// adminBackupPreview 收下备份、暂存,跳到差异页。这里不校验备份内容——
// 差异页反正是要解析一遍的,失败时让它报错即可,免得同一份包解两次。
func (s *Server) adminBackupPreview(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxUploadBytes+1<<20)
if err := r.ParseMultipartForm(32 << 20); err != nil {
s.adminFlash(w, r, "err", "读取上传内容失败:"+err.Error())
return
}
file, hdr, err := r.FormFile("backup")
if err != nil {
s.adminFlash(w, r, "err", "请选择要分析的 .rondbackup 文件")
return
}
defer file.Close()
name := filepath.Base(hdr.Filename)
lower := strings.ToLower(name)
if !strings.HasSuffix(lower, ".rondbackup") && !strings.HasSuffix(lower, ".zip") {
s.adminFlash(w, r, "err", "文件类型不正确,请上传 rond 导出的 .rondbackup 备份")
return
}
data, err := io.ReadAll(io.LimitReader(file, maxUploadBytes+1))
if err != nil {
s.adminFlash(w, r, "err", "读取文件失败:"+err.Error())
return
}
if int64(len(data)) > maxUploadBytes {
s.adminFlash(w, r, "err", "备份文件过大,疑似不是 rond 备份")
return
}
token, err := stashPut(s.backupStashDir(), backupStashExt, data)
if err != nil {
s.serverError(w, r, err)
return
}
http.Redirect(w, r, "/admin/preview/"+token, http.StatusSeeOther)
}
// adminBackupPreviewPage 渲染差异页。每次都重新算:库里的数据随时可能被别处改掉。
func (s *Server) adminBackupPreviewPage(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
p, err := s.pageBase(r, "admin", "备份差异预览")
if err != nil {
s.serverError(w, r, err)
return
}
token := r.PathValue("token")
path, ok := stashPath(s.backupStashDir(), backupStashExt, token)
if !ok {
s.adminFlash(w, r, "err", "预览链接无效,请重新上传备份")
return
}
if _, err := os.Stat(path); err != nil {
s.adminFlash(w, r, "err", "这次预览已过期,请重新上传备份")
return
}
ds, err := s.datasetFor(ctx)
if err != nil {
s.serverError(w, r, err)
return
}
if ds == nil {
s.adminFlash(w, r, "err", "还没有可比较的数据集,请先直接导入一份备份")
return
}
tmp, err := ingest.TempDir(s.cfg.DataDir, "preview")
if err != nil {
s.serverError(w, r, err)
return
}
defer os.RemoveAll(tmp)
sqlDB, err := rond.Open(path, tmp)
if err != nil {
s.adminFlash(w, r, "err", "解析备份失败:"+err.Error())
return
}
backup, perr := rond.Parse(sqlDB)
sqlDB.Close()
if perr != nil {
s.adminFlash(w, r, "err", "读取备份数据失败:"+perr.Error())
return
}
storedVisits, err := s.loadStoredVisits(ctx, ds.ID)
if err != nil {
s.serverError(w, r, err)
return
}
storedPlaces, err := s.loadStoredPlaces(ctx, ds.ID)
if err != nil {
s.serverError(w, r, err)
return
}
storedActs, err := s.loadStoredActivities(ctx, ds.ID)
if err != nil {
s.serverError(w, r, err)
return
}
vd := rond.DiffVisits(storedVisits, backup)
pd := rond.DiffPlaces(storedPlaces, backup)
ad := rond.DiffActivities(storedActs, backup)
d := BackupDiffData{
Page: p, Token: token,
DatasetName: ds.Name,
VisitCounts: rond.VisitCounts(vd),
PlaceCounts: rond.PlaceCounts(pd),
ActCounts: rond.ActivityCounts(ad),
PlaceAdded: limitSlice(pd.Added, diffListLimit),
PlaceGone: limitSlice(pd.Removed, diffListLimit),
PlaceClash: limitSlice(pd.Changed, diffListLimit),
ActAdded: limitSlice(ad.Added, diffListLimit),
ActGone: limitSlice(ad.Removed, diffListLimit),
ActClash: limitSlice(ad.Changed, diffListLimit),
BackupVisits: len(backup.Visits), BackupPlaces: len(backup.Locations),
BackupActs: len(backup.Activities), BackupMovements: len(backup.Movements),
BackupWeather: len(backup.Weather), BackupRaw: backup.RawVisitNum,
StoredVisits: len(storedVisits), StoredPlaces: len(storedPlaces),
StoredActs: len(storedActs),
}
d.VisitAdded, d.VisitGone, d.VisitClash = visitRows(vd, backup, diffListLimit)
d.Truncated = anyTruncated(vd, pd, ad)
d.StoredMovements, d.StoredWeather, err = s.countOther(ctx, ds.ID)
if err != nil {
s.serverError(w, r, err)
return
}
d.Identical = d.VisitCounts.Added+d.VisitCounts.Removed+d.VisitCounts.Changed+
d.PlaceCounts.Added+d.PlaceCounts.Removed+d.PlaceCounts.Changed+
d.ActCounts.Added+d.ActCounts.Removed+d.ActCounts.Changed == 0
s.render(w, "admin/backup-preview", d)
}
// visitRows 把三类到访差异摊平成展示行。备份里的地点/活动只有编号,要借备份自己的
// 表翻成名称;库里那侧的名称在加载时就一并取出来了。
func visitRows(vd rond.VisitDiff, b *rond.Backup, limit int) (added, gone, clash []VisitRow) {
place, act := map[int]string{}, map[int]string{}
for _, l := range b.Locations {
place[l.SrcPK] = placeLabel(l.Name, l.Lat, l.Lon)
}
for _, a := range b.Activities {
act[a.SrcPK] = a.Name
}
fromBackup := func(v rond.Visit) VisitRow {
r := VisitRow{Arrival: v.Arrival, Departure: v.Departure, Remark: v.Remark, Emoji: v.Emoji}
if v.LocationSrcPK != nil {
r.Place = place[*v.LocationSrcPK]
}
if v.ActivitySrcPK != nil {
r.Activity = act[*v.ActivitySrcPK]
}
return r
}
for _, v := range limitSlice(vd.Added, limit) {
added = append(added, fromBackup(v))
}
for _, sv := range limitSlice(vd.Removed, limit) {
gone = append(gone, VisitRow{SrcPK: sv.SrcPK, Arrival: sv.Arrival, Departure: sv.Departure,
Place: sv.PlaceName, Activity: sv.ActivityName, Remark: sv.Remark, Emoji: sv.Emoji,
UserAdded: sv.UserAdded})
}
for _, c := range limitSlice(vd.Changed, limit) {
r := fromBackup(c.Backup)
r.Key = c.DB.UUID
r.UserAdded = c.DB.UserAdded
r.Fields = c.Fields
clash = append(clash, r)
}
return
}
func placeLabel(name string, lat, lon float64) string {
if name != "" {
return name
}
return fmt.Sprintf("%.5f, %.5f", lat, lon)
}
// anyTruncated 报告是否有哪一类差异多到被截断了。
func anyTruncated(vd rond.VisitDiff, pd rond.PlaceDiff, ad rond.ActivityDiff) bool {
for _, n := range []int{len(vd.Added), len(vd.Removed), len(vd.Changed),
len(pd.Added), len(pd.Removed), len(pd.Changed),
len(ad.Added), len(ad.Removed), len(ad.Changed)} {
if n > diffListLimit {
return true
}
}
return false
}
func limitSlice[T any](in []T, n int) []T {
if len(in) <= n {
return in
}
return in[:n]
}
func (s *Server) countOther(ctx context.Context, datasetID int64) (movements, weather int, err error) {
err = s.db.QueryRowContext(ctx, `SELECT
(SELECT count(*) FROM movements WHERE dataset_id=$1),
(SELECT count(*) FROM weather WHERE dataset_id=$1)`, datasetID).Scan(&movements, &weather)
return
}
// loadStoredVisits 读出该数据集的全部到访,并把原始行留档里的 UUID 解出来当对齐键。
// 没有留档的行就是站点新建的(真机不会给它 UUID),对齐键留空。
func (s *Server) loadStoredVisits(ctx context.Context, datasetID int64) ([]rond.StoredVisit, error) {
rows, err := s.db.QueryContext(ctx, `
SELECT v.src_pk, v.arrival, v.departure, COALESCE(v.remark,''), COALESCE(v.emoji,''),
v.bookmarked, v.is_user_added, COALESCE(p.src_pk,0),
COALESCE(p.name,''), COALESCE(p.lat,0), COALESCE(p.lon,0), COALESCE(a.name,''),
COALESCE(er.raw #>> '{ZIDENTIFIER_,$blob}','')
FROM visits v
LEFT JOIN places p ON p.id = v.place_id
LEFT JOIN activities a ON a.id = v.activity_id
LEFT JOIN entity_raw er ON er.dataset_id = v.dataset_id
AND er.entity = 'ZVISIT' AND er.src_pk = v.src_pk
WHERE v.dataset_id = $1`, datasetID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []rond.StoredVisit
for rows.Next() {
var sv rond.StoredVisit
var arr time.Time
var dep sql.NullTime
var placePK int
var placeName string
var lat, lon float64
var b64 string
if err := rows.Scan(&sv.SrcPK, &arr, &dep, &sv.Remark, &sv.Emoji, &sv.Bookmarked,
&sv.UserAdded, &placePK, &placeName, &lat, &lon, &sv.ActivityName, &b64); err != nil {
return nil, err
}
sv.Arrival = &arr
if dep.Valid {
sv.Departure = &dep.Time
}
sv.PlaceName = placeName
// 没有坐标的地点不能都落到 "0.000000|0.000000" 上,否则两个地点会互相认领
if placePK > 0 && (lat != 0 || lon != 0) {
sv.PlaceKey = rond.PlaceCoordKey(lat, lon)
}
sv.UUID = uuidFromBase64(b64)
out = append(out, sv)
}
return out, rows.Err()
}
func (s *Server) loadStoredPlaces(ctx context.Context, datasetID int64) ([]rond.StoredPlace, error) {
rows, err := s.db.QueryContext(ctx, `SELECT src_pk, COALESCE(name,''), COALESCE(lat,0), COALESCE(lon,0),
COALESCE(province,''), COALESCE(city,''), COALESCE(poi_category,''), visit_count
FROM places WHERE dataset_id=$1`, datasetID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []rond.StoredPlace
for rows.Next() {
var sp rond.StoredPlace
if err := rows.Scan(&sp.SrcPK, &sp.Name, &sp.Lat, &sp.Lon,
&sp.Province, &sp.City, &sp.POICategory, &sp.VisitCount); err != nil {
return nil, err
}
if sp.Lat != 0 || sp.Lon != 0 {
sp.Key = rond.PlaceCoordKey(sp.Lat, sp.Lon)
}
out = append(out, sp)
}
return out, rows.Err()
}
// loadStoredActivities 读活动类型;UUID 同样从留档里取,用于识别「改了名但其实是同一个」。
func (s *Server) loadStoredActivities(ctx context.Context, datasetID int64) ([]rond.StoredActivity, error) {
rows, err := s.db.QueryContext(ctx, `SELECT a.src_pk, a.name,
COALESCE(er.raw #>> '{ZUID_,$blob}','')
FROM activities a
LEFT JOIN entity_raw er ON er.dataset_id = a.dataset_id
AND er.entity = 'ZACTIVITY' AND er.src_pk = a.src_pk
WHERE a.dataset_id=$1`, datasetID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []rond.StoredActivity
for rows.Next() {
var sa rond.StoredActivity
var b64 string
if err := rows.Scan(&sa.SrcPK, &sa.Name, &b64); err != nil {
return nil, err
}
sa.UUID = uuidFromBase64(b64)
out = append(out, sa)
}
return out, rows.Err()
}
// uuidFromBase64 把留档里的 $blob(base64 的 16 字节 UUID)转成小写十六进制,
// 好跟备份侧 hex(ZIDENTIFIER_) 的写法对上。
func uuidFromBase64(s string) string {
if s == "" {
return ""
}
b, err := base64.StdEncoding.DecodeString(s)
if err != nil || len(b) != 16 {
log.Printf("留档里的 UUID 不是 16 字节,忽略: %q", s)
return ""
}
return hex.EncodeToString(b)
}
+164
View File
@@ -0,0 +1,164 @@
package web
import (
"bytes"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"testing"
)
// previewBackup 走真实的上传处理函数:POST → 303 → GET 差异页,返回渲染结果。
func previewBackup(t *testing.T, f *movementFixture, path string) string {
t.Helper()
_, page := previewBackupToken(t, f, path)
return page
}
// previewBackupToken 同上,但把令牌也交出来,供后续的合并步骤使用。
func previewBackupToken(t *testing.T, f *movementFixture, path string) (string, string) {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("读取 %s 失败: %v", path, err)
}
var buf bytes.Buffer
mw := multipart.NewWriter(&buf)
fw, err := mw.CreateFormFile("backup", filepath.Base(path))
if err != nil {
t.Fatalf("构造上传表单失败: %v", err)
}
if _, err := fw.Write(data); err != nil {
t.Fatalf("写入上传内容失败: %v", err)
}
mw.Close()
rec := httptest.NewRecorder()
req := httptest.NewRequest("POST", "/admin/preview", &buf)
req.Header.Set("Content-Type", mw.FormDataContentType())
f.srv.adminBackupPreview(rec, req.WithContext(f.ctx))
if rec.Code != http.StatusSeeOther {
t.Fatalf("上传应跳转到差异页: code=%d body=%s", rec.Code, rec.Body.String())
}
loc := rec.Header().Get("Location")
token := strings.TrimPrefix(loc, "/admin/preview/")
if token == loc || len(token) != stashTokenLen {
t.Fatalf("差异页地址不对: %q", loc)
}
rec = httptest.NewRecorder()
req = httptest.NewRequest("GET", loc, nil).WithContext(f.ctx)
req.SetPathValue("token", token)
f.srv.adminBackupPreviewPage(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("差异页渲染失败: code=%d body=%s", rec.Code, rec.Body.String())
}
return token, rec.Body.String()
}
// visitCounts 从概览表里取出「到访」那一行的 (备份新增, 仅站内有, 冲突)。
// 不能全页 Contains 数字——地点行也有同样的 data-label,会张冠李戴。
func visitCounts(t *testing.T, page string) (added, gone, clash int) {
t.Helper()
row := regexp.MustCompile(`(?s)<td>到访</td>(.*?)</tr>`).FindStringSubmatch(page)
if row == nil {
t.Fatal("概览表里找不到「到访」行")
}
nums := regexp.MustCompile(`data-label="(备份新增|仅站内有|冲突)">(\d+)<`).FindAllStringSubmatch(row[1], -1)
if len(nums) != 3 {
t.Fatalf("到访行应有三个计数,实际 %d", len(nums))
}
for _, n := range nums {
v, _ := strconv.Atoi(n[2])
switch n[1] {
case "备份新增":
added = v
case "仅站内有":
gone = v
case "冲突":
clash = v
}
}
return
}
// 同一份备份再传一次:UUID 对得上、字段也没变,到访应当一条不差。
//
// 这条也顺带钉住「按 UUID 而不是编号对齐」:库内那些行的编号与备份里完全不同
// (rond 重排过),只按编号比会让 800 多条全变冲突。
//
// 只断言到访,不断言「完全一致」整句:导入后站点会反查补全地点(改名称/地址),
// 地点层面因此真的存在差异,那是预期行为,不该由这个用例来管。
func TestBackupDiffIdentical(t *testing.T) {
f := newMovementFixture(t)
page := previewBackup(t, f, f.src)
if a, g, c := visitCounts(t, page); a != 0 || g != 0 || c != 0 {
t.Fatalf("同一份备份的到访不该有差异: 新增 %d 仅站内有 %d 冲突 %d", a, g, c)
}
}
// 站内改过备注、又补录了一条到访,差异页要把这两件事分别认出来。
//
// 这里的关键是「按 UUID 对齐」:库里那些行的 Z_PK 与备份里完全不同
// (rond 重排过),只按编号比会把 800 多条全判成冲突。
func TestBackupDiffDetectsSiteEdits(t *testing.T) {
f := newMovementFixture(t)
var pk int
if err := f.pg.QueryRowContext(f.ctx, `UPDATE visits SET remark='站点改过的备注'
WHERE id = (SELECT id FROM visits WHERE dataset_id=$1 ORDER BY src_pk LIMIT 1)
RETURNING src_pk`, f.dsID).Scan(&pk); err != nil {
t.Fatalf("改备注失败: %v", err)
}
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO visits
(dataset_id, src_pk, arrival, departure, is_user_added, remark)
VALUES ($1, 999999, now(), now(), true, '站点补录的一条')`, f.dsID); err != nil {
t.Fatalf("插入补录到访失败: %v", err)
}
page := previewBackup(t, f, f.src)
// 「站点改过的备注」只可能从冲突行的字段差异里渲染出来;「站内补录」是那条补录行的来源标记
for _, want := range []string{"备份差异预览", "站点改过的备注", "站点补录的一条", ">站内补录<"} {
if !strings.Contains(page, want) {
t.Errorf("差异页缺少 %q", want)
}
}
a, g, c := visitCounts(t, page)
if c != 1 {
t.Errorf("改过一条备注,到访冲突数应为 1,实际 %d", c)
}
if g != 1 {
t.Errorf("补录了一条,到访「仅站内有」应为 1,实际 %d", g)
}
if a != 0 {
t.Errorf("备份里没有多出来的到访,实际 %d", a)
}
}
// 差异页是只读的:渲染前后库里的行数必须一模一样。
func TestBackupDiffPreviewWritesNothing(t *testing.T) {
f := newMovementFixture(t)
count := func() (visits, places, raw int) {
t.Helper()
if err := f.pg.QueryRowContext(f.ctx, `SELECT
(SELECT count(*) FROM visits WHERE dataset_id=$1),
(SELECT count(*) FROM places WHERE dataset_id=$1),
(SELECT count(*) FROM entity_raw WHERE dataset_id=$1)`, f.dsID).
Scan(&visits, &places, &raw); err != nil {
t.Fatalf("统计失败: %v", err)
}
return
}
v0, p0, r0 := count()
previewBackup(t, f, f.src)
v1, p1, r1 := count()
if v0 != v1 || p0 != p1 || r0 != r1 {
t.Fatalf("差异页不该写库: 到访 %d→%d 地点 %d→%d 留档 %d→%d", v0, v1, p0, p1, r0, r1)
}
}
+753
View File
@@ -0,0 +1,753 @@
package web
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"log"
"net/http"
"os"
"strconv"
"strings"
"rond-with-you/internal/geo"
"rond-with-you/internal/ingest"
"rond-with-you/internal/rond"
)
// 差异页确认后的合并执行。
//
// 全部改动在一个事务里完成:半合并的数据集比没合并更难收拾(比如地点建了、
// 到访没建,地图上就多一个永远没人去的点)。删除不在这里——删地点会级联删掉
// 它的全部到访,那件事单独再议。
//
// 编号是这里最容易出错的地方:**库里那侧的编号空间才是唯一有效的**。
// 新插入的行拿 nextSrcPK 发的新号,于是位移的 from_visit_src、天气的 visit_src
// 都必须写「库里的新号」,而不是备份里的原号。
type mergeStats struct {
Places, Activities, Tags int
Visits, Movements, Weather int
UpdPlaces, UpdActs, UpdVisits int
DelPlaces, DelActs, DelVisits int
}
func (s *Server) adminBackupMerge(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
ds, err := s.datasetFor(ctx)
if err != nil {
s.serverError(w, r, err)
return
}
if ds == nil {
s.adminFlash(w, r, "err", "没有可合并的数据集")
return
}
token := r.PathValue("token")
path, ok := stashPath(s.backupStashDir(), backupStashExt, token)
if !ok {
s.adminFlash(w, r, "err", "合并链接无效,请重新上传备份")
return
}
if _, err := os.Stat(path); err != nil {
s.adminFlash(w, r, "err", "这次预览已过期,请重新上传备份")
return
}
if err := r.ParseForm(); err != nil {
s.serverError(w, r, err)
return
}
backup, err := s.parseBackupFile(path)
if err != nil {
s.adminFlash(w, r, "err", err.Error())
return
}
visits, places, acts, err := s.loadStoredAll(ctx, ds.ID)
if err != nil {
s.serverError(w, r, err)
return
}
// 重新算一遍差异再执行:从渲染差异页到点下按钮之间,库可能已经被别处改过
vd := rond.DiffVisits(visits, backup)
pd := rond.DiffPlaces(places, backup)
ad := rond.DiffActivities(acts, backup)
addNew := r.PostFormValue("add_new") == "1"
plan := rond.BuildMergePlan(vd, pd, ad, rond.Choices{
AddVisits: addNew, AddPlaces: addNew, AddActs: addNew,
VisitClash: pickForm(r, "vc_"),
PlaceClash: pickForm(r, "pc_"),
ActClash: pickForm(r, "ac_"),
DelVisits: pickIntForm(r, "dv_"),
DelPlaces: pickIntForm(r, "dp_"),
DelActs: pickIntForm(r, "da_"),
})
if plan.Empty() {
s.adminFlash(w, r, "ok", "按当前的选择没有要写入的改动,数据未变")
return
}
st, err := s.applyMergePlan(ctx, ds, backup, plan, visits)
if err != nil {
log.Printf("合并失败: %v", err)
s.adminFlash(w, r, "err", "合并失败,已整体回滚、数据未变:"+err.Error())
return
}
stashDrop(s.backupStashDir(), backupStashExt, token)
msg := fmt.Sprintf("合并完成:新增 %d 个地点 / %d 个活动 / %d 条到访;更新 %d 个地点 / %d 个活动 / %d 条到访",
st.Places, st.Activities, st.Visits, st.UpdPlaces, st.UpdActs, st.UpdVisits)
if st.DelVisits+st.DelActs+st.DelPlaces > 0 {
msg += fmt.Sprintf(";删除 %d 条到访 / %d 个活动 / %d 个地点", st.DelVisits, st.DelActs, st.DelPlaces)
}
if st.Tags > 0 {
msg += fmt.Sprintf(";顺带建了 %d 个标签", st.Tags)
}
if st.Movements+st.Weather > 0 {
msg += fmt.Sprintf(";随到访带入 %d 条位移、%d 条天气", st.Movements, st.Weather)
}
s.adminFlash(w, r, "ok", msg)
}
// pickIntForm 收集「前缀 + 整数」形式的勾选,比如 dv_825=1(要删除的到访)。
func pickIntForm(r *http.Request, prefix string) map[int]bool {
out := map[int]bool{}
for k, vs := range r.PostForm {
if len(vs) == 0 || vs[0] == "" || !strings.HasPrefix(k, prefix) {
continue
}
if n, err := strconv.Atoi(strings.TrimPrefix(k, prefix)); err == nil {
out[n] = true
}
}
return out
}
// pickForm 收集「前缀 + 键」形式的逐条选择,比如 vc_<uuid>=backup。
func pickForm(r *http.Request, prefix string) map[string]string {
out := map[string]string{}
for k, vs := range r.PostForm {
if len(vs) == 0 || !strings.HasPrefix(k, prefix) {
continue
}
out[strings.TrimPrefix(k, prefix)] = vs[0]
}
return out
}
func (s *Server) loadStoredAll(ctx context.Context, dsID int64) ([]rond.StoredVisit, []rond.StoredPlace, []rond.StoredActivity, error) {
visits, err := s.loadStoredVisits(ctx, dsID)
if err != nil {
return nil, nil, nil, err
}
places, err := s.loadStoredPlaces(ctx, dsID)
if err != nil {
return nil, nil, nil, err
}
acts, err := s.loadStoredActivities(ctx, dsID)
if err != nil {
return nil, nil, nil, err
}
return visits, places, acts, nil
}
func (s *Server) parseBackupFile(path string) (*rond.Backup, error) {
tmp, err := ingest.TempDir(s.cfg.DataDir, "merge")
if err != nil {
return nil, err
}
defer os.RemoveAll(tmp)
db, err := rond.Open(path, tmp)
if err != nil {
return nil, fmt.Errorf("解析备份失败:%w", err)
}
defer db.Close()
b, err := rond.Parse(db)
if err != nil {
return nil, fmt.Errorf("读取备份数据失败:%w", err)
}
return b, nil
}
func (s *Server) applyMergePlan(ctx context.Context, ds *Dataset, b *rond.Backup, plan rond.MergePlan,
storedVisits []rond.StoredVisit) (*mergeStats, error) {
st := &mergeStats{}
err := s.withTx(ctx, func(ex execer) error {
placeID, err := s.mergePlaces(ctx, ex, ds.ID, b, plan, st)
if err != nil {
return err
}
actID, err := s.mergeActivities(ctx, ex, ds.ID, b, plan, st)
if err != nil {
return err
}
tagID, err := s.mergeTags(ctx, ex, ds.ID, b, st)
if err != nil {
return err
}
if err := s.mergeVisits(ctx, ex, ds.ID, b, plan, placeID, actID, tagID, storedVisits, st); err != nil {
return err
}
// 删除放在最后:先让新增/更新落定,再按最终状态删
if err := s.applyDeletions(ctx, ex, ds.ID, plan, st); err != nil {
return err
}
return s.recountPlaces(ctx, ex, ds.ID)
})
if err != nil {
return nil, err
}
return st, nil
}
// mergePlaces 先把备份地点映射到库里的 places.id(已存在的直接映射),
// 再处理冲突更新与新增,返回「备份地点编号 -> 库 id」。
func (s *Server) mergePlaces(ctx context.Context, ex execer, dsID int64, b *rond.Backup,
plan rond.MergePlan, st *mergeStats) (map[int]int64, error) {
byCoord, byName := map[string]int64{}, map[string]int64{}
rows, err := ex.QueryContext(ctx, `SELECT id, COALESCE(name,''), COALESCE(lat,0), COALESCE(lon,0)
FROM places WHERE dataset_id=$1`, dsID)
if err != nil {
return nil, err
}
for rows.Next() {
var id int64
var name string
var lat, lon float64
if err := rows.Scan(&id, &name, &lat, &lon); err != nil {
rows.Close()
return nil, err
}
if lat != 0 || lon != 0 {
byCoord[rond.PlaceCoordKey(lat, lon)] = id
}
if _, dup := byName[name]; !dup && name != "" {
byName[name] = id
}
}
rows.Close()
if err := rows.Err(); err != nil {
return nil, err
}
placeID := map[int]int64{}
for _, l := range b.Locations {
if l.Lat != 0 || l.Lon != 0 {
if id, ok := byCoord[rond.PlaceCoordKey(l.Lat, l.Lon)]; ok {
placeID[l.SrcPK] = id
continue
}
}
if l.Name != "" {
if id, ok := byName[l.Name]; ok {
placeID[l.SrcPK] = id
}
}
}
for _, up := range plan.UpdPlaces {
id := placeID[up.L.SrcPK]
if id == 0 {
id = byCoord[up.CoordKey]
}
if id == 0 {
id = byName[up.L.Name]
}
if id == 0 {
return nil, fmt.Errorf("找不到要更新的地点(%s)", up.L.Name)
}
if _, err := ex.ExecContext(ctx, `UPDATE places SET name=$1, poi_category=$2, lat=$3, lon=$4,
country_code=$5, province=$6, city=$7, district=$8, sublocality=$9, thoroughfare=$10, timezone=$11
WHERE id=$12`,
nullStr(up.L.Name), nullStr(up.L.POICategory), up.L.Lat, up.L.Lon,
nullStr(up.L.CountryCode), nullStr(up.L.Province), nullStr(up.L.City), nullStr(up.L.District),
nullStr(up.L.Sublocality), nullStr(up.L.Thoroughfare), nullStr(up.L.Timezone), id); err != nil {
return nil, fmt.Errorf("更新地点: %w", err)
}
placeID[up.L.SrcPK] = id
st.UpdPlaces++
}
for _, l := range plan.AddPlaces {
pk, err := s.nextSrcPK(ctx, ex, dsID, placeSeq)
if err != nil {
return nil, err
}
var id int64
if err := ex.QueryRowContext(ctx, `INSERT INTO places
(dataset_id, src_pk, name, poi_category, lat, lon, country_code, province, city,
district, sublocality, thoroughfare, timezone)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13) RETURNING id`,
dsID, pk, nullStr(l.Name), nullStr(l.POICategory), l.Lat, l.Lon, nullStr(l.CountryCode),
nullStr(l.Province), nullStr(l.City), nullStr(l.District), nullStr(l.Sublocality),
nullStr(l.Thoroughfare), nullStr(l.Timezone)).Scan(&id); err != nil {
return nil, fmt.Errorf("新增地点: %w", err)
}
placeID[l.SrcPK] = id
st.Places++
if err := writeRawRow(ctx, ex, dsID, "ZLOCATION", pk, b.RawRows["ZLOCATION"][l.SrcPK]); err != nil {
return nil, err
}
}
return placeID, nil
}
func (s *Server) mergeActivities(ctx context.Context, ex execer, dsID int64, b *rond.Backup,
plan rond.MergePlan, st *mergeStats) (map[int]int64, error) {
byUUID, byName := map[string]int64{}, map[string]int64{}
rows, err := ex.QueryContext(ctx, `SELECT a.id, a.name, COALESCE(er.raw #>> '{ZUID_,$blob}','')
FROM activities a
LEFT JOIN entity_raw er ON er.dataset_id = a.dataset_id
AND er.entity = 'ZACTIVITY' AND er.src_pk = a.src_pk
WHERE a.dataset_id=$1`, dsID)
if err != nil {
return nil, err
}
for rows.Next() {
var id int64
var name, b64 string
if err := rows.Scan(&id, &name, &b64); err != nil {
rows.Close()
return nil, err
}
if u := uuidFromBase64(b64); u != "" {
byUUID[u] = id
}
if _, dup := byName[name]; !dup && name != "" {
byName[name] = id
}
}
rows.Close()
if err := rows.Err(); err != nil {
return nil, err
}
actID := map[int]int64{}
for _, a := range b.Activities {
if u := b.ActivityUUID[a.SrcPK]; u != "" {
if id, ok := byUUID[u]; ok {
actID[a.SrcPK] = id
continue
}
}
if id, ok := byName[a.Name]; ok {
actID[a.SrcPK] = id
}
}
for _, up := range plan.UpdActs {
id := actID[up.A.SrcPK]
if id == 0 {
id = byName[up.A.Name]
}
if id == 0 {
return nil, fmt.Errorf("找不到要更新的活动(%s)", up.A.Name)
}
if _, err := ex.ExecContext(ctx, `UPDATE activities SET name=$1, color=$2, icon=$3,
is_home=$4, is_work=$5, is_excluded=$6, is_archived=$7 WHERE id=$8`,
up.A.Name, nullStr(up.A.Color), nullStr(up.A.Icon), up.A.IsHome, up.A.IsWork,
up.A.IsExcluded, up.A.IsArchived, id); err != nil {
return nil, fmt.Errorf("更新活动: %w", err)
}
st.UpdActs++
}
for _, a := range plan.AddActs {
pk, err := s.nextSrcPK(ctx, ex, dsID, activitySeq)
if err != nil {
return nil, err
}
var id int64
if err := ex.QueryRowContext(ctx, `INSERT INTO activities
(dataset_id, src_pk, name, color, icon, is_home, is_work, is_excluded, is_archived)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) RETURNING id`,
dsID, pk, a.Name, nullStr(a.Color), nullStr(a.Icon),
a.IsHome, a.IsWork, a.IsExcluded, a.IsArchived).Scan(&id); err != nil {
return nil, fmt.Errorf("新增活动: %w", err)
}
actID[a.SrcPK] = id
st.Activities++
if err := writeRawRow(ctx, ex, dsID, "ZACTIVITY", pk, b.RawRows["ZACTIVITY"][a.SrcPK]); err != nil {
return nil, err
}
}
return actID, nil
}
// mergeTags 只做「库里有同名的就复用、没有就建」——标签是纯名称实体,没有别的标识。
func (s *Server) mergeTags(ctx context.Context, ex execer, dsID int64, b *rond.Backup, st *mergeStats) (map[int]int64, error) {
byName := map[string]int64{}
rows, err := ex.QueryContext(ctx, `SELECT id, name FROM tags WHERE dataset_id=$1`, dsID)
if err != nil {
return nil, err
}
for rows.Next() {
var id int64
var name string
if err := rows.Scan(&id, &name); err != nil {
rows.Close()
return nil, err
}
byName[name] = id
}
rows.Close()
if err := rows.Err(); err != nil {
return nil, err
}
tagID := map[int]int64{}
for _, t := range b.Tags {
if id, ok := byName[t.Name]; ok {
tagID[t.SrcPK] = id
continue
}
pk, err := s.nextSrcPK(ctx, ex, dsID, tagSeq)
if err != nil {
return nil, err
}
var id int64
if err := ex.QueryRowContext(ctx, `INSERT INTO tags (dataset_id, src_pk, name, color, group_name)
VALUES ($1,$2,$3,$4,NULLIF($5,'')) RETURNING id`,
dsID, pk, t.Name, nullStr(t.Color), t.GroupName).Scan(&id); err != nil {
return nil, fmt.Errorf("新增标签: %w", err)
}
byName[t.Name] = id
tagID[t.SrcPK] = id
st.Tags++
if err := writeRawRow(ctx, ex, dsID, "ZTAG", pk, b.RawRows["ZTAG"][t.SrcPK]); err != nil {
return nil, err
}
}
return tagID, nil
}
func (s *Server) mergeVisits(ctx context.Context, ex execer, dsID int64, b *rond.Backup, plan rond.MergePlan,
placeID, actID, tagID map[int]int64, storedVisits []rond.StoredVisit, st *mergeStats) error {
// 备份到访编号 -> 库里的 visits.src_pk:位移的 from/to_visit_src 与天气的
// visit_src 写的都是**库里的编号**,端点在备份里的原号在这里换成库里的号。
visSrc := map[int]int{}
storedByUUID := map[string]rond.StoredVisit{}
for _, sv := range storedVisits {
if sv.UUID != "" {
storedByUUID[sv.UUID] = sv
}
}
for _, v := range b.Visits {
if u := b.VisitUUID[v.SrcPK]; u != "" {
if sv, ok := storedByUUID[u]; ok {
visSrc[v.SrcPK] = sv.SrcPK
}
}
}
activityByPK := map[int]rond.Activity{}
for _, a := range b.Activities {
activityByPK[a.SrcPK] = a
}
visitPlace := map[int]int{}
for _, v := range b.Visits {
if v.LocationSrcPK != nil {
visitPlace[v.SrcPK] = *v.LocationSrcPK
}
}
resolve := func(v rond.Visit) (pid, aid any, isHome, isWork bool) {
if v.LocationSrcPK != nil {
if x, ok := placeID[*v.LocationSrcPK]; ok {
pid = x
}
}
if v.ActivitySrcPK != nil {
if x, ok := actID[*v.ActivitySrcPK]; ok {
aid = x
}
if a, ok := activityByPK[*v.ActivitySrcPK]; ok {
isHome, isWork = a.IsHome, a.IsWork
}
}
return
}
for _, up := range plan.UpdVisits {
sv, ok := storedByUUID[up.UUID]
if !ok {
return fmt.Errorf("找不到要更新的到访(%s)", up.UUID)
}
pid, aid, isHome, isWork := resolve(up.V)
if _, err := ex.ExecContext(ctx, `UPDATE visits SET place_id=$1, activity_id=$2, arrival=$3, departure=$4,
duration_min=$5, is_home=$6, is_work=$7, bookmarked=$8, remark=NULLIF($9,''), emoji=NULLIF($10,'')
WHERE dataset_id=$11 AND src_pk=$12`,
pid, aid, up.V.Arrival, up.V.Departure, visitDuration(up.V), isHome, isWork,
up.V.Bookmarked, up.V.Remark, up.V.Emoji, dsID, sv.SrcPK); err != nil {
return fmt.Errorf("更新到访: %w", err)
}
st.UpdVisits++
}
addedVisit := map[int]bool{}
for _, v := range plan.AddVisits {
pid, aid, isHome, isWork := resolve(v)
pk, err := s.nextSrcPK(ctx, ex, dsID, visitSeq)
if err != nil {
return err
}
var id int64
if err := ex.QueryRowContext(ctx, `INSERT INTO visits
(dataset_id, src_pk, place_id, activity_id, arrival, departure, duration_min,
is_home, is_work, bookmarked, is_user_added, remark, emoji, weather_symbol)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,NULLIF($12,''),NULLIF($13,''),NULLIF($14,''))
ON CONFLICT DO NOTHING RETURNING id`,
dsID, pk, pid, aid, v.Arrival, v.Departure, visitDuration(v), isHome, isWork,
v.Bookmarked, v.UserAdded, v.Remark, v.Emoji, v.WeatherSymbol).Scan(&id); err != nil {
if err == sql.ErrNoRows {
continue
}
return fmt.Errorf("新增到访: %w", err)
}
visSrc[v.SrcPK] = pk
addedVisit[v.SrcPK] = true
st.Visits++
// 留档必须写:下次再比对时库侧就是靠这里的 UUID 认出这一条,
// 少了它只能退到「到达时间 + 坐标」兜底。
if err := writeRawRow(ctx, ex, dsID, "ZVISIT", pk, b.RawRows["ZVISIT"][v.SrcPK]); err != nil {
return err
}
for _, tpk := range v.TagSrcPKs {
if tid, ok := tagID[tpk]; ok {
if _, err := ex.ExecContext(ctx, `INSERT INTO visit_tags (visit_id, tag_id)
VALUES ($1,$2) ON CONFLICT DO NOTHING`, id, tid); err != nil {
return fmt.Errorf("写入到访标签: %w", err)
}
}
}
}
transName, transColor, transIcon := map[int]string{}, map[int]string{}, map[int]string{}
for _, t := range b.Transports {
transName[t.SrcPK], transColor[t.SrcPK], transIcon[t.SrcPK] = t.Name, t.Color, t.Icon
}
coordOf := map[int][2]float64{}
for _, l := range b.Locations {
coordOf[l.SrcPK] = [2]float64{l.Lat, l.Lon}
}
// 位移与天气跟着**新插入的到访**走:只插这一轮真正补进来的那些,
// 否则库里已有的位移会被整份复制一遍。另一端可以是老到访(库里已经有)。
for _, m := range b.Movements {
if m.FromVisitSrc == nil || m.ToVisitSrc == nil {
continue
}
if !addedVisit[*m.FromVisitSrc] && !addedVisit[*m.ToVisitSrc] {
continue
}
fsrc, ok1 := visSrc[*m.FromVisitSrc]
tsrc, ok2 := visSrc[*m.ToVisitSrc]
if !ok1 || !ok2 {
continue
}
var fromPID, toPID any
if fp, ok := visitPlace[*m.FromVisitSrc]; ok {
if x, ok := placeID[fp]; ok {
fromPID = x
}
}
if tp, ok := visitPlace[*m.ToVisitSrc]; ok {
if x, ok := placeID[tp]; ok {
toPID = x
}
}
var dist any
if fp, ok1 := visitPlace[*m.FromVisitSrc]; ok1 {
if tp, ok2 := visitPlace[*m.ToVisitSrc]; ok2 {
c1, c2 := coordOf[fp], coordOf[tp]
if c1[0] != 0 && c2[0] != 0 {
dist = geo.Haversine(c1[0], c1[1], c2[0], c2[1])
}
}
}
var tname, tcolor, ticon any
if m.TransportSrc != nil {
tname = nullStr(transName[*m.TransportSrc])
tcolor = nullStr(transColor[*m.TransportSrc])
ticon = nullStr(transIcon[*m.TransportSrc])
}
var dur any
if m.Start != nil && m.End != nil {
if d := int(m.End.Sub(*m.Start).Minutes()); d >= 0 {
dur = d
}
}
pk, err := s.nextSrcPK(ctx, ex, dsID, movementSeq)
if err != nil {
return err
}
if _, err := ex.ExecContext(ctx, `INSERT INTO movements
(dataset_id, src_pk, transport_src, transport_name, transport_color, transport_icon,
from_visit_src, to_visit_src, from_place_id, to_place_id, started_at, ended_at, duration_min, distance_km)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT DO NOTHING`,
dsID, pk, m.TransportSrc, tname, tcolor, ticon, fsrc, tsrc,
fromPID, toPID, m.Start, m.End, dur, dist); err != nil {
return fmt.Errorf("新增位移: %w", err)
}
st.Movements++
if err := writeRawRow(ctx, ex, dsID, "ZMOVEMENT", pk, b.RawRows["ZMOVEMENT"][m.SrcPK]); err != nil {
return err
}
}
for _, w := range b.Weather {
if w.VisitSrc == nil || w.At == nil {
continue
}
if !addedVisit[*w.VisitSrc] {
continue
}
vsrc, ok := visSrc[*w.VisitSrc]
if !ok {
continue
}
// 站内删到访时会把天气的归属置空而保留记录(见 editVisitDelete)。
// 这类「同一时刻、没有归属」的天气先认领回来,否则同一条到访找回来之后
// 会多出一份重复的天气。
var claimed int64
if err := ex.QueryRowContext(ctx, `UPDATE weather SET visit_src=$1
WHERE dataset_id=$2 AND at=$3 AND visit_src IS NULL RETURNING id`,
vsrc, dsID, w.At).Scan(&claimed); err == nil {
continue
}
pk, err := s.nextSrcPK(ctx, ex, dsID, weatherSeq)
if err != nil {
return err
}
if _, err := ex.ExecContext(ctx, `INSERT INTO weather
(dataset_id, src_pk, visit_src, at, is_daylight, temperature_c, apparent_c, humidity,
precipitation_amount, precipitation_chance, wind_speed, visibility, uv_index,
condition, symbol, uv_category, wind_direction)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,NULLIF($14,''),NULLIF($15,''),NULLIF($16,''),NULLIF($17,''))
ON CONFLICT DO NOTHING`,
dsID, pk, vsrc, w.At, w.IsDaylight, w.TempC, w.ApparentC, w.Humidity,
w.PrecipAmt, w.PrecipPct, w.WindSpeed, w.Visibility, w.UVIndex,
w.Condition, w.Symbol, w.UVCategory, w.WindDir); err != nil {
return fmt.Errorf("新增天气: %w", err)
}
st.Weather++
if err := writeRawRow(ctx, ex, dsID, "ZHOURLYWEATHER", pk, b.RawRows["ZHOURLYWEATHER"][w.SrcPK]); err != nil {
return err
}
}
return nil
}
// applyDeletions 处理「仅站内有」里被勾选删除的记录。顺序是到访 → 活动 → 地点:
// 删地点会级联删掉它名下的全部到访(外键 ON DELETE CASCADE),所以反过来会误删;
// 而且**有到访引用时直接拒绝**,让站长先处理到访,而不是悄悄带走一批。
func (s *Server) applyDeletions(ctx context.Context, ex execer, dsID int64, plan rond.MergePlan, st *mergeStats) error {
for _, sv := range plan.DelVisits {
if _, err := ex.ExecContext(ctx, `DELETE FROM movements WHERE dataset_id=$1
AND (from_visit_src=$2 OR to_visit_src=$2)`, dsID, sv.SrcPK); err != nil {
return fmt.Errorf("删除到访的位移: %w", err)
}
// 天气保留、只解除归属——与站点 editVisitDelete 的处理一致
if _, err := ex.ExecContext(ctx, `UPDATE weather SET visit_src=NULL
WHERE dataset_id=$1 AND visit_src=$2`, dsID, sv.SrcPK); err != nil {
return fmt.Errorf("解除天气归属: %w", err)
}
if _, err := ex.ExecContext(ctx, `DELETE FROM visits WHERE dataset_id=$1 AND src_pk=$2`,
dsID, sv.SrcPK); err != nil {
return fmt.Errorf("删除到访: %w", err)
}
if _, err := ex.ExecContext(ctx, `DELETE FROM entity_raw WHERE dataset_id=$1
AND entity='ZVISIT' AND src_pk=$2`, dsID, sv.SrcPK); err != nil {
return err
}
st.DelVisits++
}
for _, sa := range plan.DelActs {
// 到访的 activity_id 是 ON DELETE SET NULL,删活动不会带走到访
if _, err := ex.ExecContext(ctx, `DELETE FROM activities WHERE dataset_id=$1 AND src_pk=$2`,
dsID, sa.SrcPK); err != nil {
return fmt.Errorf("删除活动: %w", err)
}
if _, err := ex.ExecContext(ctx, `DELETE FROM entity_raw WHERE dataset_id=$1
AND entity='ZACTIVITY' AND src_pk=$2`, dsID, sa.SrcPK); err != nil {
return err
}
st.DelActs++
}
for _, sp := range plan.DelPlaces {
var n int
if err := ex.QueryRowContext(ctx, `SELECT count(*) FROM visits v
JOIN places p ON p.id = v.place_id
WHERE p.dataset_id=$1 AND p.src_pk=$2`, dsID, sp.SrcPK).Scan(&n); err != nil {
return err
}
if n > 0 {
return fmt.Errorf("地点「%s」名下还有 %d 条到访,删它会连带删掉那些到访——请先处理这些到访",
sp.Name, n)
}
if _, err := ex.ExecContext(ctx, `DELETE FROM places WHERE dataset_id=$1 AND src_pk=$2`,
dsID, sp.SrcPK); err != nil {
return fmt.Errorf("删除地点: %w", err)
}
if _, err := ex.ExecContext(ctx, `DELETE FROM entity_raw WHERE dataset_id=$1
AND entity='ZLOCATION' AND src_pk=$2`, dsID, sp.SrcPK); err != nil {
return err
}
st.DelPlaces++
}
return nil
}
// recountPlaces 重算地点聚合与数据集汇总。全量重算而不是只动受影响的行:
// 367 个地点两条 SQL 就够,省得漏掉任何一条派生列。
func (s *Server) recountPlaces(ctx context.Context, ex execer, dsID int64) error {
if _, err := ex.ExecContext(ctx, `UPDATE places SET visit_count=0, dwell_minutes=0,
first_visit_at=NULL, last_visit_at=NULL WHERE dataset_id=$1`, dsID); err != nil {
return err
}
if _, err := ex.ExecContext(ctx, `UPDATE places p SET visit_count=sub.c, dwell_minutes=sub.d,
first_visit_at=sub.f, last_visit_at=sub.l
FROM (SELECT place_id, count(*) c,
COALESCE(sum(LEAST(COALESCE(duration_min,0),10080)),0) d,
min(arrival) f, max(arrival) l
FROM visits WHERE dataset_id=$1 AND place_id IS NOT NULL GROUP BY place_id) sub
WHERE p.id = sub.place_id`, dsID); err != nil {
return err
}
_, err := ex.ExecContext(ctx, `UPDATE datasets SET
visit_count=(SELECT count(*) FROM visits WHERE dataset_id=$1),
place_count=(SELECT count(*) FROM places WHERE dataset_id=$1),
first_visit_at=(SELECT min(arrival) FROM visits WHERE dataset_id=$1),
last_visit_at=(SELECT max(arrival) FROM visits WHERE dataset_id=$1)
WHERE id=$1`, dsID)
return err
}
// writeRawRow 给新插入的行留一份原始行:导出时要靠它逐列还原,
// 再次比对时要靠它认出 UUID(否则只能退到「时间 + 坐标」兜底)。
func writeRawRow(ctx context.Context, ex execer, dsID int64, entity string, pk int, raw map[string]any) error {
if len(raw) == 0 {
return nil
}
blob, err := json.Marshal(raw)
if err != nil {
log.Printf("留档 %s#%d 序列化失败: %v", entity, pk, err)
return nil
}
_, err = ex.ExecContext(ctx, `INSERT INTO entity_raw (dataset_id, entity, src_pk, raw, skipped)
VALUES ($1,$2,$3,$4,false)
ON CONFLICT (dataset_id, entity, src_pk) DO UPDATE SET raw=EXCLUDED.raw`,
dsID, entity, pk, string(blob))
return err
}
func visitDuration(v rond.Visit) any {
if v.Arrival == nil || v.Departure == nil {
return nil
}
m := int(v.Departure.Sub(*v.Arrival).Minutes())
if m < 0 {
m = 0
}
return m
}
+273
View File
@@ -0,0 +1,273 @@
package web
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
)
// mergeBackup 走真实的合并处理函数;失败(重定向里带 err=)直接判错。
func mergeBackup(t *testing.T, f *movementFixture, token, body string) {
t.Helper()
if loc := mergeBackupAllowErr(t, f, token, body); strings.Contains(loc, "err=") {
t.Fatalf("合并失败: %s", loc)
}
}
// mergeBackupAllowErr 同上,但把落点交出来,供「预期会失败」的用例断言。
func mergeBackupAllowErr(t *testing.T, f *movementFixture, token, body string) string {
t.Helper()
rec := httptest.NewRecorder()
req := httptest.NewRequest("POST", "/admin/preview/"+token+"/merge", strings.NewReader(body))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.SetPathValue("token", token)
f.srv.adminBackupMerge(rec, req.WithContext(f.ctx))
if rec.Code != http.StatusSeeOther {
t.Fatalf("合并应跳回后台: code=%d body=%s", rec.Code, rec.Body.String())
}
return rec.Header().Get("Location")
}
// addExtraVisit 造一条「仅站内有」的到访:没有留档,于是没有 UUID,比对时必然落在仅站内有里。
func (f *movementFixture) addExtraVisit(t *testing.T, pk int, remark string) {
t.Helper()
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO visits
(dataset_id, src_pk, arrival, departure, is_user_added, remark)
VALUES ($1,$2,now(),now(),true,$3)`, f.dsID, pk, remark); err != nil {
t.Fatalf("插入补录到访失败: %v", err)
}
}
func (f *movementFixture) scalar(t *testing.T, q string, args ...any) int {
t.Helper()
var n int
if err := f.pg.QueryRowContext(f.ctx, q, args...).Scan(&n); err != nil {
t.Fatalf("查询失败: %v", err)
}
return n
}
// 合并把备份里多出来的记录写进去,而且写完要「复原」——同一份备份再比一次必须一条不差。
//
// 这条测试同时钉住三件事:
// - 新增行用 nextSrcPK 发新号,且**写进 entity_raw**(否则下次比对认不出 UUID);
// - 位移/天气的外键指向库里的新编号(写错就成孤儿);
// - 合并是幂等的:再传同一份备份,差异归零。
func TestBackupMergeRestoresMissingVisits(t *testing.T) {
f := newMovementFixture(t)
visitCount := func() int {
return f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1`, f.dsID)
}
all := visitCount()
movCount := f.scalar(t, `SELECT count(*) FROM movements WHERE dataset_id=$1`, f.dsID)
weatherCount := f.scalar(t, `SELECT count(*) FROM weather WHERE dataset_id=$1`, f.dsID)
// 原始数据里就有指向「因缺到达时间而没入库」的到访的位移,先记下基线再比
orphanQ := `SELECT count(*) FROM movements m WHERE m.dataset_id=$1
AND (m.from_visit_src IS NULL OR NOT EXISTS (
SELECT 1 FROM visits v WHERE v.dataset_id=$1 AND v.src_pk=m.from_visit_src))`
orphans0 := f.scalar(t, orphanQ, f.dsID)
// 制造差异:按**站点删到访的方式**删掉最早的两条(见 editVisitDelete)——
// 位移删掉、天气的归属置空、到访删掉。直接 DELETE visits 会留下孤儿天气,
// 那是测试手法造成的,不是产品行为。
var pk1, pk2 int
if err := f.pg.QueryRowContext(f.ctx, `SELECT
(SELECT src_pk FROM visits WHERE dataset_id=$1 ORDER BY arrival LIMIT 1),
(SELECT src_pk FROM visits WHERE dataset_id=$1 ORDER BY arrival OFFSET 1 LIMIT 1)`,
f.dsID).Scan(&pk1, &pk2); err != nil {
t.Fatalf("取待删到访失败: %v", err)
}
for _, q := range []string{
`DELETE FROM movements WHERE dataset_id=$1 AND (from_visit_src IN ($2,$3) OR to_visit_src IN ($2,$3))`,
`UPDATE weather SET visit_src=NULL WHERE dataset_id=$1 AND visit_src IN ($2,$3)`,
`DELETE FROM visits WHERE dataset_id=$1 AND src_pk IN ($2,$3)`,
} {
if _, err := f.pg.ExecContext(f.ctx, q, f.dsID, pk1, pk2); err != nil {
t.Fatalf("预置删除失败: %v", err)
}
}
if got := visitCount(); got != all-2 {
t.Fatalf("预置失败:应有 %d 条,实际 %d", all-2, got)
}
token, page := previewBackupToken(t, f, f.src)
if a, _, _ := visitCounts(t, page); a != 2 {
t.Fatalf("差异页应报 2 条新增,实际 %d", a)
}
mergeBackup(t, f, token, "add_new=1")
if got := visitCount(); got != all {
t.Fatalf("合并后到访数应回到 %d,实际 %d", all, got)
}
// 新补进来的那两条必须各有一份留档(下次比对就靠它认 UUID)。
// 被删掉的行会留下孤儿留档——那是老数据,不影响,所以只查新编号。
if got := f.scalar(t, `SELECT count(*) FROM entity_raw er
WHERE er.dataset_id=$1 AND er.entity='ZVISIT' AND EXISTS (
SELECT 1 FROM visits v WHERE v.dataset_id=$1 AND v.src_pk=er.src_pk)`, f.dsID); got != all {
t.Errorf("每条在库的到访都该有原始行留档,实际 %d / %d", got, all)
}
if got := f.scalar(t, `SELECT count(*) FROM movements WHERE dataset_id=$1`, f.dsID); got != movCount {
t.Errorf("位移应恢复到 %d,实际 %d", movCount, got)
}
if got := f.scalar(t, `SELECT count(*) FROM weather WHERE dataset_id=$1`, f.dsID); got != weatherCount {
t.Errorf("天气应恢复到 %d,实际 %d", weatherCount, got)
}
// 合并不该新增孤儿位移(原始数据里本来就有一些,见 orphanQ 的注释)
if orphans := f.scalar(t, orphanQ, f.dsID); orphans > orphans0 {
t.Errorf("孤儿位移变多了:%d → %d", orphans0, orphans)
}
// 幂等:再比一次应当完全一致
page = previewBackup(t, f, f.src)
if a, g, c := visitCounts(t, page); a != 0 || g != 0 || c != 0 {
t.Fatalf("合并后再比对应当没有差异,实际 新增 %d / 仅站内有 %d / 冲突 %d", a, g, c)
}
}
// 冲突逐条选「用实机」就把备份的值写进去,选「用站内」就原样不动。
func TestBackupMergeConflictChoice(t *testing.T) {
f := newMovementFixture(t)
var pk int
if err := f.pg.QueryRowContext(f.ctx, `UPDATE visits SET remark='站内改过的备注'
WHERE id = (SELECT id FROM visits WHERE dataset_id=$1 ORDER BY src_pk LIMIT 1)
RETURNING src_pk`, f.dsID).Scan(&pk); err != nil {
t.Fatalf("改备注失败: %v", err)
}
// 这条的 UUID 就是页面上的冲突键
var b64 string
if err := f.pg.QueryRowContext(f.ctx, `SELECT COALESCE(raw #>> '{ZIDENTIFIER_,$blob}','')
FROM entity_raw WHERE dataset_id=$1 AND entity='ZVISIT' AND src_pk=$2`, f.dsID, pk).Scan(&b64); err != nil {
t.Fatalf("取 UUID 失败: %v", err)
}
uuid := uuidFromBase64(b64)
if uuid == "" {
t.Fatal("这条到访应当有 UUID")
}
remark := func() string {
var s string
if err := f.pg.QueryRowContext(f.ctx, `SELECT COALESCE(remark,'') FROM visits
WHERE dataset_id=$1 AND src_pk=$2`, f.dsID, pk).Scan(&s); err != nil {
t.Fatalf("读备注失败: %v", err)
}
return s
}
// 用站内:备注保持站里那份
token, _ := previewBackupToken(t, f, f.src)
mergeBackup(t, f, token, "vc_"+uuid+"=db")
if got := remark(); got != "站内改过的备注" {
t.Fatalf("选「用站内」不该改动,实际 %q", got)
}
// 用实机:备注被备份里的值(空)覆盖回去
token, _ = previewBackupToken(t, f, f.src)
mergeBackup(t, f, token, "vc_"+uuid+"=backup")
if got := remark(); got != "" {
t.Fatalf("选「用实机」应覆盖成备份里的值,实际 %q", got)
}
}
// 什么都不勾、冲突全跳过时,合并应当什么都不写。
func TestBackupMergeNoopKeepsData(t *testing.T) {
f := newMovementFixture(t)
before := f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1`, f.dsID)
token, _ := previewBackupToken(t, f, f.src)
mergeBackup(t, f, token, "add_new=0")
if got := f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1`, f.dsID); got != before {
t.Fatalf("没勾任何改动却动了数据: %d → %d", before, got)
}
}
// 删除必须「勾了才删」:没勾的一个都不动,勾了的那条(连带它的位移)才消失。
func TestBackupMergeDeletesOnlyChecked(t *testing.T) {
f := newMovementFixture(t)
visitCount := func() int {
return f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1`, f.dsID)
}
before := visitCount()
f.addExtraVisit(t, 900001, "稍后删掉")
f.addExtraVisit(t, 900002, "留着不动")
if got := visitCount(); got != before+2 {
t.Fatalf("预置失败: %d", got)
}
// 只勾一条
token, page := previewBackupToken(t, f, f.src)
if _, g, _ := visitCounts(t, page); g != 2 {
t.Fatalf("差异页应报 2 条仅站内有,实际 %d", g)
}
mergeBackup(t, f, token, "dv_900001=1")
if got := visitCount(); got != before+1 {
t.Fatalf("应只删掉勾选的那条,实际剩 %d(原 %d)", got, before)
}
if n := f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1 AND src_pk=900002`, f.dsID); n != 1 {
t.Error("没勾的那条不该被动")
}
// 删掉的到访不该留下留档
if n := f.scalar(t, `SELECT count(*) FROM entity_raw WHERE dataset_id=$1
AND entity='ZVISIT' AND src_pk=900001`, f.dsID); n != 0 {
t.Error("删除到访时要连它的原始行留档一起清掉")
}
// 再合并一次、一条都不勾:剩下那条必须还在
token, _ = previewBackupToken(t, f, f.src)
mergeBackup(t, f, token, "add_new=0")
if got := visitCount(); got != before+1 {
t.Fatalf("不勾就不该删,实际 %d", got)
}
}
// 名下有到访的地点必须拒绝删除——那个外键是 ON DELETE CASCADE,
// 真删下去会把到访一起带走。拒绝后还要整体回滚,别留下半截。
func TestBackupMergeRefusesDeletingPlaceInUse(t *testing.T) {
f := newMovementFixture(t)
visitCount := func() int {
return f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1`, f.dsID)
}
beforeVisits := visitCount()
// 一个没有到访的空地点:允许删
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO places (dataset_id, src_pk, name, lat, lon)
VALUES ($1, 910001, '临时空地点', 30.0, 120.0)`, f.dsID); err != nil {
t.Fatalf("插入地点失败: %v", err)
}
// 一个有到访的地点:必须拒绝
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO places (dataset_id, src_pk, name, lat, lon)
VALUES ($1, 910002, '有到访的地点', 31.0, 121.0)`, f.dsID); err != nil {
t.Fatalf("插入地点失败: %v", err)
}
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO visits (dataset_id, src_pk, place_id, arrival, departure)
VALUES ($1, 910003, (SELECT id FROM places WHERE dataset_id=$1 AND src_pk=910002), now(), now())`,
f.dsID); err != nil {
t.Fatalf("插入到访失败: %v", err)
}
// 先删空地点:应当成功
token, _ := previewBackupToken(t, f, f.src)
mergeBackup(t, f, token, "dp_910001=1")
if n := f.scalar(t, `SELECT count(*) FROM places WHERE dataset_id=$1 AND src_pk=910001`, f.dsID); n != 0 {
t.Fatal("空地点应当被删掉")
}
// 再删有到访的地点:应当被拒绝
token, _ = previewBackupToken(t, f, f.src)
loc := mergeBackupAllowErr(t, f, token, "dp_910002=1")
if !strings.Contains(loc, "err=") {
t.Fatal("名下有到访的地点应当拒绝删除")
}
if n := f.scalar(t, `SELECT count(*) FROM places WHERE dataset_id=$1 AND src_pk=910002`, f.dsID); n != 1 {
t.Error("被拒绝后地点不该消失")
}
if got := visitCount(); got != beforeVisits+1 {
t.Errorf("被拒绝后到访数不该变,实际 %d(期望 %d)", got, beforeVisits+1)
}
}
+151 -18
View File
@@ -3,11 +3,13 @@ package web
import ( import (
"bytes" "bytes"
"context" "context"
"errors"
"fmt" "fmt"
"image" "image"
"image/png" "image/png"
"io" "io"
"log" "log"
"math"
"net/http" "net/http"
"path/filepath" "path/filepath"
"sort" "sort"
@@ -111,41 +113,57 @@ func fogTileCacheControl(isAdmin bool) string {
return "public, max-age=3600" return "public, max-age=3600"
} }
// adminFogUpload 导入 .fwss:mode=overlay 按位或合并进现有迷雾, // readFogSnapshot 从 multipart 表单里读出 .fwss 的字节并做基本校验。
// mode=cover 清空后整体替换。 func readFogSnapshot(w http.ResponseWriter, r *http.Request) ([]byte, error) {
func (s *Server) adminFogUpload(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
u := userFrom(ctx)
r.Body = http.MaxBytesReader(w, r.Body, maxFogUploadBytes+1<<20) r.Body = http.MaxBytesReader(w, r.Body, maxFogUploadBytes+1<<20)
if err := r.ParseMultipartForm(32 << 20); err != nil { if err := r.ParseMultipartForm(32 << 20); err != nil {
s.adminFlash(w, r, "err", "读取上传内容失败:"+err.Error()) return nil, fmt.Errorf("读取上传内容失败:%w", err)
return
} }
file, hdr, err := r.FormFile("fwss") file, hdr, err := r.FormFile("fwss")
if err != nil { if err != nil {
s.adminFlash(w, r, "err", "请选择要上传的 .fwss 迷雾快照") return nil, errors.New("请选择要上传的 .fwss 迷雾快照")
return
} }
defer file.Close() defer file.Close()
name := strings.ToLower(filepath.Base(hdr.Filename)) name := strings.ToLower(filepath.Base(hdr.Filename))
if !strings.HasSuffix(name, ".fwss") && !strings.HasSuffix(name, ".zip") { if !strings.HasSuffix(name, ".fwss") && !strings.HasSuffix(name, ".zip") {
s.adminFlash(w, r, "err", "文件类型不正确,请上传 Fog of World 导出的 .fwss 快照") return nil, errors.New("文件类型不正确,请上传 Fog of World 导出的 .fwss 快照")
return
} }
data, err := io.ReadAll(io.LimitReader(file, maxFogUploadBytes+1))
if err != nil {
return nil, fmt.Errorf("读取文件失败:%w", err)
}
if int64(len(data)) > maxFogUploadBytes {
return nil, errors.New("迷雾快照超过 64MB,疑似文件有误")
}
return data, nil
}
// adminFogUpload 导入 .fwss:mode=overlay 按位或合并进现有迷雾,
// mode=cover 清空后整体替换。
//
// 带 stash 令牌的请求来自差异页的确认按钮,快照已经在暂存目录里,
// 不用让浏览器把几十 MB 再传一遍。
func (s *Server) adminFogUpload(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
u := userFrom(ctx)
mode := r.PostFormValue("mode") mode := r.PostFormValue("mode")
if mode != "overlay" && mode != "cover" { if mode != "overlay" && mode != "cover" {
mode = "overlay" mode = "overlay"
} }
data, err := io.ReadAll(io.LimitReader(file, maxFogUploadBytes+1)) token := r.URL.Query().Get("stash")
if err != nil { var data []byte
s.adminFlash(w, r, "err", "读取文件失败:"+err.Error()) var err error
return if token != "" {
} if data, err = s.loadStashedFog(token); err != nil {
if int64(len(data)) > maxFogUploadBytes { s.adminFlash(w, r, "err", err.Error())
s.adminFlash(w, r, "err", "迷雾快照超过 64MB,疑似文件有误") return
}
} else if data, err = readFogSnapshot(w, r); err != nil {
s.adminFlash(w, r, "err", err.Error())
return return
} }
blocks, cells, err := fog.Decode(data) blocks, cells, err := fog.Decode(data)
if err != nil { if err != nil {
s.adminFlash(w, r, "err", "解析迷雾快照失败:"+err.Error()) s.adminFlash(w, r, "err", "解析迷雾快照失败:"+err.Error())
@@ -181,10 +199,125 @@ func (s *Server) adminFogUpload(w http.ResponseWriter, r *http.Request) {
if mode == "cover" { if mode == "cover" {
verb = "整体替换" verb = "整体替换"
} }
// 导入成功才丢掉暂存件;失败时留着,站长可以回差异页重试
if token != "" {
s.dropStashedFog(token)
}
s.adminFlash(w, r, "ok", fmt.Sprintf( s.adminFlash(w, r, "ok", fmt.Sprintf(
"迷雾导入成功(%s):本次写入 %d 块 / %d 格,当前共 %d 块 / %d 格", verb, applied, cells, b, c)) "迷雾导入成功(%s):本次写入 %d 块 / %d 格,当前共 %d 块 / %d 格", verb, applied, cells, b, c))
} }
// adminFogPreview 先把快照收下暂存,不写库,跳到差异页——
// 叠加能补多少、整体替换会丢多少,都得让站长在下手之前看见。
func (s *Server) adminFogPreview(w http.ResponseWriter, r *http.Request) {
data, err := readFogSnapshot(w, r)
if err != nil {
s.adminFlash(w, r, "err", err.Error())
return
}
// 先解一遍确认是真快照,免得把坏文件暂存下来、到差异页才发现
if _, _, err := fog.Decode(data); err != nil {
s.adminFlash(w, r, "err", "解析迷雾快照失败:"+err.Error())
return
}
token, err := s.stashFog(data)
if err != nil {
s.serverError(w, r, err)
return
}
http.Redirect(w, r, "/admin/fog/preview/"+token, http.StatusSeeOther)
}
// adminFogPreviewPage 渲染差异页。每次都重新算一遍:库里可能刚被别的操作改过。
func (s *Server) adminFogPreviewPage(w http.ResponseWriter, r *http.Request) {
p, err := s.pageBase(r, "admin", "迷雾导入预览")
if err != nil {
s.serverError(w, r, err)
return
}
token := r.PathValue("token")
data, err := s.loadStashedFog(token)
if err != nil {
s.adminFlash(w, r, "err", err.Error())
return
}
blocks, _, err := fog.Decode(data)
if err != nil {
s.adminFlash(w, r, "err", "解析迷雾快照失败:"+err.Error())
return
}
if len(blocks) == 0 {
s.adminFlash(w, r, "err", "快照里没有找到迷雾瓦片数据")
return
}
layers, err := fog.RawLayers(data)
if err != nil {
log.Printf("解析快照的其它图层失败: %v", err)
}
ix, err := s.fogIndex()
if err != nil {
s.serverError(w, r, err)
return
}
var cur []fog.Block
ix.View(func(m map[[2]int][]byte) {
cur = make([]fog.Block, 0, len(m))
for k, bm := range m {
cur = append(cur, fog.Block{GX: k[0], GY: k[1], Bitmap: bm})
}
})
tilde, hash, other := fogLayerCounts(layers)
d := FogPreviewData{
Page: p, Token: token, Diff: fog.DiffBlocks(cur, blocks),
TildeLayers: tilde, HashLayers: hash, OtherLayers: other,
}
d.MinLat, d.MinLon, d.MaxLat, d.MaxLon, d.HasBounds = diffBounds(cur, blocks, s.cfg.MapCRS)
// 布局按这个标记引入 Leaflet,漏了地图整块不出现
d.Page.HasMap = d.HasBounds
s.render(w, "admin/fog-preview", d)
}
// diffBounds 返回两边块集覆盖范围的包围盒,供差异地图定位。迷雾网格是 WGS-84,
// 高德这类底图按 GCJ-02 摆放,所以坐标要跟着底图走,否则视野会偏出几百米。
func diffBounds(cur, inc []fog.Block, crs string) (minLat, minLon, maxLat, maxLon float64, ok bool) {
visit := func(gbx, gby int) {
lat, lon := fog.BlockLonLat(gbx, gby)
if crs == "gcj02" {
lat, lon = geo.WGS84ToGCJ02(lat, lon)
}
if !ok {
minLat, maxLat, minLon, maxLon, ok = lat, lat, lon, lon, true
return
}
minLat, maxLat = math.Min(minLat, lat), math.Max(maxLat, lat)
minLon, maxLon = math.Min(minLon, lon), math.Max(maxLon, lon)
}
for _, b := range cur {
visit(b.GX, b.GY)
}
for _, b := range inc {
visit(b.GX, b.GY)
}
return
}
// fogLayerCounts 按前缀归类快照里「已探索层之外」的条目。
// Model/~ 与 Model/# 两层站点不解析、只原样留存,但整体替换会把它们一并换掉,
// 差异页要把这件事说清楚。
func fogLayerCounts(layers map[string][]byte) (tilde, hash, other int) {
for p := range layers {
switch {
case strings.HasPrefix(p, "Model/~/") || strings.HasPrefix(p, "Model~/"):
tilde++
case strings.HasPrefix(p, "Model/#/") || strings.HasPrefix(p, "Model#/"):
hash++
default:
other++
}
}
return
}
// ---------- 迷雾覆盖率 ---------- // ---------- 迷雾覆盖率 ----------
// FogCity 是单个城市(或省份)的迷雾点亮统计:地点里有多少个落在已探索格内。 // FogCity 是单个城市(或省份)的迷雾点亮统计:地点里有多少个落在已探索格内。
+94
View File
@@ -0,0 +1,94 @@
package web
import (
"bytes"
"image/png"
"net/http"
"strconv"
"sync"
"time"
"rond-with-you/internal/fog"
)
// 差异瓦片要反复读同一份快照、同一份库内迷雾,单槽缓存住两边的块集即可——
// 站长一次只会看一份预览。缓存里的 map 加载完就只读,渲染时并发遍历是安全的。
var (
diffSetsMu sync.Mutex
diffSetsToken string
diffSetsAt time.Time
diffSetsCur map[[2]int][]byte
diffSetsInc map[[2]int][]byte
)
// diffSetsTTL 短于暂存的 2 小时:库内迷雾随时可能被别的操作改掉,
// 缓存太久会让差异图与页面上那组数字对不上。
const diffSetsTTL = 10 * time.Minute
func (s *Server) fogDiffSets(token string) (cur, inc map[[2]int][]byte, err error) {
diffSetsMu.Lock()
defer diffSetsMu.Unlock()
if token == diffSetsToken && time.Since(diffSetsAt) < diffSetsTTL {
return diffSetsCur, diffSetsInc, nil
}
data, err := s.loadStashedFog(token)
if err != nil {
return nil, nil, err
}
blocks, _, err := fog.Decode(data)
if err != nil {
return nil, nil, err
}
ix, err := s.fogIndex()
if err != nil {
return nil, nil, err
}
cur = map[[2]int][]byte{}
ix.View(func(m map[[2]int][]byte) {
for k, bm := range m {
cur[k] = bm
}
})
inc = make(map[[2]int][]byte, len(blocks))
for _, b := range blocks {
inc[[2]int{b.GX, b.GY}] = b.Bitmap
}
diffSetsToken, diffSetsCur, diffSetsInc, diffSetsAt = token, cur, inc, time.Now()
return cur, inc, nil
}
// adminFogDiffTile 渲染一张差异瓦片:绿=叠加会新增、灰=两边都有、红=整体替换会丢。
// 只给站长看,不涉及遮罩。
func (s *Server) adminFogDiffTile(w http.ResponseWriter, r *http.Request) {
z, err1 := strconv.Atoi(r.PathValue("z"))
x, err2 := strconv.Atoi(r.PathValue("x"))
y, err3 := strconv.Atoi(r.PathValue("y"))
if err1 != nil || err2 != nil || err3 != nil || z < 0 || z > 22 ||
x < 0 || y < 0 || x >= 1<<uint(z) || y >= 1<<uint(z) {
http.NotFound(w, r)
return
}
cur, inc, err := s.fogDiffSets(r.PathValue("token"))
if err != nil {
http.NotFound(w, r)
return
}
crs := r.URL.Query().Get("crs")
if crs != "gcj02" && crs != "wgs84" {
crs = s.cfg.MapCRS
}
img := fog.RenderDiffTile(cur, inc, z, x, y, crs)
if img == nil {
http.NotFound(w, r)
return
}
var buf bytes.Buffer
if err := png.Encode(&buf, img); err != nil {
s.serverError(w, r, err)
return
}
w.Header().Set("Content-Type", "image/png")
// 令牌是一次性的,内容也只给站长看,不落共享缓存
w.Header().Set("Cache-Control", "private, max-age=300")
w.Write(buf.Bytes())
}
+422
View File
@@ -0,0 +1,422 @@
package web
import (
"bytes"
"context"
"database/sql"
"fmt"
"image"
"image/png"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
"rond-with-you/internal/auth"
"rond-with-you/internal/config"
"rond-with-you/internal/db"
"rond-with-you/internal/fog"
)
// fogPreviewFixture 是一个迷雾为空、带一次性账号的测试库。
//
// ROND_TEST_CONF=conf/app-test.ini go test ./internal/web -run FogPreview -v
//
// 迷雾是站点级数据(公开读路径不区分归属),所以测试必须从空库出发;
// 用的是独立测试库(rond_fog_test),不会碰到开发库里的迷雾。
type fogPreviewFixture struct {
cfg *config.Config
pg *sql.DB
srv *Server
ctx context.Context
uid int64
ix *fog.Index
}
func newFogPreviewFixture(t *testing.T) *fogPreviewFixture {
t.Helper()
confPath := os.Getenv("ROND_TEST_CONF")
if confPath == "" {
t.Skip("设置 ROND_TEST_CONF 后运行迷雾导入校验")
}
chdirModuleRoot(t)
cfg, err := config.Load(confPath)
if err != nil {
t.Fatalf("加载配置 %s 失败: %v", confPath, err)
}
ctx := context.Background()
pg, err := db.Open(cfg.DSN)
if err != nil {
t.Fatalf("连接数据库失败: %v", err)
}
t.Cleanup(func() { pg.Close() })
if err := db.Migrate(ctx, pg); err != nil {
t.Fatalf("初始化数据库失败: %v", err)
}
user := fmt.Sprintf("e2e-fogpreview-%d", time.Now().UnixNano())
hash, err := auth.HashPassword("e2e-password")
if err != nil {
t.Fatalf("生成口令哈希失败: %v", err)
}
var uid int64
if err := pg.QueryRowContext(ctx, `INSERT INTO users (username, password_hash, display_name)
VALUES ($1,$2,'e2e') RETURNING id`, user, hash).Scan(&uid); err != nil {
t.Fatalf("创建测试账号失败: %v", err)
}
clearFog(t, ctx, pg)
t.Cleanup(func() {
clearFog(t, ctx, pg)
pg.ExecContext(ctx, `DELETE FROM users WHERE id=$1`, uid)
os.RemoveAll(filepath.Join(cfg.DataDir, "fog-staging"))
fogResetIndex()
})
srv, err := New(cfg, pg)
if err != nil {
t.Fatalf("初始化服务失败: %v", err)
}
f := &fogPreviewFixture{cfg: cfg, pg: pg, srv: srv, uid: uid,
ctx: ctxWithUser(&auth.User{ID: uid, Username: user})}
f.ix = f.reloadIndex(t)
return f
}
func clearFog(t *testing.T, ctx context.Context, pg *sql.DB) {
t.Helper()
for _, q := range []string{`DELETE FROM fog_blocks`, `DELETE FROM fog_raw_files`} {
if _, err := pg.ExecContext(ctx, q); err != nil {
t.Fatalf("清空迷雾失败: %v", err)
}
}
}
// reloadIndex 让内存索引与库对齐:fogIndex 的首次加载只跑一次(sync.Once),
// 测试里反复清库写库,必须手动同步,否则差异页算的是上一轮的残留。
func (f *fogPreviewFixture) reloadIndex(t *testing.T) *fog.Index {
t.Helper()
ix, err := f.srv.fogIndex()
if err != nil {
t.Fatalf("获取迷雾索引失败: %v", err)
}
if err := ix.LoadAll(f.pg); err != nil {
t.Fatalf("重载迷雾索引失败: %v", err)
}
return ix
}
// upload 走后台真实的上传处理函数,返回差异页的令牌。
func (f *fogPreviewFixture) upload(t *testing.T, data []byte) string {
t.Helper()
rec := httptest.NewRecorder()
f.srv.adminFogPreview(rec, fogUploadRequest(t, data).WithContext(f.ctx))
if rec.Code != http.StatusSeeOther {
t.Fatalf("上传应跳转到差异页: code=%d body=%s", rec.Code, rec.Body.String())
}
loc := rec.Header().Get("Location")
token := strings.TrimPrefix(loc, "/admin/fog/preview/")
if token == loc || len(token) != fogStashTokenLen {
t.Fatalf("差异页地址不对: %q", loc)
}
return token
}
// preview 渲染差异页。
func (f *fogPreviewFixture) preview(t *testing.T, token string) string {
t.Helper()
rec := httptest.NewRecorder()
req := httptest.NewRequest("GET", "/admin/fog/preview/"+token, nil).WithContext(f.ctx)
req.SetPathValue("token", token)
f.srv.adminFogPreviewPage(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("差异页渲染失败: code=%d body=%s", rec.Code, rec.Body.String())
}
return rec.Body.String()
}
// confirm 走差异页确认按钮那条路径:带 stash 令牌、只提交 mode。
func (f *fogPreviewFixture) confirm(t *testing.T, token, mode string) {
t.Helper()
rec := httptest.NewRecorder()
req := httptest.NewRequest("POST", "/admin/fog?stash="+token, strings.NewReader("mode="+mode))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
f.srv.adminFogUpload(rec, req.WithContext(f.ctx))
if rec.Code != http.StatusSeeOther {
t.Fatalf("确认导入应跳回后台: code=%d body=%s", rec.Code, rec.Body.String())
}
if loc := rec.Header().Get("Location"); !strings.HasPrefix(loc, "/admin?ok=") {
t.Fatalf("确认导入应报成功: %s", loc)
}
}
func (f *fogPreviewFixture) cells(t *testing.T) int {
t.Helper()
_, cells := f.reloadIndex(t).Stats()
return cells
}
func (f *fogPreviewFixture) blocks(t *testing.T) int {
t.Helper()
blocks, _ := f.reloadIndex(t).Stats()
return blocks
}
// TestFogPreviewShowsDiffBeforeWriting 端到端验证「上传 → 差异页 → 确认导入」:
// 差异页必须先把两种模式的后果摆出来,而且在上传那一刻格子里一行都不能写。
func TestFogPreviewShowsDiffBeforeWriting(t *testing.T) {
f := newFogPreviewFixture(t)
// 库里先有一块 3 格的区域
if _, err := f.ix.Import(f.pg, f.uid, []fog.Block{
newFogBlock(100, 200, [2]int{0, 0}, [2]int{1, 0}, [2]int{2, 0}),
}, false); err != nil {
t.Fatalf("预置迷雾失败: %v", err)
}
// 新快照:同一块补亮 2 格,外加一整块新的
inc := []fog.Block{
newFogBlock(100, 200, [2]int{0, 0}, [2]int{1, 0}, [2]int{2, 0}, [2]int{5, 5}, [2]int{6, 5}),
newFogBlock(101, 200, [2]int{1, 1}),
}
data, err := fog.Encode(inc, map[string][]byte{
"Model/~/aaa": {1, 2, 3},
"Model/#/bbb": {4, 5, 6},
})
if err != nil {
t.Fatalf("构造快照失败: %v", err)
}
token := f.upload(t, data)
// 差异页只是预览:格数必须还是 3
if got := f.cells(t); got != 3 {
t.Fatalf("上传阶段不该写库,当前格数 %d", got)
}
page := f.preview(t, token)
for _, want := range []string{
"迷雾导入预览",
"叠加导入",
"整体替换",
"会补上当前迷雾没有的", // 叠加卡片给出的实际收益
"替换不会丢数据", // 库里的 3 格都在快照里
"Model/~", // 图层提示
"/admin/fog?stash=" + token,
`value="overlay"`,
`value="cover"`,
} {
if !strings.Contains(page, want) {
t.Errorf("差异页缺少 %q", want)
}
}
if strings.Contains(page, "完全覆盖") {
t.Error("这份快照有新内容,不该报「已被完全覆盖」")
}
// 确认叠加:3 格 + 补亮的 2 格 + 新块 1 格 = 6 格,两块
f.confirm(t, token, "overlay")
if got := f.cells(t); got != 6 {
t.Fatalf("叠加后应有 6 格,实际 %d", got)
}
if got := f.blocks(t); got != 2 {
t.Fatalf("叠加后应有 2 块,实际 %d", got)
}
}
// TestFogPreviewWarnsBeforeCoverLosesData 整体替换必须在下手前算出「会丢多少」,
// 并且差异页要看得出这份快照已经被完全覆盖过(叠加没有意义)。
func TestFogPreviewWarnsBeforeCoverLosesData(t *testing.T) {
f := newFogPreviewFixture(t)
// 库里两块共 5 格
if _, err := f.ix.Import(f.pg, f.uid, []fog.Block{
newFogBlock(100, 200, [2]int{0, 0}, [2]int{1, 0}, [2]int{2, 0}),
newFogBlock(101, 200, [2]int{7, 7}, [2]int{8, 8}),
}, false); err != nil {
t.Fatalf("预置迷雾失败: %v", err)
}
// 快照只有第一块、且一格不多:叠加零收益,替换会丢掉第二块的两格
inc := []fog.Block{newFogBlock(100, 200, [2]int{0, 0}, [2]int{1, 0}, [2]int{2, 0})}
data, err := fog.Encode(inc, nil)
if err != nil {
t.Fatalf("构造快照失败: %v", err)
}
token := f.upload(t, data)
page := f.preview(t, token)
for _, want := range []string{
"已经被当前迷雾", // 叠加无收益
"从点亮变回未探索",
"Model/~", // 快照没有这两层,替换会清掉现有的
} {
if !strings.Contains(page, want) {
t.Errorf("差异页缺少 %q", want)
}
}
// 确认整体替换:只该剩下快照里的 1 块 3 格
f.confirm(t, token, "cover")
if got := f.cells(t); got != 3 {
t.Fatalf("替换后应有 3 格,实际 %d", got)
}
if got := f.blocks(t); got != 1 {
t.Fatalf("替换后应有 1 块,实际 %d", got)
}
// 暂存件用掉就该删掉,且过期令牌不该还能用
if _, err := os.Stat(filepath.Join(f.cfg.DataDir, "fog-staging", token+".fwss")); !os.IsNotExist(err) {
t.Error("确认导入后应删除暂存文件")
}
rec := httptest.NewRecorder()
req := httptest.NewRequest("GET", "/admin/fog/preview/"+token, nil).WithContext(f.ctx)
req.SetPathValue("token", token)
f.srv.adminFogPreviewPage(rec, req)
if rec.Code != http.StatusSeeOther {
t.Fatalf("过期令牌应跳回后台报错: code=%d", rec.Code)
}
}
// 差异瓦片要按请求坐标渲染出正确的三色,并挡住越界坐标与过期令牌。
func TestFogDiffTileColors(t *testing.T) {
f := newFogPreviewFixture(t)
if _, err := f.ix.Import(f.pg, f.uid, []fog.Block{
newFogBlock(100, 200, [2]int{0, 0}, [2]int{4, 0}),
}, false); err != nil {
t.Fatalf("预置迷雾失败: %v", err)
}
data, err := fog.Encode([]fog.Block{newFogBlock(100, 200, [2]int{0, 0}, [2]int{1, 0})}, nil)
if err != nil {
t.Fatalf("构造快照失败: %v", err)
}
token := f.upload(t, data)
// z=18 时一格 16 像素,块 (100,200) 完整落在瓦片 (400,800) 里
img := f.diffTile(t, token, 18, 400, 800)
if got := kindOf(pixAtImg(img, 8, 8)); got != "both" {
t.Errorf("两边都有的格应为灰色,实际 %s", got)
}
if got := kindOf(pixAtImg(img, 24, 8)); got != "gain" {
t.Errorf("只有快照有的格应为绿色,实际 %s", got)
}
if got := kindOf(pixAtImg(img, 72, 8)); got != "lost" {
t.Errorf("只有库里有、替换会丢的格应为红色,实际 %s", got)
}
if got := kindOf(pixAtImg(img, 200, 200)); got != "none" {
t.Errorf("没探索过的格应为透明,实际 %s", got)
}
// 越界坐标与过期令牌都只给 404,不能穿透到渲染
for _, c := range []struct{ z, x, y string }{
{"18", "262144", "0"}, // x 已达该级别的上限
{"18", "0", "-1"},
{"99", "0", "0"}, // z 越界
} {
rec := httptest.NewRecorder()
req := httptest.NewRequest("GET", "/admin/fog/preview/"+token+"/x", nil).WithContext(f.ctx)
req.SetPathValue("token", token)
req.SetPathValue("z", c.z)
req.SetPathValue("x", c.x)
req.SetPathValue("y", c.y)
f.srv.adminFogDiffTile(rec, req)
if rec.Code != http.StatusNotFound {
t.Errorf("越界坐标 %v 应给 404,实际 %d", c, rec.Code)
}
}
rec := httptest.NewRecorder()
req := httptest.NewRequest("GET", "/admin/fog/preview/deadbeef/18/400/800", nil).WithContext(f.ctx)
req.SetPathValue("token", strings.Repeat("f", 32))
req.SetPathValue("z", "18")
req.SetPathValue("x", "400")
req.SetPathValue("y", "800")
f.srv.adminFogDiffTile(rec, req)
if rec.Code != http.StatusNotFound {
t.Errorf("无效令牌应给 404,实际 %d", rec.Code)
}
}
func (f *fogPreviewFixture) diffTile(t *testing.T, token string, z, x, y int) image.Image {
t.Helper()
rec := httptest.NewRecorder()
req := httptest.NewRequest("GET", fmt.Sprintf("/admin/fog/preview/%s/%d/%d/%d?crs=wgs84", token, z, x, y), nil).
WithContext(f.ctx)
req.SetPathValue("token", token)
req.SetPathValue("z", strconv.Itoa(z))
req.SetPathValue("x", strconv.Itoa(x))
req.SetPathValue("y", strconv.Itoa(y))
f.srv.adminFogDiffTile(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("差异瓦片渲染失败: code=%d", rec.Code)
}
if ct := rec.Header().Get("Content-Type"); ct != "image/png" {
t.Fatalf("差异瓦片应为 PNG,实际 %q", ct)
}
img, err := png.Decode(bytes.NewReader(rec.Body.Bytes()))
if err != nil {
t.Fatalf("解码差异瓦片失败: %v", err)
}
return img
}
// kindOf 按「哪个通道最亮」判类别,不去比对精确色值——PNG 解码回来的是
// 预乘 alpha 的像素,硬比色值会被 alpha 换算绕进去。
func kindOf(c [4]byte) string {
switch {
case c[3] == 0:
return "none"
case c[1] > c[0] && c[1] > c[2]:
return "gain"
case c[0] > c[1] && c[0] > c[2]:
return "lost"
}
return "both"
}
func pixAtImg(img image.Image, px, py int) [4]byte {
r, g, b, a := img.At(px, py).RGBA()
return [4]byte{byte(r >> 8), byte(g >> 8), byte(b >> 8), byte(a >> 8)}
}
// 令牌来自 URL,必须挡死路径穿越。
func TestFogStashTokenRejectsTraversal(t *testing.T) {
s := &Server{cfg: &config.Config{DataDir: t.TempDir()}}
for _, bad := range []string{"", "..", "../../etc/passwd", strings.Repeat("a", 32) + "/../x", strings.ToUpper(strings.Repeat("a", 32))} {
if p, ok := s.fogStashPath(bad); ok {
t.Errorf("令牌 %q 不该通过校验,得到 %s", bad, p)
}
}
if _, ok := s.fogStashPath(strings.Repeat("0", 32)); !ok {
t.Error("32 位十六进制令牌应当通过")
}
}
func newFogBlock(gx, gy int, cells ...[2]int) fog.Block {
bm := make([]byte, 512)
for _, c := range cells {
bm[c[1]*8+c[0]/8] |= 0x80 >> (c[0] % 8)
}
return fog.Block{GX: gx, GY: gy, Bitmap: bm}
}
func fogUploadRequest(t *testing.T, data []byte) *http.Request {
t.Helper()
var buf bytes.Buffer
mw := multipart.NewWriter(&buf)
fw, err := mw.CreateFormFile("fwss", "Snapshot-test.fwss")
if err != nil {
t.Fatalf("构造上传表单失败: %v", err)
}
if _, err := fw.Write(data); err != nil {
t.Fatalf("写入上传内容失败: %v", err)
}
mw.Close()
req := httptest.NewRequest("POST", "/admin/fog/preview", &buf)
req.Header.Set("Content-Type", mw.FormDataContentType())
return req
}
+116
View File
@@ -0,0 +1,116 @@
package web
import (
"crypto/rand"
"encoding/hex"
"fmt"
"os"
"path/filepath"
"strings"
"time"
)
// 待确认的上传(迷雾快照、rond 备份)都先落到磁盘暂存,让站长看完差异再决定怎么导入;
// 看完直接走人的靠 TTL 回收,不至于把几十 MB 的文件一直留在盘上。
//
// 令牌只认固定长度的十六进制串,这样即便来自 URL 也拼不出 ../ 之类的路径。
const (
stashTokenLen = 32 // 16 字节的十六进制
stashTTL = 2 * time.Hour
)
func stashPut(dir, ext string, data []byte) (string, error) {
if err := os.MkdirAll(dir, 0o700); err != nil {
return "", err
}
var raw [stashTokenLen / 2]byte
if _, err := rand.Read(raw[:]); err != nil {
return "", err
}
token := hex.EncodeToString(raw[:])
if err := os.WriteFile(filepath.Join(dir, token+ext), data, 0o600); err != nil {
return "", err
}
stashSweep(dir, ext)
return token, nil
}
func stashPath(dir, ext, token string) (string, bool) {
if len(token) != stashTokenLen {
return "", false
}
for i := 0; i < len(token); i++ {
c := token[i]
if (c < '0' || c > '9') && (c < 'a' || c > 'f') {
return "", false
}
}
return filepath.Join(dir, token+ext), true
}
func stashRead(dir, ext, token, what string) ([]byte, error) {
p, ok := stashPath(dir, ext, token)
if !ok {
return nil, fmt.Errorf("预览链接无效,请重新上传%s", what)
}
data, err := os.ReadFile(p)
if err != nil {
return nil, fmt.Errorf("这次预览已过期,请重新上传%s", what)
}
return data, nil
}
func stashDrop(dir, ext, token string) {
if p, ok := stashPath(dir, ext, token); ok {
os.Remove(p)
}
}
func stashSweep(dir, ext string) {
entries, err := os.ReadDir(dir)
if err != nil {
return
}
cutoff := time.Now().Add(-stashTTL)
for _, e := range entries {
if e.IsDir() || !strings.HasSuffix(e.Name(), ext) {
continue
}
if info, err := e.Info(); err == nil && info.ModTime().After(cutoff) {
continue
}
os.Remove(filepath.Join(dir, e.Name()))
}
}
// ---------- 迷雾快照 ----------
// fogStashTokenLen 是迷雾预览令牌的长度,测试与调用方按它校验。
const fogStashTokenLen = stashTokenLen
const fogStashExt = ".fwss"
func (s *Server) fogStashDir() string {
return filepath.Join(s.cfg.DataDir, "fog-staging")
}
func (s *Server) stashFog(data []byte) (string, error) {
return stashPut(s.fogStashDir(), fogStashExt, data)
}
func (s *Server) fogStashPath(token string) (string, bool) {
return stashPath(s.fogStashDir(), fogStashExt, token)
}
func (s *Server) loadStashedFog(token string) ([]byte, error) {
return stashRead(s.fogStashDir(), fogStashExt, token, "快照")
}
func (s *Server) dropStashedFog(token string) {
stashDrop(s.fogStashDir(), fogStashExt, token)
}
func (s *Server) sweepFogStash() {
stashSweep(s.fogStashDir(), fogStashExt)
}
+5 -5
View File
@@ -769,12 +769,12 @@ func (s *Server) notesPage(w http.ResponseWriter, r *http.Request) {
case "note": case "note":
order = "n.updated_at DESC NULLS LAST, p.name" order = "n.updated_at DESC NULLS LAST, p.name"
case "talk": case "talk":
order = "(COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.src_pk = p.src_pk),0)) DESC, p.visit_count DESC" order = "(COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.place_key = " + placeKeyExpr + "),0)) DESC, p.visit_count DESC"
} }
base := `FROM places p base := `FROM places p
LEFT JOIN place_notes n ON n.src_pk = p.src_pk AND n.dataset_id = p.dataset_id LEFT JOIN place_notes n ON n.place_key = ` + placeKeyExpr + `
LEFT JOIN place_votes v ON v.src_pk = p.src_pk AND v.dataset_id = p.dataset_id LEFT JOIN place_votes v ON v.place_key = ` + placeKeyExpr + `
WHERE ` + strings.Join(where, " AND ") WHERE ` + strings.Join(where, " AND ")
var total int var total int
@@ -790,8 +790,8 @@ func (s *Server) notesPage(w http.ResponseWriter, r *http.Request) {
limitArgs := append(append([]any{}, args...), notesPerPage, (d.PageNo-1)*notesPerPage) limitArgs := append(append([]any{}, args...), notesPerPage, (d.PageNo-1)*notesPerPage)
q2 := `SELECT p.src_pk, COALESCE(n.alias,''), COALESCE(n.note,''), COALESCE(n.rond_note,''), COALESCE(v.verdict,0), q2 := `SELECT p.src_pk, COALESCE(n.alias,''), COALESCE(n.note,''), COALESCE(n.rond_note,''), COALESCE(v.verdict,0),
COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.src_pk=p.src_pk AND vv.vote=1),0), COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.place_key = ` + placeKeyExpr + ` AND vv.vote=1),0),
COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.src_pk=p.src_pk AND vv.vote=-1),0), COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.place_key = ` + placeKeyExpr + ` AND vv.vote=-1),0),
COALESCE(n.updated_at, to_timestamp(0)), p.id, COALESCE(p.name,''), COALESCE(p.city,''), COALESCE(p.visit_count,0) COALESCE(n.updated_at, to_timestamp(0)), p.id, COALESCE(p.name,''), COALESCE(p.city,''), COALESCE(p.visit_count,0)
` + base + ` ORDER BY ` + order + fmt.Sprintf(` LIMIT $%d OFFSET $%d`, len(args)+1, len(args)+2) ` + base + ` ORDER BY ` + order + fmt.Sprintf(` LIMIT $%d OFFSET $%d`, len(args)+1, len(args)+2)
+1
View File
@@ -1742,6 +1742,7 @@ var (
activitySeq = srcPKSeq{"activities", "src_pk", "Activity", "ZACTIVITY"} activitySeq = srcPKSeq{"activities", "src_pk", "Activity", "ZACTIVITY"}
weatherSeq = srcPKSeq{"weather", "src_pk", "HourlyWeather", "ZHOURLYWEATHER"} weatherSeq = srcPKSeq{"weather", "src_pk", "HourlyWeather", "ZHOURLYWEATHER"}
movementSeq = srcPKSeq{"movements", "src_pk", "Movement", "ZMOVEMENT"} movementSeq = srcPKSeq{"movements", "src_pk", "Movement", "ZMOVEMENT"}
tagSeq = srcPKSeq{"tags", "src_pk", "Tag", "ZTAG"}
transportSeq = srcPKSeq{"movements", "transport_src", "Transport", "ZTRANSPORT"} transportSeq = srcPKSeq{"movements", "transport_src", "Transport", "ZTRANSPORT"}
) )
+40 -6
View File
@@ -3,6 +3,7 @@ package web
import ( import (
"context" "context"
"crypto/sha256" "crypto/sha256"
"database/sql"
"encoding/hex" "encoding/hex"
"fmt" "fmt"
"io" "io"
@@ -26,6 +27,11 @@ const (
imageUploadMem = 8 << 20 // multipart 解析用的内存上限 imageUploadMem = 8 << 20 // multipart 解析用的内存上限
) )
// placeKeyExpr 把 places p 那一行的坐标换算成 place_key,图片、备注、结论共用这个键。
// **必须与 rond.PlaceCoordKey 以及 notes/votes 包里的同款表达式逐位一致**:
// fmt.Sprintf("%.6f|%.6f") ↔ to_char(x,'FM999990.000000')||'|'||to_char(y,'FM999990.000000')
const placeKeyExpr = `to_char(p.lat,'FM999990.000000')||'|'||to_char(p.lon,'FM999990.000000')`
// PlaceImage 是模板里用的地点图片。 // PlaceImage 是模板里用的地点图片。
type PlaceImage struct { type PlaceImage struct {
ID int64 ID int64
@@ -57,14 +63,18 @@ func (s *Server) savePlaceImage(ctx context.Context, datasetID int64, srcPK int,
if err := s.media.Save(ctx, backend, key, data, "image/webp"); err != nil { if err := s.media.Save(ctx, backend, key, data, "image/webp"); err != nil {
return 0, "", err return 0, "", err
} }
pkey, err := s.placeKeyOf(ctx, datasetID, srcPK)
if err != nil {
return 0, "", err
}
var id int64 var id int64
if err := s.db.QueryRowContext(ctx, `INSERT INTO place_images if err := s.db.QueryRowContext(ctx, `INSERT INTO place_images
(dataset_id, src_pk, storage, object_key, width, height, bytes, position) (dataset_id, src_pk, place_key, storage, object_key, width, height, bytes, position)
VALUES ($1,$2,$3,$4,$5,$6,$7, VALUES ($1,$2,$3,$4,$5,$6,$7,$8,
COALESCE((SELECT max(position)+1 FROM place_images WHERE dataset_id=$1 AND src_pk=$2), 0)) COALESCE((SELECT max(position)+1 FROM place_images WHERE place_key=$3), 0))
ON CONFLICT (dataset_id, src_pk, object_key) DO UPDATE SET bytes=EXCLUDED.bytes ON CONFLICT (dataset_id, src_pk, object_key) DO UPDATE SET bytes=EXCLUDED.bytes
RETURNING id`, RETURNING id`,
datasetID, srcPK, backend, key, w, h, int64(len(data))).Scan(&id); err != nil { datasetID, srcPK, pkey, backend, key, w, h, int64(len(data))).Scan(&id); err != nil {
return 0, "", err return 0, "", err
} }
return id, s.media.PublicURL(backend, key), nil return id, s.media.PublicURL(backend, key), nil
@@ -95,9 +105,15 @@ func (s *Server) adminPlaceImageUpload(w http.ResponseWriter, r *http.Request) {
return return
} }
pkey, err := s.placeKeyOf(ctx, ds.ID, src)
if err != nil {
s.serverError(w, r, err)
return
}
var count int var count int
// 按坐标键数:换数据集后图片仍跟着同一个地点,配额也要跟着
if err := s.db.QueryRowContext(ctx, `SELECT count(*) FROM place_images if err := s.db.QueryRowContext(ctx, `SELECT count(*) FROM place_images
WHERE dataset_id=$1 AND src_pk=$2`, ds.ID, src).Scan(&count); err != nil { WHERE place_key=$1`, pkey).Scan(&count); err != nil {
s.serverError(w, r, err) s.serverError(w, r, err)
return return
} }
@@ -196,10 +212,28 @@ func (s *Server) imageBackend(ctx context.Context) (string, error) {
return setting.MediaBackendLocal, nil return setting.MediaBackendLocal, nil
} }
// placeKeyOf 把某个数据集里的地点编号换算成坐标键。
// 换数据集之后仍靠它认出「还是同一个地点」,所以图片、备注、结论都挂在这上面。
func (s *Server) placeKeyOf(ctx context.Context, datasetID int64, srcPK int) (string, error) {
var key string
err := s.db.QueryRowContext(ctx, `SELECT `+placeKeyExpr+` FROM places p
WHERE p.dataset_id=$1 AND p.src_pk=$2 AND p.lat IS NOT NULL AND p.lon IS NOT NULL`,
datasetID, srcPK).Scan(&key)
if err == sql.ErrNoRows {
return "", fmt.Errorf("地点不存在或没有坐标")
}
return key, err
}
// placeImagesOf 读某个地点的图片,按 position 排序。 // placeImagesOf 读某个地点的图片,按 position 排序。
// 按坐标键查而不是 (dataset_id, src_pk):换一份备份之后图片还跟着同一个地点。
func (s *Server) placeImagesOf(ctx context.Context, datasetID int64, srcPK int) ([]PlaceImage, error) { func (s *Server) placeImagesOf(ctx context.Context, datasetID int64, srcPK int) ([]PlaceImage, error) {
pkey, err := s.placeKeyOf(ctx, datasetID, srcPK)
if err != nil {
return nil, nil // 地点没有坐标就当作没有图
}
rows, err := s.db.QueryContext(ctx, `SELECT id, storage, object_key, width, height, bytes rows, err := s.db.QueryContext(ctx, `SELECT id, storage, object_key, width, height, bytes
FROM place_images WHERE dataset_id=$1 AND src_pk=$2 ORDER BY position, id`, datasetID, srcPK) FROM place_images WHERE place_key=$1 ORDER BY position, id`, pkey)
if err != nil { if err != nil {
return nil, err return nil, err
} }
+3 -3
View File
@@ -183,7 +183,7 @@ func (s *Server) attachVotes(ctx context.Context, datasetID int64, places []stat
if err != nil { if err != nil {
return return
} }
tallies, err := s.votes.Tallies(ctx) tallies, err := s.votes.Tallies(ctx, datasetID)
if err != nil { if err != nil {
return return
} }
@@ -539,8 +539,8 @@ func (s *Server) placePage(w http.ResponseWriter, r *http.Request) {
} }
} }
d.Verdict, _ = s.votes.Verdict(r.Context(), p.Dataset.ID, place.SrcPK) d.Verdict, _ = s.votes.Verdict(r.Context(), p.Dataset.ID, place.SrcPK)
d.Votes, _ = s.votes.Tally(r.Context(), place.SrcPK) d.Votes, _ = s.votes.Tally(r.Context(), p.Dataset.ID, place.SrcPK)
d.MyVote = s.votes.Voter(r.Context(), place.SrcPK, s.voterKeyFromRequest(r)) d.MyVote = s.votes.Voter(r.Context(), p.Dataset.ID, place.SrcPK, s.voterKeyFromRequest(r))
// 图片:访客也要看得到,所以不分管理员 // 图片:访客也要看得到,所以不分管理员
if imgs, err := s.placeImagesOf(r.Context(), p.Dataset.ID, place.SrcPK); err == nil { if imgs, err := s.placeImagesOf(r.Context(), p.Dataset.ID, place.SrcPK); err == nil {
d.Images = imgs d.Images = imgs
+20 -8
View File
@@ -102,12 +102,12 @@ func (s *Server) apiVotes(w http.ResponseWriter, r *http.Request) {
s.json(w, map[string]string{"error": err.Error()}) s.json(w, map[string]string{"error": err.Error()})
return return
} }
tallies, err := s.votes.Tallies(ctx) tallies, err := s.votes.Tallies(ctx, votesDatasetID)
if err != nil { if err != nil {
s.json(w, map[string]string{"error": err.Error()}) s.json(w, map[string]string{"error": err.Error()})
return return
} }
mine, err := s.votes.VoterAll(ctx, s.voterKeyFromRequest(r)) mine, err := s.votes.VoterAll(ctx, votesDatasetID, s.voterKeyFromRequest(r))
if err != nil { if err != nil {
s.json(w, map[string]string{"error": err.Error()}) s.json(w, map[string]string{"error": err.Error()})
return return
@@ -145,10 +145,16 @@ func (s *Server) apiVerdict(w http.ResponseWriter, r *http.Request) {
// apiVote 记录访客对某地点的点赞 / 点踩,返回最新票数与该访客的选择。 // apiVote 记录访客对某地点的点赞 / 点踩,返回最新票数与该访客的选择。
func (s *Server) apiVote(w http.ResponseWriter, r *http.Request) { func (s *Server) apiVote(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
if err := r.ParseForm(); err != nil { if err := r.ParseForm(); err != nil {
s.json(w, map[string]string{"error": "请求解析失败"}) s.json(w, map[string]string{"error": "请求解析失败"})
return return
} }
ds, err := s.datasetFor(ctx)
if err != nil || ds == nil {
s.json(w, map[string]string{"error": "还没有可投票的数据集"})
return
}
srcPK, err := strconv.Atoi(r.PostFormValue("src_pk")) srcPK, err := strconv.Atoi(r.PostFormValue("src_pk"))
vote, _ := strconv.Atoi(r.PostFormValue("vote")) vote, _ := strconv.Atoi(r.PostFormValue("vote"))
if err != nil || (vote != 1 && vote != -1) { if err != nil || (vote != 1 && vote != -1) {
@@ -160,7 +166,7 @@ func (s *Server) apiVote(w http.ResponseWriter, r *http.Request) {
s.json(w, map[string]string{"error": "无法识别访客"}) s.json(w, map[string]string{"error": "无法识别访客"})
return return
} }
t, mine, err := s.votes.Cast(r.Context(), srcPK, key, vote) t, mine, err := s.votes.Cast(ctx, ds.ID, srcPK, key, vote)
if err != nil { if err != nil {
s.json(w, map[string]string{"error": err.Error()}) s.json(w, map[string]string{"error": err.Error()})
return return
@@ -170,11 +176,17 @@ func (s *Server) apiVote(w http.ResponseWriter, r *http.Request) {
// voteTotals 汇总站长结论数量与访客票数,供统计页展示。 // voteTotals 汇总站长结论数量与访客票数,供统计页展示。
func (s *Server) voteTotals(ctx context.Context, datasetID int64) (rec, avoid, up, down int, err error) { func (s *Server) voteTotals(ctx context.Context, datasetID int64) (rec, avoid, up, down int, err error) {
// 两张票表都挂坐标键,统计要按「本数据集的地点」圈定范围,
// 不能再按它们自己的 dataset_id 过滤——那个只是最后写入时留下的线索。
err = s.db.QueryRowContext(ctx, `SELECT err = s.db.QueryRowContext(ctx, `SELECT
(SELECT count(*) FROM place_votes WHERE dataset_id=$1 AND verdict=1), (SELECT count(*) FROM place_votes v JOIN places p ON v.place_key = `+placeKeyExpr+`
(SELECT count(*) FROM place_votes WHERE dataset_id=$1 AND verdict=-1), WHERE p.dataset_id=$1 AND v.verdict=1),
(SELECT count(*) FROM place_visitor_votes WHERE vote=1), (SELECT count(*) FROM place_votes v JOIN places p ON v.place_key = `+placeKeyExpr+`
(SELECT count(*) FROM place_visitor_votes WHERE vote=-1)`, datasetID). WHERE p.dataset_id=$1 AND v.verdict=-1),
(SELECT count(*) FROM place_visitor_votes vv JOIN places p ON vv.place_key = `+placeKeyExpr+`
WHERE p.dataset_id=$1 AND vv.vote=1),
(SELECT count(*) FROM place_visitor_votes vv JOIN places p ON vv.place_key = `+placeKeyExpr+`
WHERE p.dataset_id=$1 AND vv.vote=-1)`, datasetID).
Scan(&rec, &avoid, &up, &down) Scan(&rec, &avoid, &up, &down)
return return
} }
@@ -183,7 +195,7 @@ func (s *Server) voteTotals(ctx context.Context, datasetID int64) (rec, avoid, u
func (s *Server) verdictPlaces(ctx context.Context, datasetID int64, verdict, limit int) ([]stats.Place, error) { func (s *Server) verdictPlaces(ctx context.Context, datasetID int64, verdict, limit int) ([]stats.Place, error) {
q := `SELECT ` + stats.PlaceCols + ` FROM places p q := `SELECT ` + stats.PlaceCols + ` FROM places p
WHERE p.dataset_id=$1 AND EXISTS (SELECT 1 FROM place_votes pv WHERE p.dataset_id=$1 AND EXISTS (SELECT 1 FROM place_votes pv
WHERE pv.src_pk=p.src_pk AND pv.dataset_id=p.dataset_id AND pv.verdict=$2) WHERE pv.place_key = ` + placeKeyExpr + ` AND pv.verdict=$2)
ORDER BY p.visit_count DESC LIMIT ` + strconv.Itoa(limit) ORDER BY p.visit_count DESC LIMIT ` + strconv.Itoa(limit)
rows, err := s.db.QueryContext(ctx, q, datasetID, verdict) rows, err := s.db.QueryContext(ctx, q, datasetID, verdict)
if err != nil { if err != nil {
+149
View File
@@ -0,0 +1,149 @@
package web
import (
"fmt"
"testing"
"time"
"rond-with-you/internal/votes"
)
// testPlace 在两个数据集里各建一个**同坐标**的地点,返回它们的编号与第二个数据集。
//
// 坐标是随手挑的专属值:备注按坐标共享,而读取按设计不按 user_id 过滤(单站长站点),
// 借用真实数据里的坐标会把别的测试残留的记录串进来。
func testPlace(t *testing.T, f *movementFixture, uid int64, lat, lon float64) (pkA, pkB int, dsB int64) {
t.Helper()
const pkAVal, pkBVal = 660001, 660002
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO places (dataset_id, src_pk, name, lat, lon)
VALUES ($1,$2,'甲侧地点',$3,$4)`, f.dsID, pkAVal, lat, lon); err != nil {
t.Fatalf("建地点失败: %v", err)
}
sha := fmt.Sprintf("%064d", time.Now().UnixNano())
if err := f.pg.QueryRowContext(f.ctx, `INSERT INTO datasets (user_id, original_name, sha256, status, is_active)
VALUES ($1,'另一个数据集',$2,'done',false) RETURNING id`, uid, sha).Scan(&dsB); err != nil {
t.Fatalf("建数据集失败: %v", err)
}
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO places (dataset_id, src_pk, name, lat, lon)
VALUES ($1,$2,'乙侧地点',$3,$4)`, dsB, pkBVal, lat, lon); err != nil {
t.Fatalf("建地点失败: %v", err)
}
key := fmt.Sprintf("%.6f|%.6f", lat, lon)
t.Cleanup(func() {
f.pg.ExecContext(f.ctx, `DELETE FROM place_notes WHERE place_key=$1`, key)
f.pg.ExecContext(f.ctx, `DELETE FROM place_votes WHERE place_key=$1`, key)
f.pg.ExecContext(f.ctx, `DELETE FROM place_visitor_votes WHERE place_key=$1`, key)
f.pg.ExecContext(f.ctx, `DELETE FROM datasets WHERE id=$1`, dsB)
})
return pkAVal, pkBVal, dsB
}
// 访客票也一样:它是站点自己的数据(rondbackup 里没有这一项),
// 但同样要认得出「是哪个地点」,所以也挂坐标键。
func TestVisitorVotesFollowPlaceAcrossDatasets(t *testing.T) {
f := newMovementFixture(t)
uid := userFrom(f.ctx).ID
pkA, pkB, dsB := testPlace(t, f, uid, 77.777777, 88.888888)
if _, mine, err := f.srv.votes.Cast(f.ctx, f.dsID, pkA, "visitor-a", 1); err != nil || mine != 1 {
t.Fatalf("投票失败: mine=%d err=%v", mine, err)
}
tally, err := f.srv.votes.Tally(f.ctx, dsB, pkB)
if err != nil {
t.Fatalf("读票数失败: %v", err)
}
if tally.Up != 1 || tally.Down != 0 {
t.Fatalf("换数据集后票数应照旧,实际 %+v", tally)
}
if v := f.srv.votes.Voter(f.ctx, dsB, pkB, "visitor-a"); v != 1 {
t.Fatalf("「我投的」也该认得出来,实际 %d", v)
}
// 再点一次是撤销:从哪一侧点都该撤掉同一张票
if _, mine, err := f.srv.votes.Cast(f.ctx, dsB, pkB, "visitor-a", 1); err != nil || mine != 0 {
t.Fatalf("再点一次应撤销: mine=%d err=%v", mine, err)
}
if tally, err := f.srv.votes.Tally(f.ctx, f.dsID, pkA); err != nil || tally.Up != 0 {
t.Fatalf("撤销后另一边也该归零: %+v err=%v", tally, err)
}
}
// 备注、结论、图片挂的是坐标键,所以**换数据集之后仍然跟着同一个地点**——
// 这正是这次迁移要解决的问题。老结构挂 (dataset_id, src_pk),
// 换一份备份同编号就是另一个地点,内容全部失联。
func TestNotesAndVerdictsFollowPlaceAcrossDatasets(t *testing.T) {
f := newMovementFixture(t)
uid := userFrom(f.ctx).ID
pkA, pkB, dsB := testPlace(t, f, uid, 11.111111, 22.222222)
if err := f.srv.notes.Save(f.ctx, uid, f.dsID, pkA, "别名甲", "备注甲"); err != nil {
t.Fatalf("写备注失败: %v", err)
}
if err := f.srv.votes.SaveVerdict(f.ctx, uid, f.dsID, pkA, votes.VerdictRecommend); err != nil {
t.Fatalf("写结论失败: %v", err)
}
// 从「另一个数据集」读:编号完全不同,但坐标一致,应当照样读得到
all, err := f.srv.notes.All(f.ctx, dsB)
if err != nil {
t.Fatalf("读备注失败: %v", err)
}
got, ok := all[pkB]
if !ok {
t.Fatal("换数据集后应当仍能读到这个地点的备注")
}
if got.Alias != "别名甲" || got.Content != "备注甲" {
t.Fatalf("备注应原样跟过来,实际 alias=%q note=%q", got.Alias, got.Content)
}
v, err := f.srv.votes.Verdict(f.ctx, dsB, pkB)
if err != nil {
t.Fatalf("读结论失败: %v", err)
}
if v != votes.VerdictRecommend {
t.Fatalf("结论应跟过来,实际 %d", v)
}
}
// 两个数据集里的编号完全不同,但备注与结论是同一份——改一处两边都变。
func TestNotesUpdateVisibleFromOtherDataset(t *testing.T) {
f := newMovementFixture(t)
uid := userFrom(f.ctx).ID
pkA, pkB, dsB := testPlace(t, f, uid, 33.333333, 44.444444)
if err := f.srv.notes.Save(f.ctx, uid, dsB, pkB, "别名乙", "备注乙"); err != nil {
t.Fatalf("在新数据集写备注失败: %v", err)
}
back, err := f.srv.notes.All(f.ctx, f.dsID)
if err != nil {
t.Fatalf("回读失败: %v", err)
}
if back[pkA].Content != "备注乙" {
t.Fatalf("备注是同一份数据,改一处两边都该变,实际 %q", back[pkA].Content)
}
}
// 坐标不同的地点绝不能互相认领——键是坐标,不是「大概位置」。
func TestNotesDoNotLeakBetweenDifferentPlaces(t *testing.T) {
f := newMovementFixture(t)
uid := userFrom(f.ctx).ID
pkA, _, dsB := testPlace(t, f, uid, 55.555555, 66.666666)
if err := f.srv.notes.Save(f.ctx, uid, f.dsID, pkA, "别名甲", "备注甲"); err != nil {
t.Fatalf("写备注失败: %v", err)
}
// 在乙数据集里放一个坐标差 0.001 度(约 100m)的地点,是完全不同的地方
const otherPK = 660003
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO places (dataset_id, src_pk, name, lat, lon)
VALUES ($1,$2,'隔壁那家', 55.556555, 66.666666)`, dsB, otherPK); err != nil {
t.Fatalf("建地点失败: %v", err)
}
all, err := f.srv.notes.All(f.ctx, dsB)
if err != nil {
t.Fatalf("读备注失败: %v", err)
}
if _, ok := all[otherPK]; ok {
t.Fatal("坐标不同的地点不该认领到别人的备注")
}
}
+6 -4
View File
@@ -317,14 +317,16 @@ func TestVisitorPlacePageCarriesImageURLs(t *testing.T) {
var placeID int64 var placeID int64
var srcPK int var srcPK int
if err := f.pg.QueryRowContext(f.ctx, `SELECT id, src_pk FROM places if err := f.pg.QueryRowContext(f.ctx, `SELECT id, src_pk FROM places
WHERE dataset_id=$1 ORDER BY id LIMIT 1`, f.dsID).Scan(&placeID, &srcPK); err != nil { WHERE dataset_id=$1 AND lat IS NOT NULL ORDER BY id LIMIT 1`, f.dsID).Scan(&placeID, &srcPK); err != nil {
t.Fatalf("找不到地点: %v", err) t.Fatalf("找不到地点: %v", err)
} }
const key = "9/visitor-page-test.webp" const key = "9/visitor-page-test.webp"
// 这里只验发下去的地址,不验能否取回字节,所以不用真的写文件 // 这里只验发下去的地址,不验能否取回字节,所以不用真的写文件。
// place_key 得照真实写入路径一并算好——读取是认坐标键的,不认 (dataset_id, src_pk)。
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO place_images if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO place_images
(dataset_id, src_pk, storage, object_key, width, height, bytes, position) (dataset_id, src_pk, place_key, storage, object_key, width, height, bytes, position)
VALUES ($1,$2,'local',$3,1200,900,1024,0) SELECT $1, $2, `+placeKeyExpr+`, 'local', $3, 1200, 900, 1024, 0
FROM places p WHERE p.dataset_id=$1 AND p.src_pk=$2
ON CONFLICT (dataset_id, src_pk, object_key) DO NOTHING`, ON CONFLICT (dataset_id, src_pk, object_key) DO NOTHING`,
f.dsID, srcPK, key); err != nil { f.dsID, srcPK, key); err != nil {
t.Fatalf("插入图片记录失败: %v", err) t.Fatalf("插入图片记录失败: %v", err)
+28 -20
View File
@@ -40,26 +40,28 @@ var pageFiles = map[string]struct {
files []string files []string
entry string entry string
}{ }{
"home": {[]string{"templates/layout.html", "templates/home.html"}, "layout"}, "home": {[]string{"templates/layout.html", "templates/home.html"}, "layout"},
"map": {[]string{"templates/layout.html", "templates/map.html"}, "layout"}, "map": {[]string{"templates/layout.html", "templates/map.html"}, "layout"},
"places": {[]string{"templates/layout.html", "templates/places.html"}, "layout"}, "places": {[]string{"templates/layout.html", "templates/places.html"}, "layout"},
"place": {[]string{"templates/layout.html", "templates/place.html"}, "layout"}, "place": {[]string{"templates/layout.html", "templates/place.html"}, "layout"},
"timeline": {[]string{"templates/layout.html", "templates/timeline.html"}, "layout"}, "timeline": {[]string{"templates/layout.html", "templates/timeline.html"}, "layout"},
"stats": {[]string{"templates/layout.html", "templates/stats.html"}, "layout"}, "stats": {[]string{"templates/layout.html", "templates/stats.html"}, "layout"},
"report": {[]string{"templates/layout.html", "templates/report.html"}, "layout"}, "report": {[]string{"templates/layout.html", "templates/report.html"}, "layout"},
"admin/index": {[]string{"templates/layout.html", "templates/admin/index.html"}, "layout"}, "admin/index": {[]string{"templates/layout.html", "templates/admin/index.html"}, "layout"},
"admin/settings": {[]string{"templates/layout.html", "templates/admin/settings.html"}, "layout"}, "admin/settings": {[]string{"templates/layout.html", "templates/admin/settings.html"}, "layout"},
"admin/password": {[]string{"templates/layout.html", "templates/admin/password.html"}, "layout"}, "admin/password": {[]string{"templates/layout.html", "templates/admin/password.html"}, "layout"},
"admin/login": {[]string{"templates/admin/login.html"}, "login"}, "admin/login": {[]string{"templates/admin/login.html"}, "login"},
"admin/twofa-login": {[]string{"templates/admin/twofa-login.html"}, "login2"}, "admin/twofa-login": {[]string{"templates/admin/twofa-login.html"}, "login2"},
"admin/twofa": {[]string{"templates/layout.html", "templates/admin/twofa.html"}, "layout"}, "admin/twofa": {[]string{"templates/layout.html", "templates/admin/twofa.html"}, "layout"},
"admin/fog-mask": {[]string{"templates/layout.html", "templates/admin/fog-mask.html"}, "layout"}, "admin/fog-mask": {[]string{"templates/layout.html", "templates/admin/fog-mask.html"}, "layout"},
"admin/notes": {[]string{"templates/layout.html", "templates/admin/notes.html"}, "layout"}, "admin/fog-preview": {[]string{"templates/layout.html", "templates/admin/fog-preview.html"}, "layout"},
"admin/edit": {[]string{"templates/layout.html", "templates/admin/edit.html"}, "layout"}, "admin/backup-preview": {[]string{"templates/layout.html", "templates/admin/backup-preview.html"}, "layout"},
"admin/topics": {[]string{"templates/layout.html", "templates/admin/topics.html"}, "layout"}, "admin/notes": {[]string{"templates/layout.html", "templates/admin/notes.html"}, "layout"},
"admin/photos": {[]string{"templates/layout.html", "templates/admin/photos.html"}, "layout"}, "admin/edit": {[]string{"templates/layout.html", "templates/admin/edit.html"}, "layout"},
"topics": {[]string{"templates/layout.html", "templates/topics.html"}, "layout"}, "admin/topics": {[]string{"templates/layout.html", "templates/admin/topics.html"}, "layout"},
"topic": {[]string{"templates/layout.html", "templates/topic.html"}, "layout"}, "admin/photos": {[]string{"templates/layout.html", "templates/admin/photos.html"}, "layout"},
"topics": {[]string{"templates/layout.html", "templates/topics.html"}, "layout"},
"topic": {[]string{"templates/layout.html", "templates/topic.html"}, "layout"},
} }
type pageSet struct { type pageSet struct {
@@ -229,6 +231,9 @@ func (s *Server) routes() {
s.mux.HandleFunc("POST /admin/twofa/disable", s.requireAdmin(s.twofaDisable)) s.mux.HandleFunc("POST /admin/twofa/disable", s.requireAdmin(s.twofaDisable))
s.mux.HandleFunc("GET /admin", s.requireAdmin(s.adminHome)) s.mux.HandleFunc("GET /admin", s.requireAdmin(s.adminHome))
s.mux.HandleFunc("POST /admin/upload", s.requireAdmin(s.adminUpload)) s.mux.HandleFunc("POST /admin/upload", s.requireAdmin(s.adminUpload))
s.mux.HandleFunc("POST /admin/preview", s.requireAdmin(s.adminBackupPreview))
s.mux.HandleFunc("GET /admin/preview/{token}", s.requireAdmin(s.adminBackupPreviewPage))
s.mux.HandleFunc("POST /admin/preview/{token}/merge", s.requireAdmin(s.adminBackupMerge))
s.mux.HandleFunc("POST /admin/datasets/{id}/activate", s.requireAdmin(s.adminActivate)) s.mux.HandleFunc("POST /admin/datasets/{id}/activate", s.requireAdmin(s.adminActivate))
s.mux.HandleFunc("POST /admin/datasets/{id}/delete", s.requireAdmin(s.adminDelete)) s.mux.HandleFunc("POST /admin/datasets/{id}/delete", s.requireAdmin(s.adminDelete))
s.mux.HandleFunc("POST /admin/place/note", s.requireAdmin(s.adminPlaceNote)) s.mux.HandleFunc("POST /admin/place/note", s.requireAdmin(s.adminPlaceNote))
@@ -261,6 +266,9 @@ func (s *Server) routes() {
s.mux.HandleFunc("POST /admin/edit/weather", s.requireAdmin(s.editWeatherCreate)) s.mux.HandleFunc("POST /admin/edit/weather", s.requireAdmin(s.editWeatherCreate))
s.mux.HandleFunc("POST /admin/gpx", s.requireAdmin(s.adminGpxUpload)) s.mux.HandleFunc("POST /admin/gpx", s.requireAdmin(s.adminGpxUpload))
s.mux.HandleFunc("POST /admin/gpx/delete", s.requireAdmin(s.adminGpxDelete)) s.mux.HandleFunc("POST /admin/gpx/delete", s.requireAdmin(s.adminGpxDelete))
s.mux.HandleFunc("POST /admin/fog/preview", s.requireAdmin(s.adminFogPreview))
s.mux.HandleFunc("GET /admin/fog/preview/{token}", s.requireAdmin(s.adminFogPreviewPage))
s.mux.HandleFunc("GET /admin/fog/preview/{token}/{z}/{x}/{y}", s.requireAdmin(s.adminFogDiffTile))
s.mux.HandleFunc("POST /admin/fog", s.requireAdmin(s.adminFogUpload)) s.mux.HandleFunc("POST /admin/fog", s.requireAdmin(s.adminFogUpload))
s.mux.HandleFunc("POST /admin/fog/clear", s.requireAdmin(s.adminFogClear)) s.mux.HandleFunc("POST /admin/fog/clear", s.requireAdmin(s.adminFogClear))
s.mux.HandleFunc("GET /admin/fog/export", s.requireAdmin(s.adminFogExport)) s.mux.HandleFunc("GET /admin/fog/export", s.requireAdmin(s.adminFogExport))
@@ -0,0 +1,310 @@
{{define "content"}}
<div class="wrap">
<div class="page-head">
<div>
<h1>备份差异预览</h1>
<div class="sub">
这份备份与当前展示的「{{.DatasetName}}」逐条比对的结果。<strong>还没有导入</strong>,站内数据一个字节都没动。
</div>
</div>
<div class="row"><a class="btn" href="/admin">← 返回后台</a></div>
</div>
{{if .Identical}}
<div class="card">
<div class="alert ok">这份备份与当前数据集<strong>完全一致</strong>:没有新增、没有被删、也没有冲突。</div>
</div>
{{end}}
{{if .Truncated}}
<div class="card"><div class="alert err">差异较多,下面每类只列出前 200 条,确切数量看概览表。</div></div>
{{end}}
<div class="card">
<h2>概览</h2>
<div class="tw">
<table class="tb tb-stack">
<thead>
<tr>
<th>实体</th><th class="num">备份</th><th class="num">站内</th>
<th class="num">备份新增</th><th class="num">仅站内有</th><th class="num">冲突</th><th class="num">一致</th>
</tr>
</thead>
<tbody>
<tr>
<td>到访</td>
<td class="num" data-label="备份">{{num .BackupVisits}}</td>
<td class="num" data-label="站内">{{num .StoredVisits}}</td>
<td class="num" data-label="备份新增">{{num .VisitCounts.Added}}</td>
<td class="num" data-label="仅站内有">{{num .VisitCounts.Removed}}</td>
<td class="num" data-label="冲突">{{num .VisitCounts.Changed}}</td>
<td class="num" data-label="一致">{{num .VisitCounts.Same}}</td>
</tr>
<tr>
<td>地点</td>
<td class="num" data-label="备份">{{num .BackupPlaces}}</td>
<td class="num" data-label="站内">{{num .StoredPlaces}}</td>
<td class="num" data-label="备份新增">{{num .PlaceCounts.Added}}</td>
<td class="num" data-label="仅站内有">{{num .PlaceCounts.Removed}}</td>
<td class="num" data-label="冲突">{{num .PlaceCounts.Changed}}</td>
<td class="num" data-label="一致">{{num .PlaceCounts.Same}}</td>
</tr>
<tr>
<td>活动类型</td>
<td class="num" data-label="备份">{{num .BackupActs}}</td>
<td class="num" data-label="站内">{{num .StoredActs}}</td>
<td class="num" data-label="备份新增">{{num .ActCounts.Added}}</td>
<td class="num" data-label="仅站内有">{{num .ActCounts.Removed}}</td>
<td class="num" data-label="冲突">{{num .ActCounts.Changed}}</td>
<td class="num" data-label="一致">{{num .ActCounts.Same}}</td>
</tr>
</tbody>
</table>
</div>
<div class="card-sub" style="margin-top:10px">
对齐按每条记录的 <strong>UUID</strong> 做:rond 每次导出都会重排记录编号(同样的 822 条到访,两次导出的编号 100% 不同,UUID 却逐条一模一样),
所以只有 UUID 认得出来是同一条;地点没有 UUID,退到「名称 + 坐标」。
站内补录的记录不会出现在真机备份里,一律归到「仅站内有」。
</div>
</div>
<div class="card">
<h2>到访</h2>
<div class="card-sub"><strong>备份里多出来的({{num .VisitCounts.Added}})</strong> —— 目前站内没有这些记录</div>
{{if .VisitAdded}}
<div class="tw">
<table class="tb tb-stack">
<thead><tr><th>到达</th><th>离开</th><th>地点</th><th>活动</th><th>备注</th></tr></thead>
<tbody>
{{range .VisitAdded}}
<tr>
<td class="mono">{{dtt .Arrival}}</td>
<td class="mono" data-label="离开">{{dtt .Departure}}</td>
<td data-label="地点">{{if .Place}}{{.Place}}{{else}}<span class="muted">—</span>{{end}}</td>
<td data-label="活动">{{if .Activity}}{{.Activity}}{{else}}<span class="muted">—</span>{{end}}</td>
<td data-label="备注">{{if .Remark}}{{.Remark}}{{else}}<span class="muted">—</span>{{end}}</td>
</tr>
{{end}}
</tbody>
</table>
</div>
{{else}}<div class="muted" style="margin-bottom:14px">没有</div>{{end}}
<div class="card-sub" style="margin-top:16px">
<strong>只在站内有({{num .VisitCounts.Removed}})</strong> —— 备份里找不到;可能是你在这里补录的,也可能是实机上删掉的。
<strong>默认一条都不删</strong>;确实想清掉的,勾上左列的「合并时删除」。
</div>
{{if .VisitGone}}
<div class="tw">
<table class="tb tb-stack">
<thead><tr><th>到达</th><th>离开</th><th>地点</th><th>活动</th><th>备注</th><th>来源</th><th>合并时删除</th></tr></thead>
<tbody>
{{range .VisitGone}}
<tr>
<td class="mono">{{dtt .Arrival}}</td>
<td class="mono" data-label="离开">{{dtt .Departure}}</td>
<td data-label="地点">{{if .Place}}{{.Place}}{{else}}<span class="muted">—</span>{{end}}</td>
<td data-label="活动">{{if .Activity}}{{.Activity}}{{else}}<span class="muted">—</span>{{end}}</td>
<td data-label="备注">{{if .Remark}}{{.Remark}}{{else}}<span class="muted">—</span>{{end}}</td>
<td data-label="来源">{{if .UserAdded}}<span>站内补录</span>{{else}}<span class="muted">备份里已无</span>{{end}}</td>
<td data-label="合并时删除">
<input type="checkbox" form="merge-form" name="dv_{{.SrcPK}}" value="1" title="勾上=合并时删掉这条到访">
</td>
</tr>
{{end}}
</tbody>
</table>
</div>
{{else}}<div class="muted" style="margin-bottom:14px">没有</div>{{end}}
<div class="card-sub" style="margin-top:16px"><strong>两边都有但不一样({{num .VisitCounts.Changed}})</strong> —— 左边是站内当前的值,右边是备份里的值</div>
{{if .VisitClash}}
<div class="tw">
<table class="tb tb-stack">
<thead><tr><th>到达</th><th>地点</th><th>活动</th><th>差异</th><th>合并时</th></tr></thead>
<tbody>
{{range .VisitClash}}
<tr>
<td class="mono">{{dtt .Arrival}}</td>
<td data-label="地点">{{if .Place}}{{.Place}}{{else}}<span class="muted">—</span>{{end}}</td>
<td data-label="活动">{{if .Activity}}{{.Activity}}{{else}}<span class="muted">—</span>{{end}}</td>
<td data-label="差异">
{{range .Fields}}
<div style="font-size:12.5px;line-height:1.7">
<span class="muted">{{.Label}}</span>
<span style="text-decoration:line-through;opacity:.65">{{if .From}}{{.From}}{{else}}(空){{end}}</span>
<span style="opacity:.5">→</span>
<strong>{{if .To}}{{.To}}{{else}}(空){{end}}</strong>
</div>
{{end}}
</td>
<td data-label="合并时" style="white-space:nowrap">
<label style="margin-right:8px"><input type="radio" form="merge-form" name="vc_{{.Key}}" value="backup" checked> 用实机</label>
<label style="margin-right:8px"><input type="radio" form="merge-form" name="vc_{{.Key}}" value="db"> 用站内</label>
<label><input type="radio" form="merge-form" name="vc_{{.Key}}" value="skip"> 跳过</label>
</td>
</tr>
{{end}}
</tbody>
</table>
</div>
{{else}}<div class="muted">没有</div>{{end}}
</div>
<div class="card">
<h2>地点</h2>
<div class="card-sub"><strong>备份里多出来的({{num .PlaceCounts.Added}})</strong></div>
{{if .PlaceAdded}}
<div class="tw">
<table class="tb tb-stack">
<thead><tr><th>名称</th><th>省 / 市</th><th>分类</th><th class="num">坐标</th></tr></thead>
<tbody>
{{range .PlaceAdded}}
<tr>
<td>{{if .Name}}{{.Name}}{{else}}<span class="muted">(未命名)</span>{{end}}</td>
<td data-label="省 / 市">{{.Province}} {{.City}}</td>
<td data-label="分类">{{if .POICategory}}{{.POICategory}}{{else}}<span class="muted">—</span>{{end}}</td>
<td class="num mono" data-label="坐标">{{printf "%.5f" .Lat}}, {{printf "%.5f" .Lon}}</td>
</tr>
{{end}}
</tbody>
</table>
</div>
{{else}}<div class="muted" style="margin-bottom:14px">没有</div>{{end}}
<div class="card-sub" style="margin-top:16px">
<strong>只在站内有({{num .PlaceCounts.Removed}})</strong>
—— 名下还有到访的地点删不掉(会连带删掉那些到访),会直接报错让你先处理。
</div>
{{if .PlaceGone}}
<div class="tw">
<table class="tb tb-stack">
<thead><tr><th>名称</th><th>省 / 市</th><th class="num">到访</th><th class="num">坐标</th><th>合并时删除</th></tr></thead>
<tbody>
{{range .PlaceGone}}
<tr>
<td>{{if .Name}}{{.Name}}{{else}}<span class="muted">(未命名)</span>{{end}}</td>
<td data-label="省 / 市">{{.Province}} {{.City}}</td>
<td class="num" data-label="到访">{{num .VisitCount}}</td>
<td class="num mono" data-label="坐标">{{printf "%.5f" .Lat}}, {{printf "%.5f" .Lon}}</td>
<td data-label="合并时删除">
{{if gt .VisitCount 0}}
<span class="muted" style="font-size:12.5px">还有到访,不能删</span>
{{else}}
<input type="checkbox" form="merge-form" name="dp_{{.SrcPK}}" value="1">
{{end}}
</td>
</tr>
{{end}}
</tbody>
</table>
</div>
{{else}}<div class="muted" style="margin-bottom:14px">没有</div>{{end}}
<div class="card-sub" style="margin-top:16px"><strong>两边都有但地址不一样({{num .PlaceCounts.Changed}})</strong></div>
{{if .PlaceClash}}
<div class="tw">
<table class="tb tb-stack">
<thead><tr><th>名称</th><th>差异</th><th>合并时</th></tr></thead>
<tbody>
{{range .PlaceClash}}
<tr>
<td>{{if .DB.Name}}{{.DB.Name}}{{else}}<span class="muted">(未命名)</span>{{end}}</td>
<td data-label="差异">
{{range .Fields}}
<div style="font-size:12.5px;line-height:1.7">
<span class="muted">{{.Label}}</span>
<span style="text-decoration:line-through;opacity:.65">{{if .From}}{{.From}}{{else}}(空){{end}}</span>
<span style="opacity:.5">→</span>
<strong>{{if .To}}{{.To}}{{else}}(空){{end}}</strong>
</div>
{{end}}
</td>
<td data-label="合并时" style="white-space:nowrap">
<label style="margin-right:8px"><input type="radio" form="merge-form" name="pc_{{.DB.Key}}" value="backup" checked> 用实机</label>
<label style="margin-right:8px"><input type="radio" form="merge-form" name="pc_{{.DB.Key}}" value="db"> 用站内</label>
<label><input type="radio" form="merge-form" name="pc_{{.DB.Key}}" value="skip"> 跳过</label>
</td>
</tr>
{{end}}
</tbody>
</table>
</div>
{{else}}<div class="muted">没有</div>{{end}}
</div>
<div class="card">
<h2>活动类型</h2>
<div class="card-sub">
新增 {{num .ActCounts.Added}} · 仅站内有 {{num .ActCounts.Removed}} · 冲突 {{num .ActCounts.Changed}} · 一致 {{num .ActCounts.Same}}
</div>
<div class="row" style="gap:24px;align-items:flex-start;flex-wrap:wrap">
<div style="flex:1 1 220px">
<div class="muted" style="font-size:12.5px;margin-bottom:4px">备份新增</div>
{{range .ActAdded}}<div>{{.Name}}</div>{{else}}<div class="muted">没有</div>{{end}}
</div>
<div style="flex:1 1 220px">
<div class="muted" style="font-size:12.5px;margin-bottom:4px">仅站内有(勾上=合并时删除)</div>
{{range .ActGone}}
<label style="display:block"><input type="checkbox" form="merge-form" name="da_{{.SrcPK}}" value="1"> {{.Name}}</label>
{{else}}<div class="muted">没有</div>{{end}}
</div>
<div style="flex:1 1 260px">
<div class="muted" style="font-size:12.5px;margin-bottom:4px">名称不同</div>
{{range .ActClash}}
<div style="margin-bottom:6px">
{{range .Fields}}<div><span style="text-decoration:line-through;opacity:.65">{{.From}}</span> → <strong>{{.To}}</strong></div>{{end}}
<div style="margin-top:4px">
<label style="margin-right:8px"><input type="radio" form="merge-form" name="ac_{{actKey .DB}}" value="backup" checked> 用实机</label>
<label style="margin-right:8px"><input type="radio" form="merge-form" name="ac_{{actKey .DB}}" value="db"> 用站内</label>
<label><input type="radio" form="merge-form" name="ac_{{actKey .DB}}" value="skip"> 跳过</label>
</div>
</div>
{{else}}<div class="muted">没有</div>{{end}}
</div>
</div>
</div>
<div class="card">
<h2>其它实体(只列数量)</h2>
<div class="tw">
<table class="tb tb-stack">
<thead><tr><th>实体</th><th class="num">备份</th><th class="num">站内</th></tr></thead>
<tbody>
<tr><td>位移</td><td class="num" data-label="备份">{{num .BackupMovements}}</td><td class="num" data-label="站内">{{num .StoredMovements}}</td></tr>
<tr><td>小时天气</td><td class="num" data-label="备份">{{num .BackupWeather}}</td><td class="num" data-label="站内">{{num .StoredWeather}}</td></tr>
<tr><td>原始定位点</td><td class="num" data-label="备份">{{num .BackupRaw}}</td><td class="num" data-label="站内"><span class="muted">不单独落库</span></td></tr>
</tbody>
</table>
</div>
<div class="card-sub" style="margin-top:10px">
位移跟着它的起止到访走、天气跟着到访走,所以这两项按到访对齐结果一并处理,不单独做逐条比对。
</div>
</div>
<div class="card">
<h2>执行合并</h2>
<div class="card-sub">
把选中的改动写进当前数据集「{{.DatasetName}}」。整个过程跑在一个事务里,任何一步失败都会<strong>整体回滚</strong>,数据不会半改。
<br>「仅站内有」的记录<strong>默认一条都不删</strong>;确实要清掉的,在上面各表里逐条勾「合并时删除」。
名下有到访的地点会被拒绝删除(否则会连带删掉那些到访),先处理到访再来。
</div>
<form id="merge-form" method="post" action="/admin/preview/{{.Token}}/merge"></form>
<label style="display:flex;align-items:flex-start;gap:8px;margin:4px 0 12px">
<input type="checkbox" form="merge-form" name="add_new" value="1" checked style="margin-top:3px">
<span>把备份里多出来的
<strong>{{num .VisitCounts.Added}}</strong> 条到访、
<strong>{{num .PlaceCounts.Added}}</strong> 个地点、
<strong>{{num .ActCounts.Added}}</strong> 个活动并进来
<span class="muted">(新到访会连它的地点、活动、标签、位移与天气一起带过来)</span>
</span>
</label>
<button class="btn primary" form="merge-form" type="submit"
onclick="return confirm('确认把选中的改动写进当前数据集?出错会自动整体回滚。')">执行合并</button>
<div class="card-sub" style="margin:10px 0 0">
合并后再传一次同一份备份,差异应当变成「完全一致」——那说明对齐键(UUID 与坐标)都写进去了。
</div>
</div>
</div>
{{end}}
@@ -0,0 +1,174 @@
{{define "content"}}
<div class="wrap">
<div class="page-head">
<div>
<h1>迷雾导入预览</h1>
<div class="sub">这份快照<strong>还没有写入</strong>。先看清两种导入方式各自的后果,再决定用哪一种。</div>
</div>
<div class="row">
<a class="btn" href="/admin">← 返回后台</a>
</div>
</div>
<div class="card">
<h2>数量对比</h2>
<div class="tw">
<table class="tb tb-stack">
<thead>
<tr><th>状态</th><th class="num">块</th><th class="num">格</th><th class="num">面积</th></tr>
</thead>
<tbody>
<tr>
<td>当前迷雾</td>
<td class="num" data-label="块">{{num .Diff.CurBlocks}}</td>
<td class="num" data-label="格">{{num .Diff.CurCells}}</td>
<td class="num" data-label="面积">{{printf "%.1f" .Diff.CurAreaKm2}} km²</td>
</tr>
<tr>
<td>这份快照</td>
<td class="num" data-label="块">{{num .Diff.IncBlocks}}</td>
<td class="num" data-label="格">{{num .Diff.IncCells}}</td>
<td class="num" data-label="面积">{{printf "%.1f" .Diff.IncAreaKm2}} km²</td>
</tr>
<tr>
<td>叠加导入后</td>
<td class="num" data-label="块">{{num .Diff.CurBlocks}}<span class="muted"> +{{num .Diff.GainBlocks}}</span></td>
<td class="num" data-label="格">{{num .Diff.CurCells}}<span class="muted"> +{{num .Diff.GainCells}}</span></td>
<td class="num" data-label="面积">{{printf "%.1f" .Diff.MergeAreaKm2}} km²</td>
</tr>
<tr>
<td>整体替换后</td>
<td class="num" data-label="块">{{num .Diff.IncBlocks}}</td>
<td class="num" data-label="格">{{num .Diff.IncCells}}</td>
<td class="num" data-label="面积">{{printf "%.1f" .Diff.CoverAreaKm2}} km²</td>
</tr>
</tbody>
</table>
</div>
<div class="card-sub" style="margin-top:10px">
两个块坐标相同就只按位合并,所以「叠加后」的块数不会等于两边相加——快照里有 {{num .Diff.OverlapBlocks}} 个块与当前迷雾落在同一块,
其中 {{num .Diff.SameBlocks}} 个一格都不会变。
</div>
</div>
{{if .HasBounds}}
<div class="card">
<h2>差异地图</h2>
<div class="card-sub">在真实底图上核对差异落在哪些区域,可以缩放平移。</div>
<div class="row" style="gap:16px;margin:2px 0 10px;font-size:12.5px;flex-wrap:wrap">
<span style="display:flex;align-items:center;gap:6px">
<i style="width:13px;height:13px;border-radius:3px;background:#24a45a;display:inline-block"></i>叠加会新增
</span>
<span style="display:flex;align-items:center;gap:6px">
<i style="width:13px;height:13px;border-radius:3px;background:#7e8692;display:inline-block"></i>两边都有
</span>
<span style="display:flex;align-items:center;gap:6px">
<i style="width:13px;height:13px;border-radius:3px;background:#d63838;display:inline-block"></i>整体替换会丢掉
</span>
</div>
<div id="fog-diff-map" style="height:400px;border-radius:10px;background:var(--line-soft)"
data-token="{{.Token}}"
data-min-lat="{{printf "%.6f" .MinLat}}" data-min-lon="{{printf "%.6f" .MinLon}}"
data-max-lat="{{printf "%.6f" .MaxLat}}" data-max-lon="{{printf "%.6f" .MaxLon}}"></div>
</div>
{{end}}
<div class="card">
<h2>叠加导入</h2>
{{if .Diff.Changed}}
<div class="alert ok">
会补上当前迷雾没有的 <strong>{{num .Diff.GainBlocks}}</strong> 块、<strong>{{num .Diff.GainCells}}</strong> 格
(约 {{printf "%.1f" .Diff.GainAreaKm2}} km²)。叠加只做按位合并,<strong>不会丢失任何已有数据</strong>。
</div>
{{else}}
<div class="alert err">
这份快照的点亮区域已经被当前迷雾<strong>完全覆盖</strong>——叠加导入不会带来任何变化,
多半是同一批数据又导出了一次。
</div>
{{end}}
<form method="post" action="/admin/fog?stash={{.Token}}"
onsubmit="this.querySelector('button').disabled=true">
<input type="hidden" name="mode" value="overlay">
<button class="btn primary" type="submit">叠加导入</button>
</form>
</div>
<div class="card">
<h2>整体替换</h2>
{{if gt .Diff.LostCells 0}}
<div class="alert err">
⚠️ 快照里<strong>没有</strong>当前迷雾的 <strong>{{num .Diff.LostBlocks}}</strong> 块、<strong>{{num .Diff.LostCells}}</strong> 格
(约 {{printf "%.1f" .Diff.LostAreaKm2}} km²)。整体替换后,这些区域会<strong>从点亮变回未探索</strong>,且无法恢复。
</div>
{{else}}
<div class="alert ok">
这份快照完整包含了当前迷雾的全部点亮区域,替换不会丢数据。
</div>
{{end}}
{{if and (eq .TildeLayers 0) (eq .HashLayers 0)}}
<div class="alert err">
⚠️ 快照里没有 <span class="mono">Model/~</span> 与 <span class="mono">Model/#</span> 两层数据,替换会把现有的这两层一并清掉。
</div>
{{else}}
<div class="muted" style="font-size:12.5px;margin-bottom:8px">
另外,站点不解析但会留存的 <span class="mono">Model/~</span>({{.TildeLayers}} 项)与
<span class="mono">Model/#</span>({{.HashLayers}} 项)也会一并换成这份快照里的{{if .OtherLayers}},另有 {{.OtherLayers}} 项其它条目{{end}}。
</div>
{{end}}
<form method="post" action="/admin/fog?stash={{.Token}}"
onsubmit="return confirm('整体替换会清空当前迷雾。{{if gt .Diff.LostCells 0}}快照里缺 {{num .Diff.LostCells}} 格,替换后这些区域会变回未探索,且不可恢复。{{end}}确认替换?')">
<input type="hidden" name="mode" value="cover">
<button class="btn danger" type="submit">整体替换</button>
</form>
</div>
</div>
<script>
(function () {
// Leaflet 由 layout 在 content 之后引入,脚本执行时 window.L 还没有,
// 必须等 load 之后再初始化,否则地图整个不出现(与迷雾遮罩页同一条约定)
function whenReady(cb) {
if (document.readyState === 'complete') { setTimeout(cb, 0); return; }
window.addEventListener('load', cb);
}
var el = document.getElementById('fog-diff-map');
if (!el) return;
// 底图走站点配置(与迷雾遮罩页同一套取法):带密钥的图层由服务端转发,
// 免密钥的是浏览器直连的 {s} 模板,不能硬写 /tiles/ 路由
var fallback = {
def: { url: 'https://webrd0{s}.is.autonavi.com/appmaptile?lang=zh_cn&size=1&scale=1&style=8&x={x}&y={y}&z={z}', maxZoom: 18, attr: '© 高德地图', crs: 'gcj02' },
subs: ['1', '2', '3', '4']
};
function init(cfg) {
var def = cfg.def;
var map = L.map(el, { scrollWheelZoom: false });
L.tileLayer(def.url, { subdomains: cfg.subs, maxZoom: def.maxZoom || 18, minZoom: 3, attribution: def.attr || '' }).addTo(map);
// 差异瓦片必须和底图同坐标系:迷雾网格是 WGS-84,服务端按 crs 逐块纠偏
L.tileLayer('/admin/fog/preview/' + el.dataset.token + '/{z}/{x}/{y}?crs=' + (def.crs || 'gcj02'), {
minZoom: 3, maxZoom: 18, opacity: 0.95
}).addTo(map);
var b = [
[parseFloat(el.dataset.minLat), parseFloat(el.dataset.minLon)],
[parseFloat(el.dataset.maxLat), parseFloat(el.dataset.maxLon)]
];
// 只有一个块时两个角重合,不设上限会一路放大到最大级别
map.fitBounds(b, { padding: [24, 24], maxZoom: 13 });
L.control.scale({ imperial: false }).addTo(map);
}
whenReady(function () {
if (!window.L) return;
fetch('/api/map-config', { headers: { 'Accept': 'application/json' } })
.then(function (r) { return r.json(); })
.then(function (c) {
var def = (c && c.base && c.base.length) ? c.base[0] : fallback.def;
init({ def: def, subs: (c && c.subs && c.subs.length) ? c.subs : fallback.subs });
})
.catch(function () { init(fallback); });
});
})();
</script>
{{end}}
+11 -10
View File
@@ -29,6 +29,7 @@
<b>上传是以备份为准重建数据</b>:站点里删掉的地点、改过的记录,只要下次传一份 <b>上传是以备份为准重建数据</b>:站点里删掉的地点、改过的记录,只要下次传一份
<b>内容不同</b>的备份,就会被备份里的原样覆盖回来(重复传同一份文件不会,会提示「完全一致」)。 <b>内容不同</b>的备份,就会被备份里的原样覆盖回来(重复传同一份文件不会,会提示「完全一致」)。
在站点上新加的地点属于当时那个数据集,传新备份后要看它们,可在下面的数据集列表里切回。 在站点上新加的地点属于当时那个数据集,传新备份后要看它们,可在下面的数据集列表里切回。
<br><b>拿不准就先用「分析差异」</b>:它会逐条比对这份备份与站内当前数据,告诉你哪些是新增、哪些只在站内有、哪些两边不一样,看完再决定要不要导入。
</div> </div>
<form method="post" action="/admin/upload" enctype="multipart/form-data"> <form method="post" action="/admin/upload" enctype="multipart/form-data">
<label class="f" style="max-width:520px"> <label class="f" style="max-width:520px">
@@ -36,7 +37,10 @@
<input type="file" name="backup" accept=".rondbackup,.zip" required <input type="file" name="backup" accept=".rondbackup,.zip" required
style="padding:8px;border:1px dashed var(--line);border-radius:8px;background:var(--bg-2)"> style="padding:8px;border:1px dashed var(--line);border-radius:8px;background:var(--bg-2)">
</label> </label>
<button class="btn primary" type="submit">上传并导入</button> <div class="row">
<button class="btn primary" type="submit" formaction="/admin/preview">分析差异</button>
<button class="btn" type="submit" formaction="/admin/upload">直接导入</button>
</div>
</form> </form>
</div> </div>
@@ -53,21 +57,18 @@
<b>尚未导入迷雾</b>——把 Fog of World 的 <span class="mono">.fwss</span>(在 App 内导出,通常在 zip 里)上传即可点亮地图。 <b>尚未导入迷雾</b>——把 Fog of World 的 <span class="mono">.fwss</span>(在 App 内导出,通常在 zip 里)上传即可点亮地图。
{{end}} {{end}}
</div> </div>
<form method="post" action="/admin/fog" enctype="multipart/form-data"> <form method="post" action="/admin/fog/preview" enctype="multipart/form-data">
<label class="f" style="max-width:520px"> <label class="f" style="max-width:520px">
<span>选择快照</span> <span>选择快照</span>
<input type="file" name="fwss" accept=".fwss,.zip" required <input type="file" name="fwss" accept=".fwss,.zip" required
style="padding:8px;border:1px dashed var(--line);border-radius:8px;background:var(--bg-2)"> style="padding:8px;border:1px dashed var(--line);border-radius:8px;background:var(--bg-2)">
</label> </label>
<label class="f" style="max-width:520px"> <button class="btn primary" type="submit">分析差异</button>
<span>导入方式</span>
<select name="mode">
<option value="overlay" selected>叠加 — 保留现有迷雾,把新路线合并进来</option>
<option value="cover">覆盖 — 清空现有迷雾,整体替换</option>
</select>
</label>
<button class="btn primary" type="submit">导入迷雾</button>
</form> </form>
<div class="muted" style="font-size:12.5px;margin-top:8px">
上传后<strong>先不写入</strong>:会先给出这份快照能补多少新区域、以及「整体替换」会丢掉多少已有区域,
确认之后才真正导入。
</div>
{{if gt .FogBlocks 0}} {{if gt .FogBlocks 0}}
<div style="margin:10px 0 0;display:flex;gap:8px;flex-wrap:wrap"> <div style="margin:10px 0 0;display:flex;gap:8px;flex-wrap:wrap">
<a class="btn" href="/admin/fog/export">导出当前迷雾(.fwss)</a> <a class="btn" href="/admin/fog/export">导出当前迷雾(.fwss)</a>
+71
View File
@@ -14,6 +14,7 @@ import (
"rond-with-you/internal/auth" "rond-with-you/internal/auth"
"rond-with-you/internal/fog" "rond-with-you/internal/fog"
"rond-with-you/internal/rond"
"rond-with-you/internal/setting" "rond-with-you/internal/setting"
"rond-with-you/internal/stats" "rond-with-you/internal/stats"
"rond-with-you/internal/votes" "rond-with-you/internal/votes"
@@ -637,6 +638,74 @@ type AdminData struct {
DefaultPW bool DefaultPW bool
} }
// VisitRow 是一条到访摊平后的展示行。地点与活动写名称而不是编号——
// 两边的编号空间完全不同,编号对人没有任何意义。
type VisitRow struct {
// Key 是这条冲突的对齐键(到访 UUID),合并表单拿它当字段名
Key string
// SrcPK 只有「仅站内有」的行有意义:要删哪条就勾哪条
SrcPK int
Arrival *time.Time
Departure *time.Time
Place string
Activity string
Remark string
Emoji string
UserAdded bool // 站点补录的行
Fields []rond.FieldChange
}
// BackupDiffData 是「备份差异预览」页的数据。
//
// 上限说明:列表按 diffListLimit 截断,页面上只提示「还有更多」,不做分页——
// 预演是给人下判断用的,看前 200 条足够判断要不要继续;真正要逐条取舍的是
// 后面那个合并流程(那里才有逐条操作)。
type BackupDiffData struct {
Page
Token string
DatasetName string
VisitCounts rond.Counts
PlaceCounts rond.Counts
ActCounts rond.Counts
VisitAdded, VisitGone, VisitClash []VisitRow
PlaceAdded []rond.Location
PlaceGone []rond.StoredPlace
PlaceClash []rond.PlaceConflict
ActAdded []rond.Activity
ActGone []rond.StoredActivity
ActClash []rond.ActivityConflict
// 两侧总量,用于「有没有少数据」的直觉判断
BackupVisits, StoredVisits int
BackupPlaces, StoredPlaces int
BackupActs, StoredActs int
BackupMovements, StoredMovements int
BackupWeather, StoredWeather int
BackupRaw int
Truncated bool
Identical bool
}
// FogPreviewData 是「迷雾导入预览」页的数据:先把差异摆出来,再决定叠加还是整体替换。
type FogPreviewData struct {
Page
Token string
Diff fog.Diff
// 快照里不参与渲染、但整体替换会一并换掉的图层条目数
TildeLayers int
HashLayers int
OtherLayers int
// 差异地图的初始视野:两边块集覆盖范围的包围盒(已按底图坐标系换算)。
// 没有任何块时 HasBounds 为假,模板不画地图。
HasBounds bool
MinLat, MinLon float64
MaxLat, MaxLon float64
}
type NotesData struct { type NotesData struct {
Page Page
Rows []NoteRow Rows []NoteRow
@@ -1247,6 +1316,8 @@ func templateFuncs() template.FuncMap {
"km": func(f float64) string { return fmt.Sprintf("%.1f", f) }, "km": func(f float64) string { return fmt.Sprintf("%.1f", f) },
"f1": func(f float64) string { return fmt.Sprintf("%.1f", f) }, "f1": func(f float64) string { return fmt.Sprintf("%.1f", f) },
"dt": func(t time.Time) string { return t.Local().Format("2006-01-02 15:04") }, "dt": func(t time.Time) string { return t.Local().Format("2006-01-02 15:04") },
// actKey 给活动冲突生成表单字段名,与处理函数共用同一套键
"actKey": rond.ActivityKey,
// dtt 接受 *time.Time,nil 返回空串,便于可选时间字段直接渲染 // dtt 接受 *time.Time,nil 返回空串,便于可选时间字段直接渲染
"dtt": func(t *time.Time) string { "dtt": func(t *time.Time) string {
if t == nil { if t == nil {
+89
View File
@@ -0,0 +1,89 @@
// 一次性探针:验证 ZLOCATION.ZPLACEID / ZVISIT.ZEVENTIDENTIFIER 是否跨备份稳定。
package main
import (
"fmt"
"os"
"rond-with-you/internal/rond"
)
type snap struct {
placeID map[string]int
eventID map[string]int
nulls int
total int
evTotal int
}
func load(p string) *snap {
dir, err := os.MkdirTemp("", "colprobe")
if err != nil {
panic(err)
}
defer os.RemoveAll(dir)
db, err := rond.Open(p, dir)
if err != nil {
panic(err)
}
defer db.Close()
s := &snap{placeID: map[string]int{}, eventID: map[string]int{}}
rows, err := db.Query(`SELECT Z_PK, COALESCE(ZPLACEID,'') FROM ZLOCATION`)
if err != nil {
panic(err)
}
for rows.Next() {
var pk int
var id string
rows.Scan(&pk, &id)
s.total++
if id == "" {
s.nulls++
} else {
s.placeID[id] = pk
}
}
rows.Close()
rows, err = db.Query(`SELECT COALESCE(ZEVENTIDENTIFIER,'') FROM ZVISIT`)
if err != nil {
panic(err)
}
for rows.Next() {
var id string
rows.Scan(&id)
if id != "" {
s.eventID[id] = 1
s.evTotal++
}
}
rows.Close()
fmt.Printf("%s: ZLOCATION %d 个(ZPLACEID 空 %d),ZEVENTIDENTIFIER 非空 %d\n",
p, s.total, s.nulls, s.evTotal)
return s
}
func main() {
a := load(os.Args[1])
b := load(os.Args[2])
inter, samePK := 0, 0
for k, pa := range a.placeID {
if pb, ok := b.placeID[k]; ok {
inter++
if pa == pb {
samePK++
}
}
}
fmt.Printf("\nZPLACEID 交集 %d(A 独有 %d,B 独有 %d),其中 Z_PK 相同 %d\n",
inter, len(a.placeID)-inter, len(b.placeID)-inter, samePK)
ei := 0
for k := range a.eventID {
if _, ok := b.eventID[k]; ok {
ei++
}
}
fmt.Printf("ZEVENTIDENTIFIER 交集 %d(A %d / B %d)\n", ei, len(a.eventID), len(b.eventID))
}
+438
View File
@@ -0,0 +1,438 @@
// 一次性探针:把一份 .rondbackup 与 PG 里某个数据集的记录逐条对齐,输出差异画像。
// 用完即可删,不属于站点功能。
package main
import (
"context"
"database/sql"
"fmt"
"os"
"sort"
"time"
_ "github.com/jackc/pgx/v5/stdlib"
"rond-with-you/internal/rond"
)
func main() {
if len(os.Args) < 4 {
fmt.Println("用法: diffprobe <archive.rondbackup> <dsn> <datasetID>")
os.Exit(2)
}
archive, dsn := os.Args[1], os.Args[2]
var datasetID int64
fmt.Sscanf(os.Args[3], "%d", &datasetID)
tmp, err := os.MkdirTemp("", "diffprobe")
must(err)
defer os.RemoveAll(tmp)
sqlDB, err := rond.Open(archive, tmp)
must(err)
b, err := rond.Parse(sqlDB)
sqlDB.Close()
must(err)
pg, err := sql.Open("pgx", dsn)
must(err)
defer pg.Close()
ctx := context.Background()
must(pg.PingContext(ctx))
fmt.Printf("=== 备份 %s ===\n", archive)
fmt.Printf("备份: 到访 %d / 地点 %d / 位移 %d / 天气 %d / 活动 %d / 标签 %d / 原始定位 %d\n",
len(b.Visits), len(b.Locations), len(b.Movements), len(b.Weather),
len(b.Activities), len(b.Tags), b.RawVisitNum)
var bFirst, bLast sql.NullTime
for _, v := range b.Visits {
if v.Arrival == nil {
continue
}
if !bFirst.Valid || v.Arrival.Before(bFirst.Time) {
bFirst = sql.NullTime{Time: *v.Arrival, Valid: true}
}
if !bLast.Valid || v.Arrival.After(bLast.Time) {
bLast = sql.NullTime{Time: *v.Arrival, Valid: true}
}
}
fmt.Printf("备份时间跨度: %s ~ %s\n", ts(bFirst), ts(bLast))
fmt.Printf("\n=== 库内 dataset #%d ===\n", datasetID)
var name string
var cnt, pcnt, raw int
var gFirst, gLast sql.NullTime
must(pg.QueryRowContext(ctx, `SELECT original_name, visit_count, place_count, raw_visit_count,
first_visit_at, last_visit_at FROM datasets WHERE id=$1`, datasetID).
Scan(&name, &cnt, &pcnt, &raw, &gFirst, &gLast))
fmt.Printf("%s: 到访 %d / 地点 %d / 原始定位 %d,跨度 %s ~ %s\n", name, cnt, pcnt, raw, ts(gFirst), ts(gLast))
diffVisits(ctx, pg, datasetID, b)
diffPlaces(ctx, pg, datasetID, b)
diffMovements(ctx, pg, datasetID, b)
diffWeather(ctx, pg, datasetID, b)
diffActivities(ctx, pg, datasetID, b)
diffTags(ctx, pg, datasetID, b)
}
func ts(t sql.NullTime) string {
if !t.Valid {
return "-"
}
return t.Time.Local().Format("2006-01-02 15:04")
}
type visitRow struct {
pk, placeSrc, actSrc int
arrival, departure *time.Time
remark, emoji string
bookmarked, userAdd bool
}
func diffVisits(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
fmt.Printf("\n---------- 到访 ----------\n")
rows, err := pg.QueryContext(ctx, `SELECT v.src_pk, COALESCE(p.src_pk,0), COALESCE(a.src_pk,0),
v.arrival, v.departure, COALESCE(v.remark,''), COALESCE(v.emoji,''), v.bookmarked, v.is_user_added
FROM visits v LEFT JOIN places p ON p.id=v.place_id LEFT JOIN activities a ON a.id=v.activity_id
WHERE v.dataset_id=$1`, dsID)
must(err)
db := map[int]visitRow{}
for rows.Next() {
var r visitRow
var arr time.Time
var dep sql.NullTime
must(rows.Scan(&r.pk, &r.placeSrc, &r.actSrc, &arr, &dep, &r.remark, &r.emoji, &r.bookmarked, &r.userAdd))
r.arrival = &arr
if dep.Valid {
r.departure = &dep.Time
}
db[r.pk] = r
}
rows.Close()
bk := map[int]rond.Visit{}
for _, v := range b.Visits {
bk[v.SrcPK] = v
}
var onlyBackup, onlyDB, bothDiff, same []int
for pk, v := range bk {
r, ok := db[pk]
if !ok {
onlyBackup = append(onlyBackup, pk)
continue
}
if vdiff(v, r) != "" {
bothDiff = append(bothDiff, pk)
continue
}
same = append(same, pk)
}
for pk := range db {
if _, ok := bk[pk]; !ok {
onlyDB = append(onlyDB, pk)
}
}
sort.Ints(onlyBackup)
sort.Ints(onlyDB)
sort.Ints(bothDiff)
fmt.Printf("同号且一致: %d\n", len(same))
fmt.Printf("仅备份有(实机新增,库里没有): %d\n", len(onlyBackup))
fmt.Printf("仅库里有(站点侧新增/实机已删除): %d\n", len(onlyDB))
fmt.Printf("同号但内容不同(冲突): %d\n", len(bothDiff))
// 按到达时间做多重集匹配:判断「同号冲突」到底是同批数据换了主键,还是两批无关数据
byTime := map[string][]int{}
for pk, r := range db {
if r.arrival != nil {
byTime[r.arrival.UTC().Format(time.RFC3339)] = append(byTime[r.arrival.UTC().Format(time.RFC3339)], pk)
}
}
matched, miss := 0, 0
pair := [][2]int{}
for pk, v := range bk {
if v.Arrival == nil {
continue
}
k := v.Arrival.UTC().Format(time.RFC3339)
if list := byTime[k]; len(list) > 0 {
matched++
pair = append(pair, [2]int{pk, list[0]})
byTime[k] = list[1:]
} else {
miss++
}
}
fmt.Printf("\n[时间对齐] 备份到访按「到达时刻」能在库内找到同刻记录: %d 条,找不到: %d 条\n", matched, miss)
sort.Slice(pair, func(i, j int) bool { return pair[i][0] < pair[j][0] })
fmt.Printf("[时间对齐] 同刻对的 (备份pk→库pk) 前 12 组: ")
for i, p := range pair {
if i >= 12 {
break
}
fmt.Printf("%d→%d ", p[0], p[1])
}
fmt.Println()
nDiff := 0
for _, p := range pair {
if p[0] != p[1] {
nDiff++
}
}
fmt.Printf("[时间对齐] 同刻但主键不同: %d / %d\n", nDiff, len(pair))
show := func(label string, pks []int, f func(pk int) string) {
if len(pks) == 0 {
return
}
fmt.Printf("\n[%s] 共 %d 条,前 8 条:\n", label, len(pks))
for i, pk := range pks {
if i >= 8 {
break
}
fmt.Printf(" #%d %s\n", pk, f(pk))
}
}
show("仅备份有", onlyBackup, func(pk int) string {
v := bk[pk]
return fmt.Sprintf("到访 %s 离开 %s 地点src=%s", ft(v.Arrival), ft(v.Departure), ps(v.LocationSrcPK))
})
show("仅库里有", onlyDB, func(pk int) string {
r := db[pk]
return fmt.Sprintf("到访 %s 离开 %s 地点src=%d 站点新增=%v 备注=%q",
ft(r.arrival), ftp(r.departure), r.placeSrc, r.userAdd, r.remark)
})
show("冲突", bothDiff, func(pk int) string {
return vdiff(bk[pk], db[pk])
})
}
func vdiff(v rond.Visit, r visitRow) string {
var d string
if !eq(v.Arrival, r.arrival) {
d += fmt.Sprintf("到达 %s→%s ", ft(v.Arrival), ft(r.arrival))
}
if !eq(v.Departure, r.departure) {
d += fmt.Sprintf("离开 %s→%s ", ft(v.Departure), ft(r.departure))
}
if pv(v.LocationSrcPK) != r.placeSrc {
d += fmt.Sprintf("地点 %s→%d ", ps(v.LocationSrcPK), r.placeSrc)
}
if av(v.ActivitySrcPK) != r.actSrc {
d += fmt.Sprintf("活动 %s→%d ", ps(v.ActivitySrcPK), r.actSrc)
}
if v.Remark != r.remark {
d += fmt.Sprintf("备注 %q→%q ", v.Remark, r.remark)
}
if v.Emoji != r.emoji {
d += fmt.Sprintf("表情 %q→%q ", v.Emoji, r.emoji)
}
if v.Bookmarked != r.bookmarked {
d += fmt.Sprintf("收藏 %v→%v ", v.Bookmarked, r.bookmarked)
}
return d
}
func diffPlaces(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
fmt.Printf("\n---------- 地点 ----------\n")
type prow struct {
name, poi, cc, prov, city, dist, sub, thor, tz string
lat, lon float64
}
rows, err := pg.QueryContext(ctx, `SELECT src_pk, COALESCE(name,''), COALESCE(poi_category,''),
COALESCE(country_code,''), COALESCE(province,''), COALESCE(city,''), COALESCE(district,''),
COALESCE(sublocality,''), COALESCE(thoroughfare,''), COALESCE(timezone,''),
COALESCE(lat,0), COALESCE(lon,0) FROM places WHERE dataset_id=$1`, dsID)
must(err)
db := map[int]prow{}
for rows.Next() {
var pk int
var r prow
must(rows.Scan(&pk, &r.name, &r.poi, &r.cc, &r.prov, &r.city, &r.dist, &r.sub, &r.thor, &r.tz, &r.lat, &r.lon))
db[pk] = r
}
rows.Close()
bk := map[int]rond.Location{}
for _, l := range b.Locations {
bk[l.SrcPK] = l
}
var onlyB, onlyD, diff []int
for pk, l := range bk {
r, ok := db[pk]
if !ok {
onlyB = append(onlyB, pk)
continue
}
if l.Name != r.name || l.POICategory != r.poi || l.Province != r.prov || l.City != r.city ||
l.District != r.dist || l.Sublocality != r.sub || l.Thoroughfare != r.thor || l.Timezone != r.tz ||
l.CountryCode != r.cc || !near(l.Lat, r.lat) || !near(l.Lon, r.lon) {
diff = append(diff, pk)
}
}
for pk := range db {
if _, ok := bk[pk]; !ok {
onlyD = append(onlyD, pk)
}
}
sort.Ints(onlyB)
sort.Ints(onlyD)
sort.Ints(diff)
fmt.Printf("仅备份有: %d,仅库里有: %d,同号内容不同: %d\n", len(onlyB), len(onlyD), len(diff))
for i, pk := range onlyB {
if i >= 6 {
break
}
fmt.Printf(" 仅备份 #%d %s (%s %s %s)\n", pk, bk[pk].Name, bk[pk].Province, bk[pk].City, bk[pk].POICategory)
}
for i, pk := range onlyD {
if i >= 6 {
break
}
fmt.Printf(" 仅库内 #%d %s (%s %s)\n", pk, db[pk].name, db[pk].prov, db[pk].city)
}
for i, pk := range diff {
if i >= 6 {
break
}
l, r := bk[pk], db[pk]
fmt.Printf(" 冲突 #%d 备份[name=%q city=%q %.6f,%.6f] 库[name=%q city=%q %.6f,%.6f]\n",
pk, l.Name, l.City, l.Lat, l.Lon, r.name, r.city, r.lat, r.lon)
}
}
func diffMovements(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
fmt.Printf("\n---------- 位移 ----------\n")
rows, err := pg.QueryContext(ctx, `SELECT src_pk, COALESCE(transport_src,0),
COALESCE(from_visit_src,0), COALESCE(to_visit_src,0), started_at, ended_at
FROM movements WHERE dataset_id=$1`, dsID)
must(err)
type mrow struct {
t, f, to int
s, e *time.Time
}
db := map[int]mrow{}
for rows.Next() {
var pk int
var r mrow
var s, e sql.NullTime
must(rows.Scan(&pk, &r.t, &r.f, &r.to, &s, &e))
if s.Valid {
r.s = &s.Time
}
if e.Valid {
r.e = &e.Time
}
db[pk] = r
}
rows.Close()
var onlyB, onlyD int
for _, m := range b.Movements {
if _, ok := db[m.SrcPK]; !ok {
onlyB++
}
}
for pk := range db {
found := false
for _, m := range b.Movements {
if m.SrcPK == pk {
found = true
break
}
}
if !found {
onlyD++
}
}
fmt.Printf("备份 %d 条 / 库内 %d 条;仅备份有 %d,仅库里有 %d\n", len(b.Movements), len(db), onlyB, onlyD)
}
func diffWeather(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
var n int
must(pg.QueryRowContext(ctx, `SELECT count(*) FROM weather WHERE dataset_id=$1`, dsID).Scan(&n))
fmt.Printf("\n---------- 天气 ----------\n备份 %d 条 / 库内 %d 条\n", len(b.Weather), n)
}
func diffActivities(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
fmt.Printf("\n---------- 活动类型 ----------\n")
rows, err := pg.QueryContext(ctx, `SELECT src_pk, name FROM activities WHERE dataset_id=$1`, dsID)
must(err)
db := map[int]string{}
for rows.Next() {
var pk int
var n string
must(rows.Scan(&pk, &n))
db[pk] = n
}
rows.Close()
for _, a := range b.Activities {
n, ok := db[a.SrcPK]
if !ok {
fmt.Printf(" 仅备份 #%d %q\n", a.SrcPK, a.Name)
} else if n != a.Name {
fmt.Printf(" 冲突 #%d 备份=%q 库=%q\n", a.SrcPK, a.Name, n)
}
}
for pk, n := range db {
found := false
for _, a := range b.Activities {
if a.SrcPK == pk {
found = true
}
}
if !found {
fmt.Printf(" 仅库内 #%d %q\n", pk, n)
}
}
}
func diffTags(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
var n int
must(pg.QueryRowContext(ctx, `SELECT count(*) FROM tags WHERE dataset_id=$1`, dsID).Scan(&n))
fmt.Printf("\n---------- 标签 ----------\n备份 %d 条 / 库内 %d 条\n", len(b.Tags), n)
}
func eq(a, b *time.Time) bool {
if a == nil || b == nil {
return a == nil && b == nil
}
return a.Equal(*b)
}
func pv(p *int) int {
if p == nil {
return 0
}
return *p
}
func av(p *int) int { return pv(p) }
func ps(p *int) string {
if p == nil {
return "-"
}
return fmt.Sprint(*p)
}
func ft(t *time.Time) string {
if t == nil {
return "-"
}
return t.Local().Format("01-02 15:04")
}
func ftp(t *time.Time) string { return ft(t) }
func near(a, b float64) bool {
d := a - b
return d < 1e-9 && d > -1e-9
}
func must(err error) {
if err != nil {
fmt.Fprintln(os.Stderr, "错误:", err)
os.Exit(1)
}
}
+40
View File
@@ -0,0 +1,40 @@
// 一次性探针:调用站点的导出逻辑产出一份 .rondbackup,用于核对
// 「站点导出 → 导入 rond → rond 再导出」这条链路上 UUID 与 Z_PK 是否还连贯。
package main
import (
"context"
"database/sql"
"fmt"
"os"
"strconv"
_ "github.com/jackc/pgx/v5/stdlib"
"rond-with-you/internal/rond"
)
func main() {
if len(os.Args) < 5 {
fmt.Println("用法: exportprobe <dsn> <datasetID> <baseline.sqlite> <out.rondbackup>")
os.Exit(2)
}
dsID, err := strconv.ParseInt(os.Args[2], 10, 64)
must(err)
db, err := sql.Open("pgx", os.Args[1])
must(err)
defer db.Close()
f, err := os.Create(os.Args[4])
must(err)
defer f.Close()
must(rond.ExportRondbackup(context.Background(), db, dsID, os.Args[3], f))
fmt.Printf("已导出到 %s\n", os.Args[4])
}
func must(err error) {
if err != nil {
fmt.Fprintln(os.Stderr, "错误:", err)
os.Exit(1)
}
}
+133
View File
@@ -0,0 +1,133 @@
// 一次性探针:把一份 .fwss 与 PG 里的 fog_blocks 逐块比对,量化「叠加 / 替换」两种模式的影响。
package main
import (
"database/sql"
"encoding/hex"
"fmt"
"os"
"strconv"
"strings"
_ "github.com/jackc/pgx/v5/stdlib"
"rond-with-you/internal/fog"
)
func main() {
if len(os.Args) < 3 {
fmt.Println("用法: fogprobe <snapshot.fwss> <dsn> [userID]")
os.Exit(2)
}
uid := int64(1)
if len(os.Args) > 3 {
uid, _ = strconv.ParseInt(os.Args[3], 10, 64)
}
data, err := os.ReadFile(os.Args[1])
must(err)
inc, incCells, err := fog.Decode(data)
must(err)
fmt.Printf("快照 %s: %d 块 / %d 格\n", os.Args[1], len(inc), incCells)
pg, err := sql.Open("pgx", os.Args[2])
must(err)
defer pg.Close()
rows, err := pg.Query(`SELECT gbx, gby, bitmap FROM fog_blocks WHERE user_id=$1`, uid)
must(err)
local := map[[2]int][]byte{}
localCells := 0
for rows.Next() {
var gx, gy int
var bm []byte
must(rows.Scan(&gx, &gy, &bm))
local[[2]int{gx, gy}] = bm
localCells += popcount(bm)
}
rows.Close()
fmt.Printf("库内: %d 块 / %d 格\n", len(local), localCells)
var newBlocks, newCells int
var novBlock, overlap int
var overlapGain int
for _, b := range inc {
k := [2]int{b.GX, b.GY}
old, ok := local[k]
if !ok {
newBlocks++
newCells += popcount(b.Bitmap)
continue
}
novBlock++
gain := 0
for i := range b.Bitmap {
gain += poptab[b.Bitmap[i]&^old[i]]
}
if gain > 0 {
overlap++
overlapGain += gain
}
}
var lostBlocks, lostCells, lostFull int
for k, bm := range local {
found := false
for _, b := range inc {
if b.GX == k[0] && b.GY == k[1] {
found = true
break
}
}
if !found {
lostBlocks++
lostCells += popcount(bm)
lostFull++
}
}
fmt.Printf("\n【叠加合并 overlay】\n")
fmt.Printf(" 新增整块: %d 块(+%d 格)\n", newBlocks, newCells)
fmt.Printf(" 与已有块重叠: %d 块,其中 %d 块含新格(+%d 格)\n", novBlock, overlap, overlapGain)
fmt.Printf(" 合计新增格: %d,合并后总量: %d 块 / %d 格\n",
newCells+overlapGain, len(local)+newBlocks, localCells+newCells+overlapGain)
fmt.Printf("\n【整体替换 cover】\n")
fmt.Printf(" 库里现有 %d 块会被丢弃(快照里完全没有这些块),损失 %d 格\n", lostFull, lostCells)
fmt.Printf(" 替换后总量: %d 块 / %d 格\n", len(inc), incCells)
fmt.Printf(" 净变化: %+d 块 / %+d 格\n", len(inc)-len(local), incCells-localCells)
fmt.Printf("\n【其它图层】\n")
layers, err := fog.RawLayers(data)
if err == nil {
for k, v := range layers {
fmt.Printf(" %s (%d 字节, %s)\n", k, len(v), hex.EncodeToString(v[:min(8, len(v))]))
}
}
}
func min(a, b int) int {
if a < b {
return a
}
return b
}
var poptab = func() [256]int {
var t [256]int
for i := range t {
t[i] = strings.Count(strconv.FormatUint(uint64(i), 2), "1")
}
return t
}()
func popcount(b []byte) int {
n := 0
for _, v := range b {
n += poptab[v]
}
return n
}
func must(err error) {
if err != nil {
fmt.Fprintln(os.Stderr, "错误:", err)
os.Exit(1)
}
}
+171
View File
@@ -0,0 +1,171 @@
// 一次性探针:检查 rond 备份里是否存在「跨备份稳定」的对齐键。
// 用完即可删。
package main
import (
"database/sql"
"fmt"
"os"
"strings"
"rond-with-you/internal/rond"
)
func main() {
if len(os.Args) < 3 {
fmt.Println("用法: idprobe <a.rondbackup> <b.rondbackup>")
os.Exit(2)
}
a := load(os.Args[1])
b := load(os.Args[2])
fmt.Printf("\n=== A=%s ===\n=== B=%s ===\n", os.Args[1], os.Args[2])
fmt.Printf("\n---------- ZVISIT.ZIDENTIFIER_ 稳定性 ----------\n")
fmt.Printf("A 有 UUID %d / %d,B 有 UUID %d / %d\n", len(a.visitUUID), len(a.visitPK), len(b.visitUUID), len(b.visitPK))
inter := 0
for u := range a.visitUUID {
if _, ok := b.visitUUID[u]; ok {
inter++
}
}
fmt.Printf("两次备份 UUID 交集: %d(A 独有 %d,B 独有 %d)\n",
inter, len(a.visitUUID)-inter, len(b.visitUUID)-inter)
// UUID 相同则对比 pk 是否相同
samePK, diffPK := 0, 0
for u, pa := range a.visitUUID {
if pb, ok := b.visitUUID[u]; ok {
if pa == pb {
samePK++
} else {
diffPK++
}
}
}
fmt.Printf("UUID 相同者中:Z_PK 也相同 %d,Z_PK 已变 %d\n", samePK, diffPK)
fmt.Printf("\n---------- ZACTIVITY.ZUID_ ----------\n")
fmt.Printf("A UUID %d / %d,B UUID %d / %d\n", len(a.actUUID), len(a.actPK), len(b.actUUID), len(b.actPK))
same, dif := 0, 0
for u, pa := range a.actUUID {
if pb, ok := b.actUUID[u]; ok {
if pa == pb {
same++
} else {
dif++
}
}
}
fmt.Printf("交集配对:Z_PK 相同 %d,Z_PK 已变 %d\n", same, dif)
fmt.Printf("\n---------- ZLOCATION 可用稳定列 ----------\n")
fmt.Printf("A 列: %s\n", strings.Join(a.locCols, ", "))
fmt.Printf("A 地点 %d,B 地点 %d\n", len(a.locPK), len(b.locPK))
// 名称+粗坐标是否可作键
type lk struct {
name string
lat float64
lon float64
}
keyA := map[string][]int{}
for pk, l := range a.locPK {
k := fmt.Sprintf("%s|%.5f|%.5f", l.name, l.lat, l.lon)
keyA[k] = append(keyA[k], pk)
}
hit, miss := 0, 0
for pk, l := range b.locPK {
k := fmt.Sprintf("%s|%.5f|%.5f", l.name, l.lat, l.lon)
if _, ok := keyA[k]; ok {
hit++
} else {
miss++
if miss <= 5 {
fmt.Printf(" B 独有地点 #%d %q %.6f,%.6f\n", pk, l.name, l.lat, l.lon)
}
}
}
fmt.Printf("按「名称+坐标(1e-5)」键:B 命中 A %d,未命中 %d\n", hit, miss)
}
type loc struct {
name string
lat, lon float64
}
type snap struct {
visitUUID map[string]int
visitPK map[int]bool
actUUID map[string]int
actPK map[int]bool
locPK map[int]loc
locCols []string
}
func load(path string) *snap {
dir, err := os.MkdirTemp("", "idprobe")
must(err)
defer os.RemoveAll(dir)
db, err := rond.Open(path, dir)
must(err)
defer db.Close()
s := &snap{visitUUID: map[string]int{}, visitPK: map[int]bool{},
actUUID: map[string]int{}, actPK: map[int]bool{}, locPK: map[int]loc{}}
rows, err := db.Query(`SELECT Z_PK, hex(ZIDENTIFIER_) FROM ZVISIT`)
must(err)
for rows.Next() {
var pk int
var u sql.NullString
must(rows.Scan(&pk, &u))
s.visitPK[pk] = true
if u.Valid && u.String != "" && strings.Trim(u.String, "0") != "" {
s.visitUUID[u.String] = pk
}
}
rows.Close()
rows, err = db.Query(`SELECT Z_PK, hex(ZUID_), COALESCE(ZNAME_,'') FROM ZACTIVITY`)
must(err)
for rows.Next() {
var pk int
var u sql.NullString
var n string
must(rows.Scan(&pk, &u, &n))
s.actPK[pk] = true
if u.Valid && u.String != "" && strings.Trim(u.String, "0") != "" {
s.actUUID[u.String] = pk
}
}
rows.Close()
rows, err = db.Query(`PRAGMA table_info(ZLOCATION)`)
must(err)
for rows.Next() {
var cid int
var name, typ string
var notnull int
var dflt sql.NullString
var pk int
must(rows.Scan(&cid, &name, &typ, &notnull, &dflt, &pk))
s.locCols = append(s.locCols, name)
}
rows.Close()
rows, err = db.Query(`SELECT Z_PK, COALESCE(ZNAME_,''), COALESCE(ZLATITUDE,0), COALESCE(ZLONGITUDE,0) FROM ZLOCATION`)
must(err)
for rows.Next() {
var pk int
var l loc
must(rows.Scan(&pk, &l.name, &l.lat, &l.lon))
s.locPK[pk] = l
}
rows.Close()
return s
}
func must(err error) {
if err != nil {
fmt.Fprintln(os.Stderr, "错误:", err)
os.Exit(1)
}
}
+57
View File
@@ -0,0 +1,57 @@
// 一次性探针:在 rond 备份的 SQLite 上跑一条查询并打印结果(BLOB 打十六进制)。
package main
import (
"fmt"
"os"
"strings"
"rond-with-you/internal/rond"
)
func main() {
if len(os.Args) < 3 {
fmt.Println(`用法: sqlprobe <backup> "SELECT ..."`)
os.Exit(2)
}
dir, err := os.MkdirTemp("", "sqlprobe")
if err != nil {
panic(err)
}
defer os.RemoveAll(dir)
db, err := rond.Open(os.Args[1], dir)
if err != nil {
panic(err)
}
defer db.Close()
rows, err := db.Query(os.Args[2])
if err != nil {
panic(err)
}
defer rows.Close()
cols, _ := rows.Columns()
fmt.Println(strings.Join(cols, " | "))
for rows.Next() {
vals := make([]any, len(cols))
ptrs := make([]any, len(cols))
for i := range vals {
ptrs[i] = &vals[i]
}
if err := rows.Scan(ptrs...); err != nil {
panic(err)
}
parts := make([]string, len(cols))
for i, v := range vals {
switch t := v.(type) {
case nil:
parts[i] = "-"
case []byte:
parts[i] = fmt.Sprintf("%x", t)
default:
parts[i] = fmt.Sprint(t)
}
}
fmt.Println(strings.Join(parts, " | "))
}
}