备份与迷雾的差异分析、合并执行,附属数据改挂坐标键
备份(.rondbackup)与迷雾(.fwss)各加一层「先看差异再动手」: - 迷雾:上传先出差异页,说清叠加能补多少、整体替换会丢多少,并在真实底图上 用绿/灰/红标出会新增 / 不变 / 会丢的区域 - 备份:与当前数据集按 UUID 逐条比对(rond 每次导出都会重排 Z_PK,编号认不出人), 新增合入、冲突逐条选、仅站内有逐条勾选删除,全在一个事务里执行; 合入的行写回 entity_raw,保证再传同一份备份时差异归零 - 地点按坐标对齐:站点的「补全地址」会把地点改名,拿名称当键会让一整个地点的 到访全变成假冲突 - 附属数据(备注/结论/访客票/图片)从 dataset_id+src_pk 换成坐标键,换数据集不再失联; 访客票没有 dataset_id,回填按活跃数据集认领 - 修正 contentHash 拿 Z_PK 当指纹的隐患,以及两处「src_pk 跨快照稳定」的错误注释 导出格式(.rondbackup / .fwss)保持原样,往返测试通过。
This commit is contained in:
+150
-13
@@ -171,38 +171,50 @@ CREATE TABLE IF NOT EXISTS fog_blocks (
|
||||
PRIMARY KEY (user_id, gbx, gby)
|
||||
);
|
||||
|
||||
-- 地点备注与别名。按 src_pk(rond 内部地点 ID,跨快照稳定)而不是 places.id 存,
|
||||
-- 这样重新上传备份、数据集换了新 id 之后备注依然挂着。
|
||||
-- dataset_id 是后加的:src_pk 只在单个数据集内唯一,缺了它切数据集时同编号的
|
||||
-- 地点会把别人的备注/别名认领过去。
|
||||
-- 地点备注与别名。挂**坐标键**(place_key)而不是 src_pk:
|
||||
--
|
||||
-- src_pk(rond 的 Z_PK)只在单个数据集内有效——rond 每次导出/同步都会重排编号
|
||||
-- (实测两份备份里 822 条到访的 UUID 完全一致、Z_PK 100% 不同),所以换一份备份、
|
||||
-- 换个数据集之后,同编号的完全是另一个地点,备注就再也挂不回去了。
|
||||
-- 坐标是地点的地理事实,跨备份稳定:place_key = 经纬度各取 6 位小数拼起来
|
||||
-- (约 0.1m 精度),与 Go 侧的 rond.PlaceCoordKey 必须逐位一致。
|
||||
-- dataset_id / src_pk 保留下来只作「最后一次写入时挂在哪儿」的排查线索。
|
||||
CREATE TABLE IF NOT EXISTS place_notes (
|
||||
user_id BIGINT NOT NULL,
|
||||
place_key TEXT NOT NULL,
|
||||
dataset_id BIGINT NOT NULL DEFAULT 0,
|
||||
src_pk INT NOT NULL,
|
||||
src_pk INT NOT NULL DEFAULT 0,
|
||||
alias TEXT NOT NULL DEFAULT '',
|
||||
note TEXT NOT NULL DEFAULT '',
|
||||
rond_note TEXT NOT NULL DEFAULT '',
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, dataset_id, src_pk)
|
||||
PRIMARY KEY (user_id, place_key)
|
||||
);
|
||||
|
||||
-- 站长对地点的「官方结论」:推荐(1) / 踩雷(-1),与自由文本备注分开存,同样按 src_pk。
|
||||
-- 站长对地点的「官方结论」:推荐(1) / 踩雷(-1),与备注一样按坐标键存。
|
||||
CREATE TABLE IF NOT EXISTS place_votes (
|
||||
user_id BIGINT NOT NULL,
|
||||
place_key TEXT NOT NULL,
|
||||
dataset_id BIGINT NOT NULL DEFAULT 0,
|
||||
src_pk INT NOT NULL,
|
||||
src_pk INT NOT NULL DEFAULT 0,
|
||||
verdict SMALLINT NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, dataset_id, src_pk)
|
||||
PRIMARY KEY (user_id, place_key)
|
||||
);
|
||||
|
||||
-- 访客对地点的投票:一人一票、可改可撤。voter_key 由匿名 cookie + IP 派生,
|
||||
-- 与数据集无关(换快照不丢),故不挂 dataset_id。
|
||||
-- 访客对地点的投票:一人一票、可改可撤。voter_key 由匿名 cookie + IP 派生。
|
||||
--
|
||||
-- 这是站点自己的数据(rondbackup 里没有这一项),但同样要认得出「是哪个地点」:
|
||||
-- 挂 place_key 而不是 src_pk,理由与备注、结论一样。src_pk 只留作排查线索。
|
||||
-- 迁移时这张表没有 dataset_id 可依,只能按**活跃数据集**认领那个编号属于哪个地点
|
||||
-- (见文件末尾的迁移块)。
|
||||
CREATE TABLE IF NOT EXISTS place_visitor_votes (
|
||||
src_pk INT NOT NULL,
|
||||
place_key TEXT NOT NULL,
|
||||
voter_key TEXT NOT NULL,
|
||||
vote SMALLINT NOT NULL,
|
||||
src_pk INT NOT NULL DEFAULT 0,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (src_pk, voter_key)
|
||||
PRIMARY KEY (place_key, voter_key)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_pvv_src ON place_visitor_votes(src_pk);
|
||||
@@ -309,9 +321,17 @@ UPDATE place_notes SET dataset_id = COALESCE((SELECT id FROM datasets WHERE is_a
|
||||
UPDATE place_votes SET dataset_id = COALESCE((SELECT id FROM datasets WHERE is_active ORDER BY id DESC LIMIT 1), 0)
|
||||
WHERE dataset_id = 0;
|
||||
|
||||
-- ⚠️ 这一段只在「还没搬到坐标键」的库上跑:文件末尾那次迁移会把主键换成
|
||||
-- (user_id, place_key),这里要是也跟着改回去,两张表就会在每次启动时来回翻,
|
||||
-- 永远停在最后一个执行的块上。
|
||||
DO $$
|
||||
DECLARE cols text;
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'place_notes' AND column_name = 'place_key') THEN
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
SELECT string_agg(a.attname, ',' ORDER BY array_position(c.conkey, a.attnum))
|
||||
INTO cols
|
||||
FROM pg_constraint c
|
||||
@@ -395,6 +415,7 @@ ALTER TABLE places ADD COLUMN IF NOT EXISTS src_lon DOUBLE PRECISION;
|
||||
-- SQL 里写不出来。启动时由 ingest.BackfillAllCoordSource 统一处理,那里同时能识别出
|
||||
-- 「站点导出的备份给新建地点合成的 ZLOCATION 行」,不会把它误当成真机的原始 GPS。
|
||||
|
||||
|
||||
-- 地点图片:站长给地点配的图。字节由**浏览器**转成 WebP 后再上传(服务端只负责收下与转存),
|
||||
-- 1 核机器不用扛解码。storage 记这一张实际放在哪、object_key 是本地相对路径或对象存储 key:
|
||||
-- 换了存储后端之后旧图仍按各自那一列去取,不会因为改了设置就全变破图。
|
||||
@@ -402,6 +423,9 @@ CREATE TABLE IF NOT EXISTS place_images (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
dataset_id BIGINT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE,
|
||||
src_pk INT NOT NULL,
|
||||
-- place_key 与备注、结论共用同一套坐标键:换数据集后图片还挂得回原地点。
|
||||
-- 对象键里仍然带 dataset_id(那只是存储路径,与「认不认得出是同一个地点」无关)。
|
||||
place_key TEXT NOT NULL DEFAULT '',
|
||||
storage TEXT NOT NULL DEFAULT 'local',
|
||||
object_key TEXT NOT NULL,
|
||||
width INT NOT NULL DEFAULT 0,
|
||||
@@ -412,3 +436,116 @@ CREATE TABLE IF NOT EXISTS place_images (
|
||||
UNIQUE (dataset_id, src_pk, object_key)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_place_images_place ON place_images(dataset_id, src_pk, position);
|
||||
|
||||
-- ---------- 附属数据改挂坐标键(一次性迁移)----------
|
||||
-- 备注 / 结论 / 图片原本挂 (dataset_id, src_pk),换个数据集就全看不见了。
|
||||
-- 这里把它们搬到坐标键上;只在「还没有 place_key 列」时执行,所以新库直接走上面的
|
||||
-- CREATE TABLE,老库走这一段。
|
||||
--
|
||||
-- 坐标键的算法必须与 Go 侧 rond.PlaceCoordKey 逐位一致:
|
||||
-- fmt.Sprintf("%.6f|%.6f", lat, lon) ↔ to_char(lat,'FM999990.000000')||'|'||to_char(lon,'FM999990.000000')
|
||||
-- 已在真实数据上逐条比对过(368/368 相同)。改任何一边都要重新比。
|
||||
DO $$
|
||||
DECLARE
|
||||
cols text;
|
||||
BEGIN
|
||||
-- 判据是**主键**而不是「列在不在」:半迁移过的库(列补上了、主键还是老的三列)
|
||||
-- 也得能修好,否则这张表会一直卡在中间状态。
|
||||
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'place_notes' AND column_name = 'place_key') THEN
|
||||
ALTER TABLE place_notes ADD COLUMN place_key TEXT;
|
||||
END IF;
|
||||
SELECT string_agg(a.attname, ',' ORDER BY array_position(c.conkey, a.attnum))
|
||||
INTO cols FROM pg_constraint c
|
||||
JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = ANY(c.conkey)
|
||||
WHERE c.conname = 'place_notes_pkey';
|
||||
IF cols IS DISTINCT FROM 'user_id,place_key' THEN
|
||||
UPDATE place_notes n SET place_key = (
|
||||
SELECT to_char(p.lat, 'FM999990.000000') || '|' || to_char(p.lon, 'FM999990.000000')
|
||||
FROM places p
|
||||
WHERE p.dataset_id = n.dataset_id AND p.src_pk = n.src_pk
|
||||
AND p.lat IS NOT NULL AND p.lon IS NOT NULL)
|
||||
WHERE n.place_key IS NULL;
|
||||
-- 地点已经不在库里的(被删过),留一个不会撞的孤儿键,宁可留着也别丢
|
||||
UPDATE place_notes SET place_key = 'orphan:' || dataset_id || ':' || src_pk
|
||||
WHERE place_key IS NULL;
|
||||
-- 同一坐标上有多条时只留最新的一条,否则加不上主键
|
||||
DELETE FROM place_notes a USING place_notes b
|
||||
WHERE a.user_id = b.user_id AND a.place_key = b.place_key
|
||||
AND (a.updated_at, a.ctid) < (b.updated_at, b.ctid);
|
||||
ALTER TABLE place_notes ALTER COLUMN place_key SET NOT NULL;
|
||||
ALTER TABLE place_notes DROP CONSTRAINT IF EXISTS place_notes_pkey;
|
||||
ALTER TABLE place_notes ADD PRIMARY KEY (user_id, place_key);
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'place_votes' AND column_name = 'place_key') THEN
|
||||
ALTER TABLE place_votes ADD COLUMN place_key TEXT;
|
||||
END IF;
|
||||
SELECT string_agg(a.attname, ',' ORDER BY array_position(c.conkey, a.attnum))
|
||||
INTO cols FROM pg_constraint c
|
||||
JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = ANY(c.conkey)
|
||||
WHERE c.conname = 'place_votes_pkey';
|
||||
IF cols IS DISTINCT FROM 'user_id,place_key' THEN
|
||||
UPDATE place_votes n SET place_key = (
|
||||
SELECT to_char(p.lat, 'FM999990.000000') || '|' || to_char(p.lon, 'FM999990.000000')
|
||||
FROM places p
|
||||
WHERE p.dataset_id = n.dataset_id AND p.src_pk = n.src_pk
|
||||
AND p.lat IS NOT NULL AND p.lon IS NOT NULL)
|
||||
WHERE n.place_key IS NULL;
|
||||
UPDATE place_votes SET place_key = 'orphan:' || dataset_id || ':' || src_pk
|
||||
WHERE place_key IS NULL;
|
||||
DELETE FROM place_votes a USING place_votes b
|
||||
WHERE a.user_id = b.user_id AND a.place_key = b.place_key
|
||||
AND (a.updated_at, a.ctid) < (b.updated_at, b.ctid);
|
||||
ALTER TABLE place_votes ALTER COLUMN place_key SET NOT NULL;
|
||||
ALTER TABLE place_votes DROP CONSTRAINT IF EXISTS place_votes_pkey;
|
||||
ALTER TABLE place_votes ADD PRIMARY KEY (user_id, place_key);
|
||||
END IF;
|
||||
|
||||
-- 图片自己有 id 主键,只需补列 + 回填;查询走文件末尾那个索引
|
||||
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'place_images' AND column_name = 'place_key') THEN
|
||||
ALTER TABLE place_images ADD COLUMN place_key TEXT NOT NULL DEFAULT '';
|
||||
END IF;
|
||||
UPDATE place_images i SET place_key = COALESCE((
|
||||
SELECT to_char(p.lat, 'FM999990.000000') || '|' || to_char(p.lon, 'FM999990.000000')
|
||||
FROM places p
|
||||
WHERE p.dataset_id = i.dataset_id AND p.src_pk = i.src_pk
|
||||
AND p.lat IS NOT NULL AND p.lon IS NOT NULL),
|
||||
'orphan:' || i.dataset_id || ':' || i.src_pk)
|
||||
WHERE i.place_key = '';
|
||||
|
||||
-- 访客票:这张表没有 dataset_id,回填时只能按**活跃数据集**认领那个编号属于哪个地点。
|
||||
-- 两个数据集里都有同一编号时也只认活跃的那个——这是唯一说得通的猜法,
|
||||
-- 认不出来的留孤儿键,票不丢。
|
||||
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'place_visitor_votes' AND column_name = 'place_key') THEN
|
||||
ALTER TABLE place_visitor_votes ADD COLUMN place_key TEXT;
|
||||
END IF;
|
||||
SELECT string_agg(a.attname, ',' ORDER BY array_position(c.conkey, a.attnum))
|
||||
INTO cols FROM pg_constraint c
|
||||
JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = ANY(c.conkey)
|
||||
WHERE c.conname = 'place_visitor_votes_pkey';
|
||||
IF cols IS DISTINCT FROM 'place_key,voter_key' THEN
|
||||
UPDATE place_visitor_votes vv SET place_key = COALESCE((
|
||||
SELECT to_char(p.lat, 'FM999990.000000') || '|' || to_char(p.lon, 'FM999990.000000')
|
||||
FROM places p JOIN datasets d ON d.id = p.dataset_id
|
||||
WHERE d.is_active AND p.src_pk = vv.src_pk
|
||||
AND p.lat IS NOT NULL AND p.lon IS NOT NULL
|
||||
ORDER BY p.id LIMIT 1),
|
||||
'orphan:' || vv.src_pk)
|
||||
WHERE vv.place_key IS NULL;
|
||||
-- 同一个访客对同一坐标只剩一条:留最新的
|
||||
DELETE FROM place_visitor_votes a USING place_visitor_votes b
|
||||
WHERE a.voter_key = b.voter_key AND a.place_key = b.place_key
|
||||
AND (a.updated_at, a.ctid) < (b.updated_at, b.ctid);
|
||||
ALTER TABLE place_visitor_votes ALTER COLUMN place_key SET NOT NULL;
|
||||
ALTER TABLE place_visitor_votes DROP CONSTRAINT IF EXISTS place_visitor_votes_pkey;
|
||||
ALTER TABLE place_visitor_votes ADD PRIMARY KEY (place_key, voter_key);
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_place_images_key ON place_images(place_key, position);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_pvv_key ON place_visitor_votes(place_key);
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package fog
|
||||
|
||||
// Diff 是一份待导入快照与当前迷雾之间的差异画像。
|
||||
//
|
||||
// 导入有两种语义完全不同的模式:叠加只做按位或、不可能丢数据;整体替换会先清空,
|
||||
// 快照里没有的块一律消失。以前两种模式在界面上只是一个下拉框,站长在下手之前
|
||||
// 看不到「这次能补多少」和「这次会丢多少」,实测里整体替换静默丢掉过 14% 的已探索区域。
|
||||
type Diff struct {
|
||||
CurBlocks, CurCells int // 库里现有
|
||||
IncBlocks, IncCells int // 快照本身
|
||||
|
||||
// GainBlocks/GainCells 是叠加模式真正新增的部分:快照里当前完全没有的整块,
|
||||
// 加上重叠块里补亮的格。
|
||||
GainBlocks int
|
||||
GainCells int
|
||||
|
||||
// OverlapBlocks 是与现有块坐标相同的块数,其中 SameBlocks 是一格都不会变的。
|
||||
OverlapBlocks int
|
||||
SameBlocks int
|
||||
|
||||
// LostBlocks/LostCells 是快照里完全没有、只有整体替换才会消失的部分。
|
||||
LostBlocks int
|
||||
LostCells int
|
||||
|
||||
curAreaM2, incAreaM2, gainAreaM2, lostAreaM2 float64
|
||||
}
|
||||
|
||||
// DiffBlocks 比对当前块集与一份待导入快照。入参块集不要求有序。
|
||||
func DiffBlocks(cur, inc []Block) Diff {
|
||||
var d Diff
|
||||
d.CurBlocks, d.IncBlocks = len(cur), len(inc)
|
||||
|
||||
curMap := make(map[[2]int][]byte, len(cur))
|
||||
for _, b := range cur {
|
||||
curMap[[2]int{b.GX, b.GY}] = b.Bitmap
|
||||
n := popcountSum(b.Bitmap)
|
||||
d.CurCells += n
|
||||
d.curAreaM2 += float64(n) * cellAreaM2(b.GX, b.GY)
|
||||
}
|
||||
|
||||
incSet := make(map[[2]int]struct{}, len(inc))
|
||||
for _, b := range inc {
|
||||
incSet[[2]int{b.GX, b.GY}] = struct{}{}
|
||||
n := popcountSum(b.Bitmap)
|
||||
d.IncCells += n
|
||||
d.incAreaM2 += float64(n) * cellAreaM2(b.GX, b.GY)
|
||||
|
||||
old, ok := curMap[[2]int{b.GX, b.GY}]
|
||||
if !ok {
|
||||
d.GainBlocks++
|
||||
d.GainCells += n
|
||||
d.gainAreaM2 += float64(n) * cellAreaM2(b.GX, b.GY)
|
||||
continue
|
||||
}
|
||||
d.OverlapBlocks++
|
||||
gain := 0
|
||||
for i := range b.Bitmap {
|
||||
gain += int(popcount[b.Bitmap[i]&^old[i]])
|
||||
}
|
||||
if gain == 0 {
|
||||
d.SameBlocks++
|
||||
continue
|
||||
}
|
||||
d.GainCells += gain
|
||||
d.gainAreaM2 += float64(gain) * cellAreaM2(b.GX, b.GY)
|
||||
}
|
||||
|
||||
for _, b := range cur {
|
||||
if _, ok := incSet[[2]int{b.GX, b.GY}]; ok {
|
||||
continue
|
||||
}
|
||||
n := popcountSum(b.Bitmap)
|
||||
d.LostBlocks++
|
||||
d.LostCells += n
|
||||
d.lostAreaM2 += float64(n) * cellAreaM2(b.GX, b.GY)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func (d Diff) CurAreaKm2() float64 { return d.curAreaM2 / 1e6 }
|
||||
func (d Diff) IncAreaKm2() float64 { return d.incAreaM2 / 1e6 }
|
||||
func (d Diff) GainAreaKm2() float64 { return d.gainAreaM2 / 1e6 }
|
||||
func (d Diff) LostAreaKm2() float64 { return d.lostAreaM2 / 1e6 }
|
||||
|
||||
// MergeAreaKm2 是叠加后的总量,CoverAreaKm2 是整体替换后的总量。
|
||||
func (d Diff) MergeAreaKm2() float64 { return (d.curAreaM2 + d.gainAreaM2) / 1e6 }
|
||||
func (d Diff) CoverAreaKm2() float64 { return d.incAreaM2 / 1e6 }
|
||||
|
||||
// Changed 报告叠加是否会让地图有任何变化——为假说明这份快照已经被完全覆盖过。
|
||||
func (d Diff) Changed() bool { return d.GainCells > 0 }
|
||||
@@ -0,0 +1,88 @@
|
||||
package fog
|
||||
|
||||
import "testing"
|
||||
|
||||
// blk 造一个块,cells 是块内的 (列, 行) 坐标。
|
||||
func blk(gx, gy int, cells ...[2]int) Block {
|
||||
bm := make([]byte, bitmapSize)
|
||||
for _, c := range cells {
|
||||
bm[c[1]*8+c[0]/8] |= 0x80 >> (c[0] % 8)
|
||||
}
|
||||
return Block{GX: gx, GY: gy, Bitmap: bm}
|
||||
}
|
||||
|
||||
func TestDiffEmptyBase(t *testing.T) {
|
||||
d := DiffBlocks(nil, []Block{blk(10, 20, [2]int{0, 0}, [2]int{1, 0})})
|
||||
if d.CurBlocks != 0 || d.CurCells != 0 {
|
||||
t.Fatalf("空库不该有现有数据: %+v", d)
|
||||
}
|
||||
if d.GainBlocks != 1 || d.GainCells != 2 || d.IncCells != 2 {
|
||||
t.Fatalf("空库叠加应为全部新增: %+v", d)
|
||||
}
|
||||
if d.LostBlocks != 0 || d.LostCells != 0 || !d.Changed() {
|
||||
t.Fatalf("空库不该有丢失: %+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiffIdentical(t *testing.T) {
|
||||
cur := []Block{blk(10, 20, [2]int{0, 0}, [2]int{1, 0}), blk(11, 20, [2]int{5, 5})}
|
||||
d := DiffBlocks(cur, cur)
|
||||
if d.GainCells != 0 || d.Changed() {
|
||||
t.Fatalf("同一份快照再叠一次不该有任何新增: %+v", d)
|
||||
}
|
||||
if d.LostBlocks != 0 || d.LostCells != 0 {
|
||||
t.Fatalf("同一份快照不该有可丢的部分: %+v", d)
|
||||
}
|
||||
if d.OverlapBlocks != 2 || d.SameBlocks != 2 {
|
||||
t.Fatalf("两个块都应算作「重叠且无变化」: %+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiffOverlapGain(t *testing.T) {
|
||||
d := DiffBlocks(
|
||||
[]Block{blk(10, 20, [2]int{0, 0})},
|
||||
[]Block{blk(10, 20, [2]int{0, 0}, [2]int{3, 4})})
|
||||
if d.GainBlocks != 0 || d.GainCells != 1 {
|
||||
t.Fatalf("重叠块应只补亮多出的那一格: %+v", d)
|
||||
}
|
||||
if d.OverlapBlocks != 1 || d.SameBlocks != 0 {
|
||||
t.Fatalf("重叠但补亮了,不该记为无变化: %+v", d)
|
||||
}
|
||||
if d.LostBlocks != 0 {
|
||||
t.Fatalf("同一个块不该被判为会丢失: %+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiffDisjoint(t *testing.T) {
|
||||
d := DiffBlocks(
|
||||
[]Block{blk(10, 20, [2]int{0, 0})},
|
||||
[]Block{blk(30, 40, [2]int{1, 1})})
|
||||
if d.GainBlocks != 1 || d.GainCells != 1 {
|
||||
t.Fatalf("另一块应是新增: %+v", d)
|
||||
}
|
||||
if d.LostBlocks != 1 || d.LostCells != 1 {
|
||||
t.Fatalf("库里那块在快照里没有,整体替换会丢掉: %+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
// 面积按块中心纬度做 cos 修正,越靠近两极越小;总量关系也要自洽。
|
||||
func TestDiffAreas(t *testing.T) {
|
||||
equator := DiffBlocks(nil, []Block{blk(100, WorldCells/BlockCells/2, [2]int{1, 1})})
|
||||
high := DiffBlocks(nil, []Block{blk(100, 20000, [2]int{1, 1})})
|
||||
if high.GainAreaKm2() <= 0 {
|
||||
t.Fatalf("高纬单格面积应为正: %v", high.GainAreaKm2())
|
||||
}
|
||||
if !(equator.GainAreaKm2() > high.GainAreaKm2()) {
|
||||
t.Fatalf("赤道单格应大于高纬: %v vs %v", equator.GainAreaKm2(), high.GainAreaKm2())
|
||||
}
|
||||
|
||||
m := DiffBlocks(
|
||||
[]Block{blk(10, 20, [2]int{0, 0})},
|
||||
[]Block{blk(10, 20, [2]int{0, 0}, [2]int{9, 9})})
|
||||
if diff := m.MergeAreaKm2() - (m.CurAreaKm2() + m.GainAreaKm2()); diff > 1e-12 || diff < -1e-12 {
|
||||
t.Fatalf("叠加后总量应等于现有加新增: %v", diff)
|
||||
}
|
||||
if m.CoverAreaKm2() != m.IncAreaKm2() {
|
||||
t.Fatalf("整体替换后总量应等于快照本身: %v vs %v", m.CoverAreaKm2(), m.IncAreaKm2())
|
||||
}
|
||||
}
|
||||
@@ -180,6 +180,12 @@ func mercY(lat float64) float64 {
|
||||
return (math.Pi - math.Asinh(math.Tan(rad))) / (2 * math.Pi) * WorldCells
|
||||
}
|
||||
|
||||
// BlockLonLat 返回某块中心的 WGS-84 经纬度,供差异地图定位视野。
|
||||
func BlockLonLat(gbx, gby int) (lat, lon float64) {
|
||||
lon, lat = cellLonLat(gbx*BlockCells+BlockCells/2, gby*BlockCells+BlockCells/2)
|
||||
return
|
||||
}
|
||||
|
||||
func cellLonLat(cx, cy int) (float64, float64) {
|
||||
lon := (float64(cx)+0.5)/WorldCells*360.0 - 180.0
|
||||
n := 2 * math.Pi * (float64(cy) + 0.5) / WorldCells
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
package fog
|
||||
|
||||
import (
|
||||
"image"
|
||||
"math"
|
||||
)
|
||||
|
||||
// 差异图的三种归属。改这里的色值要同步 templates/admin/fog-preview.html 的图例。
|
||||
var (
|
||||
DiffBothColor = [4]byte{126, 134, 146, 148} // 两边都有:导入不影响
|
||||
DiffGainColor = [4]byte{36, 164, 90, 205} // 只有快照有:叠加会新增
|
||||
DiffLostColor = [4]byte{214, 56, 56, 210} // 只有库里有:整体替换会丢
|
||||
)
|
||||
|
||||
const (
|
||||
diffNone uint8 = iota
|
||||
diffBoth
|
||||
diffGain
|
||||
diffLost
|
||||
)
|
||||
|
||||
// RenderDiffTile 渲染一张 256x256 的差异瓦片,供站长在真实底图上核对
|
||||
// 「这份快照能补上哪里」与「整体替换会丢掉哪里」。
|
||||
//
|
||||
// 与 RenderTile 一样按底图坐标系做 WGS84→GCJ02 纠偏;不做描边——三种颜色
|
||||
// 已经够区分,再描边在粗缩放级下会糊成一片。
|
||||
//
|
||||
// 逐格判定只在放大到「一格不小于一像素」(z>=14)时做;更粗的级别下一个块还不到
|
||||
// 一个像素,按块内占多数的类别整块涂色。绘制顺序 both→gain→lost,让「会新增」
|
||||
// 和「会丢」压住底色,重叠处不至于被灰吃掉。
|
||||
func RenderDiffTile(cur, inc map[[2]int][]byte, z, x, y int, crs string) *image.NRGBA {
|
||||
if z < 0 || z > 22 {
|
||||
return nil
|
||||
}
|
||||
step := WorldCells >> z
|
||||
scale := 256.0 / float64(step)
|
||||
ox, oy := float64(x*step), float64(y*step)
|
||||
|
||||
layer := make([]uint8, 256*256)
|
||||
mark := func(px, py, w, h int, kind uint8) {
|
||||
for r := 0; r < h; r++ {
|
||||
yy := py + r
|
||||
if yy < 0 || yy > 255 {
|
||||
continue
|
||||
}
|
||||
row := yy * 256
|
||||
for c := 0; c < w; c++ {
|
||||
xx := px + c
|
||||
if xx < 0 || xx > 255 {
|
||||
continue
|
||||
}
|
||||
layer[row+xx] = kind
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 直接遍历块表而不是全球块范围:低缩放级下范围循环会膨胀到数十亿次空查询。
|
||||
// GCJ 偏移最多约 700m(74 格),粗裁剪时两侧各放宽两块保证不漏。
|
||||
const margin = 2 * BlockCells
|
||||
lo, hi := float64(x*step-margin), float64((x+1)*step+margin)
|
||||
rlo, rhi := float64(y*step-margin), float64((y+1)*step+margin)
|
||||
shift := crs == "gcj02"
|
||||
|
||||
draw := func(pass uint8, gbx, gby int, cb, ib []byte) {
|
||||
bx, by := float64(gbx*BlockCells), float64(gby*BlockCells)
|
||||
if bx+BlockCells <= lo || bx >= hi || by+BlockCells <= rlo || by >= rhi {
|
||||
return
|
||||
}
|
||||
ddx, ddy := 0.0, 0.0
|
||||
if shift {
|
||||
ddx, ddy = blockShift(gbx, gby)
|
||||
}
|
||||
u0 := (bx + ddx - ox) * scale
|
||||
v0 := (by + ddy - oy) * scale
|
||||
|
||||
if scale >= 1 {
|
||||
w := int(math.Ceil(scale))
|
||||
for row := 0; row < BlockCells; row++ {
|
||||
base := row * 8
|
||||
for colB := 0; colB < 8; colB++ {
|
||||
var cbv, ibv byte
|
||||
if cb != nil {
|
||||
cbv = cb[base+colB]
|
||||
}
|
||||
if ib != nil {
|
||||
ibv = ib[base+colB]
|
||||
}
|
||||
if cbv|ibv == 0 {
|
||||
continue
|
||||
}
|
||||
py := int(math.Floor(v0 + float64(row)*scale))
|
||||
for bit := 0; bit < 8; bit++ {
|
||||
m := byte(0x80 >> bit)
|
||||
kind := cellKind(cbv&m != 0, ibv&m != 0)
|
||||
if kind != pass {
|
||||
continue
|
||||
}
|
||||
px := int(math.Floor(u0 + float64(colB*8+bit)*scale))
|
||||
mark(px, py, w, w, kind)
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// 一格不足一像素:整块按多数类涂,三种混在一起的块以数量最多的为准
|
||||
if dominantKind(blockDiffCounts(cb, ib)) != pass {
|
||||
return
|
||||
}
|
||||
u1 := (bx + BlockCells + ddx - ox) * scale
|
||||
v1 := (by + BlockCells + ddy - oy) * scale
|
||||
px0, py0 := int(math.Floor(u0)), int(math.Floor(v0))
|
||||
px1, py1 := int(math.Ceil(u1)), int(math.Ceil(v1))
|
||||
if px1-px0 < 1 {
|
||||
px1 = px0 + 1
|
||||
}
|
||||
if py1-py0 < 1 {
|
||||
py1 = py0 + 1
|
||||
}
|
||||
mark(px0, py0, px1-px0, py1-py0, pass)
|
||||
}
|
||||
|
||||
for _, pass := range []uint8{diffBoth, diffGain, diffLost} {
|
||||
for key, cb := range cur {
|
||||
draw(pass, key[0], key[1], cb, inc[key])
|
||||
}
|
||||
// 只在快照里的块单独走一遍,两边都有的块不再重复画
|
||||
for key, ib := range inc {
|
||||
if _, dup := cur[key]; dup {
|
||||
continue
|
||||
}
|
||||
draw(pass, key[0], key[1], nil, ib)
|
||||
}
|
||||
}
|
||||
|
||||
img := image.NewNRGBA(image.Rect(0, 0, 256, 256))
|
||||
for i, kind := range layer {
|
||||
if kind == diffNone {
|
||||
continue
|
||||
}
|
||||
var c [4]byte
|
||||
switch kind {
|
||||
case diffBoth:
|
||||
c = DiffBothColor
|
||||
case diffGain:
|
||||
c = DiffGainColor
|
||||
default:
|
||||
c = DiffLostColor
|
||||
}
|
||||
o := i * 4
|
||||
img.Pix[o], img.Pix[o+1], img.Pix[o+2], img.Pix[o+3] = c[0], c[1], c[2], c[3]
|
||||
}
|
||||
return img
|
||||
}
|
||||
|
||||
func cellKind(inCur, inInc bool) uint8 {
|
||||
switch {
|
||||
case inCur && inInc:
|
||||
return diffBoth
|
||||
case inInc:
|
||||
return diffGain
|
||||
case inCur:
|
||||
return diffLost
|
||||
}
|
||||
return diffNone
|
||||
}
|
||||
|
||||
// blockDiffCounts 数一个块里三类各占多少格。两个位图都可能缺失(只有一边有这个块)。
|
||||
func blockDiffCounts(cb, ib []byte) (both, gain, lost int) {
|
||||
n := len(cb)
|
||||
if len(ib) > n {
|
||||
n = len(ib)
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
var c, in byte
|
||||
if i < len(cb) {
|
||||
c = cb[i]
|
||||
}
|
||||
if i < len(ib) {
|
||||
in = ib[i]
|
||||
}
|
||||
both += int(popcount[c&in])
|
||||
gain += int(popcount[in&^c])
|
||||
lost += int(popcount[c&^in])
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// dominantKind 取三类里占得最多的一类,用来在粗缩放级下决定整块的着色。
|
||||
func dominantKind(both, gain, lost int) uint8 {
|
||||
switch {
|
||||
case gain > 0 && gain >= both && gain >= lost:
|
||||
return diffGain
|
||||
case lost > 0 && lost >= both:
|
||||
return diffLost
|
||||
case both > 0:
|
||||
return diffBoth
|
||||
}
|
||||
return diffNone
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package fog
|
||||
|
||||
import (
|
||||
"image"
|
||||
"math"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func bmOf(cells ...[2]int) []byte {
|
||||
bm := make([]byte, bitmapSize)
|
||||
for _, c := range cells {
|
||||
bm[c[1]*8+c[0]/8] |= 0x80 >> (c[0] % 8)
|
||||
}
|
||||
return bm
|
||||
}
|
||||
|
||||
func pixAt(img *image.NRGBA, px, py int) [4]byte {
|
||||
i := (py*256 + px) * 4
|
||||
return [4]byte{img.Pix[i], img.Pix[i+1], img.Pix[i+2], img.Pix[i+3]}
|
||||
}
|
||||
|
||||
// z=18 时一格正好 16 像素,块 (100,200) 完整落在瓦片 (400,800) 里,
|
||||
// 格 (col,row) 的中心像素是 (col*16+8, row*16+8)。
|
||||
func TestRenderDiffTilePerCell(t *testing.T) {
|
||||
key := [2]int{100, 200}
|
||||
cur := map[[2]int][]byte{key: bmOf([2]int{0, 0}, [2]int{4, 0})}
|
||||
inc := map[[2]int][]byte{key: bmOf([2]int{0, 0}, [2]int{1, 0})}
|
||||
img := RenderDiffTile(cur, inc, 18, 400, 800, "wgs84")
|
||||
if img == nil {
|
||||
t.Fatal("瓦片不该为空")
|
||||
}
|
||||
if got := pixAt(img, 8, 8); got != DiffBothColor {
|
||||
t.Errorf("两边都有的格应为灰色,实际 %v", got)
|
||||
}
|
||||
if got := pixAt(img, 24, 8); got != DiffGainColor {
|
||||
t.Errorf("只有快照有的格应为绿色,实际 %v", got)
|
||||
}
|
||||
if got := pixAt(img, 72, 8); got != DiffLostColor {
|
||||
t.Errorf("只有库里有、整体替换会丢的格应为红色,实际 %v", got)
|
||||
}
|
||||
if got := pixAt(img, 200, 200); got[3] != 0 {
|
||||
t.Errorf("没被探索的格应保持透明,实际 %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderDiffTileSingleSided(t *testing.T) {
|
||||
key := [2]int{100, 200}
|
||||
onlyCur := RenderDiffTile(map[[2]int][]byte{key: bmOf([2]int{3, 3})}, nil, 18, 400, 800, "wgs84")
|
||||
if got := pixAt(onlyCur, 56, 56); got != DiffLostColor {
|
||||
t.Errorf("快照里完全没有的块应整块为红,实际 %v", got)
|
||||
}
|
||||
onlyInc := RenderDiffTile(nil, map[[2]int][]byte{key: bmOf([2]int{3, 3})}, 18, 400, 800, "wgs84")
|
||||
if got := pixAt(onlyInc, 56, 56); got != DiffGainColor {
|
||||
t.Errorf("库里完全没有的块应整块为绿,实际 %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// z=10 时一格不足一像素,块只占 4x4 像素,按块内多数类涂色。
|
||||
func TestRenderDiffTileCoarseZoomUsesDominantKind(t *testing.T) {
|
||||
key := [2]int{100, 200}
|
||||
cur := map[[2]int][]byte{key: bmOf([2]int{0, 0})}
|
||||
inc := map[[2]int][]byte{key: bmOf([2]int{1, 1}, [2]int{2, 2}, [2]int{3, 3})}
|
||||
img := RenderDiffTile(cur, inc, 10, 1, 3, "wgs84")
|
||||
|
||||
const step, scale = WorldCells >> 10, 256.0 / (WorldCells >> 10)
|
||||
bx, by := float64(key[0]*BlockCells), float64(key[1]*BlockCells)
|
||||
px := int(math.Floor((bx - 1*step) * scale))
|
||||
py := int(math.Floor((by - 3*step) * scale))
|
||||
if got := pixAt(img, px+1, py+1); got != DiffGainColor {
|
||||
t.Errorf("三格新增、一格不变的块应以绿色为主,实际 %v", got)
|
||||
}
|
||||
if w := int(math.Ceil(float64(BlockCells) * scale)); w != 4 {
|
||||
t.Fatalf("z=10 下块应占 4 像素,实际 %d", w)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderDiffTileEdgeCases(t *testing.T) {
|
||||
if RenderDiffTile(nil, nil, 18, 400, 800, "wgs84") == nil {
|
||||
t.Error("两边都空也应返回一张空瓦片")
|
||||
}
|
||||
if RenderDiffTile(nil, nil, 23, 0, 0, "wgs84") != nil {
|
||||
t.Error("超出缩放范围应返回 nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDominantKind(t *testing.T) {
|
||||
cases := []struct {
|
||||
both, gain, lost int
|
||||
want uint8
|
||||
}{
|
||||
{0, 0, 0, diffNone},
|
||||
{5, 1, 0, diffBoth},
|
||||
{1, 5, 0, diffGain},
|
||||
{1, 0, 5, diffLost},
|
||||
{2, 2, 0, diffGain}, // 平局让「会新增」胜出
|
||||
{3, 1, 1, diffBoth},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := dominantKind(c.both, c.gain, c.lost); got != c.want {
|
||||
t.Errorf("dominantKind(%d,%d,%d)=%d,期望 %d", c.both, c.gain, c.lost, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlockDiffCounts(t *testing.T) {
|
||||
both, gain, lost := blockDiffCounts(bmOf([2]int{0, 0}, [2]int{1, 0}), bmOf([2]int{0, 0}, [2]int{2, 0}))
|
||||
if both != 1 || gain != 1 || lost != 1 {
|
||||
t.Errorf("一块里应各有一格:both=%d gain=%d lost=%d", both, gain, lost)
|
||||
}
|
||||
both, gain, lost = blockDiffCounts(nil, bmOf([2]int{5, 5}))
|
||||
if both != 0 || gain != 1 || lost != 0 {
|
||||
t.Errorf("只有快照有该块:both=%d gain=%d lost=%d", both, gain, lost)
|
||||
}
|
||||
}
|
||||
@@ -67,15 +67,41 @@ func insertEntityRaw(ctx context.Context, tx *sql.Tx, datasetID int64, rows map[
|
||||
}
|
||||
|
||||
// contentHash 给备份内容算指纹,用来识别「内容没变、只是重新导出了一次」的上传。
|
||||
// rond 每次导出连 WAL 一起打包,字节必然不同,文件摘要挡不住这种重复;
|
||||
// 但同一批到访的 Z_PK 是稳定的,所以按到访自然键排序后取摘要即可。
|
||||
// rond 每次导出连 WAL 一起打包,字节必然不同,文件摘要挡不住这种重复。
|
||||
//
|
||||
// 指纹按**到访/活动/地点的稳定标识**算,绝不能掺 Z_PK:rond 每次导出/同步都会重排
|
||||
// 编号(实测两份备份里 822 条到访的 UUID 完全一致、Z_PK 100% 不同),拿编号当指纹会把
|
||||
// 「同一批数据重导一次」判成新内容 —— 于是新建数据集,而备注与投票是按 dataset_id
|
||||
// 隔离的,换了数据集就再也看不见。
|
||||
//
|
||||
// 没有 UUID 的老备份退回「到达时间 + 地点坐标」,仍比编号可靠。
|
||||
func contentHash(b *rond.Backup) string {
|
||||
keys := make([]string, 0, len(b.Visits))
|
||||
locCoord := make(map[int]string, len(b.Locations))
|
||||
keys := make([]string, 0, len(b.Visits)+len(b.Locations)+len(b.Activities))
|
||||
for _, l := range b.Locations {
|
||||
locCoord[l.SrcPK] = fmt.Sprintf("%.6f/%.6f", l.Lat, l.Lon)
|
||||
keys = append(keys, "L"+l.Name+"/"+locCoord[l.SrcPK])
|
||||
}
|
||||
for _, v := range b.Visits {
|
||||
if v.Arrival == nil {
|
||||
continue // 导入时就被跳过的行,不该影响指纹
|
||||
}
|
||||
if u := b.VisitUUID[v.SrcPK]; u != "" {
|
||||
keys = append(keys, "V"+u)
|
||||
continue
|
||||
}
|
||||
keys = append(keys, fmt.Sprintf("%d|%d", v.SrcPK, v.Arrival.Unix()))
|
||||
loc := ""
|
||||
if v.LocationSrcPK != nil {
|
||||
loc = locCoord[*v.LocationSrcPK]
|
||||
}
|
||||
keys = append(keys, fmt.Sprintf("T%d/%s", v.Arrival.Unix(), loc))
|
||||
}
|
||||
for _, a := range b.Activities {
|
||||
if u := b.ActivityUUID[a.SrcPK]; u != "" {
|
||||
keys = append(keys, "A"+u)
|
||||
continue
|
||||
}
|
||||
keys = append(keys, "A"+a.Name)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
h := sha256.New()
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"rond-with-you/internal/rond"
|
||||
)
|
||||
|
||||
// sampleBackup 造一份最小的备份:到访 + 地点 + 活动,编号与 UUID 分别给定。
|
||||
func sampleBackup(pks []int, uuids []string) *rond.Backup {
|
||||
b := &rond.Backup{VisitUUID: map[int]string{}, ActivityUUID: map[int]string{}}
|
||||
for i, pk := range pks {
|
||||
at := time.Unix(int64(1700000000+i*3600), 0)
|
||||
b.Visits = append(b.Visits, rond.Visit{SrcPK: pk, Arrival: &at, LocationSrcPK: intp(1)})
|
||||
b.VisitUUID[pk] = uuids[i]
|
||||
}
|
||||
b.Locations = []rond.Location{{SrcPK: 1, Name: "家", Lat: 24.1, Lon: 109.1}}
|
||||
b.Activities = []rond.Activity{{SrcPK: 9, Name: "工作"}}
|
||||
b.ActivityUUID[9] = "cccccccccccccccccccccccccccccccc"
|
||||
return b
|
||||
}
|
||||
|
||||
func intp(v int) *int { return &v }
|
||||
|
||||
// 同一批数据重新导出一次:UUID 一个没变、Z_PK 全被打乱(顺序也变了),
|
||||
// 指纹必须一样。否则会被判成「新内容」→ 新建数据集 → 按 dataset_id 隔离的
|
||||
// 备注与投票全部失联。
|
||||
func TestContentHashIgnoresPKReorder(t *testing.T) {
|
||||
a := sampleBackup([]int{1, 2, 3}, []string{
|
||||
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
||||
"dddddddddddddddddddddddddddddddd",
|
||||
})
|
||||
b := sampleBackup([]int{555, 276, 88}, []string{
|
||||
"dddddddddddddddddddddddddddddddd",
|
||||
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
||||
})
|
||||
if contentHash(a) != contentHash(b) {
|
||||
t.Fatal("编号重排不该改变指纹")
|
||||
}
|
||||
}
|
||||
|
||||
// 内容真的变了(多一条到访)就得算不同的指纹,否则新数据会被当成重复上传而丢掉。
|
||||
func TestContentHashDetectsNewVisit(t *testing.T) {
|
||||
a := sampleBackup([]int{1, 2}, []string{
|
||||
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
||||
})
|
||||
b := sampleBackup([]int{1, 2, 3}, []string{
|
||||
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
||||
"dddddddddddddddddddddddddddddddd",
|
||||
})
|
||||
if contentHash(a) == contentHash(b) {
|
||||
t.Fatal("多了一条到访就是不同内容")
|
||||
}
|
||||
}
|
||||
|
||||
// 只改了地点名(到访一条没动)也算内容变了:站点是靠地名认地点的。
|
||||
func TestContentHashDetectsPlaceRename(t *testing.T) {
|
||||
a := sampleBackup([]int{1}, []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"})
|
||||
b := sampleBackup([]int{1}, []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"})
|
||||
b.Locations[0].Name = "老家"
|
||||
if contentHash(a) == contentHash(b) {
|
||||
t.Fatal("地点改名应改变指纹")
|
||||
}
|
||||
}
|
||||
|
||||
// 没有 UUID 的老备份退回「到达时间 + 地点坐标」,同一批数据重导仍要认出是同一份。
|
||||
func TestContentHashFallsBackWithoutUUID(t *testing.T) {
|
||||
a := sampleBackup([]int{1, 2}, []string{"", ""})
|
||||
b := sampleBackup([]int{700, 900}, []string{"", ""})
|
||||
if contentHash(a) != contentHash(b) {
|
||||
t.Fatal("没有 UUID 时应按到达时间与地点兜底,不该受编号影响")
|
||||
}
|
||||
c := sampleBackup([]int{1, 2}, []string{"", ""})
|
||||
c.Visits[1].Arrival = ptrTime(time.Unix(1700000000, 0)) // 改掉第二条的时间
|
||||
if contentHash(a) == contentHash(c) {
|
||||
t.Fatal("时间不同应算不同内容")
|
||||
}
|
||||
}
|
||||
|
||||
// 没有到达时间的行导入时就被跳过,不该参与指纹。
|
||||
func TestContentHashSkipsRowsWithoutArrival(t *testing.T) {
|
||||
a := sampleBackup([]int{1}, []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"})
|
||||
b := sampleBackup([]int{1}, []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"})
|
||||
b.Visits = append(b.Visits, rond.Visit{SrcPK: 42})
|
||||
if contentHash(a) != contentHash(b) {
|
||||
t.Fatal("缺到达时间的行不该影响指纹")
|
||||
}
|
||||
}
|
||||
|
||||
// 指纹是稳定的十六进制摘要,不是随手拼的字符串。
|
||||
func TestContentHashShape(t *testing.T) {
|
||||
h := contentHash(sampleBackup([]int{1}, []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}))
|
||||
if len(h) != 64 {
|
||||
t.Fatalf("应是 64 位 sha256 十六进制,实际 %d 位: %s", len(h), h)
|
||||
}
|
||||
if _, err := fmt.Sscanf(h, "%x", new(string)); err != nil {
|
||||
t.Fatalf("不是纯十六进制: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func ptrTime(t time.Time) *time.Time { return &t }
|
||||
+62
-37
@@ -1,11 +1,14 @@
|
||||
// Package notes 存放站长为地点加的别名与备注。
|
||||
// 备注按 rond 的地点 ID(src_pk)而非数据库自增 id 存储,
|
||||
// 因此重新上传备份、地点换了新 id 之后内容依然挂在原地。
|
||||
//
|
||||
// 备注挂的是**坐标键**(place_key)而不是 rond 的地点编号:src_pk 只在单个数据集内有效
|
||||
// (rond 每次导出都会重排 Z_PK,实测两份备份里 822 条到访的 UUID 一致而 Z_PK 100% 不同),
|
||||
// 换个数据集同编号就是另一个地点,备注会全部失联。坐标是地理事实,跨数据集稳定。
|
||||
package notes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -20,15 +23,26 @@ type Note struct {
|
||||
|
||||
type Store struct{ DB *sql.DB }
|
||||
|
||||
// All 返回某个数据集的全站备注(站点只有一个站长账号,读的时候不按 user_id 过滤,
|
||||
// 这样匿名访客也能看到别名;写的时候才需要登录用户的 id)。
|
||||
// 后写覆盖先写,保证同一地点只保留一条。必须按 dataset_id 过滤:
|
||||
// src_pk 只在单个数据集内唯一,跨数据集同编号是完全不同的地点。
|
||||
// placeKeyExpr 把某个 places 别名下的坐标换算成 place_key。
|
||||
// **必须与 Go 侧的 rond.PlaceCoordKey 逐位一致**:
|
||||
//
|
||||
// fmt.Sprintf("%.6f|%.6f") ↔ to_char(x,'FM999990.000000')||'|'||to_char(y,'FM999990.000000')
|
||||
//
|
||||
// 已在真实数据上逐条比对(368/368 相同)。改任何一边都要重新比。
|
||||
func placeKeyExpr(alias string) string {
|
||||
return fmt.Sprintf(
|
||||
"to_char(%[1]s.lat,'FM999990.000000')||'|'||to_char(%[1]s.lon,'FM999990.000000')", alias)
|
||||
}
|
||||
|
||||
// All 返回某个数据集里「有备注的地点」,键是地点在该数据集内的编号。
|
||||
// 站点只有一个站长账号,读的时候不按 user_id 过滤(匿名访客也要看到别名)。
|
||||
func (s *Store) All(ctx context.Context, datasetID int64) (map[int]Note, error) {
|
||||
out := map[int]Note{}
|
||||
rows, err := s.DB.QueryContext(ctx,
|
||||
`SELECT src_pk, alias, note, rond_note, updated_at FROM place_notes
|
||||
WHERE dataset_id=$1 ORDER BY updated_at DESC`, datasetID)
|
||||
rows, err := s.DB.QueryContext(ctx, `
|
||||
SELECT p.src_pk, n.alias, n.note, n.rond_note, n.updated_at
|
||||
FROM places p
|
||||
JOIN place_notes n ON n.place_key = `+placeKeyExpr("p")+`
|
||||
WHERE p.dataset_id = $1 AND p.lat IS NOT NULL AND p.lon IS NOT NULL`, datasetID)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
@@ -46,9 +60,9 @@ func (s *Store) All(ctx context.Context, datasetID int64) (map[int]Note, error)
|
||||
// SetNote 只改备注、保留别名(批量「清空备注」用);两者都为空则删除记录。
|
||||
func (s *Store) SetNote(ctx context.Context, userID, datasetID int64, srcPK int, content string) error {
|
||||
var alias string
|
||||
err := s.DB.QueryRowContext(ctx,
|
||||
`SELECT alias FROM place_notes WHERE user_id=$1 AND dataset_id=$2 AND src_pk=$3`,
|
||||
userID, datasetID, srcPK).Scan(&alias)
|
||||
err := s.DB.QueryRowContext(ctx, `SELECT n.alias FROM place_notes n
|
||||
JOIN places p ON n.place_key = `+placeKeyExpr("p")+`
|
||||
WHERE n.user_id=$1 AND p.dataset_id=$2 AND p.src_pk=$3`, userID, datasetID, srcPK).Scan(&alias)
|
||||
if err != nil && err != sql.ErrNoRows {
|
||||
return err
|
||||
}
|
||||
@@ -57,8 +71,9 @@ func (s *Store) SetNote(ctx context.Context, userID, datasetID int64, srcPK int,
|
||||
|
||||
// Delete 删除某地点的整条备注与别名。
|
||||
func (s *Store) Delete(ctx context.Context, userID, datasetID int64, srcPK int) error {
|
||||
_, err := s.DB.ExecContext(ctx,
|
||||
`DELETE FROM place_notes WHERE user_id=$1 AND dataset_id=$2 AND src_pk=$3`, userID, datasetID, srcPK)
|
||||
_, err := s.DB.ExecContext(ctx, `DELETE FROM place_notes n USING places p
|
||||
WHERE n.user_id=$1 AND p.dataset_id=$2 AND p.src_pk=$3
|
||||
AND n.place_key = `+placeKeyExpr("p"), userID, datasetID, srcPK)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -68,21 +83,28 @@ func (s *Store) Save(ctx context.Context, userID, datasetID int64, srcPK int, al
|
||||
content = strings.TrimSpace(content)
|
||||
if alias == "" && content == "" {
|
||||
// 没有 rond 备注时直接删行;有则只清空手写部分,别把回填结果一起删掉
|
||||
_, err := s.DB.ExecContext(ctx,
|
||||
`DELETE FROM place_notes WHERE user_id=$1 AND dataset_id=$2 AND src_pk=$3 AND rond_note=''`,
|
||||
userID, datasetID, srcPK)
|
||||
if err != nil {
|
||||
if _, err := s.DB.ExecContext(ctx, `DELETE FROM place_notes n USING places p
|
||||
WHERE n.user_id=$1 AND p.dataset_id=$2 AND p.src_pk=$3
|
||||
AND n.place_key = `+placeKeyExpr("p")+` AND n.rond_note=''`,
|
||||
userID, datasetID, srcPK); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.DB.ExecContext(ctx,
|
||||
`UPDATE place_notes SET alias='', note='', updated_at=now()
|
||||
WHERE user_id=$1 AND dataset_id=$2 AND src_pk=$3`, userID, datasetID, srcPK)
|
||||
_, err := s.DB.ExecContext(ctx, `UPDATE place_notes n
|
||||
SET alias='', note='', updated_at=now()
|
||||
FROM places p
|
||||
WHERE n.user_id=$1 AND p.dataset_id=$2 AND p.src_pk=$3
|
||||
AND n.place_key = `+placeKeyExpr("p"), userID, datasetID, srcPK)
|
||||
return err
|
||||
}
|
||||
_, err := s.DB.ExecContext(ctx, `INSERT INTO place_notes (user_id, dataset_id, src_pk, alias, note, updated_at)
|
||||
VALUES ($1,$2,$3,$4,$5,now())
|
||||
ON CONFLICT (user_id, dataset_id, src_pk)
|
||||
DO UPDATE SET alias=EXCLUDED.alias, note=EXCLUDED.note, updated_at=now()`,
|
||||
// 先按坐标键算出这条地点该挂在哪,找不到该地点就什么也不写
|
||||
_, err := s.DB.ExecContext(ctx, `INSERT INTO place_notes
|
||||
(user_id, place_key, dataset_id, src_pk, alias, note, updated_at)
|
||||
SELECT $1, `+placeKeyExpr("p")+`, $2, $3, $4, $5, now()
|
||||
FROM places p
|
||||
WHERE p.dataset_id=$2 AND p.src_pk=$3 AND p.lat IS NOT NULL AND p.lon IS NOT NULL
|
||||
ON CONFLICT (user_id, place_key) DO UPDATE SET
|
||||
alias=EXCLUDED.alias, note=EXCLUDED.note,
|
||||
dataset_id=EXCLUDED.dataset_id, src_pk=EXCLUDED.src_pk, updated_at=now()`,
|
||||
userID, datasetID, srcPK, alias, content)
|
||||
return err
|
||||
}
|
||||
@@ -92,20 +114,22 @@ func (s *Store) Save(ctx context.Context, userID, datasetID int64, srcPK int, al
|
||||
// 只重写 rond_note 一列,站长手写的 note 一概不动。返回写了多少行。
|
||||
func (s *Store) BackfillRond(ctx context.Context, userID, datasetID int64) (int, error) {
|
||||
// DISTINCT 在「日期 + 内容」这一层做,同一天重复写的同一句只留一条
|
||||
const agg = `
|
||||
agg := `
|
||||
WITH raw AS (
|
||||
SELECT p.src_pk, to_char(v.arrival, 'YYYY-MM-DD') || ' ' || btrim(v.remark) AS line
|
||||
SELECT ` + placeKeyExpr("p") + ` AS key,
|
||||
to_char(v.arrival, 'YYYY-MM-DD') || ' ' || btrim(v.remark) AS line
|
||||
FROM visits v JOIN places p ON p.id = v.place_id
|
||||
WHERE v.dataset_id = $1 AND btrim(COALESCE(v.remark, '')) <> ''
|
||||
AND p.lat IS NOT NULL AND p.lon IS NOT NULL
|
||||
), uniq AS (
|
||||
SELECT DISTINCT src_pk, line, left(line, 10) AS day FROM raw
|
||||
SELECT DISTINCT key, line, left(line, 10) AS day FROM raw
|
||||
), agg AS (
|
||||
SELECT src_pk, string_agg(line, E'\n' ORDER BY day, line) AS txt FROM uniq GROUP BY src_pk
|
||||
SELECT key, string_agg(line, E'\n' ORDER BY day, line) AS txt FROM uniq GROUP BY key
|
||||
)`
|
||||
res, err := s.DB.ExecContext(ctx, agg+`
|
||||
INSERT INTO place_notes (user_id, dataset_id, src_pk, alias, note, rond_note, updated_at)
|
||||
SELECT $2, $1, src_pk, '', '', txt, now() FROM agg
|
||||
ON CONFLICT (user_id, dataset_id, src_pk) DO UPDATE SET rond_note = EXCLUDED.rond_note`,
|
||||
INSERT INTO place_notes (user_id, place_key, dataset_id, src_pk, alias, note, rond_note, updated_at)
|
||||
SELECT $2, key, $1, 0, '', '', txt, now() FROM agg
|
||||
ON CONFLICT (user_id, place_key) DO UPDATE SET rond_note = EXCLUDED.rond_note`,
|
||||
datasetID, userID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
@@ -113,15 +137,16 @@ func (s *Store) BackfillRond(ctx context.Context, userID, datasetID int64) (int,
|
||||
affected, _ := res.RowsAffected()
|
||||
|
||||
// 备注被删掉的地点要清掉旧值,否则会一直留着过期的 rond 内容。
|
||||
// 限定在本数据集的地点里:datasetID 不合法时子查询为空,一行都不会动,
|
||||
// 不会把别的数据集的回填结果误清掉。
|
||||
// 限定在本数据集的地点里:datasetID 不合法时子查询为空,一行都不会动。
|
||||
if _, err := s.DB.ExecContext(ctx, `
|
||||
UPDATE place_notes n SET rond_note = ''
|
||||
WHERE n.user_id = $2 AND n.dataset_id = $1 AND n.rond_note <> ''
|
||||
AND n.src_pk IN (SELECT src_pk FROM places WHERE dataset_id = $1)
|
||||
WHERE n.user_id = $2 AND n.rond_note <> ''
|
||||
AND n.place_key IN (SELECT `+placeKeyExpr("p")+`
|
||||
FROM places p WHERE p.dataset_id = $1 AND p.lat IS NOT NULL AND p.lon IS NOT NULL)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM visits v JOIN places p ON p.id = v.place_id
|
||||
WHERE v.dataset_id = $1 AND p.src_pk = n.src_pk AND btrim(COALESCE(v.remark, '')) <> ''
|
||||
WHERE v.dataset_id = $1 AND `+placeKeyExpr("p")+` = n.place_key
|
||||
AND btrim(COALESCE(v.remark, '')) <> ''
|
||||
)`, datasetID, userID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,390 @@
|
||||
package rond
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
// 两份数据的比对。
|
||||
//
|
||||
// 对齐键的选择是这个文件的全部要点:rond 每次导出/同步都会重排 Z_PK(实测两份备份里
|
||||
// 822 条到访的 UUID 完全一致、Z_PK 100% 不同),所以 src_pk 只能在一个数据集内部使用。
|
||||
// 跨备份对齐一律走真机写的 UUID;地点/标签/交通方式没有 UUID,退到「名称 + 坐标」。
|
||||
//
|
||||
// 这里只做判定,不碰数据库、也不写任何东西。
|
||||
|
||||
// StoredVisit 是库里的一条到访,外加从 entity_raw 留档取出的对齐键。
|
||||
type StoredVisit struct {
|
||||
SrcPK int
|
||||
UUID string // 空 = 站点新建的行(真机不会给它 UUID)
|
||||
Arrival *time.Time
|
||||
Departure *time.Time
|
||||
Remark string
|
||||
Emoji string
|
||||
Bookmarked bool
|
||||
UserAdded bool
|
||||
PlaceKey string // 地点对齐键,空 = 该到访没有地点
|
||||
PlaceName string // 仅用于展示
|
||||
ActivityName string
|
||||
}
|
||||
|
||||
// StoredPlace 是库里的一个地点。
|
||||
type StoredPlace struct {
|
||||
SrcPK int
|
||||
Key string
|
||||
Name string
|
||||
Lat, Lon float64
|
||||
Province string
|
||||
City string
|
||||
POICategory string
|
||||
VisitCount int
|
||||
}
|
||||
|
||||
// StoredActivity 是库里的一个活动类型。
|
||||
type StoredActivity struct {
|
||||
SrcPK int
|
||||
Name string
|
||||
UUID string
|
||||
}
|
||||
|
||||
// FieldChange 是冲突里的一项字段差异。From 是库里当前的值,To 是备份里的值。
|
||||
type FieldChange struct {
|
||||
Label string
|
||||
From string
|
||||
To string
|
||||
}
|
||||
|
||||
// VisitConflict 是同一条到访(UUID 相同)两边字段不一致。
|
||||
type VisitConflict struct {
|
||||
Backup Visit
|
||||
DB StoredVisit
|
||||
Fields []FieldChange
|
||||
}
|
||||
|
||||
// VisitDiff 是到访的差异画像。
|
||||
type VisitDiff struct {
|
||||
Added []Visit // 备份有、库里没有
|
||||
Removed []StoredVisit // 库里有、备份里没有
|
||||
Changed []VisitConflict
|
||||
Same int // 对齐上且完全一致
|
||||
}
|
||||
|
||||
// PlaceDiff 是地点的差异画像。
|
||||
type PlaceDiff struct {
|
||||
Added []Location
|
||||
Removed []StoredPlace
|
||||
Changed []PlaceConflict
|
||||
Same int
|
||||
}
|
||||
|
||||
type PlaceConflict struct {
|
||||
Backup Location
|
||||
DB StoredPlace
|
||||
Fields []FieldChange
|
||||
}
|
||||
|
||||
// ActivityDiff 是活动类型的差异画像(按 UUID 对齐,退不到就按名称)。
|
||||
type ActivityDiff struct {
|
||||
Added []Activity
|
||||
Removed []StoredActivity
|
||||
Changed []ActivityConflict
|
||||
Same int
|
||||
}
|
||||
|
||||
type ActivityConflict struct {
|
||||
Backup Activity
|
||||
DB StoredActivity
|
||||
Fields []FieldChange
|
||||
}
|
||||
|
||||
// PlaceCoordKey 是地点的跨备份对齐键:**只认坐标,不认名称**。
|
||||
//
|
||||
// 地点没有 UUID,名称又会被站点自己的「补全缺失地址」改掉(实测:站内把
|
||||
// 「体育馆美食街」反查成了「河池市国民体质监测中心」,坐标一字未动,于是 17 条
|
||||
// 到访全被误报成「换了地点」)。坐标取 6 位小数(约 0.1m)——足以吸收浮点往返误差,
|
||||
// 又不会把挨着的两个地点并成一个。
|
||||
func PlaceCoordKey(lat, lon float64) string {
|
||||
return fmt.Sprintf("%.6f|%.6f", lat, lon)
|
||||
}
|
||||
|
||||
func coordStr(v float64) string { return fmt.Sprintf("%.6f", v) }
|
||||
|
||||
// DiffVisits 按 UUID 对齐两边的到访。
|
||||
//
|
||||
// 真机偶尔有条目没写 UUID(实测 824 行到访里有 2 行),只按 UUID 配对会让同一条既算
|
||||
// 「新增」又算「仅站内有」;所以没有 UUID 的行退到「到达时间 + 地点」兜底配对。
|
||||
// 库里那些连兜底也配不上的(站点新建的行)一律算「库里有、备份没有」。
|
||||
func DiffVisits(stored []StoredVisit, b *Backup) VisitDiff {
|
||||
placeKey, placeName := map[int]string{}, map[int]string{}
|
||||
for _, l := range b.Locations {
|
||||
// 没有坐标的地点不参与坐标对齐,否则它们会共用 "0.000000|0.000000" 互相认领
|
||||
if l.Lat != 0 || l.Lon != 0 {
|
||||
placeKey[l.SrcPK] = PlaceCoordKey(l.Lat, l.Lon)
|
||||
}
|
||||
placeName[l.SrcPK] = placeLabel(l.Name, l.Lat, l.Lon)
|
||||
}
|
||||
actName := map[int]string{}
|
||||
for _, a := range b.Activities {
|
||||
actName[a.SrcPK] = a.Name
|
||||
}
|
||||
|
||||
used := make([]bool, len(stored))
|
||||
byUUID := make(map[string]int, len(stored))
|
||||
byTime := map[string][]int{}
|
||||
for i, sv := range stored {
|
||||
if sv.UUID != "" {
|
||||
byUUID[sv.UUID] = i
|
||||
continue
|
||||
}
|
||||
if sv.Arrival != nil {
|
||||
k := timeKey(*sv.Arrival, sv.PlaceKey)
|
||||
byTime[k] = append(byTime[k], i)
|
||||
}
|
||||
}
|
||||
|
||||
var d VisitDiff
|
||||
classify := func(i int, v Visit) {
|
||||
used[i] = true
|
||||
sv := stored[i]
|
||||
if fields := visitFieldChanges(sv, v, placeKey, placeName, actName); len(fields) > 0 {
|
||||
d.Changed = append(d.Changed, VisitConflict{Backup: v, DB: sv, Fields: fields})
|
||||
} else {
|
||||
d.Same++
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range b.Visits {
|
||||
if v.Arrival == nil {
|
||||
continue // 没有到达时间的行导入时就被跳过了,不该被算成「新增」
|
||||
}
|
||||
if uuid := b.VisitUUID[v.SrcPK]; uuid != "" {
|
||||
if i, ok := byUUID[uuid]; ok {
|
||||
classify(i, v)
|
||||
continue
|
||||
}
|
||||
d.Added = append(d.Added, v)
|
||||
continue
|
||||
}
|
||||
bpk := ""
|
||||
if v.LocationSrcPK != nil {
|
||||
bpk = placeKey[*v.LocationSrcPK]
|
||||
}
|
||||
if list := byTime[timeKey(*v.Arrival, bpk)]; len(list) > 0 {
|
||||
byTime[timeKey(*v.Arrival, bpk)] = list[1:]
|
||||
classify(list[0], v)
|
||||
continue
|
||||
}
|
||||
d.Added = append(d.Added, v)
|
||||
}
|
||||
for i, sv := range stored {
|
||||
if !used[i] {
|
||||
d.Removed = append(d.Removed, sv)
|
||||
}
|
||||
}
|
||||
sortVisits(d)
|
||||
return d
|
||||
}
|
||||
|
||||
// timeKey 是没有 UUID 时的兜底配对键。只到秒:苹果时间戳转过来是浮点秒,
|
||||
// 而 PG 存的是微秒,逐位比会永远配不上。
|
||||
func timeKey(t time.Time, place string) string {
|
||||
return fmt.Sprintf("%d|%s", t.Unix(), place)
|
||||
}
|
||||
|
||||
func visitFieldChanges(sv StoredVisit, v Visit, placeKey, placeName, actName map[int]string) []FieldChange {
|
||||
var out []FieldChange
|
||||
add := func(label, from, to string) {
|
||||
if from != to {
|
||||
out = append(out, FieldChange{Label: label, From: from, To: to})
|
||||
}
|
||||
}
|
||||
if !sameTime(sv.Arrival, v.Arrival) {
|
||||
add("到达时间", fmtTime(sv.Arrival), fmtTime(v.Arrival))
|
||||
}
|
||||
if !sameTime(sv.Departure, v.Departure) {
|
||||
add("离开时间", fmtTime(sv.Departure), fmtTime(v.Departure))
|
||||
}
|
||||
// 地点按坐标判、显示名称;活动按名称判。都不能比 src_pk——两边编号空间不同。
|
||||
bp, bn := "", ""
|
||||
if v.LocationSrcPK != nil {
|
||||
bp, bn = placeKey[*v.LocationSrcPK], placeName[*v.LocationSrcPK]
|
||||
}
|
||||
// 地点按坐标判、显示名称:名称被「补全地址」反查改过,不算换了地点
|
||||
if sv.PlaceKey != bp {
|
||||
out = append(out, FieldChange{Label: "地点", From: sv.PlaceName, To: bn})
|
||||
}
|
||||
bl := ""
|
||||
if v.ActivitySrcPK != nil {
|
||||
bl = actName[*v.ActivitySrcPK]
|
||||
}
|
||||
add("活动", sv.ActivityName, bl)
|
||||
add("备注", sv.Remark, v.Remark)
|
||||
add("表情", sv.Emoji, v.Emoji)
|
||||
add("收藏", fmtBool(sv.Bookmarked), fmtBool(v.Bookmarked))
|
||||
return out
|
||||
}
|
||||
|
||||
// placeLabel 给没有名字的地点一个可读的占位。
|
||||
func placeLabel(name string, lat, lon float64) string {
|
||||
if name != "" {
|
||||
return name
|
||||
}
|
||||
return fmt.Sprintf("%.5f, %.5f", lat, lon)
|
||||
}
|
||||
|
||||
// DiffPlaces 对齐地点。坐标优先(改名不影响配对),配不上再按名称找一遍
|
||||
// (覆盖「站点把坐标纠偏过」这种),两样都对不上才算新增。
|
||||
func DiffPlaces(stored []StoredPlace, b *Backup) PlaceDiff {
|
||||
byCoord := make(map[string]StoredPlace, len(stored))
|
||||
byName := map[string]StoredPlace{}
|
||||
for _, sp := range stored {
|
||||
if sp.Lat != 0 || sp.Lon != 0 {
|
||||
byCoord[PlaceCoordKey(sp.Lat, sp.Lon)] = sp
|
||||
}
|
||||
if _, dup := byName[sp.Name]; !dup && sp.Name != "" {
|
||||
byName[sp.Name] = sp
|
||||
}
|
||||
}
|
||||
used := map[int]bool{}
|
||||
var d PlaceDiff
|
||||
classify := func(sp StoredPlace, l Location) {
|
||||
used[sp.SrcPK] = true
|
||||
if fields := placeFieldChanges(sp, l); len(fields) > 0 {
|
||||
d.Changed = append(d.Changed, PlaceConflict{Backup: l, DB: sp, Fields: fields})
|
||||
} else {
|
||||
d.Same++
|
||||
}
|
||||
}
|
||||
for _, l := range b.Locations {
|
||||
if sp, ok := byCoord[PlaceCoordKey(l.Lat, l.Lon)]; ok && !used[sp.SrcPK] {
|
||||
classify(sp, l)
|
||||
continue
|
||||
}
|
||||
if sp, ok := byName[l.Name]; ok && !used[sp.SrcPK] {
|
||||
classify(sp, l)
|
||||
continue
|
||||
}
|
||||
d.Added = append(d.Added, l)
|
||||
}
|
||||
for _, sp := range stored {
|
||||
if !used[sp.SrcPK] {
|
||||
d.Removed = append(d.Removed, sp)
|
||||
}
|
||||
}
|
||||
sort.Slice(d.Added, func(i, j int) bool { return d.Added[i].Name < d.Added[j].Name })
|
||||
sort.Slice(d.Removed, func(i, j int) bool { return d.Removed[i].Name < d.Removed[j].Name })
|
||||
sort.Slice(d.Changed, func(i, j int) bool { return d.Changed[i].DB.Name < d.Changed[j].DB.Name })
|
||||
return d
|
||||
}
|
||||
|
||||
// placeFieldChanges 比地点上除对齐键之外的字段。名称也算——按坐标配上之后,
|
||||
// 「名字不一样」正是站长最想看到的那类差异。
|
||||
func placeFieldChanges(sp StoredPlace, l Location) []FieldChange {
|
||||
var out []FieldChange
|
||||
add := func(label, from, to string) {
|
||||
if from != to {
|
||||
out = append(out, FieldChange{Label: label, From: from, To: to})
|
||||
}
|
||||
}
|
||||
add("名称", sp.Name, l.Name)
|
||||
add("纬度", coordStr(sp.Lat), coordStr(l.Lat))
|
||||
add("经度", coordStr(sp.Lon), coordStr(l.Lon))
|
||||
add("省份", sp.Province, l.Province)
|
||||
add("城市", sp.City, l.City)
|
||||
add("分类", sp.POICategory, l.POICategory)
|
||||
return out
|
||||
}
|
||||
|
||||
// DiffActivities 优先按 UUID 对齐;库里没有 UUID 的(站点新建)退回名称匹配,
|
||||
// 匹配不上就算「库里有、备份没有」。
|
||||
func DiffActivities(stored []StoredActivity, b *Backup) ActivityDiff {
|
||||
byUUID := map[string]StoredActivity{}
|
||||
byName := map[string]StoredActivity{}
|
||||
for _, sa := range stored {
|
||||
if sa.UUID != "" {
|
||||
byUUID[sa.UUID] = sa
|
||||
}
|
||||
if _, dup := byName[sa.Name]; !dup {
|
||||
byName[sa.Name] = sa
|
||||
}
|
||||
}
|
||||
var d ActivityDiff
|
||||
matched := map[int]bool{}
|
||||
for _, a := range b.Activities {
|
||||
sa, ok := byUUID[b.ActivityUUID[a.SrcPK]]
|
||||
if !ok {
|
||||
if sa, ok = byName[a.Name]; !ok {
|
||||
d.Added = append(d.Added, a)
|
||||
continue
|
||||
}
|
||||
}
|
||||
matched[sa.SrcPK] = true
|
||||
if sa.Name != a.Name {
|
||||
d.Changed = append(d.Changed, ActivityConflict{Backup: a, DB: sa,
|
||||
Fields: []FieldChange{{Label: "名称", From: sa.Name, To: a.Name}}})
|
||||
} else {
|
||||
d.Same++
|
||||
}
|
||||
}
|
||||
for _, sa := range stored {
|
||||
if !matched[sa.SrcPK] {
|
||||
d.Removed = append(d.Removed, sa)
|
||||
}
|
||||
}
|
||||
sort.Slice(d.Added, func(i, j int) bool { return d.Added[i].Name < d.Added[j].Name })
|
||||
sort.Slice(d.Removed, func(i, j int) bool { return d.Removed[i].Name < d.Removed[j].Name })
|
||||
return d
|
||||
}
|
||||
|
||||
func sortVisits(d VisitDiff) {
|
||||
tv := func(t *time.Time) int64 {
|
||||
if t == nil {
|
||||
return 0
|
||||
}
|
||||
return t.Unix()
|
||||
}
|
||||
sort.Slice(d.Added, func(i, j int) bool { return tv(d.Added[i].Arrival) < tv(d.Added[j].Arrival) })
|
||||
sort.Slice(d.Removed, func(i, j int) bool { return tv(d.Removed[i].Arrival) < tv(d.Removed[j].Arrival) })
|
||||
sort.Slice(d.Changed, func(i, j int) bool { return tv(d.Changed[i].Backup.Arrival) < tv(d.Changed[j].Backup.Arrival) })
|
||||
}
|
||||
|
||||
func sameTime(a, b *time.Time) bool {
|
||||
if a == nil || b == nil {
|
||||
return a == nil && b == nil
|
||||
}
|
||||
return a.Equal(*b)
|
||||
}
|
||||
|
||||
func fmtTime(t *time.Time) string {
|
||||
if t == nil {
|
||||
return "—"
|
||||
}
|
||||
return t.Local().Format("2006-01-02 15:04")
|
||||
}
|
||||
|
||||
func fmtBool(v bool) string {
|
||||
if v {
|
||||
return "是"
|
||||
}
|
||||
return "否"
|
||||
}
|
||||
|
||||
// Counts 汇总各实体的三类条数,供概览展示。
|
||||
type Counts struct {
|
||||
Added, Removed, Changed, Same int
|
||||
}
|
||||
|
||||
func (c Counts) Total() int { return c.Added + c.Removed + c.Changed + c.Same }
|
||||
|
||||
func VisitCounts(d VisitDiff) Counts {
|
||||
return Counts{len(d.Added), len(d.Removed), len(d.Changed), d.Same}
|
||||
}
|
||||
|
||||
func PlaceCounts(d PlaceDiff) Counts {
|
||||
return Counts{len(d.Added), len(d.Removed), len(d.Changed), d.Same}
|
||||
}
|
||||
|
||||
func ActivityCounts(d ActivityDiff) Counts {
|
||||
return Counts{len(d.Added), len(d.Removed), len(d.Changed), d.Same}
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
package rond
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func tp(s string) *time.Time {
|
||||
t, err := time.Parse("2006-01-02 15:04", s)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &t
|
||||
}
|
||||
|
||||
func ip(v int) *int { return &v }
|
||||
|
||||
// 两组编号刻意完全不同:rond 每次导出都会重排 Z_PK,对齐只能靠 UUID。
|
||||
const (
|
||||
uuidA = "11111111111111111111111111111111"
|
||||
uuidB = "22222222222222222222222222222222"
|
||||
uuidC = "33333333333333333333333333333333"
|
||||
)
|
||||
|
||||
func sampleBackup() *Backup {
|
||||
return &Backup{
|
||||
Visits: []Visit{
|
||||
{SrcPK: 555, Arrival: tp("2026-09-01 10:00"), Departure: tp("2026-09-01 11:00"),
|
||||
LocationSrcPK: ip(3), ActivitySrcPK: ip(7)},
|
||||
{SrcPK: 556, Arrival: tp("2026-09-02 10:00"), LocationSrcPK: ip(3)},
|
||||
},
|
||||
Locations: []Location{
|
||||
{SrcPK: 3, Name: "家", Lat: 24.1, Lon: 109.1, City: "柳州市"},
|
||||
{SrcPK: 4, Name: "公司", Lat: 24.2, Lon: 109.2, City: "柳州市"},
|
||||
},
|
||||
Activities: []Activity{
|
||||
{SrcPK: 7, Name: "工作"},
|
||||
{SrcPK: 8, Name: "散步"},
|
||||
},
|
||||
VisitUUID: map[int]string{555: uuidA, 556: uuidB},
|
||||
ActivityUUID: map[int]string{7: uuidC},
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiffVisitsAlignsByUUIDNotPK(t *testing.T) {
|
||||
stored := []StoredVisit{
|
||||
{SrcPK: 1, UUID: uuidA, Arrival: tp("2026-09-01 10:00"), Departure: tp("2026-09-01 11:00"),
|
||||
PlaceKey: PlaceCoordKey(24.1, 109.1), ActivityName: "工作"},
|
||||
{SrcPK: 2, UUID: uuidB, Arrival: tp("2026-09-02 10:00"),
|
||||
PlaceKey: PlaceCoordKey(24.1, 109.1)},
|
||||
}
|
||||
d := DiffVisits(stored, sampleBackup())
|
||||
if len(d.Added) != 0 || len(d.Removed) != 0 || len(d.Changed) != 0 {
|
||||
t.Fatalf("两边一致时不该有差异: +%d -%d ~%d", len(d.Added), len(d.Removed), len(d.Changed))
|
||||
}
|
||||
if d.Same != 2 {
|
||||
t.Fatalf("应配上 2 条,实际 %d", d.Same)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiffVisitsFieldChanges(t *testing.T) {
|
||||
stored := []StoredVisit{{
|
||||
SrcPK: 1, UUID: uuidA, Arrival: tp("2026-09-01 09:00"), Departure: tp("2026-09-01 11:00"),
|
||||
PlaceKey: PlaceCoordKey(24.1, 109.1), ActivityName: "工作", Remark: "站点备注",
|
||||
}}
|
||||
d := DiffVisits(stored, sampleBackup())
|
||||
if len(d.Changed) != 1 {
|
||||
t.Fatalf("到达时间不同应算冲突,实际 %d 条", len(d.Changed))
|
||||
}
|
||||
f := d.Changed[0].Fields
|
||||
if len(f) != 2 {
|
||||
t.Fatalf("应报出到达时间与备注两项,实际 %v", f)
|
||||
}
|
||||
if f[0].Label != "到达时间" || f[1].Label != "备注" {
|
||||
t.Fatalf("字段名不对: %v", f)
|
||||
}
|
||||
if f[1].From != "站点备注" || f[1].To != "" {
|
||||
t.Fatalf("方向搞反了(From 应是库里的值): %+v", f[1])
|
||||
}
|
||||
}
|
||||
|
||||
// 站点新建的到访没有 UUID,而且它的 src_pk 很可能与真机新行的编号撞上——
|
||||
// 绝不能因为编号相同就当成同一条。
|
||||
func TestDiffVisitsIgnoresPKCollision(t *testing.T) {
|
||||
stored := []StoredVisit{{
|
||||
SrcPK: 555, UUID: "", Arrival: tp("2026-08-01 08:00"), UserAdded: true,
|
||||
PlaceKey: PlaceCoordKey(30.0, 120.0),
|
||||
}}
|
||||
d := DiffVisits(stored, sampleBackup())
|
||||
if len(d.Added) != 2 {
|
||||
t.Fatalf("备份里两条都是新增,实际 %d", len(d.Added))
|
||||
}
|
||||
if len(d.Removed) != 1 {
|
||||
t.Fatalf("站点补录那条应算「仅库里有」,实际 %d", len(d.Removed))
|
||||
}
|
||||
if !d.Removed[0].UserAdded {
|
||||
t.Error("站点补录的标记要带上,页面上要区分开")
|
||||
}
|
||||
}
|
||||
|
||||
// 真机偶尔有不带 UUID 的到访,只按 UUID 配对会让同一条既算新增又算仅站内有。
|
||||
func TestDiffVisitsFallsBackWhenUUIDMissing(t *testing.T) {
|
||||
b := &Backup{
|
||||
Visits: []Visit{
|
||||
{SrcPK: 900, Arrival: tp("2026-09-01 10:00"), LocationSrcPK: ip(3)},
|
||||
{SrcPK: 901, Arrival: tp("2026-09-02 10:00"), LocationSrcPK: ip(3)},
|
||||
},
|
||||
Locations: []Location{{SrcPK: 3, Name: "家", Lat: 24.1, Lon: 109.1}},
|
||||
VisitUUID: map[int]string{}, // 一条都没有 UUID
|
||||
ActivityUUID: map[int]string{},
|
||||
}
|
||||
stored := []StoredVisit{
|
||||
{SrcPK: 1, Arrival: tp("2026-09-01 10:00"), PlaceKey: PlaceCoordKey(24.1, 109.1)},
|
||||
{SrcPK: 2, Arrival: tp("2026-09-02 10:00"), PlaceKey: PlaceCoordKey(24.1, 109.1)},
|
||||
}
|
||||
d := DiffVisits(stored, b)
|
||||
if len(d.Added) != 0 || len(d.Removed) != 0 {
|
||||
t.Fatalf("没有 UUID 时应按「到达时间 + 地点」兜底配对: +%d -%d", len(d.Added), len(d.Removed))
|
||||
}
|
||||
if d.Same != 2 {
|
||||
t.Fatalf("两条都该配上,实际 %d", d.Same)
|
||||
}
|
||||
}
|
||||
|
||||
// 兜底配对要认得出「同一条但时间被改过」——时间不同就配不上,此时按新增 + 仅站内有报出,
|
||||
// 总比静默当成同一条好。
|
||||
func TestDiffVisitsFallbackNeedsSameTime(t *testing.T) {
|
||||
b := &Backup{
|
||||
Visits: []Visit{{SrcPK: 900, Arrival: tp("2026-09-01 10:00"), LocationSrcPK: ip(3)}},
|
||||
Locations: []Location{{SrcPK: 3, Name: "家", Lat: 24.1, Lon: 109.1}},
|
||||
VisitUUID: map[int]string{},
|
||||
}
|
||||
stored := []StoredVisit{{SrcPK: 1, Arrival: tp("2026-09-01 11:00"), PlaceKey: PlaceCoordKey(24.1, 109.1)}}
|
||||
d := DiffVisits(stored, b)
|
||||
if len(d.Added) != 1 || len(d.Removed) != 1 {
|
||||
t.Fatalf("时间对不上时该分别报出: +%d -%d", len(d.Added), len(d.Removed))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiffVisitsSkipsRowsWithoutArrival(t *testing.T) {
|
||||
b := sampleBackup()
|
||||
b.Visits = append(b.Visits, Visit{SrcPK: 557, LocationSrcPK: ip(3)})
|
||||
d := DiffVisits(nil, b)
|
||||
if len(d.Added) != 2 {
|
||||
t.Fatalf("没有到达时间的行导入时就被跳过,不该算新增,实际 %d", len(d.Added))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiffPlacesByKey(t *testing.T) {
|
||||
b := sampleBackup()
|
||||
stored := []StoredPlace{
|
||||
{SrcPK: 10, Key: PlaceCoordKey(24.1, 109.1), Name: "家", Lat: 24.1, Lon: 109.1, City: "柳州市"},
|
||||
}
|
||||
d := DiffPlaces(stored, b)
|
||||
if len(d.Added) != 1 || d.Added[0].Name != "公司" {
|
||||
t.Fatalf("应只有「公司」是新增: %+v", d.Added)
|
||||
}
|
||||
if len(d.Removed) != 0 || len(d.Changed) != 0 || d.Same != 1 {
|
||||
t.Fatalf("「家」应配上且无差异: same=%d changed=%d removed=%d", d.Same, len(d.Changed), len(d.Removed))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiffPlacesReportsFieldChanges(t *testing.T) {
|
||||
b := sampleBackup()
|
||||
stored := []StoredPlace{
|
||||
{SrcPK: 10, Key: PlaceCoordKey(24.1, 109.1), Name: "家", Lat: 24.1, Lon: 109.1, City: "桂林市"},
|
||||
}
|
||||
d := DiffPlaces(stored, b)
|
||||
if len(d.Changed) != 1 {
|
||||
t.Fatalf("城市不同应算冲突,实际 %d", len(d.Changed))
|
||||
}
|
||||
if f := d.Changed[0].Fields; len(f) != 1 || f[0].Label != "城市" || f[0].From != "桂林市" || f[0].To != "柳州市" {
|
||||
t.Fatalf("字段差异不对: %+v", f)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiffActivities(t *testing.T) {
|
||||
b := sampleBackup()
|
||||
stored := []StoredActivity{
|
||||
{SrcPK: 20, Name: "工作", UUID: uuidC},
|
||||
{SrcPK: 21, Name: "站点自建"},
|
||||
}
|
||||
d := DiffActivities(stored, b)
|
||||
if len(d.Added) != 1 || d.Added[0].Name != "散步" {
|
||||
t.Fatalf("「散步」应是新增: %+v", d.Added)
|
||||
}
|
||||
if len(d.Removed) != 1 || d.Removed[0].Name != "站点自建" {
|
||||
t.Fatalf("站点自建的活动应算仅库里有: %+v", d.Removed)
|
||||
}
|
||||
if d.Same != 1 {
|
||||
t.Fatalf("「工作」应配上,实际 same=%d", d.Same)
|
||||
}
|
||||
}
|
||||
|
||||
// 活动改了名:UUID 还在,就该认出来是同一个,只报名称变化,而不是「删一个加一个」。
|
||||
func TestDiffActivitiesRenameByUUID(t *testing.T) {
|
||||
b := sampleBackup()
|
||||
stored := []StoredActivity{{SrcPK: 20, Name: "上班", UUID: uuidC}}
|
||||
d := DiffActivities(stored, b)
|
||||
if len(d.Added) != 1 || len(d.Removed) != 0 {
|
||||
t.Fatalf("只该多出「散步」,「上班」应认出是同一个: +%d -%d", len(d.Added), len(d.Removed))
|
||||
}
|
||||
if len(d.Changed) != 1 || d.Changed[0].Fields[0].From != "上班" || d.Changed[0].Fields[0].To != "工作" {
|
||||
t.Fatalf("应按 UUID 认出重命名: %+v", d.Changed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlaceCoordKeyStable(t *testing.T) {
|
||||
a := PlaceCoordKey(24.1, 109.1)
|
||||
if a != PlaceCoordKey(24.1, 109.1) {
|
||||
t.Fatal("同样的输入应得到同样的键")
|
||||
}
|
||||
if a == PlaceCoordKey(24.1, 109.2) {
|
||||
t.Fatal("坐标不同不该算同一个地点")
|
||||
}
|
||||
}
|
||||
|
||||
// 站点把地点反查改名(坐标没动)时,到访层面不该报「换了地点」——
|
||||
// 实测里这一条会让 17 条到访全变成假冲突。
|
||||
func TestDiffVisitsIgnoresPlaceRename(t *testing.T) {
|
||||
b := &Backup{
|
||||
Visits: []Visit{{SrcPK: 5, Arrival: tp("2026-09-01 10:00"), LocationSrcPK: ip(3)}},
|
||||
Locations: []Location{{SrcPK: 3, Name: "河池市国民体质监测中心", Lat: 24.691827, Lon: 108.035268}},
|
||||
VisitUUID: map[int]string{5: uuidA},
|
||||
}
|
||||
stored := []StoredVisit{{
|
||||
SrcPK: 1, UUID: uuidA, Arrival: tp("2026-09-01 10:00"),
|
||||
PlaceKey: PlaceCoordKey(24.691827, 108.035268), PlaceName: "体育馆美食街",
|
||||
}}
|
||||
d := DiffVisits(stored, b)
|
||||
if len(d.Changed) != 0 {
|
||||
t.Fatalf("只改了地点名不该算到访冲突: %+v", d.Changed[0].Fields)
|
||||
}
|
||||
if d.Same != 1 {
|
||||
t.Fatalf("应算作一致,实际 same=%d", d.Same)
|
||||
}
|
||||
}
|
||||
|
||||
// 反之,地点层面要把改名报出来。
|
||||
func TestDiffPlacesReportsRename(t *testing.T) {
|
||||
b := &Backup{Locations: []Location{
|
||||
{SrcPK: 3, Name: "河池市国民体质监测中心", Lat: 24.691827, Lon: 108.035268, City: "河池市"},
|
||||
}}
|
||||
stored := []StoredPlace{{
|
||||
SrcPK: 10, Key: PlaceCoordKey(24.691827, 108.035268), Name: "体育馆美食街",
|
||||
Lat: 24.691827, Lon: 108.035268, City: "河池市",
|
||||
}}
|
||||
d := DiffPlaces(stored, b)
|
||||
if len(d.Added) != 0 || len(d.Removed) != 0 {
|
||||
t.Fatalf("坐标一样就是同一个地点: +%d -%d", len(d.Added), len(d.Removed))
|
||||
}
|
||||
if len(d.Changed) != 1 {
|
||||
t.Fatalf("改名应报成一条冲突,实际 %d", len(d.Changed))
|
||||
}
|
||||
f := d.Changed[0].Fields
|
||||
if len(f) != 1 || f[0].Label != "名称" || f[0].From != "体育馆美食街" || f[0].To != "河池市国民体质监测中心" {
|
||||
t.Fatalf("字段差异不对: %+v", f)
|
||||
}
|
||||
}
|
||||
|
||||
// 没有坐标的地点不能都落到同一个键上,否则两个不同地点会互相认领。
|
||||
func TestDiffPlacesWithoutCoordDoesNotCollide(t *testing.T) {
|
||||
b := &Backup{Locations: []Location{{SrcPK: 3, Name: "甲地"}}}
|
||||
stored := []StoredPlace{
|
||||
{SrcPK: 10, Name: "甲地"},
|
||||
{SrcPK: 11, Name: "乙地"},
|
||||
}
|
||||
d := DiffPlaces(stored, b)
|
||||
if len(d.Added) != 0 || len(d.Removed) != 1 {
|
||||
t.Fatalf("应只配上同名的「甲地」: +%d -%d", len(d.Added), len(d.Removed))
|
||||
}
|
||||
if d.Removed[0].Name != "乙地" {
|
||||
t.Fatalf("没配上的应是「乙地」,实际 %q", d.Removed[0].Name)
|
||||
}
|
||||
if d.Same != 1 {
|
||||
t.Fatalf("「甲地」应算一致,实际 %d", d.Same)
|
||||
}
|
||||
}
|
||||
|
||||
// 站点把坐标纠偏过(名称没变)也要认得出是同一个地点。
|
||||
func TestDiffPlacesMatchesByNameWhenCoordMoved(t *testing.T) {
|
||||
b := &Backup{Locations: []Location{{SrcPK: 3, Name: "家", Lat: 24.100001, Lon: 109.100001}}}
|
||||
stored := []StoredPlace{{
|
||||
SrcPK: 10, Key: PlaceCoordKey(24.1, 109.1), Name: "家", Lat: 24.1, Lon: 109.1,
|
||||
}}
|
||||
d := DiffPlaces(stored, b)
|
||||
if len(d.Added) != 0 || len(d.Removed) != 0 {
|
||||
t.Fatalf("名称一样就该配上: +%d -%d", len(d.Added), len(d.Removed))
|
||||
}
|
||||
if len(d.Changed) != 1 || d.Changed[0].Fields[0].Label != "纬度" {
|
||||
t.Fatalf("应报出坐标差异: %+v", d.Changed)
|
||||
}
|
||||
}
|
||||
@@ -1332,9 +1332,12 @@ var uuidNamespace = [16]byte{0x6b, 0x1c, 0x9a, 0x2e, 0x42, 0x7d, 0x4f, 0x16,
|
||||
// uuidV5 由命名空间与名字生成 RFC 4122 版本 5 的 UUID(16 字节原始形式)。
|
||||
//
|
||||
// rond 给每条到访、每个活动都存一个 16 字节 UUID(ZIDENTIFIER_ / ZUID_,真机里全是 v4),
|
||||
// 站点这边没有对应字段。这里用确定性算法补:名字只取表名 + src_pk(src_pk 跨快照稳定),
|
||||
// 于是同一条记录每次导出拿到的都是同一个 UUID——若改成随机,每次导出的包都会让 rond
|
||||
// 把同一批记录当成新的。
|
||||
// 站点新建的行没有真机 UUID 可继承。这里用确定性算法补:名字取表名 + src_pk
|
||||
// (src_pk 在**同一个数据集内**稳定,够用了),于是同一条记录每次导出拿到的都是同一个
|
||||
// UUID——若改成随机,每次导出的包都会让 rond 把同一批记录当成新的。
|
||||
//
|
||||
// 注意这只保证「同一数据集内幂等」,不是跨备份的稳定标识:rond 那边每次导入都会重排
|
||||
// Z_PK,真正的跨备份对齐键是它原生的 ZIDENTIFIER_(见 diff.go)。
|
||||
func uuidV5(name string) []byte {
|
||||
h := sha1.New()
|
||||
h.Write(uuidNamespace[:])
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
package rond
|
||||
|
||||
// 合并计划:把「差异画像 + 站长在页面上做的选择」翻译成一份可执行的改动清单。
|
||||
// 这里只做判定,不碰数据库、也不写任何东西。
|
||||
|
||||
// 站长对一条冲突的处理方式。
|
||||
const (
|
||||
PickBackup = "backup" // 用备份(实机)的值覆盖站内
|
||||
PickDB = "db" // 保留站内的值
|
||||
PickSkip = "skip" // 这条不动
|
||||
)
|
||||
|
||||
// Choices 是差异页提交上来的选择。冲突是逐条选的,键分别是到访 UUID、
|
||||
// 地点坐标键、活动 UUID。
|
||||
type Choices struct {
|
||||
AddVisits bool // 把备份里多出来的到访并进来
|
||||
AddPlaces bool
|
||||
AddActs bool
|
||||
|
||||
VisitClash map[string]string
|
||||
PlaceClash map[string]string
|
||||
ActClash map[string]string
|
||||
|
||||
// 删除:「仅站内有」的记录按库里编号勾选,默认一个都不删。
|
||||
// 键是 src_pk 而不是 UUID——站点补录的行压根没有 UUID。
|
||||
DelVisits map[int]bool
|
||||
DelPlaces map[int]bool
|
||||
DelActs map[int]bool
|
||||
}
|
||||
|
||||
func (c Choices) visitPick(key string) string { return pickOf(c.VisitClash, key) }
|
||||
func (c Choices) placePick(key string) string { return pickOf(c.PlaceClash, key) }
|
||||
func (c Choices) activityPick(key string) string { return pickOf(c.ActClash, key) }
|
||||
|
||||
func pickOf(m map[string]string, key string) string {
|
||||
switch m[key] {
|
||||
case PickBackup, PickDB, PickSkip:
|
||||
return m[key]
|
||||
}
|
||||
return PickDB // 没提交或提交了怪值:默认保留站内的,不误伤
|
||||
}
|
||||
|
||||
// VisitPatch 表示用备份侧的值覆盖库里的那一条。
|
||||
type VisitPatch struct {
|
||||
UUID string // 对齐键
|
||||
V Visit
|
||||
}
|
||||
|
||||
// PlacePatch 同理,键是坐标键。
|
||||
type PlacePatch struct {
|
||||
CoordKey string
|
||||
L Location
|
||||
}
|
||||
|
||||
// ActivityPatch 键是活动 UUID(没有 UUID 时退回名称)。
|
||||
type ActivityPatch struct {
|
||||
Key string
|
||||
A Activity
|
||||
}
|
||||
|
||||
// MergePlan 是一次合并要做的全部改动。
|
||||
type MergePlan struct {
|
||||
AddPlaces []Location
|
||||
UpdPlaces []PlacePatch
|
||||
DelPlaces []StoredPlace
|
||||
AddActs []Activity
|
||||
UpdActs []ActivityPatch
|
||||
DelActs []StoredActivity
|
||||
AddVisits []Visit
|
||||
UpdVisits []VisitPatch
|
||||
DelVisits []StoredVisit
|
||||
}
|
||||
|
||||
// Empty 报告这份计划什么事都不用做。
|
||||
func (p MergePlan) Empty() bool {
|
||||
return len(p.AddPlaces)+len(p.UpdPlaces)+len(p.DelPlaces)+
|
||||
len(p.AddActs)+len(p.UpdActs)+len(p.DelActs)+
|
||||
len(p.AddVisits)+len(p.UpdVisits)+len(p.DelVisits) == 0
|
||||
}
|
||||
|
||||
// BuildMergePlan 按选择把差异筛成改动清单。
|
||||
func BuildMergePlan(vd VisitDiff, pd PlaceDiff, ad ActivityDiff, c Choices) MergePlan {
|
||||
var p MergePlan
|
||||
|
||||
if c.AddPlaces {
|
||||
p.AddPlaces = pd.Added
|
||||
}
|
||||
for _, cf := range pd.Changed {
|
||||
if c.placePick(cf.DB.Key) == PickBackup {
|
||||
p.UpdPlaces = append(p.UpdPlaces, PlacePatch{CoordKey: cf.DB.Key, L: cf.Backup})
|
||||
}
|
||||
}
|
||||
for _, sp := range pd.Removed {
|
||||
if c.DelPlaces[sp.SrcPK] {
|
||||
p.DelPlaces = append(p.DelPlaces, sp)
|
||||
}
|
||||
}
|
||||
|
||||
if c.AddActs {
|
||||
p.AddActs = ad.Added
|
||||
}
|
||||
for _, cf := range ad.Changed {
|
||||
if c.activityPick(activityKey(cf.DB)) == PickBackup {
|
||||
p.UpdActs = append(p.UpdActs, ActivityPatch{Key: activityKey(cf.DB), A: cf.Backup})
|
||||
}
|
||||
}
|
||||
for _, sa := range ad.Removed {
|
||||
if c.DelActs[sa.SrcPK] {
|
||||
p.DelActs = append(p.DelActs, sa)
|
||||
}
|
||||
}
|
||||
|
||||
if c.AddVisits {
|
||||
p.AddVisits = vd.Added
|
||||
// 新到访可能挂在一个「备份里新出现、库里还没有」的地点/活动上:
|
||||
// 不把它们一起建出来,到访的 place_id / activity_id 就无从映射,只能整条丢掉。
|
||||
p.AddPlaces = appendMissingPlaces(p.AddPlaces, pd.Added)
|
||||
p.AddActs = appendMissingActs(p.AddActs, ad.Added)
|
||||
}
|
||||
for _, cf := range vd.Changed {
|
||||
if c.visitPick(cf.DB.UUID) == PickBackup {
|
||||
p.UpdVisits = append(p.UpdVisits, VisitPatch{UUID: cf.DB.UUID, V: cf.Backup})
|
||||
}
|
||||
}
|
||||
for _, sv := range vd.Removed {
|
||||
if c.DelVisits[sv.SrcPK] {
|
||||
p.DelVisits = append(p.DelVisits, sv)
|
||||
}
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func appendMissingPlaces(have, more []Location) []Location {
|
||||
seen := make(map[int]bool, len(have))
|
||||
for _, l := range have {
|
||||
seen[l.SrcPK] = true
|
||||
}
|
||||
for _, l := range more {
|
||||
if !seen[l.SrcPK] {
|
||||
have = append(have, l)
|
||||
seen[l.SrcPK] = true
|
||||
}
|
||||
}
|
||||
return have
|
||||
}
|
||||
|
||||
func appendMissingActs(have, more []Activity) []Activity {
|
||||
seen := make(map[int]bool, len(have))
|
||||
for _, a := range have {
|
||||
seen[a.SrcPK] = true
|
||||
}
|
||||
for _, a := range more {
|
||||
if !seen[a.SrcPK] {
|
||||
have = append(have, a)
|
||||
seen[a.SrcPK] = true
|
||||
}
|
||||
}
|
||||
return have
|
||||
}
|
||||
|
||||
// activityKey 是活动冲突在页面上的标识:优先 UUID(改名也能对上),没有才用名称。
|
||||
func activityKey(sa StoredActivity) string {
|
||||
if sa.UUID != "" {
|
||||
return sa.UUID
|
||||
}
|
||||
return "n:" + sa.Name
|
||||
}
|
||||
|
||||
// ActivityKey 供页面与处理函数共用同一套键。
|
||||
func ActivityKey(sa StoredActivity) string { return activityKey(sa) }
|
||||
@@ -0,0 +1,76 @@
|
||||
package rond
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestBuildMergePlanPicks(t *testing.T) {
|
||||
vd := VisitDiff{
|
||||
Added: []Visit{{SrcPK: 900, Arrival: tp("2026-09-09 10:00")}},
|
||||
Removed: []StoredVisit{{SrcPK: 5, UUID: uuidA}},
|
||||
Changed: []VisitConflict{
|
||||
{Backup: Visit{SrcPK: 901, Arrival: tp("2026-09-01 12:00")}, DB: StoredVisit{SrcPK: 1, UUID: uuidA}},
|
||||
{Backup: Visit{SrcPK: 902, Arrival: tp("2026-09-02 12:00")}, DB: StoredVisit{SrcPK: 2, UUID: uuidB}},
|
||||
},
|
||||
}
|
||||
pd := PlaceDiff{Added: []Location{{SrcPK: 3, Name: "新地点"}}}
|
||||
ad := ActivityDiff{Added: []Activity{{SrcPK: 8, Name: "散步"}}}
|
||||
|
||||
p := BuildMergePlan(vd, pd, ad, Choices{
|
||||
AddVisits: true,
|
||||
AddPlaces: true,
|
||||
AddActs: false,
|
||||
VisitClash: map[string]string{uuidA: PickBackup, uuidB: PickDB},
|
||||
})
|
||||
if len(p.AddVisits) != 1 || p.AddVisits[0].SrcPK != 900 {
|
||||
t.Fatalf("新增到访应合入一条: %+v", p.AddVisits)
|
||||
}
|
||||
if len(p.UpdVisits) != 1 || p.UpdVisits[0].UUID != uuidA {
|
||||
t.Fatalf("只有选了「用实机」的那条才该更新: %+v", p.UpdVisits)
|
||||
}
|
||||
if len(p.AddPlaces) != 1 {
|
||||
t.Fatalf("勾了「并进来」就该带上新增地点: %d", len(p.AddPlaces))
|
||||
}
|
||||
// 即使没单独勾活动,合入到访时也要把它可能挂着的「库里还没有的活动」带上,
|
||||
// 否则那条到访的 activity_id 无从映射,只能整条丢掉。
|
||||
if len(p.AddActs) != 1 || p.AddActs[0].Name != "散步" {
|
||||
t.Fatalf("合入到访应连带它需要的活动: %+v", p.AddActs)
|
||||
}
|
||||
// 「仅站内有」永远不进计划:删除是单独的事,绝不顺手做
|
||||
if len(p.UpdPlaces)+len(p.UpdActs) != 0 {
|
||||
t.Fatal("不该有地点/活动更新")
|
||||
}
|
||||
}
|
||||
|
||||
// 没提交的键、或提交了怪值,一律当成「保留站内」——不误伤。
|
||||
func TestBuildMergePlanDefaultsToKeepDB(t *testing.T) {
|
||||
vd := VisitDiff{Changed: []VisitConflict{
|
||||
{Backup: Visit{SrcPK: 901}, DB: StoredVisit{SrcPK: 1, UUID: uuidA}},
|
||||
}}
|
||||
p := BuildMergePlan(vd, PlaceDiff{}, ActivityDiff{}, Choices{VisitClash: map[string]string{uuidA: "wat"}})
|
||||
if len(p.UpdVisits) != 0 {
|
||||
t.Fatalf("怪值不该被当成「用实机」: %+v", p.UpdVisits)
|
||||
}
|
||||
if !p.Empty() {
|
||||
t.Fatal("什么都没选,计划应当是空的")
|
||||
}
|
||||
}
|
||||
|
||||
// 冲突选了「跳过」也不动。
|
||||
func TestBuildMergePlanSkip(t *testing.T) {
|
||||
vd := VisitDiff{Changed: []VisitConflict{
|
||||
{Backup: Visit{SrcPK: 901}, DB: StoredVisit{SrcPK: 1, UUID: uuidA}},
|
||||
}}
|
||||
p := BuildMergePlan(vd, PlaceDiff{}, ActivityDiff{}, Choices{VisitClash: map[string]string{uuidA: PickSkip}})
|
||||
if !p.Empty() {
|
||||
t.Fatalf("跳过就不该产生改动: %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
// 活动改了名也要能用 UUID 当键对得上。
|
||||
func TestActivityKeyPrefersUUID(t *testing.T) {
|
||||
if got := ActivityKey(StoredActivity{SrcPK: 1, Name: "上班", UUID: uuidC}); got != uuidC {
|
||||
t.Fatalf("有 UUID 就用 UUID,实际 %q", got)
|
||||
}
|
||||
if got := ActivityKey(StoredActivity{SrcPK: 1, Name: "上班"}); got != "n:上班" {
|
||||
t.Fatalf("没有 UUID 退回名称,实际 %q", got)
|
||||
}
|
||||
}
|
||||
@@ -156,6 +156,13 @@ type Backup struct {
|
||||
// RawRows 是各实体在 Core Data 里的完整原始行:「表名 -> Z_PK -> 列名 -> 值」。
|
||||
// 站点只映射了其中一部分字段,留档让导出能逐列还原,不必依赖基线库补剩下的列。
|
||||
RawRows map[string]map[int]map[string]any
|
||||
|
||||
// VisitUUID / ActivityUUID 是真机给每条到访、每个活动存的 16 字节 UUID(小写十六进制)。
|
||||
// **这是跨备份唯一稳定的对齐键**:rond 每次导出/同步都会重排 Z_PK——实测两份备份里
|
||||
// 822 条到访的 UUID 完全一致、Z_PK 100% 不同——所以「这份备份」与「库里的那一份」
|
||||
// 之间只能按它对齐。地点/标签/交通方式没有 UUID,只能退到名称与坐标。
|
||||
VisitUUID map[int]string
|
||||
ActivityUUID map[int]string
|
||||
}
|
||||
|
||||
// Open 解压备份包到 dir,返回可用的 SQLite 连接。
|
||||
@@ -465,9 +472,38 @@ func Parse(db *sql.DB) (*Backup, error) {
|
||||
} else {
|
||||
log.Printf("抓取原始行留档失败(导出将退回基线补列): %v", rerr)
|
||||
}
|
||||
// 对齐键:导出不依赖它,但「两份数据比对」全靠它,缺了就只能放弃逐条对齐
|
||||
if b.VisitUUID, err = uuidColumn(db, "ZVISIT", "ZIDENTIFIER_"); err != nil {
|
||||
return nil, fmt.Errorf("读取到访 UUID: %w", err)
|
||||
}
|
||||
if b.ActivityUUID, err = uuidColumn(db, "ZACTIVITY", "ZUID_"); err != nil {
|
||||
return nil, fmt.Errorf("读取活动 UUID: %w", err)
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// uuidColumn 读出某表某一列的 UUID,返回 Z_PK -> 小写十六进制。
|
||||
// 列不存在(很老的备份)返回空表而不是错误;全零或空的当作没有。
|
||||
func uuidColumn(db *sql.DB, table, col string) (map[int]string, error) {
|
||||
rows, err := db.Query(fmt.Sprintf(`SELECT Z_PK, hex(%s) FROM %s`, col, table))
|
||||
if err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[int]string{}
|
||||
for rows.Next() {
|
||||
var pk int
|
||||
var h sql.NullString
|
||||
if err := rows.Scan(&pk, &h); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if h.Valid && strings.Trim(h.String, "0") != "" {
|
||||
out[pk] = strings.ToLower(h.String)
|
||||
}
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// rawTables 是需要整行留档的 Core Data 表(站点管理的实体 + 标签分组)。
|
||||
var rawTables = []string{"ZLOCATION", "ZVISIT", "ZMOVEMENT", "ZHOURLYWEATHER", "ZACTIVITY", "ZTAG", "ZTAGGROUP", "ZTRANSPORT"}
|
||||
|
||||
|
||||
@@ -666,7 +666,8 @@ const placeCols = `p.id, COALESCE(p.name,''), COALESCE(p.poi_category,''), COALE
|
||||
EXISTS (SELECT 1 FROM visits v3 WHERE v3.place_id=p.id AND v3.is_home),
|
||||
EXISTS (SELECT 1 FROM visits v4 WHERE v4.place_id=p.id AND v4.is_work),
|
||||
p.src_pk,
|
||||
EXISTS (SELECT 1 FROM place_images pi WHERE pi.dataset_id=p.dataset_id AND pi.src_pk=p.src_pk)`
|
||||
EXISTS (SELECT 1 FROM place_images pi WHERE pi.place_key =
|
||||
to_char(p.lat,'FM999990.000000')||'|'||to_char(p.lon,'FM999990.000000'))`
|
||||
|
||||
// PlaceCols 与 ScanPlace 配对,供包外构造只取单个地点的查询。
|
||||
const PlaceCols = placeCols
|
||||
@@ -708,11 +709,12 @@ func (d DB) Places(ctx context.Context, f Filter, sortBy string, limit, offset i
|
||||
args := append([]any{f.DatasetID}, inner.args...)
|
||||
base := `FROM places p WHERE p.dataset_id = $1 AND EXISTS (SELECT 1 FROM visits v WHERE ` + inner.expr() + `)`
|
||||
if f.Verdict != 0 {
|
||||
// 结论存在 place_votes,按 src_pk 关联;占位符接在内层条件之后。
|
||||
// dataset_id 也要对上:src_pk 只在单个数据集内唯一。
|
||||
// 结论挂坐标键(换数据集也要跟得住),按本数据集的地点圈定范围;
|
||||
// 占位符接在内层条件之后。
|
||||
args = append(args, f.Verdict)
|
||||
base += fmt.Sprintf(` AND EXISTS (SELECT 1 FROM place_votes pv
|
||||
WHERE pv.src_pk = p.src_pk AND pv.dataset_id = p.dataset_id AND pv.verdict = $%d)`, len(args))
|
||||
base += fmt.Sprintf(` AND EXISTS (SELECT 1 FROM place_votes pv WHERE pv.place_key =
|
||||
to_char(p.lat,'FM999990.000000')||'|'||to_char(p.lon,'FM999990.000000')
|
||||
AND pv.verdict = $%d)`, len(args))
|
||||
}
|
||||
q := `SELECT ` + placeCols + ` ` + base + ` ORDER BY ` + order + fmt.Sprintf(` LIMIT %d OFFSET %d`, limit, offset)
|
||||
|
||||
|
||||
+78
-34
@@ -2,14 +2,23 @@
|
||||
// - place_votes 站长对地点的官方结论(单值)
|
||||
// - place_visitor_votes 访客点赞 / 点踩(一人一票,可改可撤)
|
||||
//
|
||||
// 两者都按 rond 的地点 ID(src_pk)存储,重新上传备份、地点换了新 id 也不丢。
|
||||
// 站长的「官方结论」按**坐标键**(place_key)存储:src_pk 只在单个数据集内有效,
|
||||
// 换个数据集同编号就是另一个地点,结论会挂错人。
|
||||
// 访客票仍按 src_pk 存——那张表没有 dataset_id,无从判断票是投给哪个数据集的,
|
||||
// 迁移有歧义,留待单独处理。
|
||||
package votes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// placeKeyExpr 把 places p 这一行的坐标换算成 place_key。
|
||||
// **必须与 Go 侧的 rond.PlaceCoordKey 以及 notes 包里的同款表达式逐位一致**:
|
||||
// fmt.Sprintf("%.6f|%.6f") ↔ to_char(x,'FM999990.000000')||'|'||to_char(y,'FM999990.000000')
|
||||
const placeKeyExpr = `to_char(p.lat,'FM999990.000000')||'|'||to_char(p.lon,'FM999990.000000')`
|
||||
|
||||
// VerdictRecommend / VerdictAvoid 是官方结论的取值。
|
||||
const (
|
||||
VerdictAvoid = -1
|
||||
@@ -24,11 +33,13 @@ type Tally struct {
|
||||
Down int `json:"down"`
|
||||
}
|
||||
|
||||
// Verdicts 返回某个数据集的站长结论:src_pk -> verdict(1/-1)。
|
||||
// 必须按 dataset_id 过滤:src_pk 只在单个数据集内唯一。
|
||||
// Verdicts 返回某个数据集的站长结论:该数据集里的 src_pk -> verdict(1/-1)。
|
||||
func (s *Store) Verdicts(ctx context.Context, datasetID int64) (map[int]int, error) {
|
||||
out := map[int]int{}
|
||||
rows, err := s.DB.QueryContext(ctx, `SELECT src_pk, verdict FROM place_votes WHERE dataset_id=$1`, datasetID)
|
||||
rows, err := s.DB.QueryContext(ctx, `
|
||||
SELECT p.src_pk, v.verdict FROM places p
|
||||
JOIN place_votes v ON v.place_key = `+placeKeyExpr+`
|
||||
WHERE p.dataset_id=$1 AND p.lat IS NOT NULL AND p.lon IS NOT NULL`, datasetID)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
@@ -46,8 +57,9 @@ func (s *Store) Verdicts(ctx context.Context, datasetID int64) (map[int]int, err
|
||||
// Verdict 取单个地点的站长结论;未标注返回 0。
|
||||
func (s *Store) Verdict(ctx context.Context, datasetID int64, srcPK int) (int, error) {
|
||||
var v int
|
||||
err := s.DB.QueryRowContext(ctx,
|
||||
`SELECT verdict FROM place_votes WHERE dataset_id=$1 AND src_pk=$2`, datasetID, srcPK).Scan(&v)
|
||||
err := s.DB.QueryRowContext(ctx, `SELECT v.verdict FROM place_votes v
|
||||
JOIN places p ON v.place_key = `+placeKeyExpr+`
|
||||
WHERE p.dataset_id=$1 AND p.src_pk=$2`, datasetID, srcPK).Scan(&v)
|
||||
if err == sql.ErrNoRows {
|
||||
return 0, nil
|
||||
}
|
||||
@@ -57,24 +69,46 @@ func (s *Store) Verdict(ctx context.Context, datasetID int64, srcPK int) (int, e
|
||||
// SaveVerdict 设置站长结论;verdict 为 0 表示取消结论。
|
||||
func (s *Store) SaveVerdict(ctx context.Context, userID, datasetID int64, srcPK, verdict int) error {
|
||||
if verdict == 0 {
|
||||
_, err := s.DB.ExecContext(ctx,
|
||||
`DELETE FROM place_votes WHERE user_id=$1 AND dataset_id=$2 AND src_pk=$3`, userID, datasetID, srcPK)
|
||||
_, err := s.DB.ExecContext(ctx, `DELETE FROM place_votes v USING places p
|
||||
WHERE v.user_id=$1 AND p.dataset_id=$2 AND p.src_pk=$3
|
||||
AND v.place_key = `+placeKeyExpr, userID, datasetID, srcPK)
|
||||
return err
|
||||
}
|
||||
_, err := s.DB.ExecContext(ctx, `INSERT INTO place_votes (user_id, dataset_id, src_pk, verdict, updated_at)
|
||||
VALUES ($1,$2,$3,$4,now())
|
||||
ON CONFLICT (user_id, dataset_id, src_pk) DO UPDATE SET verdict=EXCLUDED.verdict, updated_at=now()`,
|
||||
// 先按坐标键算出这条地点该挂在哪,找不到该地点就什么也不写
|
||||
_, err := s.DB.ExecContext(ctx, `INSERT INTO place_votes
|
||||
(user_id, place_key, dataset_id, src_pk, verdict, updated_at)
|
||||
SELECT $1, `+placeKeyExpr+`, $2, $3, $4, now()
|
||||
FROM places p
|
||||
WHERE p.dataset_id=$2 AND p.src_pk=$3 AND p.lat IS NOT NULL AND p.lon IS NOT NULL
|
||||
ON CONFLICT (user_id, place_key) DO UPDATE SET
|
||||
verdict=EXCLUDED.verdict, dataset_id=EXCLUDED.dataset_id,
|
||||
src_pk=EXCLUDED.src_pk, updated_at=now()`,
|
||||
userID, datasetID, srcPK, verdict)
|
||||
return err
|
||||
}
|
||||
|
||||
// Tallies 返回全部地点的访客票数。票表规模很小(个人站点),一次读全更省事,
|
||||
// 也避免给 pgx 传数组参数带来的类型歧义。
|
||||
func (s *Store) Tallies(ctx context.Context) (map[int]Tally, error) {
|
||||
// placeKeyOf 把某个数据集里的地点编号换算成坐标键——访客票也是挂在坐标上的,
|
||||
// 换数据集之后票仍然跟着同一个地点。
|
||||
func (s *Store) placeKeyOf(ctx context.Context, datasetID int64, srcPK int) (string, error) {
|
||||
var key string
|
||||
err := s.DB.QueryRowContext(ctx, `SELECT `+placeKeyExpr+` FROM places p
|
||||
WHERE p.dataset_id=$1 AND p.src_pk=$2 AND p.lat IS NOT NULL AND p.lon IS NOT NULL`,
|
||||
datasetID, srcPK).Scan(&key)
|
||||
if err == sql.ErrNoRows {
|
||||
return "", fmt.Errorf("地点不存在或没有坐标")
|
||||
}
|
||||
return key, err
|
||||
}
|
||||
|
||||
// Tallies 返回某个数据集里各地点的访客票数:src_pk -> Tally。
|
||||
// 票表规模很小(个人站点),一次读全更省事,也避免给 pgx 传数组参数带来的类型歧义。
|
||||
func (s *Store) Tallies(ctx context.Context, datasetID int64) (map[int]Tally, error) {
|
||||
out := map[int]Tally{}
|
||||
rows, err := s.DB.QueryContext(ctx, `SELECT src_pk,
|
||||
count(*) FILTER (WHERE vote=1), count(*) FILTER (WHERE vote=-1)
|
||||
FROM place_visitor_votes GROUP BY src_pk`)
|
||||
rows, err := s.DB.QueryContext(ctx, `SELECT p.src_pk,
|
||||
count(*) FILTER (WHERE vv.vote=1), count(*) FILTER (WHERE vv.vote=-1)
|
||||
FROM places p JOIN place_visitor_votes vv ON vv.place_key = `+placeKeyExpr+`
|
||||
WHERE p.dataset_id=$1 AND p.lat IS NOT NULL AND p.lon IS NOT NULL
|
||||
GROUP BY p.src_pk`, datasetID)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
@@ -91,21 +125,24 @@ func (s *Store) Tallies(ctx context.Context) (map[int]Tally, error) {
|
||||
}
|
||||
|
||||
// Tally 取单个地点的访客票数。
|
||||
func (s *Store) Tally(ctx context.Context, srcPK int) (Tally, error) {
|
||||
func (s *Store) Tally(ctx context.Context, datasetID int64, srcPK int) (Tally, error) {
|
||||
var t Tally
|
||||
err := s.DB.QueryRowContext(ctx, `SELECT count(*) FILTER (WHERE vote=1), count(*) FILTER (WHERE vote=-1)
|
||||
FROM place_visitor_votes WHERE src_pk=$1`, srcPK).Scan(&t.Up, &t.Down)
|
||||
err := s.DB.QueryRowContext(ctx, `SELECT
|
||||
count(*) FILTER (WHERE vv.vote=1), count(*) FILTER (WHERE vv.vote=-1)
|
||||
FROM place_visitor_votes vv JOIN places p ON vv.place_key = `+placeKeyExpr+`
|
||||
WHERE p.dataset_id=$1 AND p.src_pk=$2`, datasetID, srcPK).Scan(&t.Up, &t.Down)
|
||||
return t, err
|
||||
}
|
||||
|
||||
// VoterAll 返回该访客投过的全部票:src_pk -> vote,供地图一次性标注「我投的」。
|
||||
func (s *Store) VoterAll(ctx context.Context, voterKey string) (map[int]int, error) {
|
||||
func (s *Store) VoterAll(ctx context.Context, datasetID int64, voterKey string) (map[int]int, error) {
|
||||
out := map[int]int{}
|
||||
if voterKey == "" {
|
||||
return out, nil
|
||||
}
|
||||
rows, err := s.DB.QueryContext(ctx,
|
||||
`SELECT src_pk, vote FROM place_visitor_votes WHERE voter_key=$1`, voterKey)
|
||||
rows, err := s.DB.QueryContext(ctx, `SELECT p.src_pk, vv.vote
|
||||
FROM place_visitor_votes vv JOIN places p ON vv.place_key = `+placeKeyExpr+`
|
||||
WHERE p.dataset_id=$1 AND vv.voter_key=$2`, datasetID, voterKey)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
@@ -121,13 +158,15 @@ func (s *Store) VoterAll(ctx context.Context, voterKey string) (map[int]int, err
|
||||
}
|
||||
|
||||
// Voter 返回该访客对某地点的既有投票(1/-1/0)。voterKey 为空表示拿不到身份。
|
||||
func (s *Store) Voter(ctx context.Context, srcPK int, voterKey string) int {
|
||||
func (s *Store) Voter(ctx context.Context, datasetID int64, srcPK int, voterKey string) int {
|
||||
if voterKey == "" {
|
||||
return 0
|
||||
}
|
||||
var v int
|
||||
if err := s.DB.QueryRowContext(ctx,
|
||||
`SELECT vote FROM place_visitor_votes WHERE src_pk=$1 AND voter_key=$2`, srcPK, voterKey).Scan(&v); err != nil {
|
||||
if err := s.DB.QueryRowContext(ctx, `SELECT vv.vote FROM place_visitor_votes vv
|
||||
JOIN places p ON vv.place_key = `+placeKeyExpr+`
|
||||
WHERE p.dataset_id=$1 AND p.src_pk=$2 AND vv.voter_key=$3`,
|
||||
datasetID, srcPK, voterKey).Scan(&v); err != nil {
|
||||
return 0
|
||||
}
|
||||
return v
|
||||
@@ -135,30 +174,35 @@ func (s *Store) Voter(ctx context.Context, srcPK int, voterKey string) int {
|
||||
|
||||
// Cast 记录访客投票:未投过则新增;投同一票则撤销(再点一次取消);投另一票则改为该票。
|
||||
// 返回更新后的票数与该访客最终的选择。
|
||||
func (s *Store) Cast(ctx context.Context, srcPK int, voterKey string, vote int) (Tally, int, error) {
|
||||
func (s *Store) Cast(ctx context.Context, datasetID int64, srcPK int, voterKey string, vote int) (Tally, int, error) {
|
||||
key, err := s.placeKeyOf(ctx, datasetID, srcPK)
|
||||
if err != nil {
|
||||
return Tally{}, 0, err
|
||||
}
|
||||
var cur int
|
||||
err := s.DB.QueryRowContext(ctx,
|
||||
`SELECT vote FROM place_visitor_votes WHERE src_pk=$1 AND voter_key=$2`, srcPK, voterKey).Scan(&cur)
|
||||
err = s.DB.QueryRowContext(ctx,
|
||||
`SELECT vote FROM place_visitor_votes WHERE place_key=$1 AND voter_key=$2`, key, voterKey).Scan(&cur)
|
||||
switch {
|
||||
case err == sql.ErrNoRows:
|
||||
_, err = s.DB.ExecContext(ctx,
|
||||
`INSERT INTO place_visitor_votes (src_pk, voter_key, vote) VALUES ($1,$2,$3)`, srcPK, voterKey, vote)
|
||||
`INSERT INTO place_visitor_votes (place_key, voter_key, vote, src_pk) VALUES ($1,$2,$3,$4)`,
|
||||
key, voterKey, vote, srcPK)
|
||||
cur = vote
|
||||
case err != nil:
|
||||
return Tally{}, 0, err
|
||||
case cur == vote:
|
||||
_, err = s.DB.ExecContext(ctx,
|
||||
`DELETE FROM place_visitor_votes WHERE src_pk=$1 AND voter_key=$2`, srcPK, voterKey)
|
||||
`DELETE FROM place_visitor_votes WHERE place_key=$1 AND voter_key=$2`, key, voterKey)
|
||||
cur = 0
|
||||
default:
|
||||
_, err = s.DB.ExecContext(ctx,
|
||||
`UPDATE place_visitor_votes SET vote=$3, updated_at=now() WHERE src_pk=$1 AND voter_key=$2`,
|
||||
srcPK, voterKey, vote)
|
||||
`UPDATE place_visitor_votes SET vote=$3, src_pk=$4, updated_at=now()
|
||||
WHERE place_key=$1 AND voter_key=$2`, key, voterKey, vote, srcPK)
|
||||
cur = vote
|
||||
}
|
||||
if err != nil {
|
||||
return Tally{}, 0, err
|
||||
}
|
||||
t, err := s.Tally(ctx, srcPK)
|
||||
t, err := s.Tally(ctx, datasetID, srcPK)
|
||||
return t, cur, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,346 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"rond-with-you/internal/ingest"
|
||||
"rond-with-you/internal/rond"
|
||||
)
|
||||
|
||||
// rondbackup 的差异预演:把上传的备份与当前展示的数据集按稳定键对齐,只读地摆出
|
||||
// 新增 / 仅库里有 / 冲突三类,**不写任何数据**。合并执行是下一步的事。
|
||||
|
||||
const backupStashExt = ".rondbackup"
|
||||
|
||||
// diffListLimit 是每类差异在页面上最多列出的条数。867 条全铺出来既慢又没人看,
|
||||
// 超过就只给前若干条并注明。
|
||||
const diffListLimit = 200
|
||||
|
||||
func (s *Server) backupStashDir() string {
|
||||
return filepath.Join(s.cfg.DataDir, "backup-staging")
|
||||
}
|
||||
|
||||
// adminBackupPreview 收下备份、暂存,跳到差异页。这里不校验备份内容——
|
||||
// 差异页反正是要解析一遍的,失败时让它报错即可,免得同一份包解两次。
|
||||
func (s *Server) adminBackupPreview(w http.ResponseWriter, r *http.Request) {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxUploadBytes+1<<20)
|
||||
if err := r.ParseMultipartForm(32 << 20); err != nil {
|
||||
s.adminFlash(w, r, "err", "读取上传内容失败:"+err.Error())
|
||||
return
|
||||
}
|
||||
file, hdr, err := r.FormFile("backup")
|
||||
if err != nil {
|
||||
s.adminFlash(w, r, "err", "请选择要分析的 .rondbackup 文件")
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
name := filepath.Base(hdr.Filename)
|
||||
lower := strings.ToLower(name)
|
||||
if !strings.HasSuffix(lower, ".rondbackup") && !strings.HasSuffix(lower, ".zip") {
|
||||
s.adminFlash(w, r, "err", "文件类型不正确,请上传 rond 导出的 .rondbackup 备份")
|
||||
return
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(file, maxUploadBytes+1))
|
||||
if err != nil {
|
||||
s.adminFlash(w, r, "err", "读取文件失败:"+err.Error())
|
||||
return
|
||||
}
|
||||
if int64(len(data)) > maxUploadBytes {
|
||||
s.adminFlash(w, r, "err", "备份文件过大,疑似不是 rond 备份")
|
||||
return
|
||||
}
|
||||
token, err := stashPut(s.backupStashDir(), backupStashExt, data)
|
||||
if err != nil {
|
||||
s.serverError(w, r, err)
|
||||
return
|
||||
}
|
||||
http.Redirect(w, r, "/admin/preview/"+token, http.StatusSeeOther)
|
||||
}
|
||||
|
||||
// adminBackupPreviewPage 渲染差异页。每次都重新算:库里的数据随时可能被别处改掉。
|
||||
func (s *Server) adminBackupPreviewPage(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
p, err := s.pageBase(r, "admin", "备份差异预览")
|
||||
if err != nil {
|
||||
s.serverError(w, r, err)
|
||||
return
|
||||
}
|
||||
token := r.PathValue("token")
|
||||
path, ok := stashPath(s.backupStashDir(), backupStashExt, token)
|
||||
if !ok {
|
||||
s.adminFlash(w, r, "err", "预览链接无效,请重新上传备份")
|
||||
return
|
||||
}
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
s.adminFlash(w, r, "err", "这次预览已过期,请重新上传备份")
|
||||
return
|
||||
}
|
||||
ds, err := s.datasetFor(ctx)
|
||||
if err != nil {
|
||||
s.serverError(w, r, err)
|
||||
return
|
||||
}
|
||||
if ds == nil {
|
||||
s.adminFlash(w, r, "err", "还没有可比较的数据集,请先直接导入一份备份")
|
||||
return
|
||||
}
|
||||
|
||||
tmp, err := ingest.TempDir(s.cfg.DataDir, "preview")
|
||||
if err != nil {
|
||||
s.serverError(w, r, err)
|
||||
return
|
||||
}
|
||||
defer os.RemoveAll(tmp)
|
||||
sqlDB, err := rond.Open(path, tmp)
|
||||
if err != nil {
|
||||
s.adminFlash(w, r, "err", "解析备份失败:"+err.Error())
|
||||
return
|
||||
}
|
||||
backup, perr := rond.Parse(sqlDB)
|
||||
sqlDB.Close()
|
||||
if perr != nil {
|
||||
s.adminFlash(w, r, "err", "读取备份数据失败:"+perr.Error())
|
||||
return
|
||||
}
|
||||
|
||||
storedVisits, err := s.loadStoredVisits(ctx, ds.ID)
|
||||
if err != nil {
|
||||
s.serverError(w, r, err)
|
||||
return
|
||||
}
|
||||
storedPlaces, err := s.loadStoredPlaces(ctx, ds.ID)
|
||||
if err != nil {
|
||||
s.serverError(w, r, err)
|
||||
return
|
||||
}
|
||||
storedActs, err := s.loadStoredActivities(ctx, ds.ID)
|
||||
if err != nil {
|
||||
s.serverError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
vd := rond.DiffVisits(storedVisits, backup)
|
||||
pd := rond.DiffPlaces(storedPlaces, backup)
|
||||
ad := rond.DiffActivities(storedActs, backup)
|
||||
|
||||
d := BackupDiffData{
|
||||
Page: p, Token: token,
|
||||
DatasetName: ds.Name,
|
||||
VisitCounts: rond.VisitCounts(vd),
|
||||
PlaceCounts: rond.PlaceCounts(pd),
|
||||
ActCounts: rond.ActivityCounts(ad),
|
||||
PlaceAdded: limitSlice(pd.Added, diffListLimit),
|
||||
PlaceGone: limitSlice(pd.Removed, diffListLimit),
|
||||
PlaceClash: limitSlice(pd.Changed, diffListLimit),
|
||||
ActAdded: limitSlice(ad.Added, diffListLimit),
|
||||
ActGone: limitSlice(ad.Removed, diffListLimit),
|
||||
ActClash: limitSlice(ad.Changed, diffListLimit),
|
||||
|
||||
BackupVisits: len(backup.Visits), BackupPlaces: len(backup.Locations),
|
||||
BackupActs: len(backup.Activities), BackupMovements: len(backup.Movements),
|
||||
BackupWeather: len(backup.Weather), BackupRaw: backup.RawVisitNum,
|
||||
StoredVisits: len(storedVisits), StoredPlaces: len(storedPlaces),
|
||||
StoredActs: len(storedActs),
|
||||
}
|
||||
d.VisitAdded, d.VisitGone, d.VisitClash = visitRows(vd, backup, diffListLimit)
|
||||
d.Truncated = anyTruncated(vd, pd, ad)
|
||||
d.StoredMovements, d.StoredWeather, err = s.countOther(ctx, ds.ID)
|
||||
if err != nil {
|
||||
s.serverError(w, r, err)
|
||||
return
|
||||
}
|
||||
d.Identical = d.VisitCounts.Added+d.VisitCounts.Removed+d.VisitCounts.Changed+
|
||||
d.PlaceCounts.Added+d.PlaceCounts.Removed+d.PlaceCounts.Changed+
|
||||
d.ActCounts.Added+d.ActCounts.Removed+d.ActCounts.Changed == 0
|
||||
s.render(w, "admin/backup-preview", d)
|
||||
}
|
||||
|
||||
// visitRows 把三类到访差异摊平成展示行。备份里的地点/活动只有编号,要借备份自己的
|
||||
// 表翻成名称;库里那侧的名称在加载时就一并取出来了。
|
||||
func visitRows(vd rond.VisitDiff, b *rond.Backup, limit int) (added, gone, clash []VisitRow) {
|
||||
place, act := map[int]string{}, map[int]string{}
|
||||
for _, l := range b.Locations {
|
||||
place[l.SrcPK] = placeLabel(l.Name, l.Lat, l.Lon)
|
||||
}
|
||||
for _, a := range b.Activities {
|
||||
act[a.SrcPK] = a.Name
|
||||
}
|
||||
fromBackup := func(v rond.Visit) VisitRow {
|
||||
r := VisitRow{Arrival: v.Arrival, Departure: v.Departure, Remark: v.Remark, Emoji: v.Emoji}
|
||||
if v.LocationSrcPK != nil {
|
||||
r.Place = place[*v.LocationSrcPK]
|
||||
}
|
||||
if v.ActivitySrcPK != nil {
|
||||
r.Activity = act[*v.ActivitySrcPK]
|
||||
}
|
||||
return r
|
||||
}
|
||||
for _, v := range limitSlice(vd.Added, limit) {
|
||||
added = append(added, fromBackup(v))
|
||||
}
|
||||
for _, sv := range limitSlice(vd.Removed, limit) {
|
||||
gone = append(gone, VisitRow{SrcPK: sv.SrcPK, Arrival: sv.Arrival, Departure: sv.Departure,
|
||||
Place: sv.PlaceName, Activity: sv.ActivityName, Remark: sv.Remark, Emoji: sv.Emoji,
|
||||
UserAdded: sv.UserAdded})
|
||||
}
|
||||
for _, c := range limitSlice(vd.Changed, limit) {
|
||||
r := fromBackup(c.Backup)
|
||||
r.Key = c.DB.UUID
|
||||
r.UserAdded = c.DB.UserAdded
|
||||
r.Fields = c.Fields
|
||||
clash = append(clash, r)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func placeLabel(name string, lat, lon float64) string {
|
||||
if name != "" {
|
||||
return name
|
||||
}
|
||||
return fmt.Sprintf("%.5f, %.5f", lat, lon)
|
||||
}
|
||||
|
||||
// anyTruncated 报告是否有哪一类差异多到被截断了。
|
||||
func anyTruncated(vd rond.VisitDiff, pd rond.PlaceDiff, ad rond.ActivityDiff) bool {
|
||||
for _, n := range []int{len(vd.Added), len(vd.Removed), len(vd.Changed),
|
||||
len(pd.Added), len(pd.Removed), len(pd.Changed),
|
||||
len(ad.Added), len(ad.Removed), len(ad.Changed)} {
|
||||
if n > diffListLimit {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func limitSlice[T any](in []T, n int) []T {
|
||||
if len(in) <= n {
|
||||
return in
|
||||
}
|
||||
return in[:n]
|
||||
}
|
||||
|
||||
func (s *Server) countOther(ctx context.Context, datasetID int64) (movements, weather int, err error) {
|
||||
err = s.db.QueryRowContext(ctx, `SELECT
|
||||
(SELECT count(*) FROM movements WHERE dataset_id=$1),
|
||||
(SELECT count(*) FROM weather WHERE dataset_id=$1)`, datasetID).Scan(&movements, &weather)
|
||||
return
|
||||
}
|
||||
|
||||
// loadStoredVisits 读出该数据集的全部到访,并把原始行留档里的 UUID 解出来当对齐键。
|
||||
// 没有留档的行就是站点新建的(真机不会给它 UUID),对齐键留空。
|
||||
func (s *Server) loadStoredVisits(ctx context.Context, datasetID int64) ([]rond.StoredVisit, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT v.src_pk, v.arrival, v.departure, COALESCE(v.remark,''), COALESCE(v.emoji,''),
|
||||
v.bookmarked, v.is_user_added, COALESCE(p.src_pk,0),
|
||||
COALESCE(p.name,''), COALESCE(p.lat,0), COALESCE(p.lon,0), COALESCE(a.name,''),
|
||||
COALESCE(er.raw #>> '{ZIDENTIFIER_,$blob}','')
|
||||
FROM visits v
|
||||
LEFT JOIN places p ON p.id = v.place_id
|
||||
LEFT JOIN activities a ON a.id = v.activity_id
|
||||
LEFT JOIN entity_raw er ON er.dataset_id = v.dataset_id
|
||||
AND er.entity = 'ZVISIT' AND er.src_pk = v.src_pk
|
||||
WHERE v.dataset_id = $1`, datasetID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []rond.StoredVisit
|
||||
for rows.Next() {
|
||||
var sv rond.StoredVisit
|
||||
var arr time.Time
|
||||
var dep sql.NullTime
|
||||
var placePK int
|
||||
var placeName string
|
||||
var lat, lon float64
|
||||
var b64 string
|
||||
if err := rows.Scan(&sv.SrcPK, &arr, &dep, &sv.Remark, &sv.Emoji, &sv.Bookmarked,
|
||||
&sv.UserAdded, &placePK, &placeName, &lat, &lon, &sv.ActivityName, &b64); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sv.Arrival = &arr
|
||||
if dep.Valid {
|
||||
sv.Departure = &dep.Time
|
||||
}
|
||||
sv.PlaceName = placeName
|
||||
// 没有坐标的地点不能都落到 "0.000000|0.000000" 上,否则两个地点会互相认领
|
||||
if placePK > 0 && (lat != 0 || lon != 0) {
|
||||
sv.PlaceKey = rond.PlaceCoordKey(lat, lon)
|
||||
}
|
||||
sv.UUID = uuidFromBase64(b64)
|
||||
out = append(out, sv)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Server) loadStoredPlaces(ctx context.Context, datasetID int64) ([]rond.StoredPlace, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT src_pk, COALESCE(name,''), COALESCE(lat,0), COALESCE(lon,0),
|
||||
COALESCE(province,''), COALESCE(city,''), COALESCE(poi_category,''), visit_count
|
||||
FROM places WHERE dataset_id=$1`, datasetID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []rond.StoredPlace
|
||||
for rows.Next() {
|
||||
var sp rond.StoredPlace
|
||||
if err := rows.Scan(&sp.SrcPK, &sp.Name, &sp.Lat, &sp.Lon,
|
||||
&sp.Province, &sp.City, &sp.POICategory, &sp.VisitCount); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if sp.Lat != 0 || sp.Lon != 0 {
|
||||
sp.Key = rond.PlaceCoordKey(sp.Lat, sp.Lon)
|
||||
}
|
||||
out = append(out, sp)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// loadStoredActivities 读活动类型;UUID 同样从留档里取,用于识别「改了名但其实是同一个」。
|
||||
func (s *Server) loadStoredActivities(ctx context.Context, datasetID int64) ([]rond.StoredActivity, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT a.src_pk, a.name,
|
||||
COALESCE(er.raw #>> '{ZUID_,$blob}','')
|
||||
FROM activities a
|
||||
LEFT JOIN entity_raw er ON er.dataset_id = a.dataset_id
|
||||
AND er.entity = 'ZACTIVITY' AND er.src_pk = a.src_pk
|
||||
WHERE a.dataset_id=$1`, datasetID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []rond.StoredActivity
|
||||
for rows.Next() {
|
||||
var sa rond.StoredActivity
|
||||
var b64 string
|
||||
if err := rows.Scan(&sa.SrcPK, &sa.Name, &b64); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sa.UUID = uuidFromBase64(b64)
|
||||
out = append(out, sa)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// uuidFromBase64 把留档里的 $blob(base64 的 16 字节 UUID)转成小写十六进制,
|
||||
// 好跟备份侧 hex(ZIDENTIFIER_) 的写法对上。
|
||||
func uuidFromBase64(s string) string {
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
b, err := base64.StdEncoding.DecodeString(s)
|
||||
if err != nil || len(b) != 16 {
|
||||
log.Printf("留档里的 UUID 不是 16 字节,忽略: %q", s)
|
||||
return ""
|
||||
}
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// previewBackup 走真实的上传处理函数:POST → 303 → GET 差异页,返回渲染结果。
|
||||
func previewBackup(t *testing.T, f *movementFixture, path string) string {
|
||||
t.Helper()
|
||||
_, page := previewBackupToken(t, f, path)
|
||||
return page
|
||||
}
|
||||
|
||||
// previewBackupToken 同上,但把令牌也交出来,供后续的合并步骤使用。
|
||||
func previewBackupToken(t *testing.T, f *movementFixture, path string) (string, string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("读取 %s 失败: %v", path, err)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
mw := multipart.NewWriter(&buf)
|
||||
fw, err := mw.CreateFormFile("backup", filepath.Base(path))
|
||||
if err != nil {
|
||||
t.Fatalf("构造上传表单失败: %v", err)
|
||||
}
|
||||
if _, err := fw.Write(data); err != nil {
|
||||
t.Fatalf("写入上传内容失败: %v", err)
|
||||
}
|
||||
mw.Close()
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest("POST", "/admin/preview", &buf)
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
f.srv.adminBackupPreview(rec, req.WithContext(f.ctx))
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("上传应跳转到差异页: code=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
loc := rec.Header().Get("Location")
|
||||
token := strings.TrimPrefix(loc, "/admin/preview/")
|
||||
if token == loc || len(token) != stashTokenLen {
|
||||
t.Fatalf("差异页地址不对: %q", loc)
|
||||
}
|
||||
|
||||
rec = httptest.NewRecorder()
|
||||
req = httptest.NewRequest("GET", loc, nil).WithContext(f.ctx)
|
||||
req.SetPathValue("token", token)
|
||||
f.srv.adminBackupPreviewPage(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("差异页渲染失败: code=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
return token, rec.Body.String()
|
||||
}
|
||||
|
||||
// visitCounts 从概览表里取出「到访」那一行的 (备份新增, 仅站内有, 冲突)。
|
||||
// 不能全页 Contains 数字——地点行也有同样的 data-label,会张冠李戴。
|
||||
func visitCounts(t *testing.T, page string) (added, gone, clash int) {
|
||||
t.Helper()
|
||||
row := regexp.MustCompile(`(?s)<td>到访</td>(.*?)</tr>`).FindStringSubmatch(page)
|
||||
if row == nil {
|
||||
t.Fatal("概览表里找不到「到访」行")
|
||||
}
|
||||
nums := regexp.MustCompile(`data-label="(备份新增|仅站内有|冲突)">(\d+)<`).FindAllStringSubmatch(row[1], -1)
|
||||
if len(nums) != 3 {
|
||||
t.Fatalf("到访行应有三个计数,实际 %d", len(nums))
|
||||
}
|
||||
for _, n := range nums {
|
||||
v, _ := strconv.Atoi(n[2])
|
||||
switch n[1] {
|
||||
case "备份新增":
|
||||
added = v
|
||||
case "仅站内有":
|
||||
gone = v
|
||||
case "冲突":
|
||||
clash = v
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// 同一份备份再传一次:UUID 对得上、字段也没变,到访应当一条不差。
|
||||
//
|
||||
// 这条也顺带钉住「按 UUID 而不是编号对齐」:库内那些行的编号与备份里完全不同
|
||||
// (rond 重排过),只按编号比会让 800 多条全变冲突。
|
||||
//
|
||||
// 只断言到访,不断言「完全一致」整句:导入后站点会反查补全地点(改名称/地址),
|
||||
// 地点层面因此真的存在差异,那是预期行为,不该由这个用例来管。
|
||||
func TestBackupDiffIdentical(t *testing.T) {
|
||||
f := newMovementFixture(t)
|
||||
page := previewBackup(t, f, f.src)
|
||||
if a, g, c := visitCounts(t, page); a != 0 || g != 0 || c != 0 {
|
||||
t.Fatalf("同一份备份的到访不该有差异: 新增 %d 仅站内有 %d 冲突 %d", a, g, c)
|
||||
}
|
||||
}
|
||||
|
||||
// 站内改过备注、又补录了一条到访,差异页要把这两件事分别认出来。
|
||||
//
|
||||
// 这里的关键是「按 UUID 对齐」:库里那些行的 Z_PK 与备份里完全不同
|
||||
// (rond 重排过),只按编号比会把 800 多条全判成冲突。
|
||||
func TestBackupDiffDetectsSiteEdits(t *testing.T) {
|
||||
f := newMovementFixture(t)
|
||||
|
||||
var pk int
|
||||
if err := f.pg.QueryRowContext(f.ctx, `UPDATE visits SET remark='站点改过的备注'
|
||||
WHERE id = (SELECT id FROM visits WHERE dataset_id=$1 ORDER BY src_pk LIMIT 1)
|
||||
RETURNING src_pk`, f.dsID).Scan(&pk); err != nil {
|
||||
t.Fatalf("改备注失败: %v", err)
|
||||
}
|
||||
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO visits
|
||||
(dataset_id, src_pk, arrival, departure, is_user_added, remark)
|
||||
VALUES ($1, 999999, now(), now(), true, '站点补录的一条')`, f.dsID); err != nil {
|
||||
t.Fatalf("插入补录到访失败: %v", err)
|
||||
}
|
||||
|
||||
page := previewBackup(t, f, f.src)
|
||||
// 「站点改过的备注」只可能从冲突行的字段差异里渲染出来;「站内补录」是那条补录行的来源标记
|
||||
for _, want := range []string{"备份差异预览", "站点改过的备注", "站点补录的一条", ">站内补录<"} {
|
||||
if !strings.Contains(page, want) {
|
||||
t.Errorf("差异页缺少 %q", want)
|
||||
}
|
||||
}
|
||||
a, g, c := visitCounts(t, page)
|
||||
if c != 1 {
|
||||
t.Errorf("改过一条备注,到访冲突数应为 1,实际 %d", c)
|
||||
}
|
||||
if g != 1 {
|
||||
t.Errorf("补录了一条,到访「仅站内有」应为 1,实际 %d", g)
|
||||
}
|
||||
if a != 0 {
|
||||
t.Errorf("备份里没有多出来的到访,实际 %d", a)
|
||||
}
|
||||
}
|
||||
|
||||
// 差异页是只读的:渲染前后库里的行数必须一模一样。
|
||||
func TestBackupDiffPreviewWritesNothing(t *testing.T) {
|
||||
f := newMovementFixture(t)
|
||||
|
||||
count := func() (visits, places, raw int) {
|
||||
t.Helper()
|
||||
if err := f.pg.QueryRowContext(f.ctx, `SELECT
|
||||
(SELECT count(*) FROM visits WHERE dataset_id=$1),
|
||||
(SELECT count(*) FROM places WHERE dataset_id=$1),
|
||||
(SELECT count(*) FROM entity_raw WHERE dataset_id=$1)`, f.dsID).
|
||||
Scan(&visits, &places, &raw); err != nil {
|
||||
t.Fatalf("统计失败: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
v0, p0, r0 := count()
|
||||
previewBackup(t, f, f.src)
|
||||
v1, p1, r1 := count()
|
||||
if v0 != v1 || p0 != p1 || r0 != r1 {
|
||||
t.Fatalf("差异页不该写库: 到访 %d→%d 地点 %d→%d 留档 %d→%d", v0, v1, p0, p1, r0, r1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,753 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"rond-with-you/internal/geo"
|
||||
"rond-with-you/internal/ingest"
|
||||
"rond-with-you/internal/rond"
|
||||
)
|
||||
|
||||
// 差异页确认后的合并执行。
|
||||
//
|
||||
// 全部改动在一个事务里完成:半合并的数据集比没合并更难收拾(比如地点建了、
|
||||
// 到访没建,地图上就多一个永远没人去的点)。删除不在这里——删地点会级联删掉
|
||||
// 它的全部到访,那件事单独再议。
|
||||
//
|
||||
// 编号是这里最容易出错的地方:**库里那侧的编号空间才是唯一有效的**。
|
||||
// 新插入的行拿 nextSrcPK 发的新号,于是位移的 from_visit_src、天气的 visit_src
|
||||
// 都必须写「库里的新号」,而不是备份里的原号。
|
||||
|
||||
type mergeStats struct {
|
||||
Places, Activities, Tags int
|
||||
Visits, Movements, Weather int
|
||||
UpdPlaces, UpdActs, UpdVisits int
|
||||
DelPlaces, DelActs, DelVisits int
|
||||
}
|
||||
|
||||
func (s *Server) adminBackupMerge(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
ds, err := s.datasetFor(ctx)
|
||||
if err != nil {
|
||||
s.serverError(w, r, err)
|
||||
return
|
||||
}
|
||||
if ds == nil {
|
||||
s.adminFlash(w, r, "err", "没有可合并的数据集")
|
||||
return
|
||||
}
|
||||
token := r.PathValue("token")
|
||||
path, ok := stashPath(s.backupStashDir(), backupStashExt, token)
|
||||
if !ok {
|
||||
s.adminFlash(w, r, "err", "合并链接无效,请重新上传备份")
|
||||
return
|
||||
}
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
s.adminFlash(w, r, "err", "这次预览已过期,请重新上传备份")
|
||||
return
|
||||
}
|
||||
if err := r.ParseForm(); err != nil {
|
||||
s.serverError(w, r, err)
|
||||
return
|
||||
}
|
||||
backup, err := s.parseBackupFile(path)
|
||||
if err != nil {
|
||||
s.adminFlash(w, r, "err", err.Error())
|
||||
return
|
||||
}
|
||||
visits, places, acts, err := s.loadStoredAll(ctx, ds.ID)
|
||||
if err != nil {
|
||||
s.serverError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
// 重新算一遍差异再执行:从渲染差异页到点下按钮之间,库可能已经被别处改过
|
||||
vd := rond.DiffVisits(visits, backup)
|
||||
pd := rond.DiffPlaces(places, backup)
|
||||
ad := rond.DiffActivities(acts, backup)
|
||||
addNew := r.PostFormValue("add_new") == "1"
|
||||
plan := rond.BuildMergePlan(vd, pd, ad, rond.Choices{
|
||||
AddVisits: addNew, AddPlaces: addNew, AddActs: addNew,
|
||||
VisitClash: pickForm(r, "vc_"),
|
||||
PlaceClash: pickForm(r, "pc_"),
|
||||
ActClash: pickForm(r, "ac_"),
|
||||
DelVisits: pickIntForm(r, "dv_"),
|
||||
DelPlaces: pickIntForm(r, "dp_"),
|
||||
DelActs: pickIntForm(r, "da_"),
|
||||
})
|
||||
if plan.Empty() {
|
||||
s.adminFlash(w, r, "ok", "按当前的选择没有要写入的改动,数据未变")
|
||||
return
|
||||
}
|
||||
st, err := s.applyMergePlan(ctx, ds, backup, plan, visits)
|
||||
if err != nil {
|
||||
log.Printf("合并失败: %v", err)
|
||||
s.adminFlash(w, r, "err", "合并失败,已整体回滚、数据未变:"+err.Error())
|
||||
return
|
||||
}
|
||||
stashDrop(s.backupStashDir(), backupStashExt, token)
|
||||
|
||||
msg := fmt.Sprintf("合并完成:新增 %d 个地点 / %d 个活动 / %d 条到访;更新 %d 个地点 / %d 个活动 / %d 条到访",
|
||||
st.Places, st.Activities, st.Visits, st.UpdPlaces, st.UpdActs, st.UpdVisits)
|
||||
if st.DelVisits+st.DelActs+st.DelPlaces > 0 {
|
||||
msg += fmt.Sprintf(";删除 %d 条到访 / %d 个活动 / %d 个地点", st.DelVisits, st.DelActs, st.DelPlaces)
|
||||
}
|
||||
if st.Tags > 0 {
|
||||
msg += fmt.Sprintf(";顺带建了 %d 个标签", st.Tags)
|
||||
}
|
||||
if st.Movements+st.Weather > 0 {
|
||||
msg += fmt.Sprintf(";随到访带入 %d 条位移、%d 条天气", st.Movements, st.Weather)
|
||||
}
|
||||
s.adminFlash(w, r, "ok", msg)
|
||||
}
|
||||
|
||||
// pickIntForm 收集「前缀 + 整数」形式的勾选,比如 dv_825=1(要删除的到访)。
|
||||
func pickIntForm(r *http.Request, prefix string) map[int]bool {
|
||||
out := map[int]bool{}
|
||||
for k, vs := range r.PostForm {
|
||||
if len(vs) == 0 || vs[0] == "" || !strings.HasPrefix(k, prefix) {
|
||||
continue
|
||||
}
|
||||
if n, err := strconv.Atoi(strings.TrimPrefix(k, prefix)); err == nil {
|
||||
out[n] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// pickForm 收集「前缀 + 键」形式的逐条选择,比如 vc_<uuid>=backup。
|
||||
func pickForm(r *http.Request, prefix string) map[string]string {
|
||||
out := map[string]string{}
|
||||
for k, vs := range r.PostForm {
|
||||
if len(vs) == 0 || !strings.HasPrefix(k, prefix) {
|
||||
continue
|
||||
}
|
||||
out[strings.TrimPrefix(k, prefix)] = vs[0]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Server) loadStoredAll(ctx context.Context, dsID int64) ([]rond.StoredVisit, []rond.StoredPlace, []rond.StoredActivity, error) {
|
||||
visits, err := s.loadStoredVisits(ctx, dsID)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
places, err := s.loadStoredPlaces(ctx, dsID)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
acts, err := s.loadStoredActivities(ctx, dsID)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
return visits, places, acts, nil
|
||||
}
|
||||
|
||||
func (s *Server) parseBackupFile(path string) (*rond.Backup, error) {
|
||||
tmp, err := ingest.TempDir(s.cfg.DataDir, "merge")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.RemoveAll(tmp)
|
||||
db, err := rond.Open(path, tmp)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("解析备份失败:%w", err)
|
||||
}
|
||||
defer db.Close()
|
||||
b, err := rond.Parse(db)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取备份数据失败:%w", err)
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (s *Server) applyMergePlan(ctx context.Context, ds *Dataset, b *rond.Backup, plan rond.MergePlan,
|
||||
storedVisits []rond.StoredVisit) (*mergeStats, error) {
|
||||
st := &mergeStats{}
|
||||
err := s.withTx(ctx, func(ex execer) error {
|
||||
placeID, err := s.mergePlaces(ctx, ex, ds.ID, b, plan, st)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
actID, err := s.mergeActivities(ctx, ex, ds.ID, b, plan, st)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tagID, err := s.mergeTags(ctx, ex, ds.ID, b, st)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.mergeVisits(ctx, ex, ds.ID, b, plan, placeID, actID, tagID, storedVisits, st); err != nil {
|
||||
return err
|
||||
}
|
||||
// 删除放在最后:先让新增/更新落定,再按最终状态删
|
||||
if err := s.applyDeletions(ctx, ex, ds.ID, plan, st); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.recountPlaces(ctx, ex, ds.ID)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// mergePlaces 先把备份地点映射到库里的 places.id(已存在的直接映射),
|
||||
// 再处理冲突更新与新增,返回「备份地点编号 -> 库 id」。
|
||||
func (s *Server) mergePlaces(ctx context.Context, ex execer, dsID int64, b *rond.Backup,
|
||||
plan rond.MergePlan, st *mergeStats) (map[int]int64, error) {
|
||||
byCoord, byName := map[string]int64{}, map[string]int64{}
|
||||
rows, err := ex.QueryContext(ctx, `SELECT id, COALESCE(name,''), COALESCE(lat,0), COALESCE(lon,0)
|
||||
FROM places WHERE dataset_id=$1`, dsID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var name string
|
||||
var lat, lon float64
|
||||
if err := rows.Scan(&id, &name, &lat, &lon); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
if lat != 0 || lon != 0 {
|
||||
byCoord[rond.PlaceCoordKey(lat, lon)] = id
|
||||
}
|
||||
if _, dup := byName[name]; !dup && name != "" {
|
||||
byName[name] = id
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
placeID := map[int]int64{}
|
||||
for _, l := range b.Locations {
|
||||
if l.Lat != 0 || l.Lon != 0 {
|
||||
if id, ok := byCoord[rond.PlaceCoordKey(l.Lat, l.Lon)]; ok {
|
||||
placeID[l.SrcPK] = id
|
||||
continue
|
||||
}
|
||||
}
|
||||
if l.Name != "" {
|
||||
if id, ok := byName[l.Name]; ok {
|
||||
placeID[l.SrcPK] = id
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, up := range plan.UpdPlaces {
|
||||
id := placeID[up.L.SrcPK]
|
||||
if id == 0 {
|
||||
id = byCoord[up.CoordKey]
|
||||
}
|
||||
if id == 0 {
|
||||
id = byName[up.L.Name]
|
||||
}
|
||||
if id == 0 {
|
||||
return nil, fmt.Errorf("找不到要更新的地点(%s)", up.L.Name)
|
||||
}
|
||||
if _, err := ex.ExecContext(ctx, `UPDATE places SET name=$1, poi_category=$2, lat=$3, lon=$4,
|
||||
country_code=$5, province=$6, city=$7, district=$8, sublocality=$9, thoroughfare=$10, timezone=$11
|
||||
WHERE id=$12`,
|
||||
nullStr(up.L.Name), nullStr(up.L.POICategory), up.L.Lat, up.L.Lon,
|
||||
nullStr(up.L.CountryCode), nullStr(up.L.Province), nullStr(up.L.City), nullStr(up.L.District),
|
||||
nullStr(up.L.Sublocality), nullStr(up.L.Thoroughfare), nullStr(up.L.Timezone), id); err != nil {
|
||||
return nil, fmt.Errorf("更新地点: %w", err)
|
||||
}
|
||||
placeID[up.L.SrcPK] = id
|
||||
st.UpdPlaces++
|
||||
}
|
||||
|
||||
for _, l := range plan.AddPlaces {
|
||||
pk, err := s.nextSrcPK(ctx, ex, dsID, placeSeq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var id int64
|
||||
if err := ex.QueryRowContext(ctx, `INSERT INTO places
|
||||
(dataset_id, src_pk, name, poi_category, lat, lon, country_code, province, city,
|
||||
district, sublocality, thoroughfare, timezone)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13) RETURNING id`,
|
||||
dsID, pk, nullStr(l.Name), nullStr(l.POICategory), l.Lat, l.Lon, nullStr(l.CountryCode),
|
||||
nullStr(l.Province), nullStr(l.City), nullStr(l.District), nullStr(l.Sublocality),
|
||||
nullStr(l.Thoroughfare), nullStr(l.Timezone)).Scan(&id); err != nil {
|
||||
return nil, fmt.Errorf("新增地点: %w", err)
|
||||
}
|
||||
placeID[l.SrcPK] = id
|
||||
st.Places++
|
||||
if err := writeRawRow(ctx, ex, dsID, "ZLOCATION", pk, b.RawRows["ZLOCATION"][l.SrcPK]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return placeID, nil
|
||||
}
|
||||
|
||||
func (s *Server) mergeActivities(ctx context.Context, ex execer, dsID int64, b *rond.Backup,
|
||||
plan rond.MergePlan, st *mergeStats) (map[int]int64, error) {
|
||||
byUUID, byName := map[string]int64{}, map[string]int64{}
|
||||
rows, err := ex.QueryContext(ctx, `SELECT a.id, a.name, COALESCE(er.raw #>> '{ZUID_,$blob}','')
|
||||
FROM activities a
|
||||
LEFT JOIN entity_raw er ON er.dataset_id = a.dataset_id
|
||||
AND er.entity = 'ZACTIVITY' AND er.src_pk = a.src_pk
|
||||
WHERE a.dataset_id=$1`, dsID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var name, b64 string
|
||||
if err := rows.Scan(&id, &name, &b64); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
if u := uuidFromBase64(b64); u != "" {
|
||||
byUUID[u] = id
|
||||
}
|
||||
if _, dup := byName[name]; !dup && name != "" {
|
||||
byName[name] = id
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
actID := map[int]int64{}
|
||||
for _, a := range b.Activities {
|
||||
if u := b.ActivityUUID[a.SrcPK]; u != "" {
|
||||
if id, ok := byUUID[u]; ok {
|
||||
actID[a.SrcPK] = id
|
||||
continue
|
||||
}
|
||||
}
|
||||
if id, ok := byName[a.Name]; ok {
|
||||
actID[a.SrcPK] = id
|
||||
}
|
||||
}
|
||||
|
||||
for _, up := range plan.UpdActs {
|
||||
id := actID[up.A.SrcPK]
|
||||
if id == 0 {
|
||||
id = byName[up.A.Name]
|
||||
}
|
||||
if id == 0 {
|
||||
return nil, fmt.Errorf("找不到要更新的活动(%s)", up.A.Name)
|
||||
}
|
||||
if _, err := ex.ExecContext(ctx, `UPDATE activities SET name=$1, color=$2, icon=$3,
|
||||
is_home=$4, is_work=$5, is_excluded=$6, is_archived=$7 WHERE id=$8`,
|
||||
up.A.Name, nullStr(up.A.Color), nullStr(up.A.Icon), up.A.IsHome, up.A.IsWork,
|
||||
up.A.IsExcluded, up.A.IsArchived, id); err != nil {
|
||||
return nil, fmt.Errorf("更新活动: %w", err)
|
||||
}
|
||||
st.UpdActs++
|
||||
}
|
||||
|
||||
for _, a := range plan.AddActs {
|
||||
pk, err := s.nextSrcPK(ctx, ex, dsID, activitySeq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var id int64
|
||||
if err := ex.QueryRowContext(ctx, `INSERT INTO activities
|
||||
(dataset_id, src_pk, name, color, icon, is_home, is_work, is_excluded, is_archived)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) RETURNING id`,
|
||||
dsID, pk, a.Name, nullStr(a.Color), nullStr(a.Icon),
|
||||
a.IsHome, a.IsWork, a.IsExcluded, a.IsArchived).Scan(&id); err != nil {
|
||||
return nil, fmt.Errorf("新增活动: %w", err)
|
||||
}
|
||||
actID[a.SrcPK] = id
|
||||
st.Activities++
|
||||
if err := writeRawRow(ctx, ex, dsID, "ZACTIVITY", pk, b.RawRows["ZACTIVITY"][a.SrcPK]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return actID, nil
|
||||
}
|
||||
|
||||
// mergeTags 只做「库里有同名的就复用、没有就建」——标签是纯名称实体,没有别的标识。
|
||||
func (s *Server) mergeTags(ctx context.Context, ex execer, dsID int64, b *rond.Backup, st *mergeStats) (map[int]int64, error) {
|
||||
byName := map[string]int64{}
|
||||
rows, err := ex.QueryContext(ctx, `SELECT id, name FROM tags WHERE dataset_id=$1`, dsID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var name string
|
||||
if err := rows.Scan(&id, &name); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
byName[name] = id
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tagID := map[int]int64{}
|
||||
for _, t := range b.Tags {
|
||||
if id, ok := byName[t.Name]; ok {
|
||||
tagID[t.SrcPK] = id
|
||||
continue
|
||||
}
|
||||
pk, err := s.nextSrcPK(ctx, ex, dsID, tagSeq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var id int64
|
||||
if err := ex.QueryRowContext(ctx, `INSERT INTO tags (dataset_id, src_pk, name, color, group_name)
|
||||
VALUES ($1,$2,$3,$4,NULLIF($5,'')) RETURNING id`,
|
||||
dsID, pk, t.Name, nullStr(t.Color), t.GroupName).Scan(&id); err != nil {
|
||||
return nil, fmt.Errorf("新增标签: %w", err)
|
||||
}
|
||||
byName[t.Name] = id
|
||||
tagID[t.SrcPK] = id
|
||||
st.Tags++
|
||||
if err := writeRawRow(ctx, ex, dsID, "ZTAG", pk, b.RawRows["ZTAG"][t.SrcPK]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return tagID, nil
|
||||
}
|
||||
|
||||
func (s *Server) mergeVisits(ctx context.Context, ex execer, dsID int64, b *rond.Backup, plan rond.MergePlan,
|
||||
placeID, actID, tagID map[int]int64, storedVisits []rond.StoredVisit, st *mergeStats) error {
|
||||
// 备份到访编号 -> 库里的 visits.src_pk:位移的 from/to_visit_src 与天气的
|
||||
// visit_src 写的都是**库里的编号**,端点在备份里的原号在这里换成库里的号。
|
||||
visSrc := map[int]int{}
|
||||
storedByUUID := map[string]rond.StoredVisit{}
|
||||
for _, sv := range storedVisits {
|
||||
if sv.UUID != "" {
|
||||
storedByUUID[sv.UUID] = sv
|
||||
}
|
||||
}
|
||||
for _, v := range b.Visits {
|
||||
if u := b.VisitUUID[v.SrcPK]; u != "" {
|
||||
if sv, ok := storedByUUID[u]; ok {
|
||||
visSrc[v.SrcPK] = sv.SrcPK
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
activityByPK := map[int]rond.Activity{}
|
||||
for _, a := range b.Activities {
|
||||
activityByPK[a.SrcPK] = a
|
||||
}
|
||||
visitPlace := map[int]int{}
|
||||
for _, v := range b.Visits {
|
||||
if v.LocationSrcPK != nil {
|
||||
visitPlace[v.SrcPK] = *v.LocationSrcPK
|
||||
}
|
||||
}
|
||||
|
||||
resolve := func(v rond.Visit) (pid, aid any, isHome, isWork bool) {
|
||||
if v.LocationSrcPK != nil {
|
||||
if x, ok := placeID[*v.LocationSrcPK]; ok {
|
||||
pid = x
|
||||
}
|
||||
}
|
||||
if v.ActivitySrcPK != nil {
|
||||
if x, ok := actID[*v.ActivitySrcPK]; ok {
|
||||
aid = x
|
||||
}
|
||||
if a, ok := activityByPK[*v.ActivitySrcPK]; ok {
|
||||
isHome, isWork = a.IsHome, a.IsWork
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
for _, up := range plan.UpdVisits {
|
||||
sv, ok := storedByUUID[up.UUID]
|
||||
if !ok {
|
||||
return fmt.Errorf("找不到要更新的到访(%s)", up.UUID)
|
||||
}
|
||||
pid, aid, isHome, isWork := resolve(up.V)
|
||||
if _, err := ex.ExecContext(ctx, `UPDATE visits SET place_id=$1, activity_id=$2, arrival=$3, departure=$4,
|
||||
duration_min=$5, is_home=$6, is_work=$7, bookmarked=$8, remark=NULLIF($9,''), emoji=NULLIF($10,'')
|
||||
WHERE dataset_id=$11 AND src_pk=$12`,
|
||||
pid, aid, up.V.Arrival, up.V.Departure, visitDuration(up.V), isHome, isWork,
|
||||
up.V.Bookmarked, up.V.Remark, up.V.Emoji, dsID, sv.SrcPK); err != nil {
|
||||
return fmt.Errorf("更新到访: %w", err)
|
||||
}
|
||||
st.UpdVisits++
|
||||
}
|
||||
|
||||
addedVisit := map[int]bool{}
|
||||
for _, v := range plan.AddVisits {
|
||||
pid, aid, isHome, isWork := resolve(v)
|
||||
pk, err := s.nextSrcPK(ctx, ex, dsID, visitSeq)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var id int64
|
||||
if err := ex.QueryRowContext(ctx, `INSERT INTO visits
|
||||
(dataset_id, src_pk, place_id, activity_id, arrival, departure, duration_min,
|
||||
is_home, is_work, bookmarked, is_user_added, remark, emoji, weather_symbol)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,NULLIF($12,''),NULLIF($13,''),NULLIF($14,''))
|
||||
ON CONFLICT DO NOTHING RETURNING id`,
|
||||
dsID, pk, pid, aid, v.Arrival, v.Departure, visitDuration(v), isHome, isWork,
|
||||
v.Bookmarked, v.UserAdded, v.Remark, v.Emoji, v.WeatherSymbol).Scan(&id); err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("新增到访: %w", err)
|
||||
}
|
||||
|
||||
visSrc[v.SrcPK] = pk
|
||||
addedVisit[v.SrcPK] = true
|
||||
st.Visits++
|
||||
// 留档必须写:下次再比对时库侧就是靠这里的 UUID 认出这一条,
|
||||
// 少了它只能退到「到达时间 + 坐标」兜底。
|
||||
if err := writeRawRow(ctx, ex, dsID, "ZVISIT", pk, b.RawRows["ZVISIT"][v.SrcPK]); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, tpk := range v.TagSrcPKs {
|
||||
if tid, ok := tagID[tpk]; ok {
|
||||
if _, err := ex.ExecContext(ctx, `INSERT INTO visit_tags (visit_id, tag_id)
|
||||
VALUES ($1,$2) ON CONFLICT DO NOTHING`, id, tid); err != nil {
|
||||
return fmt.Errorf("写入到访标签: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
transName, transColor, transIcon := map[int]string{}, map[int]string{}, map[int]string{}
|
||||
for _, t := range b.Transports {
|
||||
transName[t.SrcPK], transColor[t.SrcPK], transIcon[t.SrcPK] = t.Name, t.Color, t.Icon
|
||||
}
|
||||
coordOf := map[int][2]float64{}
|
||||
for _, l := range b.Locations {
|
||||
coordOf[l.SrcPK] = [2]float64{l.Lat, l.Lon}
|
||||
}
|
||||
// 位移与天气跟着**新插入的到访**走:只插这一轮真正补进来的那些,
|
||||
// 否则库里已有的位移会被整份复制一遍。另一端可以是老到访(库里已经有)。
|
||||
for _, m := range b.Movements {
|
||||
if m.FromVisitSrc == nil || m.ToVisitSrc == nil {
|
||||
continue
|
||||
}
|
||||
if !addedVisit[*m.FromVisitSrc] && !addedVisit[*m.ToVisitSrc] {
|
||||
continue
|
||||
}
|
||||
fsrc, ok1 := visSrc[*m.FromVisitSrc]
|
||||
tsrc, ok2 := visSrc[*m.ToVisitSrc]
|
||||
if !ok1 || !ok2 {
|
||||
continue
|
||||
}
|
||||
var fromPID, toPID any
|
||||
if fp, ok := visitPlace[*m.FromVisitSrc]; ok {
|
||||
if x, ok := placeID[fp]; ok {
|
||||
fromPID = x
|
||||
}
|
||||
}
|
||||
if tp, ok := visitPlace[*m.ToVisitSrc]; ok {
|
||||
if x, ok := placeID[tp]; ok {
|
||||
toPID = x
|
||||
}
|
||||
}
|
||||
var dist any
|
||||
if fp, ok1 := visitPlace[*m.FromVisitSrc]; ok1 {
|
||||
if tp, ok2 := visitPlace[*m.ToVisitSrc]; ok2 {
|
||||
c1, c2 := coordOf[fp], coordOf[tp]
|
||||
if c1[0] != 0 && c2[0] != 0 {
|
||||
dist = geo.Haversine(c1[0], c1[1], c2[0], c2[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
var tname, tcolor, ticon any
|
||||
if m.TransportSrc != nil {
|
||||
tname = nullStr(transName[*m.TransportSrc])
|
||||
tcolor = nullStr(transColor[*m.TransportSrc])
|
||||
ticon = nullStr(transIcon[*m.TransportSrc])
|
||||
}
|
||||
var dur any
|
||||
if m.Start != nil && m.End != nil {
|
||||
if d := int(m.End.Sub(*m.Start).Minutes()); d >= 0 {
|
||||
dur = d
|
||||
}
|
||||
}
|
||||
pk, err := s.nextSrcPK(ctx, ex, dsID, movementSeq)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := ex.ExecContext(ctx, `INSERT INTO movements
|
||||
(dataset_id, src_pk, transport_src, transport_name, transport_color, transport_icon,
|
||||
from_visit_src, to_visit_src, from_place_id, to_place_id, started_at, ended_at, duration_min, distance_km)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT DO NOTHING`,
|
||||
dsID, pk, m.TransportSrc, tname, tcolor, ticon, fsrc, tsrc,
|
||||
fromPID, toPID, m.Start, m.End, dur, dist); err != nil {
|
||||
return fmt.Errorf("新增位移: %w", err)
|
||||
}
|
||||
st.Movements++
|
||||
if err := writeRawRow(ctx, ex, dsID, "ZMOVEMENT", pk, b.RawRows["ZMOVEMENT"][m.SrcPK]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, w := range b.Weather {
|
||||
if w.VisitSrc == nil || w.At == nil {
|
||||
continue
|
||||
}
|
||||
if !addedVisit[*w.VisitSrc] {
|
||||
continue
|
||||
}
|
||||
vsrc, ok := visSrc[*w.VisitSrc]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
// 站内删到访时会把天气的归属置空而保留记录(见 editVisitDelete)。
|
||||
// 这类「同一时刻、没有归属」的天气先认领回来,否则同一条到访找回来之后
|
||||
// 会多出一份重复的天气。
|
||||
var claimed int64
|
||||
if err := ex.QueryRowContext(ctx, `UPDATE weather SET visit_src=$1
|
||||
WHERE dataset_id=$2 AND at=$3 AND visit_src IS NULL RETURNING id`,
|
||||
vsrc, dsID, w.At).Scan(&claimed); err == nil {
|
||||
continue
|
||||
}
|
||||
pk, err := s.nextSrcPK(ctx, ex, dsID, weatherSeq)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := ex.ExecContext(ctx, `INSERT INTO weather
|
||||
(dataset_id, src_pk, visit_src, at, is_daylight, temperature_c, apparent_c, humidity,
|
||||
precipitation_amount, precipitation_chance, wind_speed, visibility, uv_index,
|
||||
condition, symbol, uv_category, wind_direction)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,NULLIF($14,''),NULLIF($15,''),NULLIF($16,''),NULLIF($17,''))
|
||||
ON CONFLICT DO NOTHING`,
|
||||
dsID, pk, vsrc, w.At, w.IsDaylight, w.TempC, w.ApparentC, w.Humidity,
|
||||
w.PrecipAmt, w.PrecipPct, w.WindSpeed, w.Visibility, w.UVIndex,
|
||||
w.Condition, w.Symbol, w.UVCategory, w.WindDir); err != nil {
|
||||
return fmt.Errorf("新增天气: %w", err)
|
||||
}
|
||||
st.Weather++
|
||||
if err := writeRawRow(ctx, ex, dsID, "ZHOURLYWEATHER", pk, b.RawRows["ZHOURLYWEATHER"][w.SrcPK]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyDeletions 处理「仅站内有」里被勾选删除的记录。顺序是到访 → 活动 → 地点:
|
||||
// 删地点会级联删掉它名下的全部到访(外键 ON DELETE CASCADE),所以反过来会误删;
|
||||
// 而且**有到访引用时直接拒绝**,让站长先处理到访,而不是悄悄带走一批。
|
||||
func (s *Server) applyDeletions(ctx context.Context, ex execer, dsID int64, plan rond.MergePlan, st *mergeStats) error {
|
||||
for _, sv := range plan.DelVisits {
|
||||
if _, err := ex.ExecContext(ctx, `DELETE FROM movements WHERE dataset_id=$1
|
||||
AND (from_visit_src=$2 OR to_visit_src=$2)`, dsID, sv.SrcPK); err != nil {
|
||||
return fmt.Errorf("删除到访的位移: %w", err)
|
||||
}
|
||||
// 天气保留、只解除归属——与站点 editVisitDelete 的处理一致
|
||||
if _, err := ex.ExecContext(ctx, `UPDATE weather SET visit_src=NULL
|
||||
WHERE dataset_id=$1 AND visit_src=$2`, dsID, sv.SrcPK); err != nil {
|
||||
return fmt.Errorf("解除天气归属: %w", err)
|
||||
}
|
||||
if _, err := ex.ExecContext(ctx, `DELETE FROM visits WHERE dataset_id=$1 AND src_pk=$2`,
|
||||
dsID, sv.SrcPK); err != nil {
|
||||
return fmt.Errorf("删除到访: %w", err)
|
||||
}
|
||||
if _, err := ex.ExecContext(ctx, `DELETE FROM entity_raw WHERE dataset_id=$1
|
||||
AND entity='ZVISIT' AND src_pk=$2`, dsID, sv.SrcPK); err != nil {
|
||||
return err
|
||||
}
|
||||
st.DelVisits++
|
||||
}
|
||||
for _, sa := range plan.DelActs {
|
||||
// 到访的 activity_id 是 ON DELETE SET NULL,删活动不会带走到访
|
||||
if _, err := ex.ExecContext(ctx, `DELETE FROM activities WHERE dataset_id=$1 AND src_pk=$2`,
|
||||
dsID, sa.SrcPK); err != nil {
|
||||
return fmt.Errorf("删除活动: %w", err)
|
||||
}
|
||||
if _, err := ex.ExecContext(ctx, `DELETE FROM entity_raw WHERE dataset_id=$1
|
||||
AND entity='ZACTIVITY' AND src_pk=$2`, dsID, sa.SrcPK); err != nil {
|
||||
return err
|
||||
}
|
||||
st.DelActs++
|
||||
}
|
||||
for _, sp := range plan.DelPlaces {
|
||||
var n int
|
||||
if err := ex.QueryRowContext(ctx, `SELECT count(*) FROM visits v
|
||||
JOIN places p ON p.id = v.place_id
|
||||
WHERE p.dataset_id=$1 AND p.src_pk=$2`, dsID, sp.SrcPK).Scan(&n); err != nil {
|
||||
return err
|
||||
}
|
||||
if n > 0 {
|
||||
return fmt.Errorf("地点「%s」名下还有 %d 条到访,删它会连带删掉那些到访——请先处理这些到访",
|
||||
sp.Name, n)
|
||||
}
|
||||
if _, err := ex.ExecContext(ctx, `DELETE FROM places WHERE dataset_id=$1 AND src_pk=$2`,
|
||||
dsID, sp.SrcPK); err != nil {
|
||||
return fmt.Errorf("删除地点: %w", err)
|
||||
}
|
||||
if _, err := ex.ExecContext(ctx, `DELETE FROM entity_raw WHERE dataset_id=$1
|
||||
AND entity='ZLOCATION' AND src_pk=$2`, dsID, sp.SrcPK); err != nil {
|
||||
return err
|
||||
}
|
||||
st.DelPlaces++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// recountPlaces 重算地点聚合与数据集汇总。全量重算而不是只动受影响的行:
|
||||
// 367 个地点两条 SQL 就够,省得漏掉任何一条派生列。
|
||||
func (s *Server) recountPlaces(ctx context.Context, ex execer, dsID int64) error {
|
||||
if _, err := ex.ExecContext(ctx, `UPDATE places SET visit_count=0, dwell_minutes=0,
|
||||
first_visit_at=NULL, last_visit_at=NULL WHERE dataset_id=$1`, dsID); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := ex.ExecContext(ctx, `UPDATE places p SET visit_count=sub.c, dwell_minutes=sub.d,
|
||||
first_visit_at=sub.f, last_visit_at=sub.l
|
||||
FROM (SELECT place_id, count(*) c,
|
||||
COALESCE(sum(LEAST(COALESCE(duration_min,0),10080)),0) d,
|
||||
min(arrival) f, max(arrival) l
|
||||
FROM visits WHERE dataset_id=$1 AND place_id IS NOT NULL GROUP BY place_id) sub
|
||||
WHERE p.id = sub.place_id`, dsID); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := ex.ExecContext(ctx, `UPDATE datasets SET
|
||||
visit_count=(SELECT count(*) FROM visits WHERE dataset_id=$1),
|
||||
place_count=(SELECT count(*) FROM places WHERE dataset_id=$1),
|
||||
first_visit_at=(SELECT min(arrival) FROM visits WHERE dataset_id=$1),
|
||||
last_visit_at=(SELECT max(arrival) FROM visits WHERE dataset_id=$1)
|
||||
WHERE id=$1`, dsID)
|
||||
return err
|
||||
}
|
||||
|
||||
// writeRawRow 给新插入的行留一份原始行:导出时要靠它逐列还原,
|
||||
// 再次比对时要靠它认出 UUID(否则只能退到「时间 + 坐标」兜底)。
|
||||
func writeRawRow(ctx context.Context, ex execer, dsID int64, entity string, pk int, raw map[string]any) error {
|
||||
if len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
blob, err := json.Marshal(raw)
|
||||
if err != nil {
|
||||
log.Printf("留档 %s#%d 序列化失败: %v", entity, pk, err)
|
||||
return nil
|
||||
}
|
||||
_, err = ex.ExecContext(ctx, `INSERT INTO entity_raw (dataset_id, entity, src_pk, raw, skipped)
|
||||
VALUES ($1,$2,$3,$4,false)
|
||||
ON CONFLICT (dataset_id, entity, src_pk) DO UPDATE SET raw=EXCLUDED.raw`,
|
||||
dsID, entity, pk, string(blob))
|
||||
return err
|
||||
}
|
||||
|
||||
func visitDuration(v rond.Visit) any {
|
||||
if v.Arrival == nil || v.Departure == nil {
|
||||
return nil
|
||||
}
|
||||
m := int(v.Departure.Sub(*v.Arrival).Minutes())
|
||||
if m < 0 {
|
||||
m = 0
|
||||
}
|
||||
return m
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// mergeBackup 走真实的合并处理函数;失败(重定向里带 err=)直接判错。
|
||||
func mergeBackup(t *testing.T, f *movementFixture, token, body string) {
|
||||
t.Helper()
|
||||
if loc := mergeBackupAllowErr(t, f, token, body); strings.Contains(loc, "err=") {
|
||||
t.Fatalf("合并失败: %s", loc)
|
||||
}
|
||||
}
|
||||
|
||||
// mergeBackupAllowErr 同上,但把落点交出来,供「预期会失败」的用例断言。
|
||||
func mergeBackupAllowErr(t *testing.T, f *movementFixture, token, body string) string {
|
||||
t.Helper()
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest("POST", "/admin/preview/"+token+"/merge", strings.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
req.SetPathValue("token", token)
|
||||
f.srv.adminBackupMerge(rec, req.WithContext(f.ctx))
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("合并应跳回后台: code=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
return rec.Header().Get("Location")
|
||||
}
|
||||
|
||||
// addExtraVisit 造一条「仅站内有」的到访:没有留档,于是没有 UUID,比对时必然落在仅站内有里。
|
||||
func (f *movementFixture) addExtraVisit(t *testing.T, pk int, remark string) {
|
||||
t.Helper()
|
||||
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO visits
|
||||
(dataset_id, src_pk, arrival, departure, is_user_added, remark)
|
||||
VALUES ($1,$2,now(),now(),true,$3)`, f.dsID, pk, remark); err != nil {
|
||||
t.Fatalf("插入补录到访失败: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *movementFixture) scalar(t *testing.T, q string, args ...any) int {
|
||||
t.Helper()
|
||||
var n int
|
||||
if err := f.pg.QueryRowContext(f.ctx, q, args...).Scan(&n); err != nil {
|
||||
t.Fatalf("查询失败: %v", err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// 合并把备份里多出来的记录写进去,而且写完要「复原」——同一份备份再比一次必须一条不差。
|
||||
//
|
||||
// 这条测试同时钉住三件事:
|
||||
// - 新增行用 nextSrcPK 发新号,且**写进 entity_raw**(否则下次比对认不出 UUID);
|
||||
// - 位移/天气的外键指向库里的新编号(写错就成孤儿);
|
||||
// - 合并是幂等的:再传同一份备份,差异归零。
|
||||
func TestBackupMergeRestoresMissingVisits(t *testing.T) {
|
||||
f := newMovementFixture(t)
|
||||
|
||||
visitCount := func() int {
|
||||
return f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1`, f.dsID)
|
||||
}
|
||||
all := visitCount()
|
||||
movCount := f.scalar(t, `SELECT count(*) FROM movements WHERE dataset_id=$1`, f.dsID)
|
||||
weatherCount := f.scalar(t, `SELECT count(*) FROM weather WHERE dataset_id=$1`, f.dsID)
|
||||
// 原始数据里就有指向「因缺到达时间而没入库」的到访的位移,先记下基线再比
|
||||
orphanQ := `SELECT count(*) FROM movements m WHERE m.dataset_id=$1
|
||||
AND (m.from_visit_src IS NULL OR NOT EXISTS (
|
||||
SELECT 1 FROM visits v WHERE v.dataset_id=$1 AND v.src_pk=m.from_visit_src))`
|
||||
orphans0 := f.scalar(t, orphanQ, f.dsID)
|
||||
|
||||
// 制造差异:按**站点删到访的方式**删掉最早的两条(见 editVisitDelete)——
|
||||
// 位移删掉、天气的归属置空、到访删掉。直接 DELETE visits 会留下孤儿天气,
|
||||
// 那是测试手法造成的,不是产品行为。
|
||||
var pk1, pk2 int
|
||||
if err := f.pg.QueryRowContext(f.ctx, `SELECT
|
||||
(SELECT src_pk FROM visits WHERE dataset_id=$1 ORDER BY arrival LIMIT 1),
|
||||
(SELECT src_pk FROM visits WHERE dataset_id=$1 ORDER BY arrival OFFSET 1 LIMIT 1)`,
|
||||
f.dsID).Scan(&pk1, &pk2); err != nil {
|
||||
t.Fatalf("取待删到访失败: %v", err)
|
||||
}
|
||||
for _, q := range []string{
|
||||
`DELETE FROM movements WHERE dataset_id=$1 AND (from_visit_src IN ($2,$3) OR to_visit_src IN ($2,$3))`,
|
||||
`UPDATE weather SET visit_src=NULL WHERE dataset_id=$1 AND visit_src IN ($2,$3)`,
|
||||
`DELETE FROM visits WHERE dataset_id=$1 AND src_pk IN ($2,$3)`,
|
||||
} {
|
||||
if _, err := f.pg.ExecContext(f.ctx, q, f.dsID, pk1, pk2); err != nil {
|
||||
t.Fatalf("预置删除失败: %v", err)
|
||||
}
|
||||
}
|
||||
if got := visitCount(); got != all-2 {
|
||||
t.Fatalf("预置失败:应有 %d 条,实际 %d", all-2, got)
|
||||
}
|
||||
|
||||
token, page := previewBackupToken(t, f, f.src)
|
||||
if a, _, _ := visitCounts(t, page); a != 2 {
|
||||
t.Fatalf("差异页应报 2 条新增,实际 %d", a)
|
||||
}
|
||||
|
||||
mergeBackup(t, f, token, "add_new=1")
|
||||
|
||||
if got := visitCount(); got != all {
|
||||
t.Fatalf("合并后到访数应回到 %d,实际 %d", all, got)
|
||||
}
|
||||
// 新补进来的那两条必须各有一份留档(下次比对就靠它认 UUID)。
|
||||
// 被删掉的行会留下孤儿留档——那是老数据,不影响,所以只查新编号。
|
||||
if got := f.scalar(t, `SELECT count(*) FROM entity_raw er
|
||||
WHERE er.dataset_id=$1 AND er.entity='ZVISIT' AND EXISTS (
|
||||
SELECT 1 FROM visits v WHERE v.dataset_id=$1 AND v.src_pk=er.src_pk)`, f.dsID); got != all {
|
||||
t.Errorf("每条在库的到访都该有原始行留档,实际 %d / %d", got, all)
|
||||
}
|
||||
if got := f.scalar(t, `SELECT count(*) FROM movements WHERE dataset_id=$1`, f.dsID); got != movCount {
|
||||
t.Errorf("位移应恢复到 %d,实际 %d", movCount, got)
|
||||
}
|
||||
if got := f.scalar(t, `SELECT count(*) FROM weather WHERE dataset_id=$1`, f.dsID); got != weatherCount {
|
||||
t.Errorf("天气应恢复到 %d,实际 %d", weatherCount, got)
|
||||
}
|
||||
// 合并不该新增孤儿位移(原始数据里本来就有一些,见 orphanQ 的注释)
|
||||
if orphans := f.scalar(t, orphanQ, f.dsID); orphans > orphans0 {
|
||||
t.Errorf("孤儿位移变多了:%d → %d", orphans0, orphans)
|
||||
}
|
||||
|
||||
// 幂等:再比一次应当完全一致
|
||||
page = previewBackup(t, f, f.src)
|
||||
if a, g, c := visitCounts(t, page); a != 0 || g != 0 || c != 0 {
|
||||
t.Fatalf("合并后再比对应当没有差异,实际 新增 %d / 仅站内有 %d / 冲突 %d", a, g, c)
|
||||
}
|
||||
}
|
||||
|
||||
// 冲突逐条选「用实机」就把备份的值写进去,选「用站内」就原样不动。
|
||||
func TestBackupMergeConflictChoice(t *testing.T) {
|
||||
f := newMovementFixture(t)
|
||||
|
||||
var pk int
|
||||
if err := f.pg.QueryRowContext(f.ctx, `UPDATE visits SET remark='站内改过的备注'
|
||||
WHERE id = (SELECT id FROM visits WHERE dataset_id=$1 ORDER BY src_pk LIMIT 1)
|
||||
RETURNING src_pk`, f.dsID).Scan(&pk); err != nil {
|
||||
t.Fatalf("改备注失败: %v", err)
|
||||
}
|
||||
// 这条的 UUID 就是页面上的冲突键
|
||||
var b64 string
|
||||
if err := f.pg.QueryRowContext(f.ctx, `SELECT COALESCE(raw #>> '{ZIDENTIFIER_,$blob}','')
|
||||
FROM entity_raw WHERE dataset_id=$1 AND entity='ZVISIT' AND src_pk=$2`, f.dsID, pk).Scan(&b64); err != nil {
|
||||
t.Fatalf("取 UUID 失败: %v", err)
|
||||
}
|
||||
uuid := uuidFromBase64(b64)
|
||||
if uuid == "" {
|
||||
t.Fatal("这条到访应当有 UUID")
|
||||
}
|
||||
|
||||
remark := func() string {
|
||||
var s string
|
||||
if err := f.pg.QueryRowContext(f.ctx, `SELECT COALESCE(remark,'') FROM visits
|
||||
WHERE dataset_id=$1 AND src_pk=$2`, f.dsID, pk).Scan(&s); err != nil {
|
||||
t.Fatalf("读备注失败: %v", err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// 用站内:备注保持站里那份
|
||||
token, _ := previewBackupToken(t, f, f.src)
|
||||
mergeBackup(t, f, token, "vc_"+uuid+"=db")
|
||||
if got := remark(); got != "站内改过的备注" {
|
||||
t.Fatalf("选「用站内」不该改动,实际 %q", got)
|
||||
}
|
||||
|
||||
// 用实机:备注被备份里的值(空)覆盖回去
|
||||
token, _ = previewBackupToken(t, f, f.src)
|
||||
mergeBackup(t, f, token, "vc_"+uuid+"=backup")
|
||||
if got := remark(); got != "" {
|
||||
t.Fatalf("选「用实机」应覆盖成备份里的值,实际 %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 什么都不勾、冲突全跳过时,合并应当什么都不写。
|
||||
func TestBackupMergeNoopKeepsData(t *testing.T) {
|
||||
f := newMovementFixture(t)
|
||||
before := f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1`, f.dsID)
|
||||
|
||||
token, _ := previewBackupToken(t, f, f.src)
|
||||
mergeBackup(t, f, token, "add_new=0")
|
||||
|
||||
if got := f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1`, f.dsID); got != before {
|
||||
t.Fatalf("没勾任何改动却动了数据: %d → %d", before, got)
|
||||
}
|
||||
}
|
||||
|
||||
// 删除必须「勾了才删」:没勾的一个都不动,勾了的那条(连带它的位移)才消失。
|
||||
func TestBackupMergeDeletesOnlyChecked(t *testing.T) {
|
||||
f := newMovementFixture(t)
|
||||
visitCount := func() int {
|
||||
return f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1`, f.dsID)
|
||||
}
|
||||
before := visitCount()
|
||||
|
||||
f.addExtraVisit(t, 900001, "稍后删掉")
|
||||
f.addExtraVisit(t, 900002, "留着不动")
|
||||
if got := visitCount(); got != before+2 {
|
||||
t.Fatalf("预置失败: %d", got)
|
||||
}
|
||||
|
||||
// 只勾一条
|
||||
token, page := previewBackupToken(t, f, f.src)
|
||||
if _, g, _ := visitCounts(t, page); g != 2 {
|
||||
t.Fatalf("差异页应报 2 条仅站内有,实际 %d", g)
|
||||
}
|
||||
mergeBackup(t, f, token, "dv_900001=1")
|
||||
|
||||
if got := visitCount(); got != before+1 {
|
||||
t.Fatalf("应只删掉勾选的那条,实际剩 %d(原 %d)", got, before)
|
||||
}
|
||||
if n := f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1 AND src_pk=900002`, f.dsID); n != 1 {
|
||||
t.Error("没勾的那条不该被动")
|
||||
}
|
||||
// 删掉的到访不该留下留档
|
||||
if n := f.scalar(t, `SELECT count(*) FROM entity_raw WHERE dataset_id=$1
|
||||
AND entity='ZVISIT' AND src_pk=900001`, f.dsID); n != 0 {
|
||||
t.Error("删除到访时要连它的原始行留档一起清掉")
|
||||
}
|
||||
|
||||
// 再合并一次、一条都不勾:剩下那条必须还在
|
||||
token, _ = previewBackupToken(t, f, f.src)
|
||||
mergeBackup(t, f, token, "add_new=0")
|
||||
if got := visitCount(); got != before+1 {
|
||||
t.Fatalf("不勾就不该删,实际 %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 名下有到访的地点必须拒绝删除——那个外键是 ON DELETE CASCADE,
|
||||
// 真删下去会把到访一起带走。拒绝后还要整体回滚,别留下半截。
|
||||
func TestBackupMergeRefusesDeletingPlaceInUse(t *testing.T) {
|
||||
f := newMovementFixture(t)
|
||||
visitCount := func() int {
|
||||
return f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1`, f.dsID)
|
||||
}
|
||||
beforeVisits := visitCount()
|
||||
|
||||
// 一个没有到访的空地点:允许删
|
||||
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO places (dataset_id, src_pk, name, lat, lon)
|
||||
VALUES ($1, 910001, '临时空地点', 30.0, 120.0)`, f.dsID); err != nil {
|
||||
t.Fatalf("插入地点失败: %v", err)
|
||||
}
|
||||
// 一个有到访的地点:必须拒绝
|
||||
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO places (dataset_id, src_pk, name, lat, lon)
|
||||
VALUES ($1, 910002, '有到访的地点', 31.0, 121.0)`, f.dsID); err != nil {
|
||||
t.Fatalf("插入地点失败: %v", err)
|
||||
}
|
||||
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO visits (dataset_id, src_pk, place_id, arrival, departure)
|
||||
VALUES ($1, 910003, (SELECT id FROM places WHERE dataset_id=$1 AND src_pk=910002), now(), now())`,
|
||||
f.dsID); err != nil {
|
||||
t.Fatalf("插入到访失败: %v", err)
|
||||
}
|
||||
|
||||
// 先删空地点:应当成功
|
||||
token, _ := previewBackupToken(t, f, f.src)
|
||||
mergeBackup(t, f, token, "dp_910001=1")
|
||||
if n := f.scalar(t, `SELECT count(*) FROM places WHERE dataset_id=$1 AND src_pk=910001`, f.dsID); n != 0 {
|
||||
t.Fatal("空地点应当被删掉")
|
||||
}
|
||||
|
||||
// 再删有到访的地点:应当被拒绝
|
||||
token, _ = previewBackupToken(t, f, f.src)
|
||||
loc := mergeBackupAllowErr(t, f, token, "dp_910002=1")
|
||||
if !strings.Contains(loc, "err=") {
|
||||
t.Fatal("名下有到访的地点应当拒绝删除")
|
||||
}
|
||||
if n := f.scalar(t, `SELECT count(*) FROM places WHERE dataset_id=$1 AND src_pk=910002`, f.dsID); n != 1 {
|
||||
t.Error("被拒绝后地点不该消失")
|
||||
}
|
||||
if got := visitCount(); got != beforeVisits+1 {
|
||||
t.Errorf("被拒绝后到访数不该变,实际 %d(期望 %d)", got, beforeVisits+1)
|
||||
}
|
||||
}
|
||||
+151
-18
@@ -3,11 +3,13 @@ package web
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/png"
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
@@ -111,41 +113,57 @@ func fogTileCacheControl(isAdmin bool) string {
|
||||
return "public, max-age=3600"
|
||||
}
|
||||
|
||||
// adminFogUpload 导入 .fwss:mode=overlay 按位或合并进现有迷雾,
|
||||
// mode=cover 清空后整体替换。
|
||||
func (s *Server) adminFogUpload(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
u := userFrom(ctx)
|
||||
// readFogSnapshot 从 multipart 表单里读出 .fwss 的字节并做基本校验。
|
||||
func readFogSnapshot(w http.ResponseWriter, r *http.Request) ([]byte, error) {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxFogUploadBytes+1<<20)
|
||||
if err := r.ParseMultipartForm(32 << 20); err != nil {
|
||||
s.adminFlash(w, r, "err", "读取上传内容失败:"+err.Error())
|
||||
return
|
||||
return nil, fmt.Errorf("读取上传内容失败:%w", err)
|
||||
}
|
||||
file, hdr, err := r.FormFile("fwss")
|
||||
if err != nil {
|
||||
s.adminFlash(w, r, "err", "请选择要上传的 .fwss 迷雾快照")
|
||||
return
|
||||
return nil, errors.New("请选择要上传的 .fwss 迷雾快照")
|
||||
}
|
||||
defer file.Close()
|
||||
name := strings.ToLower(filepath.Base(hdr.Filename))
|
||||
if !strings.HasSuffix(name, ".fwss") && !strings.HasSuffix(name, ".zip") {
|
||||
s.adminFlash(w, r, "err", "文件类型不正确,请上传 Fog of World 导出的 .fwss 快照")
|
||||
return
|
||||
return nil, errors.New("文件类型不正确,请上传 Fog of World 导出的 .fwss 快照")
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(file, maxFogUploadBytes+1))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取文件失败:%w", err)
|
||||
}
|
||||
if int64(len(data)) > maxFogUploadBytes {
|
||||
return nil, errors.New("迷雾快照超过 64MB,疑似文件有误")
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// adminFogUpload 导入 .fwss:mode=overlay 按位或合并进现有迷雾,
|
||||
// mode=cover 清空后整体替换。
|
||||
//
|
||||
// 带 stash 令牌的请求来自差异页的确认按钮,快照已经在暂存目录里,
|
||||
// 不用让浏览器把几十 MB 再传一遍。
|
||||
func (s *Server) adminFogUpload(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
u := userFrom(ctx)
|
||||
mode := r.PostFormValue("mode")
|
||||
if mode != "overlay" && mode != "cover" {
|
||||
mode = "overlay"
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(io.LimitReader(file, maxFogUploadBytes+1))
|
||||
if err != nil {
|
||||
s.adminFlash(w, r, "err", "读取文件失败:"+err.Error())
|
||||
return
|
||||
}
|
||||
if int64(len(data)) > maxFogUploadBytes {
|
||||
s.adminFlash(w, r, "err", "迷雾快照超过 64MB,疑似文件有误")
|
||||
token := r.URL.Query().Get("stash")
|
||||
var data []byte
|
||||
var err error
|
||||
if token != "" {
|
||||
if data, err = s.loadStashedFog(token); err != nil {
|
||||
s.adminFlash(w, r, "err", err.Error())
|
||||
return
|
||||
}
|
||||
} else if data, err = readFogSnapshot(w, r); err != nil {
|
||||
s.adminFlash(w, r, "err", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
blocks, cells, err := fog.Decode(data)
|
||||
if err != nil {
|
||||
s.adminFlash(w, r, "err", "解析迷雾快照失败:"+err.Error())
|
||||
@@ -181,10 +199,125 @@ func (s *Server) adminFogUpload(w http.ResponseWriter, r *http.Request) {
|
||||
if mode == "cover" {
|
||||
verb = "整体替换"
|
||||
}
|
||||
// 导入成功才丢掉暂存件;失败时留着,站长可以回差异页重试
|
||||
if token != "" {
|
||||
s.dropStashedFog(token)
|
||||
}
|
||||
s.adminFlash(w, r, "ok", fmt.Sprintf(
|
||||
"迷雾导入成功(%s):本次写入 %d 块 / %d 格,当前共 %d 块 / %d 格", verb, applied, cells, b, c))
|
||||
}
|
||||
|
||||
// adminFogPreview 先把快照收下暂存,不写库,跳到差异页——
|
||||
// 叠加能补多少、整体替换会丢多少,都得让站长在下手之前看见。
|
||||
func (s *Server) adminFogPreview(w http.ResponseWriter, r *http.Request) {
|
||||
data, err := readFogSnapshot(w, r)
|
||||
if err != nil {
|
||||
s.adminFlash(w, r, "err", err.Error())
|
||||
return
|
||||
}
|
||||
// 先解一遍确认是真快照,免得把坏文件暂存下来、到差异页才发现
|
||||
if _, _, err := fog.Decode(data); err != nil {
|
||||
s.adminFlash(w, r, "err", "解析迷雾快照失败:"+err.Error())
|
||||
return
|
||||
}
|
||||
token, err := s.stashFog(data)
|
||||
if err != nil {
|
||||
s.serverError(w, r, err)
|
||||
return
|
||||
}
|
||||
http.Redirect(w, r, "/admin/fog/preview/"+token, http.StatusSeeOther)
|
||||
}
|
||||
|
||||
// adminFogPreviewPage 渲染差异页。每次都重新算一遍:库里可能刚被别的操作改过。
|
||||
func (s *Server) adminFogPreviewPage(w http.ResponseWriter, r *http.Request) {
|
||||
p, err := s.pageBase(r, "admin", "迷雾导入预览")
|
||||
if err != nil {
|
||||
s.serverError(w, r, err)
|
||||
return
|
||||
}
|
||||
token := r.PathValue("token")
|
||||
data, err := s.loadStashedFog(token)
|
||||
if err != nil {
|
||||
s.adminFlash(w, r, "err", err.Error())
|
||||
return
|
||||
}
|
||||
blocks, _, err := fog.Decode(data)
|
||||
if err != nil {
|
||||
s.adminFlash(w, r, "err", "解析迷雾快照失败:"+err.Error())
|
||||
return
|
||||
}
|
||||
if len(blocks) == 0 {
|
||||
s.adminFlash(w, r, "err", "快照里没有找到迷雾瓦片数据")
|
||||
return
|
||||
}
|
||||
layers, err := fog.RawLayers(data)
|
||||
if err != nil {
|
||||
log.Printf("解析快照的其它图层失败: %v", err)
|
||||
}
|
||||
ix, err := s.fogIndex()
|
||||
if err != nil {
|
||||
s.serverError(w, r, err)
|
||||
return
|
||||
}
|
||||
var cur []fog.Block
|
||||
ix.View(func(m map[[2]int][]byte) {
|
||||
cur = make([]fog.Block, 0, len(m))
|
||||
for k, bm := range m {
|
||||
cur = append(cur, fog.Block{GX: k[0], GY: k[1], Bitmap: bm})
|
||||
}
|
||||
})
|
||||
tilde, hash, other := fogLayerCounts(layers)
|
||||
d := FogPreviewData{
|
||||
Page: p, Token: token, Diff: fog.DiffBlocks(cur, blocks),
|
||||
TildeLayers: tilde, HashLayers: hash, OtherLayers: other,
|
||||
}
|
||||
d.MinLat, d.MinLon, d.MaxLat, d.MaxLon, d.HasBounds = diffBounds(cur, blocks, s.cfg.MapCRS)
|
||||
// 布局按这个标记引入 Leaflet,漏了地图整块不出现
|
||||
d.Page.HasMap = d.HasBounds
|
||||
s.render(w, "admin/fog-preview", d)
|
||||
}
|
||||
|
||||
// diffBounds 返回两边块集覆盖范围的包围盒,供差异地图定位。迷雾网格是 WGS-84,
|
||||
// 高德这类底图按 GCJ-02 摆放,所以坐标要跟着底图走,否则视野会偏出几百米。
|
||||
func diffBounds(cur, inc []fog.Block, crs string) (minLat, minLon, maxLat, maxLon float64, ok bool) {
|
||||
visit := func(gbx, gby int) {
|
||||
lat, lon := fog.BlockLonLat(gbx, gby)
|
||||
if crs == "gcj02" {
|
||||
lat, lon = geo.WGS84ToGCJ02(lat, lon)
|
||||
}
|
||||
if !ok {
|
||||
minLat, maxLat, minLon, maxLon, ok = lat, lat, lon, lon, true
|
||||
return
|
||||
}
|
||||
minLat, maxLat = math.Min(minLat, lat), math.Max(maxLat, lat)
|
||||
minLon, maxLon = math.Min(minLon, lon), math.Max(maxLon, lon)
|
||||
}
|
||||
for _, b := range cur {
|
||||
visit(b.GX, b.GY)
|
||||
}
|
||||
for _, b := range inc {
|
||||
visit(b.GX, b.GY)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// fogLayerCounts 按前缀归类快照里「已探索层之外」的条目。
|
||||
// Model/~ 与 Model/# 两层站点不解析、只原样留存,但整体替换会把它们一并换掉,
|
||||
// 差异页要把这件事说清楚。
|
||||
func fogLayerCounts(layers map[string][]byte) (tilde, hash, other int) {
|
||||
for p := range layers {
|
||||
switch {
|
||||
case strings.HasPrefix(p, "Model/~/") || strings.HasPrefix(p, "Model~/"):
|
||||
tilde++
|
||||
case strings.HasPrefix(p, "Model/#/") || strings.HasPrefix(p, "Model#/"):
|
||||
hash++
|
||||
default:
|
||||
other++
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// ---------- 迷雾覆盖率 ----------
|
||||
|
||||
// FogCity 是单个城市(或省份)的迷雾点亮统计:地点里有多少个落在已探索格内。
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image/png"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"rond-with-you/internal/fog"
|
||||
)
|
||||
|
||||
// 差异瓦片要反复读同一份快照、同一份库内迷雾,单槽缓存住两边的块集即可——
|
||||
// 站长一次只会看一份预览。缓存里的 map 加载完就只读,渲染时并发遍历是安全的。
|
||||
var (
|
||||
diffSetsMu sync.Mutex
|
||||
diffSetsToken string
|
||||
diffSetsAt time.Time
|
||||
diffSetsCur map[[2]int][]byte
|
||||
diffSetsInc map[[2]int][]byte
|
||||
)
|
||||
|
||||
// diffSetsTTL 短于暂存的 2 小时:库内迷雾随时可能被别的操作改掉,
|
||||
// 缓存太久会让差异图与页面上那组数字对不上。
|
||||
const diffSetsTTL = 10 * time.Minute
|
||||
|
||||
func (s *Server) fogDiffSets(token string) (cur, inc map[[2]int][]byte, err error) {
|
||||
diffSetsMu.Lock()
|
||||
defer diffSetsMu.Unlock()
|
||||
if token == diffSetsToken && time.Since(diffSetsAt) < diffSetsTTL {
|
||||
return diffSetsCur, diffSetsInc, nil
|
||||
}
|
||||
data, err := s.loadStashedFog(token)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
blocks, _, err := fog.Decode(data)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
ix, err := s.fogIndex()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
cur = map[[2]int][]byte{}
|
||||
ix.View(func(m map[[2]int][]byte) {
|
||||
for k, bm := range m {
|
||||
cur[k] = bm
|
||||
}
|
||||
})
|
||||
inc = make(map[[2]int][]byte, len(blocks))
|
||||
for _, b := range blocks {
|
||||
inc[[2]int{b.GX, b.GY}] = b.Bitmap
|
||||
}
|
||||
diffSetsToken, diffSetsCur, diffSetsInc, diffSetsAt = token, cur, inc, time.Now()
|
||||
return cur, inc, nil
|
||||
}
|
||||
|
||||
// adminFogDiffTile 渲染一张差异瓦片:绿=叠加会新增、灰=两边都有、红=整体替换会丢。
|
||||
// 只给站长看,不涉及遮罩。
|
||||
func (s *Server) adminFogDiffTile(w http.ResponseWriter, r *http.Request) {
|
||||
z, err1 := strconv.Atoi(r.PathValue("z"))
|
||||
x, err2 := strconv.Atoi(r.PathValue("x"))
|
||||
y, err3 := strconv.Atoi(r.PathValue("y"))
|
||||
if err1 != nil || err2 != nil || err3 != nil || z < 0 || z > 22 ||
|
||||
x < 0 || y < 0 || x >= 1<<uint(z) || y >= 1<<uint(z) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
cur, inc, err := s.fogDiffSets(r.PathValue("token"))
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
crs := r.URL.Query().Get("crs")
|
||||
if crs != "gcj02" && crs != "wgs84" {
|
||||
crs = s.cfg.MapCRS
|
||||
}
|
||||
img := fog.RenderDiffTile(cur, inc, z, x, y, crs)
|
||||
if img == nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := png.Encode(&buf, img); err != nil {
|
||||
s.serverError(w, r, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
// 令牌是一次性的,内容也只给站长看,不落共享缓存
|
||||
w.Header().Set("Cache-Control", "private, max-age=300")
|
||||
w.Write(buf.Bytes())
|
||||
}
|
||||
@@ -0,0 +1,422 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/png"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"rond-with-you/internal/auth"
|
||||
"rond-with-you/internal/config"
|
||||
"rond-with-you/internal/db"
|
||||
"rond-with-you/internal/fog"
|
||||
)
|
||||
|
||||
// fogPreviewFixture 是一个迷雾为空、带一次性账号的测试库。
|
||||
//
|
||||
// ROND_TEST_CONF=conf/app-test.ini go test ./internal/web -run FogPreview -v
|
||||
//
|
||||
// 迷雾是站点级数据(公开读路径不区分归属),所以测试必须从空库出发;
|
||||
// 用的是独立测试库(rond_fog_test),不会碰到开发库里的迷雾。
|
||||
type fogPreviewFixture struct {
|
||||
cfg *config.Config
|
||||
pg *sql.DB
|
||||
srv *Server
|
||||
ctx context.Context
|
||||
uid int64
|
||||
ix *fog.Index
|
||||
}
|
||||
|
||||
func newFogPreviewFixture(t *testing.T) *fogPreviewFixture {
|
||||
t.Helper()
|
||||
confPath := os.Getenv("ROND_TEST_CONF")
|
||||
if confPath == "" {
|
||||
t.Skip("设置 ROND_TEST_CONF 后运行迷雾导入校验")
|
||||
}
|
||||
chdirModuleRoot(t)
|
||||
cfg, err := config.Load(confPath)
|
||||
if err != nil {
|
||||
t.Fatalf("加载配置 %s 失败: %v", confPath, err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
pg, err := db.Open(cfg.DSN)
|
||||
if err != nil {
|
||||
t.Fatalf("连接数据库失败: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { pg.Close() })
|
||||
if err := db.Migrate(ctx, pg); err != nil {
|
||||
t.Fatalf("初始化数据库失败: %v", err)
|
||||
}
|
||||
|
||||
user := fmt.Sprintf("e2e-fogpreview-%d", time.Now().UnixNano())
|
||||
hash, err := auth.HashPassword("e2e-password")
|
||||
if err != nil {
|
||||
t.Fatalf("生成口令哈希失败: %v", err)
|
||||
}
|
||||
var uid int64
|
||||
if err := pg.QueryRowContext(ctx, `INSERT INTO users (username, password_hash, display_name)
|
||||
VALUES ($1,$2,'e2e') RETURNING id`, user, hash).Scan(&uid); err != nil {
|
||||
t.Fatalf("创建测试账号失败: %v", err)
|
||||
}
|
||||
clearFog(t, ctx, pg)
|
||||
t.Cleanup(func() {
|
||||
clearFog(t, ctx, pg)
|
||||
pg.ExecContext(ctx, `DELETE FROM users WHERE id=$1`, uid)
|
||||
os.RemoveAll(filepath.Join(cfg.DataDir, "fog-staging"))
|
||||
fogResetIndex()
|
||||
})
|
||||
|
||||
srv, err := New(cfg, pg)
|
||||
if err != nil {
|
||||
t.Fatalf("初始化服务失败: %v", err)
|
||||
}
|
||||
f := &fogPreviewFixture{cfg: cfg, pg: pg, srv: srv, uid: uid,
|
||||
ctx: ctxWithUser(&auth.User{ID: uid, Username: user})}
|
||||
f.ix = f.reloadIndex(t)
|
||||
return f
|
||||
}
|
||||
|
||||
func clearFog(t *testing.T, ctx context.Context, pg *sql.DB) {
|
||||
t.Helper()
|
||||
for _, q := range []string{`DELETE FROM fog_blocks`, `DELETE FROM fog_raw_files`} {
|
||||
if _, err := pg.ExecContext(ctx, q); err != nil {
|
||||
t.Fatalf("清空迷雾失败: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reloadIndex 让内存索引与库对齐:fogIndex 的首次加载只跑一次(sync.Once),
|
||||
// 测试里反复清库写库,必须手动同步,否则差异页算的是上一轮的残留。
|
||||
func (f *fogPreviewFixture) reloadIndex(t *testing.T) *fog.Index {
|
||||
t.Helper()
|
||||
ix, err := f.srv.fogIndex()
|
||||
if err != nil {
|
||||
t.Fatalf("获取迷雾索引失败: %v", err)
|
||||
}
|
||||
if err := ix.LoadAll(f.pg); err != nil {
|
||||
t.Fatalf("重载迷雾索引失败: %v", err)
|
||||
}
|
||||
return ix
|
||||
}
|
||||
|
||||
// upload 走后台真实的上传处理函数,返回差异页的令牌。
|
||||
func (f *fogPreviewFixture) upload(t *testing.T, data []byte) string {
|
||||
t.Helper()
|
||||
rec := httptest.NewRecorder()
|
||||
f.srv.adminFogPreview(rec, fogUploadRequest(t, data).WithContext(f.ctx))
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("上传应跳转到差异页: code=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
loc := rec.Header().Get("Location")
|
||||
token := strings.TrimPrefix(loc, "/admin/fog/preview/")
|
||||
if token == loc || len(token) != fogStashTokenLen {
|
||||
t.Fatalf("差异页地址不对: %q", loc)
|
||||
}
|
||||
return token
|
||||
}
|
||||
|
||||
// preview 渲染差异页。
|
||||
func (f *fogPreviewFixture) preview(t *testing.T, token string) string {
|
||||
t.Helper()
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest("GET", "/admin/fog/preview/"+token, nil).WithContext(f.ctx)
|
||||
req.SetPathValue("token", token)
|
||||
f.srv.adminFogPreviewPage(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("差异页渲染失败: code=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
return rec.Body.String()
|
||||
}
|
||||
|
||||
// confirm 走差异页确认按钮那条路径:带 stash 令牌、只提交 mode。
|
||||
func (f *fogPreviewFixture) confirm(t *testing.T, token, mode string) {
|
||||
t.Helper()
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest("POST", "/admin/fog?stash="+token, strings.NewReader("mode="+mode))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
f.srv.adminFogUpload(rec, req.WithContext(f.ctx))
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("确认导入应跳回后台: code=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if loc := rec.Header().Get("Location"); !strings.HasPrefix(loc, "/admin?ok=") {
|
||||
t.Fatalf("确认导入应报成功: %s", loc)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fogPreviewFixture) cells(t *testing.T) int {
|
||||
t.Helper()
|
||||
_, cells := f.reloadIndex(t).Stats()
|
||||
return cells
|
||||
}
|
||||
|
||||
func (f *fogPreviewFixture) blocks(t *testing.T) int {
|
||||
t.Helper()
|
||||
blocks, _ := f.reloadIndex(t).Stats()
|
||||
return blocks
|
||||
}
|
||||
|
||||
// TestFogPreviewShowsDiffBeforeWriting 端到端验证「上传 → 差异页 → 确认导入」:
|
||||
// 差异页必须先把两种模式的后果摆出来,而且在上传那一刻格子里一行都不能写。
|
||||
func TestFogPreviewShowsDiffBeforeWriting(t *testing.T) {
|
||||
f := newFogPreviewFixture(t)
|
||||
|
||||
// 库里先有一块 3 格的区域
|
||||
if _, err := f.ix.Import(f.pg, f.uid, []fog.Block{
|
||||
newFogBlock(100, 200, [2]int{0, 0}, [2]int{1, 0}, [2]int{2, 0}),
|
||||
}, false); err != nil {
|
||||
t.Fatalf("预置迷雾失败: %v", err)
|
||||
}
|
||||
|
||||
// 新快照:同一块补亮 2 格,外加一整块新的
|
||||
inc := []fog.Block{
|
||||
newFogBlock(100, 200, [2]int{0, 0}, [2]int{1, 0}, [2]int{2, 0}, [2]int{5, 5}, [2]int{6, 5}),
|
||||
newFogBlock(101, 200, [2]int{1, 1}),
|
||||
}
|
||||
data, err := fog.Encode(inc, map[string][]byte{
|
||||
"Model/~/aaa": {1, 2, 3},
|
||||
"Model/#/bbb": {4, 5, 6},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("构造快照失败: %v", err)
|
||||
}
|
||||
|
||||
token := f.upload(t, data)
|
||||
|
||||
// 差异页只是预览:格数必须还是 3
|
||||
if got := f.cells(t); got != 3 {
|
||||
t.Fatalf("上传阶段不该写库,当前格数 %d", got)
|
||||
}
|
||||
|
||||
page := f.preview(t, token)
|
||||
for _, want := range []string{
|
||||
"迷雾导入预览",
|
||||
"叠加导入",
|
||||
"整体替换",
|
||||
"会补上当前迷雾没有的", // 叠加卡片给出的实际收益
|
||||
"替换不会丢数据", // 库里的 3 格都在快照里
|
||||
"Model/~", // 图层提示
|
||||
"/admin/fog?stash=" + token,
|
||||
`value="overlay"`,
|
||||
`value="cover"`,
|
||||
} {
|
||||
if !strings.Contains(page, want) {
|
||||
t.Errorf("差异页缺少 %q", want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(page, "完全覆盖") {
|
||||
t.Error("这份快照有新内容,不该报「已被完全覆盖」")
|
||||
}
|
||||
|
||||
// 确认叠加:3 格 + 补亮的 2 格 + 新块 1 格 = 6 格,两块
|
||||
f.confirm(t, token, "overlay")
|
||||
if got := f.cells(t); got != 6 {
|
||||
t.Fatalf("叠加后应有 6 格,实际 %d", got)
|
||||
}
|
||||
if got := f.blocks(t); got != 2 {
|
||||
t.Fatalf("叠加后应有 2 块,实际 %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFogPreviewWarnsBeforeCoverLosesData 整体替换必须在下手前算出「会丢多少」,
|
||||
// 并且差异页要看得出这份快照已经被完全覆盖过(叠加没有意义)。
|
||||
func TestFogPreviewWarnsBeforeCoverLosesData(t *testing.T) {
|
||||
f := newFogPreviewFixture(t)
|
||||
|
||||
// 库里两块共 5 格
|
||||
if _, err := f.ix.Import(f.pg, f.uid, []fog.Block{
|
||||
newFogBlock(100, 200, [2]int{0, 0}, [2]int{1, 0}, [2]int{2, 0}),
|
||||
newFogBlock(101, 200, [2]int{7, 7}, [2]int{8, 8}),
|
||||
}, false); err != nil {
|
||||
t.Fatalf("预置迷雾失败: %v", err)
|
||||
}
|
||||
|
||||
// 快照只有第一块、且一格不多:叠加零收益,替换会丢掉第二块的两格
|
||||
inc := []fog.Block{newFogBlock(100, 200, [2]int{0, 0}, [2]int{1, 0}, [2]int{2, 0})}
|
||||
data, err := fog.Encode(inc, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("构造快照失败: %v", err)
|
||||
}
|
||||
token := f.upload(t, data)
|
||||
|
||||
page := f.preview(t, token)
|
||||
for _, want := range []string{
|
||||
"已经被当前迷雾", // 叠加无收益
|
||||
"从点亮变回未探索",
|
||||
"Model/~", // 快照没有这两层,替换会清掉现有的
|
||||
} {
|
||||
if !strings.Contains(page, want) {
|
||||
t.Errorf("差异页缺少 %q", want)
|
||||
}
|
||||
}
|
||||
|
||||
// 确认整体替换:只该剩下快照里的 1 块 3 格
|
||||
f.confirm(t, token, "cover")
|
||||
if got := f.cells(t); got != 3 {
|
||||
t.Fatalf("替换后应有 3 格,实际 %d", got)
|
||||
}
|
||||
if got := f.blocks(t); got != 1 {
|
||||
t.Fatalf("替换后应有 1 块,实际 %d", got)
|
||||
}
|
||||
|
||||
// 暂存件用掉就该删掉,且过期令牌不该还能用
|
||||
if _, err := os.Stat(filepath.Join(f.cfg.DataDir, "fog-staging", token+".fwss")); !os.IsNotExist(err) {
|
||||
t.Error("确认导入后应删除暂存文件")
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest("GET", "/admin/fog/preview/"+token, nil).WithContext(f.ctx)
|
||||
req.SetPathValue("token", token)
|
||||
f.srv.adminFogPreviewPage(rec, req)
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("过期令牌应跳回后台报错: code=%d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// 差异瓦片要按请求坐标渲染出正确的三色,并挡住越界坐标与过期令牌。
|
||||
func TestFogDiffTileColors(t *testing.T) {
|
||||
f := newFogPreviewFixture(t)
|
||||
if _, err := f.ix.Import(f.pg, f.uid, []fog.Block{
|
||||
newFogBlock(100, 200, [2]int{0, 0}, [2]int{4, 0}),
|
||||
}, false); err != nil {
|
||||
t.Fatalf("预置迷雾失败: %v", err)
|
||||
}
|
||||
data, err := fog.Encode([]fog.Block{newFogBlock(100, 200, [2]int{0, 0}, [2]int{1, 0})}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("构造快照失败: %v", err)
|
||||
}
|
||||
token := f.upload(t, data)
|
||||
|
||||
// z=18 时一格 16 像素,块 (100,200) 完整落在瓦片 (400,800) 里
|
||||
img := f.diffTile(t, token, 18, 400, 800)
|
||||
if got := kindOf(pixAtImg(img, 8, 8)); got != "both" {
|
||||
t.Errorf("两边都有的格应为灰色,实际 %s", got)
|
||||
}
|
||||
if got := kindOf(pixAtImg(img, 24, 8)); got != "gain" {
|
||||
t.Errorf("只有快照有的格应为绿色,实际 %s", got)
|
||||
}
|
||||
if got := kindOf(pixAtImg(img, 72, 8)); got != "lost" {
|
||||
t.Errorf("只有库里有、替换会丢的格应为红色,实际 %s", got)
|
||||
}
|
||||
if got := kindOf(pixAtImg(img, 200, 200)); got != "none" {
|
||||
t.Errorf("没探索过的格应为透明,实际 %s", got)
|
||||
}
|
||||
|
||||
// 越界坐标与过期令牌都只给 404,不能穿透到渲染
|
||||
for _, c := range []struct{ z, x, y string }{
|
||||
{"18", "262144", "0"}, // x 已达该级别的上限
|
||||
{"18", "0", "-1"},
|
||||
{"99", "0", "0"}, // z 越界
|
||||
} {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest("GET", "/admin/fog/preview/"+token+"/x", nil).WithContext(f.ctx)
|
||||
req.SetPathValue("token", token)
|
||||
req.SetPathValue("z", c.z)
|
||||
req.SetPathValue("x", c.x)
|
||||
req.SetPathValue("y", c.y)
|
||||
f.srv.adminFogDiffTile(rec, req)
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Errorf("越界坐标 %v 应给 404,实际 %d", c, rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest("GET", "/admin/fog/preview/deadbeef/18/400/800", nil).WithContext(f.ctx)
|
||||
req.SetPathValue("token", strings.Repeat("f", 32))
|
||||
req.SetPathValue("z", "18")
|
||||
req.SetPathValue("x", "400")
|
||||
req.SetPathValue("y", "800")
|
||||
f.srv.adminFogDiffTile(rec, req)
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Errorf("无效令牌应给 404,实际 %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fogPreviewFixture) diffTile(t *testing.T, token string, z, x, y int) image.Image {
|
||||
t.Helper()
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest("GET", fmt.Sprintf("/admin/fog/preview/%s/%d/%d/%d?crs=wgs84", token, z, x, y), nil).
|
||||
WithContext(f.ctx)
|
||||
req.SetPathValue("token", token)
|
||||
req.SetPathValue("z", strconv.Itoa(z))
|
||||
req.SetPathValue("x", strconv.Itoa(x))
|
||||
req.SetPathValue("y", strconv.Itoa(y))
|
||||
f.srv.adminFogDiffTile(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("差异瓦片渲染失败: code=%d", rec.Code)
|
||||
}
|
||||
if ct := rec.Header().Get("Content-Type"); ct != "image/png" {
|
||||
t.Fatalf("差异瓦片应为 PNG,实际 %q", ct)
|
||||
}
|
||||
img, err := png.Decode(bytes.NewReader(rec.Body.Bytes()))
|
||||
if err != nil {
|
||||
t.Fatalf("解码差异瓦片失败: %v", err)
|
||||
}
|
||||
return img
|
||||
}
|
||||
|
||||
// kindOf 按「哪个通道最亮」判类别,不去比对精确色值——PNG 解码回来的是
|
||||
// 预乘 alpha 的像素,硬比色值会被 alpha 换算绕进去。
|
||||
func kindOf(c [4]byte) string {
|
||||
switch {
|
||||
case c[3] == 0:
|
||||
return "none"
|
||||
case c[1] > c[0] && c[1] > c[2]:
|
||||
return "gain"
|
||||
case c[0] > c[1] && c[0] > c[2]:
|
||||
return "lost"
|
||||
}
|
||||
return "both"
|
||||
}
|
||||
|
||||
func pixAtImg(img image.Image, px, py int) [4]byte {
|
||||
r, g, b, a := img.At(px, py).RGBA()
|
||||
return [4]byte{byte(r >> 8), byte(g >> 8), byte(b >> 8), byte(a >> 8)}
|
||||
}
|
||||
|
||||
// 令牌来自 URL,必须挡死路径穿越。
|
||||
func TestFogStashTokenRejectsTraversal(t *testing.T) {
|
||||
s := &Server{cfg: &config.Config{DataDir: t.TempDir()}}
|
||||
for _, bad := range []string{"", "..", "../../etc/passwd", strings.Repeat("a", 32) + "/../x", strings.ToUpper(strings.Repeat("a", 32))} {
|
||||
if p, ok := s.fogStashPath(bad); ok {
|
||||
t.Errorf("令牌 %q 不该通过校验,得到 %s", bad, p)
|
||||
}
|
||||
}
|
||||
if _, ok := s.fogStashPath(strings.Repeat("0", 32)); !ok {
|
||||
t.Error("32 位十六进制令牌应当通过")
|
||||
}
|
||||
}
|
||||
|
||||
func newFogBlock(gx, gy int, cells ...[2]int) fog.Block {
|
||||
bm := make([]byte, 512)
|
||||
for _, c := range cells {
|
||||
bm[c[1]*8+c[0]/8] |= 0x80 >> (c[0] % 8)
|
||||
}
|
||||
return fog.Block{GX: gx, GY: gy, Bitmap: bm}
|
||||
}
|
||||
|
||||
func fogUploadRequest(t *testing.T, data []byte) *http.Request {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
mw := multipart.NewWriter(&buf)
|
||||
fw, err := mw.CreateFormFile("fwss", "Snapshot-test.fwss")
|
||||
if err != nil {
|
||||
t.Fatalf("构造上传表单失败: %v", err)
|
||||
}
|
||||
if _, err := fw.Write(data); err != nil {
|
||||
t.Fatalf("写入上传内容失败: %v", err)
|
||||
}
|
||||
mw.Close()
|
||||
req := httptest.NewRequest("POST", "/admin/fog/preview", &buf)
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
return req
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// 待确认的上传(迷雾快照、rond 备份)都先落到磁盘暂存,让站长看完差异再决定怎么导入;
|
||||
// 看完直接走人的靠 TTL 回收,不至于把几十 MB 的文件一直留在盘上。
|
||||
//
|
||||
// 令牌只认固定长度的十六进制串,这样即便来自 URL 也拼不出 ../ 之类的路径。
|
||||
|
||||
const (
|
||||
stashTokenLen = 32 // 16 字节的十六进制
|
||||
stashTTL = 2 * time.Hour
|
||||
)
|
||||
|
||||
func stashPut(dir, ext string, data []byte) (string, error) {
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var raw [stashTokenLen / 2]byte
|
||||
if _, err := rand.Read(raw[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
token := hex.EncodeToString(raw[:])
|
||||
if err := os.WriteFile(filepath.Join(dir, token+ext), data, 0o600); err != nil {
|
||||
return "", err
|
||||
}
|
||||
stashSweep(dir, ext)
|
||||
return token, nil
|
||||
}
|
||||
|
||||
func stashPath(dir, ext, token string) (string, bool) {
|
||||
if len(token) != stashTokenLen {
|
||||
return "", false
|
||||
}
|
||||
for i := 0; i < len(token); i++ {
|
||||
c := token[i]
|
||||
if (c < '0' || c > '9') && (c < 'a' || c > 'f') {
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
return filepath.Join(dir, token+ext), true
|
||||
}
|
||||
|
||||
func stashRead(dir, ext, token, what string) ([]byte, error) {
|
||||
p, ok := stashPath(dir, ext, token)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("预览链接无效,请重新上传%s", what)
|
||||
}
|
||||
data, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("这次预览已过期,请重新上传%s", what)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func stashDrop(dir, ext, token string) {
|
||||
if p, ok := stashPath(dir, ext, token); ok {
|
||||
os.Remove(p)
|
||||
}
|
||||
}
|
||||
|
||||
func stashSweep(dir, ext string) {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
cutoff := time.Now().Add(-stashTTL)
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || !strings.HasSuffix(e.Name(), ext) {
|
||||
continue
|
||||
}
|
||||
if info, err := e.Info(); err == nil && info.ModTime().After(cutoff) {
|
||||
continue
|
||||
}
|
||||
os.Remove(filepath.Join(dir, e.Name()))
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- 迷雾快照 ----------
|
||||
|
||||
// fogStashTokenLen 是迷雾预览令牌的长度,测试与调用方按它校验。
|
||||
const fogStashTokenLen = stashTokenLen
|
||||
|
||||
const fogStashExt = ".fwss"
|
||||
|
||||
func (s *Server) fogStashDir() string {
|
||||
return filepath.Join(s.cfg.DataDir, "fog-staging")
|
||||
}
|
||||
|
||||
func (s *Server) stashFog(data []byte) (string, error) {
|
||||
return stashPut(s.fogStashDir(), fogStashExt, data)
|
||||
}
|
||||
|
||||
func (s *Server) fogStashPath(token string) (string, bool) {
|
||||
return stashPath(s.fogStashDir(), fogStashExt, token)
|
||||
}
|
||||
|
||||
func (s *Server) loadStashedFog(token string) ([]byte, error) {
|
||||
return stashRead(s.fogStashDir(), fogStashExt, token, "快照")
|
||||
}
|
||||
|
||||
func (s *Server) dropStashedFog(token string) {
|
||||
stashDrop(s.fogStashDir(), fogStashExt, token)
|
||||
}
|
||||
|
||||
func (s *Server) sweepFogStash() {
|
||||
stashSweep(s.fogStashDir(), fogStashExt)
|
||||
}
|
||||
@@ -769,12 +769,12 @@ func (s *Server) notesPage(w http.ResponseWriter, r *http.Request) {
|
||||
case "note":
|
||||
order = "n.updated_at DESC NULLS LAST, p.name"
|
||||
case "talk":
|
||||
order = "(COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.src_pk = p.src_pk),0)) DESC, p.visit_count DESC"
|
||||
order = "(COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.place_key = " + placeKeyExpr + "),0)) DESC, p.visit_count DESC"
|
||||
}
|
||||
|
||||
base := `FROM places p
|
||||
LEFT JOIN place_notes n ON n.src_pk = p.src_pk AND n.dataset_id = p.dataset_id
|
||||
LEFT JOIN place_votes v ON v.src_pk = p.src_pk AND v.dataset_id = p.dataset_id
|
||||
LEFT JOIN place_notes n ON n.place_key = ` + placeKeyExpr + `
|
||||
LEFT JOIN place_votes v ON v.place_key = ` + placeKeyExpr + `
|
||||
WHERE ` + strings.Join(where, " AND ")
|
||||
|
||||
var total int
|
||||
@@ -790,8 +790,8 @@ func (s *Server) notesPage(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
limitArgs := append(append([]any{}, args...), notesPerPage, (d.PageNo-1)*notesPerPage)
|
||||
q2 := `SELECT p.src_pk, COALESCE(n.alias,''), COALESCE(n.note,''), COALESCE(n.rond_note,''), COALESCE(v.verdict,0),
|
||||
COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.src_pk=p.src_pk AND vv.vote=1),0),
|
||||
COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.src_pk=p.src_pk AND vv.vote=-1),0),
|
||||
COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.place_key = ` + placeKeyExpr + ` AND vv.vote=1),0),
|
||||
COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.place_key = ` + placeKeyExpr + ` AND vv.vote=-1),0),
|
||||
COALESCE(n.updated_at, to_timestamp(0)), p.id, COALESCE(p.name,''), COALESCE(p.city,''), COALESCE(p.visit_count,0)
|
||||
` + base + ` ORDER BY ` + order + fmt.Sprintf(` LIMIT $%d OFFSET $%d`, len(args)+1, len(args)+2)
|
||||
|
||||
|
||||
@@ -1742,6 +1742,7 @@ var (
|
||||
activitySeq = srcPKSeq{"activities", "src_pk", "Activity", "ZACTIVITY"}
|
||||
weatherSeq = srcPKSeq{"weather", "src_pk", "HourlyWeather", "ZHOURLYWEATHER"}
|
||||
movementSeq = srcPKSeq{"movements", "src_pk", "Movement", "ZMOVEMENT"}
|
||||
tagSeq = srcPKSeq{"tags", "src_pk", "Tag", "ZTAG"}
|
||||
transportSeq = srcPKSeq{"movements", "transport_src", "Transport", "ZTRANSPORT"}
|
||||
)
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ package web
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -26,6 +27,11 @@ const (
|
||||
imageUploadMem = 8 << 20 // multipart 解析用的内存上限
|
||||
)
|
||||
|
||||
// placeKeyExpr 把 places p 那一行的坐标换算成 place_key,图片、备注、结论共用这个键。
|
||||
// **必须与 rond.PlaceCoordKey 以及 notes/votes 包里的同款表达式逐位一致**:
|
||||
// fmt.Sprintf("%.6f|%.6f") ↔ to_char(x,'FM999990.000000')||'|'||to_char(y,'FM999990.000000')
|
||||
const placeKeyExpr = `to_char(p.lat,'FM999990.000000')||'|'||to_char(p.lon,'FM999990.000000')`
|
||||
|
||||
// PlaceImage 是模板里用的地点图片。
|
||||
type PlaceImage struct {
|
||||
ID int64
|
||||
@@ -57,14 +63,18 @@ func (s *Server) savePlaceImage(ctx context.Context, datasetID int64, srcPK int,
|
||||
if err := s.media.Save(ctx, backend, key, data, "image/webp"); err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
pkey, err := s.placeKeyOf(ctx, datasetID, srcPK)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
var id int64
|
||||
if err := s.db.QueryRowContext(ctx, `INSERT INTO place_images
|
||||
(dataset_id, src_pk, storage, object_key, width, height, bytes, position)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,
|
||||
COALESCE((SELECT max(position)+1 FROM place_images WHERE dataset_id=$1 AND src_pk=$2), 0))
|
||||
(dataset_id, src_pk, place_key, storage, object_key, width, height, bytes, position)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,
|
||||
COALESCE((SELECT max(position)+1 FROM place_images WHERE place_key=$3), 0))
|
||||
ON CONFLICT (dataset_id, src_pk, object_key) DO UPDATE SET bytes=EXCLUDED.bytes
|
||||
RETURNING id`,
|
||||
datasetID, srcPK, backend, key, w, h, int64(len(data))).Scan(&id); err != nil {
|
||||
datasetID, srcPK, pkey, backend, key, w, h, int64(len(data))).Scan(&id); err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
return id, s.media.PublicURL(backend, key), nil
|
||||
@@ -95,9 +105,15 @@ func (s *Server) adminPlaceImageUpload(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
pkey, err := s.placeKeyOf(ctx, ds.ID, src)
|
||||
if err != nil {
|
||||
s.serverError(w, r, err)
|
||||
return
|
||||
}
|
||||
var count int
|
||||
// 按坐标键数:换数据集后图片仍跟着同一个地点,配额也要跟着
|
||||
if err := s.db.QueryRowContext(ctx, `SELECT count(*) FROM place_images
|
||||
WHERE dataset_id=$1 AND src_pk=$2`, ds.ID, src).Scan(&count); err != nil {
|
||||
WHERE place_key=$1`, pkey).Scan(&count); err != nil {
|
||||
s.serverError(w, r, err)
|
||||
return
|
||||
}
|
||||
@@ -196,10 +212,28 @@ func (s *Server) imageBackend(ctx context.Context) (string, error) {
|
||||
return setting.MediaBackendLocal, nil
|
||||
}
|
||||
|
||||
// placeKeyOf 把某个数据集里的地点编号换算成坐标键。
|
||||
// 换数据集之后仍靠它认出「还是同一个地点」,所以图片、备注、结论都挂在这上面。
|
||||
func (s *Server) placeKeyOf(ctx context.Context, datasetID int64, srcPK int) (string, error) {
|
||||
var key string
|
||||
err := s.db.QueryRowContext(ctx, `SELECT `+placeKeyExpr+` FROM places p
|
||||
WHERE p.dataset_id=$1 AND p.src_pk=$2 AND p.lat IS NOT NULL AND p.lon IS NOT NULL`,
|
||||
datasetID, srcPK).Scan(&key)
|
||||
if err == sql.ErrNoRows {
|
||||
return "", fmt.Errorf("地点不存在或没有坐标")
|
||||
}
|
||||
return key, err
|
||||
}
|
||||
|
||||
// placeImagesOf 读某个地点的图片,按 position 排序。
|
||||
// 按坐标键查而不是 (dataset_id, src_pk):换一份备份之后图片还跟着同一个地点。
|
||||
func (s *Server) placeImagesOf(ctx context.Context, datasetID int64, srcPK int) ([]PlaceImage, error) {
|
||||
pkey, err := s.placeKeyOf(ctx, datasetID, srcPK)
|
||||
if err != nil {
|
||||
return nil, nil // 地点没有坐标就当作没有图
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT id, storage, object_key, width, height, bytes
|
||||
FROM place_images WHERE dataset_id=$1 AND src_pk=$2 ORDER BY position, id`, datasetID, srcPK)
|
||||
FROM place_images WHERE place_key=$1 ORDER BY position, id`, pkey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -183,7 +183,7 @@ func (s *Server) attachVotes(ctx context.Context, datasetID int64, places []stat
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
tallies, err := s.votes.Tallies(ctx)
|
||||
tallies, err := s.votes.Tallies(ctx, datasetID)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -539,8 +539,8 @@ func (s *Server) placePage(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
d.Verdict, _ = s.votes.Verdict(r.Context(), p.Dataset.ID, place.SrcPK)
|
||||
d.Votes, _ = s.votes.Tally(r.Context(), place.SrcPK)
|
||||
d.MyVote = s.votes.Voter(r.Context(), place.SrcPK, s.voterKeyFromRequest(r))
|
||||
d.Votes, _ = s.votes.Tally(r.Context(), p.Dataset.ID, place.SrcPK)
|
||||
d.MyVote = s.votes.Voter(r.Context(), p.Dataset.ID, place.SrcPK, s.voterKeyFromRequest(r))
|
||||
// 图片:访客也要看得到,所以不分管理员
|
||||
if imgs, err := s.placeImagesOf(r.Context(), p.Dataset.ID, place.SrcPK); err == nil {
|
||||
d.Images = imgs
|
||||
|
||||
@@ -102,12 +102,12 @@ func (s *Server) apiVotes(w http.ResponseWriter, r *http.Request) {
|
||||
s.json(w, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
tallies, err := s.votes.Tallies(ctx)
|
||||
tallies, err := s.votes.Tallies(ctx, votesDatasetID)
|
||||
if err != nil {
|
||||
s.json(w, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
mine, err := s.votes.VoterAll(ctx, s.voterKeyFromRequest(r))
|
||||
mine, err := s.votes.VoterAll(ctx, votesDatasetID, s.voterKeyFromRequest(r))
|
||||
if err != nil {
|
||||
s.json(w, map[string]string{"error": err.Error()})
|
||||
return
|
||||
@@ -145,10 +145,16 @@ func (s *Server) apiVerdict(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// apiVote 记录访客对某地点的点赞 / 点踩,返回最新票数与该访客的选择。
|
||||
func (s *Server) apiVote(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
if err := r.ParseForm(); err != nil {
|
||||
s.json(w, map[string]string{"error": "请求解析失败"})
|
||||
return
|
||||
}
|
||||
ds, err := s.datasetFor(ctx)
|
||||
if err != nil || ds == nil {
|
||||
s.json(w, map[string]string{"error": "还没有可投票的数据集"})
|
||||
return
|
||||
}
|
||||
srcPK, err := strconv.Atoi(r.PostFormValue("src_pk"))
|
||||
vote, _ := strconv.Atoi(r.PostFormValue("vote"))
|
||||
if err != nil || (vote != 1 && vote != -1) {
|
||||
@@ -160,7 +166,7 @@ func (s *Server) apiVote(w http.ResponseWriter, r *http.Request) {
|
||||
s.json(w, map[string]string{"error": "无法识别访客"})
|
||||
return
|
||||
}
|
||||
t, mine, err := s.votes.Cast(r.Context(), srcPK, key, vote)
|
||||
t, mine, err := s.votes.Cast(ctx, ds.ID, srcPK, key, vote)
|
||||
if err != nil {
|
||||
s.json(w, map[string]string{"error": err.Error()})
|
||||
return
|
||||
@@ -170,11 +176,17 @@ func (s *Server) apiVote(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// voteTotals 汇总站长结论数量与访客票数,供统计页展示。
|
||||
func (s *Server) voteTotals(ctx context.Context, datasetID int64) (rec, avoid, up, down int, err error) {
|
||||
// 两张票表都挂坐标键,统计要按「本数据集的地点」圈定范围,
|
||||
// 不能再按它们自己的 dataset_id 过滤——那个只是最后写入时留下的线索。
|
||||
err = s.db.QueryRowContext(ctx, `SELECT
|
||||
(SELECT count(*) FROM place_votes WHERE dataset_id=$1 AND verdict=1),
|
||||
(SELECT count(*) FROM place_votes WHERE dataset_id=$1 AND verdict=-1),
|
||||
(SELECT count(*) FROM place_visitor_votes WHERE vote=1),
|
||||
(SELECT count(*) FROM place_visitor_votes WHERE vote=-1)`, datasetID).
|
||||
(SELECT count(*) FROM place_votes v JOIN places p ON v.place_key = `+placeKeyExpr+`
|
||||
WHERE p.dataset_id=$1 AND v.verdict=1),
|
||||
(SELECT count(*) FROM place_votes v JOIN places p ON v.place_key = `+placeKeyExpr+`
|
||||
WHERE p.dataset_id=$1 AND v.verdict=-1),
|
||||
(SELECT count(*) FROM place_visitor_votes vv JOIN places p ON vv.place_key = `+placeKeyExpr+`
|
||||
WHERE p.dataset_id=$1 AND vv.vote=1),
|
||||
(SELECT count(*) FROM place_visitor_votes vv JOIN places p ON vv.place_key = `+placeKeyExpr+`
|
||||
WHERE p.dataset_id=$1 AND vv.vote=-1)`, datasetID).
|
||||
Scan(&rec, &avoid, &up, &down)
|
||||
return
|
||||
}
|
||||
@@ -183,7 +195,7 @@ func (s *Server) voteTotals(ctx context.Context, datasetID int64) (rec, avoid, u
|
||||
func (s *Server) verdictPlaces(ctx context.Context, datasetID int64, verdict, limit int) ([]stats.Place, error) {
|
||||
q := `SELECT ` + stats.PlaceCols + ` FROM places p
|
||||
WHERE p.dataset_id=$1 AND EXISTS (SELECT 1 FROM place_votes pv
|
||||
WHERE pv.src_pk=p.src_pk AND pv.dataset_id=p.dataset_id AND pv.verdict=$2)
|
||||
WHERE pv.place_key = ` + placeKeyExpr + ` AND pv.verdict=$2)
|
||||
ORDER BY p.visit_count DESC LIMIT ` + strconv.Itoa(limit)
|
||||
rows, err := s.db.QueryContext(ctx, q, datasetID, verdict)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"rond-with-you/internal/votes"
|
||||
)
|
||||
|
||||
// testPlace 在两个数据集里各建一个**同坐标**的地点,返回它们的编号与第二个数据集。
|
||||
//
|
||||
// 坐标是随手挑的专属值:备注按坐标共享,而读取按设计不按 user_id 过滤(单站长站点),
|
||||
// 借用真实数据里的坐标会把别的测试残留的记录串进来。
|
||||
func testPlace(t *testing.T, f *movementFixture, uid int64, lat, lon float64) (pkA, pkB int, dsB int64) {
|
||||
t.Helper()
|
||||
const pkAVal, pkBVal = 660001, 660002
|
||||
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO places (dataset_id, src_pk, name, lat, lon)
|
||||
VALUES ($1,$2,'甲侧地点',$3,$4)`, f.dsID, pkAVal, lat, lon); err != nil {
|
||||
t.Fatalf("建地点失败: %v", err)
|
||||
}
|
||||
sha := fmt.Sprintf("%064d", time.Now().UnixNano())
|
||||
if err := f.pg.QueryRowContext(f.ctx, `INSERT INTO datasets (user_id, original_name, sha256, status, is_active)
|
||||
VALUES ($1,'另一个数据集',$2,'done',false) RETURNING id`, uid, sha).Scan(&dsB); err != nil {
|
||||
t.Fatalf("建数据集失败: %v", err)
|
||||
}
|
||||
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO places (dataset_id, src_pk, name, lat, lon)
|
||||
VALUES ($1,$2,'乙侧地点',$3,$4)`, dsB, pkBVal, lat, lon); err != nil {
|
||||
t.Fatalf("建地点失败: %v", err)
|
||||
}
|
||||
key := fmt.Sprintf("%.6f|%.6f", lat, lon)
|
||||
t.Cleanup(func() {
|
||||
f.pg.ExecContext(f.ctx, `DELETE FROM place_notes WHERE place_key=$1`, key)
|
||||
f.pg.ExecContext(f.ctx, `DELETE FROM place_votes WHERE place_key=$1`, key)
|
||||
f.pg.ExecContext(f.ctx, `DELETE FROM place_visitor_votes WHERE place_key=$1`, key)
|
||||
f.pg.ExecContext(f.ctx, `DELETE FROM datasets WHERE id=$1`, dsB)
|
||||
})
|
||||
return pkAVal, pkBVal, dsB
|
||||
}
|
||||
|
||||
// 访客票也一样:它是站点自己的数据(rondbackup 里没有这一项),
|
||||
// 但同样要认得出「是哪个地点」,所以也挂坐标键。
|
||||
func TestVisitorVotesFollowPlaceAcrossDatasets(t *testing.T) {
|
||||
f := newMovementFixture(t)
|
||||
uid := userFrom(f.ctx).ID
|
||||
pkA, pkB, dsB := testPlace(t, f, uid, 77.777777, 88.888888)
|
||||
|
||||
if _, mine, err := f.srv.votes.Cast(f.ctx, f.dsID, pkA, "visitor-a", 1); err != nil || mine != 1 {
|
||||
t.Fatalf("投票失败: mine=%d err=%v", mine, err)
|
||||
}
|
||||
|
||||
tally, err := f.srv.votes.Tally(f.ctx, dsB, pkB)
|
||||
if err != nil {
|
||||
t.Fatalf("读票数失败: %v", err)
|
||||
}
|
||||
if tally.Up != 1 || tally.Down != 0 {
|
||||
t.Fatalf("换数据集后票数应照旧,实际 %+v", tally)
|
||||
}
|
||||
if v := f.srv.votes.Voter(f.ctx, dsB, pkB, "visitor-a"); v != 1 {
|
||||
t.Fatalf("「我投的」也该认得出来,实际 %d", v)
|
||||
}
|
||||
// 再点一次是撤销:从哪一侧点都该撤掉同一张票
|
||||
if _, mine, err := f.srv.votes.Cast(f.ctx, dsB, pkB, "visitor-a", 1); err != nil || mine != 0 {
|
||||
t.Fatalf("再点一次应撤销: mine=%d err=%v", mine, err)
|
||||
}
|
||||
if tally, err := f.srv.votes.Tally(f.ctx, f.dsID, pkA); err != nil || tally.Up != 0 {
|
||||
t.Fatalf("撤销后另一边也该归零: %+v err=%v", tally, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 备注、结论、图片挂的是坐标键,所以**换数据集之后仍然跟着同一个地点**——
|
||||
// 这正是这次迁移要解决的问题。老结构挂 (dataset_id, src_pk),
|
||||
// 换一份备份同编号就是另一个地点,内容全部失联。
|
||||
func TestNotesAndVerdictsFollowPlaceAcrossDatasets(t *testing.T) {
|
||||
f := newMovementFixture(t)
|
||||
uid := userFrom(f.ctx).ID
|
||||
pkA, pkB, dsB := testPlace(t, f, uid, 11.111111, 22.222222)
|
||||
|
||||
if err := f.srv.notes.Save(f.ctx, uid, f.dsID, pkA, "别名甲", "备注甲"); err != nil {
|
||||
t.Fatalf("写备注失败: %v", err)
|
||||
}
|
||||
if err := f.srv.votes.SaveVerdict(f.ctx, uid, f.dsID, pkA, votes.VerdictRecommend); err != nil {
|
||||
t.Fatalf("写结论失败: %v", err)
|
||||
}
|
||||
|
||||
// 从「另一个数据集」读:编号完全不同,但坐标一致,应当照样读得到
|
||||
all, err := f.srv.notes.All(f.ctx, dsB)
|
||||
if err != nil {
|
||||
t.Fatalf("读备注失败: %v", err)
|
||||
}
|
||||
got, ok := all[pkB]
|
||||
if !ok {
|
||||
t.Fatal("换数据集后应当仍能读到这个地点的备注")
|
||||
}
|
||||
if got.Alias != "别名甲" || got.Content != "备注甲" {
|
||||
t.Fatalf("备注应原样跟过来,实际 alias=%q note=%q", got.Alias, got.Content)
|
||||
}
|
||||
|
||||
v, err := f.srv.votes.Verdict(f.ctx, dsB, pkB)
|
||||
if err != nil {
|
||||
t.Fatalf("读结论失败: %v", err)
|
||||
}
|
||||
if v != votes.VerdictRecommend {
|
||||
t.Fatalf("结论应跟过来,实际 %d", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 两个数据集里的编号完全不同,但备注与结论是同一份——改一处两边都变。
|
||||
func TestNotesUpdateVisibleFromOtherDataset(t *testing.T) {
|
||||
f := newMovementFixture(t)
|
||||
uid := userFrom(f.ctx).ID
|
||||
pkA, pkB, dsB := testPlace(t, f, uid, 33.333333, 44.444444)
|
||||
|
||||
if err := f.srv.notes.Save(f.ctx, uid, dsB, pkB, "别名乙", "备注乙"); err != nil {
|
||||
t.Fatalf("在新数据集写备注失败: %v", err)
|
||||
}
|
||||
back, err := f.srv.notes.All(f.ctx, f.dsID)
|
||||
if err != nil {
|
||||
t.Fatalf("回读失败: %v", err)
|
||||
}
|
||||
if back[pkA].Content != "备注乙" {
|
||||
t.Fatalf("备注是同一份数据,改一处两边都该变,实际 %q", back[pkA].Content)
|
||||
}
|
||||
}
|
||||
|
||||
// 坐标不同的地点绝不能互相认领——键是坐标,不是「大概位置」。
|
||||
func TestNotesDoNotLeakBetweenDifferentPlaces(t *testing.T) {
|
||||
f := newMovementFixture(t)
|
||||
uid := userFrom(f.ctx).ID
|
||||
pkA, _, dsB := testPlace(t, f, uid, 55.555555, 66.666666)
|
||||
|
||||
if err := f.srv.notes.Save(f.ctx, uid, f.dsID, pkA, "别名甲", "备注甲"); err != nil {
|
||||
t.Fatalf("写备注失败: %v", err)
|
||||
}
|
||||
// 在乙数据集里放一个坐标差 0.001 度(约 100m)的地点,是完全不同的地方
|
||||
const otherPK = 660003
|
||||
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO places (dataset_id, src_pk, name, lat, lon)
|
||||
VALUES ($1,$2,'隔壁那家', 55.556555, 66.666666)`, dsB, otherPK); err != nil {
|
||||
t.Fatalf("建地点失败: %v", err)
|
||||
}
|
||||
|
||||
all, err := f.srv.notes.All(f.ctx, dsB)
|
||||
if err != nil {
|
||||
t.Fatalf("读备注失败: %v", err)
|
||||
}
|
||||
if _, ok := all[otherPK]; ok {
|
||||
t.Fatal("坐标不同的地点不该认领到别人的备注")
|
||||
}
|
||||
}
|
||||
@@ -317,14 +317,16 @@ func TestVisitorPlacePageCarriesImageURLs(t *testing.T) {
|
||||
var placeID int64
|
||||
var srcPK int
|
||||
if err := f.pg.QueryRowContext(f.ctx, `SELECT id, src_pk FROM places
|
||||
WHERE dataset_id=$1 ORDER BY id LIMIT 1`, f.dsID).Scan(&placeID, &srcPK); err != nil {
|
||||
WHERE dataset_id=$1 AND lat IS NOT NULL ORDER BY id LIMIT 1`, f.dsID).Scan(&placeID, &srcPK); err != nil {
|
||||
t.Fatalf("找不到地点: %v", err)
|
||||
}
|
||||
const key = "9/visitor-page-test.webp"
|
||||
// 这里只验发下去的地址,不验能否取回字节,所以不用真的写文件
|
||||
// 这里只验发下去的地址,不验能否取回字节,所以不用真的写文件。
|
||||
// place_key 得照真实写入路径一并算好——读取是认坐标键的,不认 (dataset_id, src_pk)。
|
||||
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO place_images
|
||||
(dataset_id, src_pk, storage, object_key, width, height, bytes, position)
|
||||
VALUES ($1,$2,'local',$3,1200,900,1024,0)
|
||||
(dataset_id, src_pk, place_key, storage, object_key, width, height, bytes, position)
|
||||
SELECT $1, $2, `+placeKeyExpr+`, 'local', $3, 1200, 900, 1024, 0
|
||||
FROM places p WHERE p.dataset_id=$1 AND p.src_pk=$2
|
||||
ON CONFLICT (dataset_id, src_pk, object_key) DO NOTHING`,
|
||||
f.dsID, srcPK, key); err != nil {
|
||||
t.Fatalf("插入图片记录失败: %v", err)
|
||||
|
||||
+28
-20
@@ -40,26 +40,28 @@ var pageFiles = map[string]struct {
|
||||
files []string
|
||||
entry string
|
||||
}{
|
||||
"home": {[]string{"templates/layout.html", "templates/home.html"}, "layout"},
|
||||
"map": {[]string{"templates/layout.html", "templates/map.html"}, "layout"},
|
||||
"places": {[]string{"templates/layout.html", "templates/places.html"}, "layout"},
|
||||
"place": {[]string{"templates/layout.html", "templates/place.html"}, "layout"},
|
||||
"timeline": {[]string{"templates/layout.html", "templates/timeline.html"}, "layout"},
|
||||
"stats": {[]string{"templates/layout.html", "templates/stats.html"}, "layout"},
|
||||
"report": {[]string{"templates/layout.html", "templates/report.html"}, "layout"},
|
||||
"admin/index": {[]string{"templates/layout.html", "templates/admin/index.html"}, "layout"},
|
||||
"admin/settings": {[]string{"templates/layout.html", "templates/admin/settings.html"}, "layout"},
|
||||
"admin/password": {[]string{"templates/layout.html", "templates/admin/password.html"}, "layout"},
|
||||
"admin/login": {[]string{"templates/admin/login.html"}, "login"},
|
||||
"admin/twofa-login": {[]string{"templates/admin/twofa-login.html"}, "login2"},
|
||||
"admin/twofa": {[]string{"templates/layout.html", "templates/admin/twofa.html"}, "layout"},
|
||||
"admin/fog-mask": {[]string{"templates/layout.html", "templates/admin/fog-mask.html"}, "layout"},
|
||||
"admin/notes": {[]string{"templates/layout.html", "templates/admin/notes.html"}, "layout"},
|
||||
"admin/edit": {[]string{"templates/layout.html", "templates/admin/edit.html"}, "layout"},
|
||||
"admin/topics": {[]string{"templates/layout.html", "templates/admin/topics.html"}, "layout"},
|
||||
"admin/photos": {[]string{"templates/layout.html", "templates/admin/photos.html"}, "layout"},
|
||||
"topics": {[]string{"templates/layout.html", "templates/topics.html"}, "layout"},
|
||||
"topic": {[]string{"templates/layout.html", "templates/topic.html"}, "layout"},
|
||||
"home": {[]string{"templates/layout.html", "templates/home.html"}, "layout"},
|
||||
"map": {[]string{"templates/layout.html", "templates/map.html"}, "layout"},
|
||||
"places": {[]string{"templates/layout.html", "templates/places.html"}, "layout"},
|
||||
"place": {[]string{"templates/layout.html", "templates/place.html"}, "layout"},
|
||||
"timeline": {[]string{"templates/layout.html", "templates/timeline.html"}, "layout"},
|
||||
"stats": {[]string{"templates/layout.html", "templates/stats.html"}, "layout"},
|
||||
"report": {[]string{"templates/layout.html", "templates/report.html"}, "layout"},
|
||||
"admin/index": {[]string{"templates/layout.html", "templates/admin/index.html"}, "layout"},
|
||||
"admin/settings": {[]string{"templates/layout.html", "templates/admin/settings.html"}, "layout"},
|
||||
"admin/password": {[]string{"templates/layout.html", "templates/admin/password.html"}, "layout"},
|
||||
"admin/login": {[]string{"templates/admin/login.html"}, "login"},
|
||||
"admin/twofa-login": {[]string{"templates/admin/twofa-login.html"}, "login2"},
|
||||
"admin/twofa": {[]string{"templates/layout.html", "templates/admin/twofa.html"}, "layout"},
|
||||
"admin/fog-mask": {[]string{"templates/layout.html", "templates/admin/fog-mask.html"}, "layout"},
|
||||
"admin/fog-preview": {[]string{"templates/layout.html", "templates/admin/fog-preview.html"}, "layout"},
|
||||
"admin/backup-preview": {[]string{"templates/layout.html", "templates/admin/backup-preview.html"}, "layout"},
|
||||
"admin/notes": {[]string{"templates/layout.html", "templates/admin/notes.html"}, "layout"},
|
||||
"admin/edit": {[]string{"templates/layout.html", "templates/admin/edit.html"}, "layout"},
|
||||
"admin/topics": {[]string{"templates/layout.html", "templates/admin/topics.html"}, "layout"},
|
||||
"admin/photos": {[]string{"templates/layout.html", "templates/admin/photos.html"}, "layout"},
|
||||
"topics": {[]string{"templates/layout.html", "templates/topics.html"}, "layout"},
|
||||
"topic": {[]string{"templates/layout.html", "templates/topic.html"}, "layout"},
|
||||
}
|
||||
|
||||
type pageSet struct {
|
||||
@@ -229,6 +231,9 @@ func (s *Server) routes() {
|
||||
s.mux.HandleFunc("POST /admin/twofa/disable", s.requireAdmin(s.twofaDisable))
|
||||
s.mux.HandleFunc("GET /admin", s.requireAdmin(s.adminHome))
|
||||
s.mux.HandleFunc("POST /admin/upload", s.requireAdmin(s.adminUpload))
|
||||
s.mux.HandleFunc("POST /admin/preview", s.requireAdmin(s.adminBackupPreview))
|
||||
s.mux.HandleFunc("GET /admin/preview/{token}", s.requireAdmin(s.adminBackupPreviewPage))
|
||||
s.mux.HandleFunc("POST /admin/preview/{token}/merge", s.requireAdmin(s.adminBackupMerge))
|
||||
s.mux.HandleFunc("POST /admin/datasets/{id}/activate", s.requireAdmin(s.adminActivate))
|
||||
s.mux.HandleFunc("POST /admin/datasets/{id}/delete", s.requireAdmin(s.adminDelete))
|
||||
s.mux.HandleFunc("POST /admin/place/note", s.requireAdmin(s.adminPlaceNote))
|
||||
@@ -261,6 +266,9 @@ func (s *Server) routes() {
|
||||
s.mux.HandleFunc("POST /admin/edit/weather", s.requireAdmin(s.editWeatherCreate))
|
||||
s.mux.HandleFunc("POST /admin/gpx", s.requireAdmin(s.adminGpxUpload))
|
||||
s.mux.HandleFunc("POST /admin/gpx/delete", s.requireAdmin(s.adminGpxDelete))
|
||||
s.mux.HandleFunc("POST /admin/fog/preview", s.requireAdmin(s.adminFogPreview))
|
||||
s.mux.HandleFunc("GET /admin/fog/preview/{token}", s.requireAdmin(s.adminFogPreviewPage))
|
||||
s.mux.HandleFunc("GET /admin/fog/preview/{token}/{z}/{x}/{y}", s.requireAdmin(s.adminFogDiffTile))
|
||||
s.mux.HandleFunc("POST /admin/fog", s.requireAdmin(s.adminFogUpload))
|
||||
s.mux.HandleFunc("POST /admin/fog/clear", s.requireAdmin(s.adminFogClear))
|
||||
s.mux.HandleFunc("GET /admin/fog/export", s.requireAdmin(s.adminFogExport))
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
{{define "content"}}
|
||||
<div class="wrap">
|
||||
<div class="page-head">
|
||||
<div>
|
||||
<h1>备份差异预览</h1>
|
||||
<div class="sub">
|
||||
这份备份与当前展示的「{{.DatasetName}}」逐条比对的结果。<strong>还没有导入</strong>,站内数据一个字节都没动。
|
||||
</div>
|
||||
</div>
|
||||
<div class="row"><a class="btn" href="/admin">← 返回后台</a></div>
|
||||
</div>
|
||||
|
||||
{{if .Identical}}
|
||||
<div class="card">
|
||||
<div class="alert ok">这份备份与当前数据集<strong>完全一致</strong>:没有新增、没有被删、也没有冲突。</div>
|
||||
</div>
|
||||
{{end}}
|
||||
|
||||
{{if .Truncated}}
|
||||
<div class="card"><div class="alert err">差异较多,下面每类只列出前 200 条,确切数量看概览表。</div></div>
|
||||
{{end}}
|
||||
|
||||
<div class="card">
|
||||
<h2>概览</h2>
|
||||
<div class="tw">
|
||||
<table class="tb tb-stack">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>实体</th><th class="num">备份</th><th class="num">站内</th>
|
||||
<th class="num">备份新增</th><th class="num">仅站内有</th><th class="num">冲突</th><th class="num">一致</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>到访</td>
|
||||
<td class="num" data-label="备份">{{num .BackupVisits}}</td>
|
||||
<td class="num" data-label="站内">{{num .StoredVisits}}</td>
|
||||
<td class="num" data-label="备份新增">{{num .VisitCounts.Added}}</td>
|
||||
<td class="num" data-label="仅站内有">{{num .VisitCounts.Removed}}</td>
|
||||
<td class="num" data-label="冲突">{{num .VisitCounts.Changed}}</td>
|
||||
<td class="num" data-label="一致">{{num .VisitCounts.Same}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>地点</td>
|
||||
<td class="num" data-label="备份">{{num .BackupPlaces}}</td>
|
||||
<td class="num" data-label="站内">{{num .StoredPlaces}}</td>
|
||||
<td class="num" data-label="备份新增">{{num .PlaceCounts.Added}}</td>
|
||||
<td class="num" data-label="仅站内有">{{num .PlaceCounts.Removed}}</td>
|
||||
<td class="num" data-label="冲突">{{num .PlaceCounts.Changed}}</td>
|
||||
<td class="num" data-label="一致">{{num .PlaceCounts.Same}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>活动类型</td>
|
||||
<td class="num" data-label="备份">{{num .BackupActs}}</td>
|
||||
<td class="num" data-label="站内">{{num .StoredActs}}</td>
|
||||
<td class="num" data-label="备份新增">{{num .ActCounts.Added}}</td>
|
||||
<td class="num" data-label="仅站内有">{{num .ActCounts.Removed}}</td>
|
||||
<td class="num" data-label="冲突">{{num .ActCounts.Changed}}</td>
|
||||
<td class="num" data-label="一致">{{num .ActCounts.Same}}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="card-sub" style="margin-top:10px">
|
||||
对齐按每条记录的 <strong>UUID</strong> 做:rond 每次导出都会重排记录编号(同样的 822 条到访,两次导出的编号 100% 不同,UUID 却逐条一模一样),
|
||||
所以只有 UUID 认得出来是同一条;地点没有 UUID,退到「名称 + 坐标」。
|
||||
站内补录的记录不会出现在真机备份里,一律归到「仅站内有」。
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>到访</h2>
|
||||
|
||||
<div class="card-sub"><strong>备份里多出来的({{num .VisitCounts.Added}})</strong> —— 目前站内没有这些记录</div>
|
||||
{{if .VisitAdded}}
|
||||
<div class="tw">
|
||||
<table class="tb tb-stack">
|
||||
<thead><tr><th>到达</th><th>离开</th><th>地点</th><th>活动</th><th>备注</th></tr></thead>
|
||||
<tbody>
|
||||
{{range .VisitAdded}}
|
||||
<tr>
|
||||
<td class="mono">{{dtt .Arrival}}</td>
|
||||
<td class="mono" data-label="离开">{{dtt .Departure}}</td>
|
||||
<td data-label="地点">{{if .Place}}{{.Place}}{{else}}<span class="muted">—</span>{{end}}</td>
|
||||
<td data-label="活动">{{if .Activity}}{{.Activity}}{{else}}<span class="muted">—</span>{{end}}</td>
|
||||
<td data-label="备注">{{if .Remark}}{{.Remark}}{{else}}<span class="muted">—</span>{{end}}</td>
|
||||
</tr>
|
||||
{{end}}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{{else}}<div class="muted" style="margin-bottom:14px">没有</div>{{end}}
|
||||
|
||||
<div class="card-sub" style="margin-top:16px">
|
||||
<strong>只在站内有({{num .VisitCounts.Removed}})</strong> —— 备份里找不到;可能是你在这里补录的,也可能是实机上删掉的。
|
||||
<strong>默认一条都不删</strong>;确实想清掉的,勾上左列的「合并时删除」。
|
||||
</div>
|
||||
{{if .VisitGone}}
|
||||
<div class="tw">
|
||||
<table class="tb tb-stack">
|
||||
<thead><tr><th>到达</th><th>离开</th><th>地点</th><th>活动</th><th>备注</th><th>来源</th><th>合并时删除</th></tr></thead>
|
||||
<tbody>
|
||||
{{range .VisitGone}}
|
||||
<tr>
|
||||
<td class="mono">{{dtt .Arrival}}</td>
|
||||
<td class="mono" data-label="离开">{{dtt .Departure}}</td>
|
||||
<td data-label="地点">{{if .Place}}{{.Place}}{{else}}<span class="muted">—</span>{{end}}</td>
|
||||
<td data-label="活动">{{if .Activity}}{{.Activity}}{{else}}<span class="muted">—</span>{{end}}</td>
|
||||
<td data-label="备注">{{if .Remark}}{{.Remark}}{{else}}<span class="muted">—</span>{{end}}</td>
|
||||
<td data-label="来源">{{if .UserAdded}}<span>站内补录</span>{{else}}<span class="muted">备份里已无</span>{{end}}</td>
|
||||
<td data-label="合并时删除">
|
||||
<input type="checkbox" form="merge-form" name="dv_{{.SrcPK}}" value="1" title="勾上=合并时删掉这条到访">
|
||||
</td>
|
||||
</tr>
|
||||
{{end}}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{{else}}<div class="muted" style="margin-bottom:14px">没有</div>{{end}}
|
||||
|
||||
<div class="card-sub" style="margin-top:16px"><strong>两边都有但不一样({{num .VisitCounts.Changed}})</strong> —— 左边是站内当前的值,右边是备份里的值</div>
|
||||
{{if .VisitClash}}
|
||||
<div class="tw">
|
||||
<table class="tb tb-stack">
|
||||
<thead><tr><th>到达</th><th>地点</th><th>活动</th><th>差异</th><th>合并时</th></tr></thead>
|
||||
<tbody>
|
||||
{{range .VisitClash}}
|
||||
<tr>
|
||||
<td class="mono">{{dtt .Arrival}}</td>
|
||||
<td data-label="地点">{{if .Place}}{{.Place}}{{else}}<span class="muted">—</span>{{end}}</td>
|
||||
<td data-label="活动">{{if .Activity}}{{.Activity}}{{else}}<span class="muted">—</span>{{end}}</td>
|
||||
<td data-label="差异">
|
||||
{{range .Fields}}
|
||||
<div style="font-size:12.5px;line-height:1.7">
|
||||
<span class="muted">{{.Label}}</span>
|
||||
<span style="text-decoration:line-through;opacity:.65">{{if .From}}{{.From}}{{else}}(空){{end}}</span>
|
||||
<span style="opacity:.5">→</span>
|
||||
<strong>{{if .To}}{{.To}}{{else}}(空){{end}}</strong>
|
||||
</div>
|
||||
{{end}}
|
||||
</td>
|
||||
<td data-label="合并时" style="white-space:nowrap">
|
||||
<label style="margin-right:8px"><input type="radio" form="merge-form" name="vc_{{.Key}}" value="backup" checked> 用实机</label>
|
||||
<label style="margin-right:8px"><input type="radio" form="merge-form" name="vc_{{.Key}}" value="db"> 用站内</label>
|
||||
<label><input type="radio" form="merge-form" name="vc_{{.Key}}" value="skip"> 跳过</label>
|
||||
</td>
|
||||
</tr>
|
||||
{{end}}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{{else}}<div class="muted">没有</div>{{end}}
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>地点</h2>
|
||||
<div class="card-sub"><strong>备份里多出来的({{num .PlaceCounts.Added}})</strong></div>
|
||||
{{if .PlaceAdded}}
|
||||
<div class="tw">
|
||||
<table class="tb tb-stack">
|
||||
<thead><tr><th>名称</th><th>省 / 市</th><th>分类</th><th class="num">坐标</th></tr></thead>
|
||||
<tbody>
|
||||
{{range .PlaceAdded}}
|
||||
<tr>
|
||||
<td>{{if .Name}}{{.Name}}{{else}}<span class="muted">(未命名)</span>{{end}}</td>
|
||||
<td data-label="省 / 市">{{.Province}} {{.City}}</td>
|
||||
<td data-label="分类">{{if .POICategory}}{{.POICategory}}{{else}}<span class="muted">—</span>{{end}}</td>
|
||||
<td class="num mono" data-label="坐标">{{printf "%.5f" .Lat}}, {{printf "%.5f" .Lon}}</td>
|
||||
</tr>
|
||||
{{end}}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{{else}}<div class="muted" style="margin-bottom:14px">没有</div>{{end}}
|
||||
|
||||
<div class="card-sub" style="margin-top:16px">
|
||||
<strong>只在站内有({{num .PlaceCounts.Removed}})</strong>
|
||||
—— 名下还有到访的地点删不掉(会连带删掉那些到访),会直接报错让你先处理。
|
||||
</div>
|
||||
{{if .PlaceGone}}
|
||||
<div class="tw">
|
||||
<table class="tb tb-stack">
|
||||
<thead><tr><th>名称</th><th>省 / 市</th><th class="num">到访</th><th class="num">坐标</th><th>合并时删除</th></tr></thead>
|
||||
<tbody>
|
||||
{{range .PlaceGone}}
|
||||
<tr>
|
||||
<td>{{if .Name}}{{.Name}}{{else}}<span class="muted">(未命名)</span>{{end}}</td>
|
||||
<td data-label="省 / 市">{{.Province}} {{.City}}</td>
|
||||
<td class="num" data-label="到访">{{num .VisitCount}}</td>
|
||||
<td class="num mono" data-label="坐标">{{printf "%.5f" .Lat}}, {{printf "%.5f" .Lon}}</td>
|
||||
<td data-label="合并时删除">
|
||||
{{if gt .VisitCount 0}}
|
||||
<span class="muted" style="font-size:12.5px">还有到访,不能删</span>
|
||||
{{else}}
|
||||
<input type="checkbox" form="merge-form" name="dp_{{.SrcPK}}" value="1">
|
||||
{{end}}
|
||||
</td>
|
||||
</tr>
|
||||
{{end}}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{{else}}<div class="muted" style="margin-bottom:14px">没有</div>{{end}}
|
||||
|
||||
<div class="card-sub" style="margin-top:16px"><strong>两边都有但地址不一样({{num .PlaceCounts.Changed}})</strong></div>
|
||||
{{if .PlaceClash}}
|
||||
<div class="tw">
|
||||
<table class="tb tb-stack">
|
||||
<thead><tr><th>名称</th><th>差异</th><th>合并时</th></tr></thead>
|
||||
<tbody>
|
||||
{{range .PlaceClash}}
|
||||
<tr>
|
||||
<td>{{if .DB.Name}}{{.DB.Name}}{{else}}<span class="muted">(未命名)</span>{{end}}</td>
|
||||
<td data-label="差异">
|
||||
{{range .Fields}}
|
||||
<div style="font-size:12.5px;line-height:1.7">
|
||||
<span class="muted">{{.Label}}</span>
|
||||
<span style="text-decoration:line-through;opacity:.65">{{if .From}}{{.From}}{{else}}(空){{end}}</span>
|
||||
<span style="opacity:.5">→</span>
|
||||
<strong>{{if .To}}{{.To}}{{else}}(空){{end}}</strong>
|
||||
</div>
|
||||
{{end}}
|
||||
</td>
|
||||
<td data-label="合并时" style="white-space:nowrap">
|
||||
<label style="margin-right:8px"><input type="radio" form="merge-form" name="pc_{{.DB.Key}}" value="backup" checked> 用实机</label>
|
||||
<label style="margin-right:8px"><input type="radio" form="merge-form" name="pc_{{.DB.Key}}" value="db"> 用站内</label>
|
||||
<label><input type="radio" form="merge-form" name="pc_{{.DB.Key}}" value="skip"> 跳过</label>
|
||||
</td>
|
||||
</tr>
|
||||
{{end}}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{{else}}<div class="muted">没有</div>{{end}}
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>活动类型</h2>
|
||||
<div class="card-sub">
|
||||
新增 {{num .ActCounts.Added}} · 仅站内有 {{num .ActCounts.Removed}} · 冲突 {{num .ActCounts.Changed}} · 一致 {{num .ActCounts.Same}}
|
||||
</div>
|
||||
<div class="row" style="gap:24px;align-items:flex-start;flex-wrap:wrap">
|
||||
<div style="flex:1 1 220px">
|
||||
<div class="muted" style="font-size:12.5px;margin-bottom:4px">备份新增</div>
|
||||
{{range .ActAdded}}<div>{{.Name}}</div>{{else}}<div class="muted">没有</div>{{end}}
|
||||
</div>
|
||||
<div style="flex:1 1 220px">
|
||||
<div class="muted" style="font-size:12.5px;margin-bottom:4px">仅站内有(勾上=合并时删除)</div>
|
||||
{{range .ActGone}}
|
||||
<label style="display:block"><input type="checkbox" form="merge-form" name="da_{{.SrcPK}}" value="1"> {{.Name}}</label>
|
||||
{{else}}<div class="muted">没有</div>{{end}}
|
||||
</div>
|
||||
<div style="flex:1 1 260px">
|
||||
<div class="muted" style="font-size:12.5px;margin-bottom:4px">名称不同</div>
|
||||
{{range .ActClash}}
|
||||
<div style="margin-bottom:6px">
|
||||
{{range .Fields}}<div><span style="text-decoration:line-through;opacity:.65">{{.From}}</span> → <strong>{{.To}}</strong></div>{{end}}
|
||||
<div style="margin-top:4px">
|
||||
<label style="margin-right:8px"><input type="radio" form="merge-form" name="ac_{{actKey .DB}}" value="backup" checked> 用实机</label>
|
||||
<label style="margin-right:8px"><input type="radio" form="merge-form" name="ac_{{actKey .DB}}" value="db"> 用站内</label>
|
||||
<label><input type="radio" form="merge-form" name="ac_{{actKey .DB}}" value="skip"> 跳过</label>
|
||||
</div>
|
||||
</div>
|
||||
{{else}}<div class="muted">没有</div>{{end}}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>其它实体(只列数量)</h2>
|
||||
<div class="tw">
|
||||
<table class="tb tb-stack">
|
||||
<thead><tr><th>实体</th><th class="num">备份</th><th class="num">站内</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>位移</td><td class="num" data-label="备份">{{num .BackupMovements}}</td><td class="num" data-label="站内">{{num .StoredMovements}}</td></tr>
|
||||
<tr><td>小时天气</td><td class="num" data-label="备份">{{num .BackupWeather}}</td><td class="num" data-label="站内">{{num .StoredWeather}}</td></tr>
|
||||
<tr><td>原始定位点</td><td class="num" data-label="备份">{{num .BackupRaw}}</td><td class="num" data-label="站内"><span class="muted">不单独落库</span></td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="card-sub" style="margin-top:10px">
|
||||
位移跟着它的起止到访走、天气跟着到访走,所以这两项按到访对齐结果一并处理,不单独做逐条比对。
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>执行合并</h2>
|
||||
<div class="card-sub">
|
||||
把选中的改动写进当前数据集「{{.DatasetName}}」。整个过程跑在一个事务里,任何一步失败都会<strong>整体回滚</strong>,数据不会半改。
|
||||
<br>「仅站内有」的记录<strong>默认一条都不删</strong>;确实要清掉的,在上面各表里逐条勾「合并时删除」。
|
||||
名下有到访的地点会被拒绝删除(否则会连带删掉那些到访),先处理到访再来。
|
||||
</div>
|
||||
<form id="merge-form" method="post" action="/admin/preview/{{.Token}}/merge"></form>
|
||||
<label style="display:flex;align-items:flex-start;gap:8px;margin:4px 0 12px">
|
||||
<input type="checkbox" form="merge-form" name="add_new" value="1" checked style="margin-top:3px">
|
||||
<span>把备份里多出来的
|
||||
<strong>{{num .VisitCounts.Added}}</strong> 条到访、
|
||||
<strong>{{num .PlaceCounts.Added}}</strong> 个地点、
|
||||
<strong>{{num .ActCounts.Added}}</strong> 个活动并进来
|
||||
<span class="muted">(新到访会连它的地点、活动、标签、位移与天气一起带过来)</span>
|
||||
</span>
|
||||
</label>
|
||||
<button class="btn primary" form="merge-form" type="submit"
|
||||
onclick="return confirm('确认把选中的改动写进当前数据集?出错会自动整体回滚。')">执行合并</button>
|
||||
<div class="card-sub" style="margin:10px 0 0">
|
||||
合并后再传一次同一份备份,差异应当变成「完全一致」——那说明对齐键(UUID 与坐标)都写进去了。
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{{end}}
|
||||
@@ -0,0 +1,174 @@
|
||||
{{define "content"}}
|
||||
<div class="wrap">
|
||||
<div class="page-head">
|
||||
<div>
|
||||
<h1>迷雾导入预览</h1>
|
||||
<div class="sub">这份快照<strong>还没有写入</strong>。先看清两种导入方式各自的后果,再决定用哪一种。</div>
|
||||
</div>
|
||||
<div class="row">
|
||||
<a class="btn" href="/admin">← 返回后台</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>数量对比</h2>
|
||||
<div class="tw">
|
||||
<table class="tb tb-stack">
|
||||
<thead>
|
||||
<tr><th>状态</th><th class="num">块</th><th class="num">格</th><th class="num">面积</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>当前迷雾</td>
|
||||
<td class="num" data-label="块">{{num .Diff.CurBlocks}}</td>
|
||||
<td class="num" data-label="格">{{num .Diff.CurCells}}</td>
|
||||
<td class="num" data-label="面积">{{printf "%.1f" .Diff.CurAreaKm2}} km²</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>这份快照</td>
|
||||
<td class="num" data-label="块">{{num .Diff.IncBlocks}}</td>
|
||||
<td class="num" data-label="格">{{num .Diff.IncCells}}</td>
|
||||
<td class="num" data-label="面积">{{printf "%.1f" .Diff.IncAreaKm2}} km²</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>叠加导入后</td>
|
||||
<td class="num" data-label="块">{{num .Diff.CurBlocks}}<span class="muted"> +{{num .Diff.GainBlocks}}</span></td>
|
||||
<td class="num" data-label="格">{{num .Diff.CurCells}}<span class="muted"> +{{num .Diff.GainCells}}</span></td>
|
||||
<td class="num" data-label="面积">{{printf "%.1f" .Diff.MergeAreaKm2}} km²</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>整体替换后</td>
|
||||
<td class="num" data-label="块">{{num .Diff.IncBlocks}}</td>
|
||||
<td class="num" data-label="格">{{num .Diff.IncCells}}</td>
|
||||
<td class="num" data-label="面积">{{printf "%.1f" .Diff.CoverAreaKm2}} km²</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="card-sub" style="margin-top:10px">
|
||||
两个块坐标相同就只按位合并,所以「叠加后」的块数不会等于两边相加——快照里有 {{num .Diff.OverlapBlocks}} 个块与当前迷雾落在同一块,
|
||||
其中 {{num .Diff.SameBlocks}} 个一格都不会变。
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{{if .HasBounds}}
|
||||
<div class="card">
|
||||
<h2>差异地图</h2>
|
||||
<div class="card-sub">在真实底图上核对差异落在哪些区域,可以缩放平移。</div>
|
||||
<div class="row" style="gap:16px;margin:2px 0 10px;font-size:12.5px;flex-wrap:wrap">
|
||||
<span style="display:flex;align-items:center;gap:6px">
|
||||
<i style="width:13px;height:13px;border-radius:3px;background:#24a45a;display:inline-block"></i>叠加会新增
|
||||
</span>
|
||||
<span style="display:flex;align-items:center;gap:6px">
|
||||
<i style="width:13px;height:13px;border-radius:3px;background:#7e8692;display:inline-block"></i>两边都有
|
||||
</span>
|
||||
<span style="display:flex;align-items:center;gap:6px">
|
||||
<i style="width:13px;height:13px;border-radius:3px;background:#d63838;display:inline-block"></i>整体替换会丢掉
|
||||
</span>
|
||||
</div>
|
||||
<div id="fog-diff-map" style="height:400px;border-radius:10px;background:var(--line-soft)"
|
||||
data-token="{{.Token}}"
|
||||
data-min-lat="{{printf "%.6f" .MinLat}}" data-min-lon="{{printf "%.6f" .MinLon}}"
|
||||
data-max-lat="{{printf "%.6f" .MaxLat}}" data-max-lon="{{printf "%.6f" .MaxLon}}"></div>
|
||||
</div>
|
||||
{{end}}
|
||||
|
||||
<div class="card">
|
||||
<h2>叠加导入</h2>
|
||||
{{if .Diff.Changed}}
|
||||
<div class="alert ok">
|
||||
会补上当前迷雾没有的 <strong>{{num .Diff.GainBlocks}}</strong> 块、<strong>{{num .Diff.GainCells}}</strong> 格
|
||||
(约 {{printf "%.1f" .Diff.GainAreaKm2}} km²)。叠加只做按位合并,<strong>不会丢失任何已有数据</strong>。
|
||||
</div>
|
||||
{{else}}
|
||||
<div class="alert err">
|
||||
这份快照的点亮区域已经被当前迷雾<strong>完全覆盖</strong>——叠加导入不会带来任何变化,
|
||||
多半是同一批数据又导出了一次。
|
||||
</div>
|
||||
{{end}}
|
||||
<form method="post" action="/admin/fog?stash={{.Token}}"
|
||||
onsubmit="this.querySelector('button').disabled=true">
|
||||
<input type="hidden" name="mode" value="overlay">
|
||||
<button class="btn primary" type="submit">叠加导入</button>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>整体替换</h2>
|
||||
{{if gt .Diff.LostCells 0}}
|
||||
<div class="alert err">
|
||||
⚠️ 快照里<strong>没有</strong>当前迷雾的 <strong>{{num .Diff.LostBlocks}}</strong> 块、<strong>{{num .Diff.LostCells}}</strong> 格
|
||||
(约 {{printf "%.1f" .Diff.LostAreaKm2}} km²)。整体替换后,这些区域会<strong>从点亮变回未探索</strong>,且无法恢复。
|
||||
</div>
|
||||
{{else}}
|
||||
<div class="alert ok">
|
||||
这份快照完整包含了当前迷雾的全部点亮区域,替换不会丢数据。
|
||||
</div>
|
||||
{{end}}
|
||||
{{if and (eq .TildeLayers 0) (eq .HashLayers 0)}}
|
||||
<div class="alert err">
|
||||
⚠️ 快照里没有 <span class="mono">Model/~</span> 与 <span class="mono">Model/#</span> 两层数据,替换会把现有的这两层一并清掉。
|
||||
</div>
|
||||
{{else}}
|
||||
<div class="muted" style="font-size:12.5px;margin-bottom:8px">
|
||||
另外,站点不解析但会留存的 <span class="mono">Model/~</span>({{.TildeLayers}} 项)与
|
||||
<span class="mono">Model/#</span>({{.HashLayers}} 项)也会一并换成这份快照里的{{if .OtherLayers}},另有 {{.OtherLayers}} 项其它条目{{end}}。
|
||||
</div>
|
||||
{{end}}
|
||||
<form method="post" action="/admin/fog?stash={{.Token}}"
|
||||
onsubmit="return confirm('整体替换会清空当前迷雾。{{if gt .Diff.LostCells 0}}快照里缺 {{num .Diff.LostCells}} 格,替换后这些区域会变回未探索,且不可恢复。{{end}}确认替换?')">
|
||||
<input type="hidden" name="mode" value="cover">
|
||||
<button class="btn danger" type="submit">整体替换</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function () {
|
||||
// Leaflet 由 layout 在 content 之后引入,脚本执行时 window.L 还没有,
|
||||
// 必须等 load 之后再初始化,否则地图整个不出现(与迷雾遮罩页同一条约定)
|
||||
function whenReady(cb) {
|
||||
if (document.readyState === 'complete') { setTimeout(cb, 0); return; }
|
||||
window.addEventListener('load', cb);
|
||||
}
|
||||
|
||||
var el = document.getElementById('fog-diff-map');
|
||||
if (!el) return;
|
||||
|
||||
// 底图走站点配置(与迷雾遮罩页同一套取法):带密钥的图层由服务端转发,
|
||||
// 免密钥的是浏览器直连的 {s} 模板,不能硬写 /tiles/ 路由
|
||||
var fallback = {
|
||||
def: { url: 'https://webrd0{s}.is.autonavi.com/appmaptile?lang=zh_cn&size=1&scale=1&style=8&x={x}&y={y}&z={z}', maxZoom: 18, attr: '© 高德地图', crs: 'gcj02' },
|
||||
subs: ['1', '2', '3', '4']
|
||||
};
|
||||
|
||||
function init(cfg) {
|
||||
var def = cfg.def;
|
||||
var map = L.map(el, { scrollWheelZoom: false });
|
||||
L.tileLayer(def.url, { subdomains: cfg.subs, maxZoom: def.maxZoom || 18, minZoom: 3, attribution: def.attr || '' }).addTo(map);
|
||||
// 差异瓦片必须和底图同坐标系:迷雾网格是 WGS-84,服务端按 crs 逐块纠偏
|
||||
L.tileLayer('/admin/fog/preview/' + el.dataset.token + '/{z}/{x}/{y}?crs=' + (def.crs || 'gcj02'), {
|
||||
minZoom: 3, maxZoom: 18, opacity: 0.95
|
||||
}).addTo(map);
|
||||
var b = [
|
||||
[parseFloat(el.dataset.minLat), parseFloat(el.dataset.minLon)],
|
||||
[parseFloat(el.dataset.maxLat), parseFloat(el.dataset.maxLon)]
|
||||
];
|
||||
// 只有一个块时两个角重合,不设上限会一路放大到最大级别
|
||||
map.fitBounds(b, { padding: [24, 24], maxZoom: 13 });
|
||||
L.control.scale({ imperial: false }).addTo(map);
|
||||
}
|
||||
|
||||
whenReady(function () {
|
||||
if (!window.L) return;
|
||||
fetch('/api/map-config', { headers: { 'Accept': 'application/json' } })
|
||||
.then(function (r) { return r.json(); })
|
||||
.then(function (c) {
|
||||
var def = (c && c.base && c.base.length) ? c.base[0] : fallback.def;
|
||||
init({ def: def, subs: (c && c.subs && c.subs.length) ? c.subs : fallback.subs });
|
||||
})
|
||||
.catch(function () { init(fallback); });
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
{{end}}
|
||||
@@ -29,6 +29,7 @@
|
||||
<b>上传是以备份为准重建数据</b>:站点里删掉的地点、改过的记录,只要下次传一份
|
||||
<b>内容不同</b>的备份,就会被备份里的原样覆盖回来(重复传同一份文件不会,会提示「完全一致」)。
|
||||
在站点上新加的地点属于当时那个数据集,传新备份后要看它们,可在下面的数据集列表里切回。
|
||||
<br><b>拿不准就先用「分析差异」</b>:它会逐条比对这份备份与站内当前数据,告诉你哪些是新增、哪些只在站内有、哪些两边不一样,看完再决定要不要导入。
|
||||
</div>
|
||||
<form method="post" action="/admin/upload" enctype="multipart/form-data">
|
||||
<label class="f" style="max-width:520px">
|
||||
@@ -36,7 +37,10 @@
|
||||
<input type="file" name="backup" accept=".rondbackup,.zip" required
|
||||
style="padding:8px;border:1px dashed var(--line);border-radius:8px;background:var(--bg-2)">
|
||||
</label>
|
||||
<button class="btn primary" type="submit">上传并导入</button>
|
||||
<div class="row">
|
||||
<button class="btn primary" type="submit" formaction="/admin/preview">分析差异</button>
|
||||
<button class="btn" type="submit" formaction="/admin/upload">直接导入</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
@@ -53,21 +57,18 @@
|
||||
<b>尚未导入迷雾</b>——把 Fog of World 的 <span class="mono">.fwss</span>(在 App 内导出,通常在 zip 里)上传即可点亮地图。
|
||||
{{end}}
|
||||
</div>
|
||||
<form method="post" action="/admin/fog" enctype="multipart/form-data">
|
||||
<form method="post" action="/admin/fog/preview" enctype="multipart/form-data">
|
||||
<label class="f" style="max-width:520px">
|
||||
<span>选择快照</span>
|
||||
<input type="file" name="fwss" accept=".fwss,.zip" required
|
||||
style="padding:8px;border:1px dashed var(--line);border-radius:8px;background:var(--bg-2)">
|
||||
</label>
|
||||
<label class="f" style="max-width:520px">
|
||||
<span>导入方式</span>
|
||||
<select name="mode">
|
||||
<option value="overlay" selected>叠加 — 保留现有迷雾,把新路线合并进来</option>
|
||||
<option value="cover">覆盖 — 清空现有迷雾,整体替换</option>
|
||||
</select>
|
||||
</label>
|
||||
<button class="btn primary" type="submit">导入迷雾</button>
|
||||
<button class="btn primary" type="submit">分析差异</button>
|
||||
</form>
|
||||
<div class="muted" style="font-size:12.5px;margin-top:8px">
|
||||
上传后<strong>先不写入</strong>:会先给出这份快照能补多少新区域、以及「整体替换」会丢掉多少已有区域,
|
||||
确认之后才真正导入。
|
||||
</div>
|
||||
{{if gt .FogBlocks 0}}
|
||||
<div style="margin:10px 0 0;display:flex;gap:8px;flex-wrap:wrap">
|
||||
<a class="btn" href="/admin/fog/export">导出当前迷雾(.fwss)</a>
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"rond-with-you/internal/auth"
|
||||
"rond-with-you/internal/fog"
|
||||
"rond-with-you/internal/rond"
|
||||
"rond-with-you/internal/setting"
|
||||
"rond-with-you/internal/stats"
|
||||
"rond-with-you/internal/votes"
|
||||
@@ -637,6 +638,74 @@ type AdminData struct {
|
||||
DefaultPW bool
|
||||
}
|
||||
|
||||
// VisitRow 是一条到访摊平后的展示行。地点与活动写名称而不是编号——
|
||||
// 两边的编号空间完全不同,编号对人没有任何意义。
|
||||
type VisitRow struct {
|
||||
// Key 是这条冲突的对齐键(到访 UUID),合并表单拿它当字段名
|
||||
Key string
|
||||
// SrcPK 只有「仅站内有」的行有意义:要删哪条就勾哪条
|
||||
SrcPK int
|
||||
Arrival *time.Time
|
||||
Departure *time.Time
|
||||
Place string
|
||||
Activity string
|
||||
Remark string
|
||||
Emoji string
|
||||
UserAdded bool // 站点补录的行
|
||||
Fields []rond.FieldChange
|
||||
}
|
||||
|
||||
// BackupDiffData 是「备份差异预览」页的数据。
|
||||
//
|
||||
// 上限说明:列表按 diffListLimit 截断,页面上只提示「还有更多」,不做分页——
|
||||
// 预演是给人下判断用的,看前 200 条足够判断要不要继续;真正要逐条取舍的是
|
||||
// 后面那个合并流程(那里才有逐条操作)。
|
||||
type BackupDiffData struct {
|
||||
Page
|
||||
Token string
|
||||
DatasetName string
|
||||
|
||||
VisitCounts rond.Counts
|
||||
PlaceCounts rond.Counts
|
||||
ActCounts rond.Counts
|
||||
|
||||
VisitAdded, VisitGone, VisitClash []VisitRow
|
||||
PlaceAdded []rond.Location
|
||||
PlaceGone []rond.StoredPlace
|
||||
PlaceClash []rond.PlaceConflict
|
||||
ActAdded []rond.Activity
|
||||
ActGone []rond.StoredActivity
|
||||
ActClash []rond.ActivityConflict
|
||||
|
||||
// 两侧总量,用于「有没有少数据」的直觉判断
|
||||
BackupVisits, StoredVisits int
|
||||
BackupPlaces, StoredPlaces int
|
||||
BackupActs, StoredActs int
|
||||
BackupMovements, StoredMovements int
|
||||
BackupWeather, StoredWeather int
|
||||
BackupRaw int
|
||||
|
||||
Truncated bool
|
||||
Identical bool
|
||||
}
|
||||
|
||||
// FogPreviewData 是「迷雾导入预览」页的数据:先把差异摆出来,再决定叠加还是整体替换。
|
||||
type FogPreviewData struct {
|
||||
Page
|
||||
Token string
|
||||
Diff fog.Diff
|
||||
// 快照里不参与渲染、但整体替换会一并换掉的图层条目数
|
||||
TildeLayers int
|
||||
HashLayers int
|
||||
OtherLayers int
|
||||
|
||||
// 差异地图的初始视野:两边块集覆盖范围的包围盒(已按底图坐标系换算)。
|
||||
// 没有任何块时 HasBounds 为假,模板不画地图。
|
||||
HasBounds bool
|
||||
MinLat, MinLon float64
|
||||
MaxLat, MaxLon float64
|
||||
}
|
||||
|
||||
type NotesData struct {
|
||||
Page
|
||||
Rows []NoteRow
|
||||
@@ -1247,6 +1316,8 @@ func templateFuncs() template.FuncMap {
|
||||
"km": func(f float64) string { return fmt.Sprintf("%.1f", f) },
|
||||
"f1": func(f float64) string { return fmt.Sprintf("%.1f", f) },
|
||||
"dt": func(t time.Time) string { return t.Local().Format("2006-01-02 15:04") },
|
||||
// actKey 给活动冲突生成表单字段名,与处理函数共用同一套键
|
||||
"actKey": rond.ActivityKey,
|
||||
// dtt 接受 *time.Time,nil 返回空串,便于可选时间字段直接渲染
|
||||
"dtt": func(t *time.Time) string {
|
||||
if t == nil {
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
// 一次性探针:验证 ZLOCATION.ZPLACEID / ZVISIT.ZEVENTIDENTIFIER 是否跨备份稳定。
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"rond-with-you/internal/rond"
|
||||
)
|
||||
|
||||
type snap struct {
|
||||
placeID map[string]int
|
||||
eventID map[string]int
|
||||
nulls int
|
||||
total int
|
||||
evTotal int
|
||||
}
|
||||
|
||||
func load(p string) *snap {
|
||||
dir, err := os.MkdirTemp("", "colprobe")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
db, err := rond.Open(p, dir)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer db.Close()
|
||||
s := &snap{placeID: map[string]int{}, eventID: map[string]int{}}
|
||||
|
||||
rows, err := db.Query(`SELECT Z_PK, COALESCE(ZPLACEID,'') FROM ZLOCATION`)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
for rows.Next() {
|
||||
var pk int
|
||||
var id string
|
||||
rows.Scan(&pk, &id)
|
||||
s.total++
|
||||
if id == "" {
|
||||
s.nulls++
|
||||
} else {
|
||||
s.placeID[id] = pk
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
rows, err = db.Query(`SELECT COALESCE(ZEVENTIDENTIFIER,'') FROM ZVISIT`)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
for rows.Next() {
|
||||
var id string
|
||||
rows.Scan(&id)
|
||||
if id != "" {
|
||||
s.eventID[id] = 1
|
||||
s.evTotal++
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
fmt.Printf("%s: ZLOCATION %d 个(ZPLACEID 空 %d),ZEVENTIDENTIFIER 非空 %d\n",
|
||||
p, s.total, s.nulls, s.evTotal)
|
||||
return s
|
||||
}
|
||||
|
||||
func main() {
|
||||
a := load(os.Args[1])
|
||||
b := load(os.Args[2])
|
||||
inter, samePK := 0, 0
|
||||
for k, pa := range a.placeID {
|
||||
if pb, ok := b.placeID[k]; ok {
|
||||
inter++
|
||||
if pa == pb {
|
||||
samePK++
|
||||
}
|
||||
}
|
||||
}
|
||||
fmt.Printf("\nZPLACEID 交集 %d(A 独有 %d,B 独有 %d),其中 Z_PK 相同 %d\n",
|
||||
inter, len(a.placeID)-inter, len(b.placeID)-inter, samePK)
|
||||
|
||||
ei := 0
|
||||
for k := range a.eventID {
|
||||
if _, ok := b.eventID[k]; ok {
|
||||
ei++
|
||||
}
|
||||
}
|
||||
fmt.Printf("ZEVENTIDENTIFIER 交集 %d(A %d / B %d)\n", ei, len(a.eventID), len(b.eventID))
|
||||
}
|
||||
@@ -0,0 +1,438 @@
|
||||
// 一次性探针:把一份 .rondbackup 与 PG 里某个数据集的记录逐条对齐,输出差异画像。
|
||||
// 用完即可删,不属于站点功能。
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
_ "github.com/jackc/pgx/v5/stdlib"
|
||||
|
||||
"rond-with-you/internal/rond"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 4 {
|
||||
fmt.Println("用法: diffprobe <archive.rondbackup> <dsn> <datasetID>")
|
||||
os.Exit(2)
|
||||
}
|
||||
archive, dsn := os.Args[1], os.Args[2]
|
||||
var datasetID int64
|
||||
fmt.Sscanf(os.Args[3], "%d", &datasetID)
|
||||
|
||||
tmp, err := os.MkdirTemp("", "diffprobe")
|
||||
must(err)
|
||||
defer os.RemoveAll(tmp)
|
||||
|
||||
sqlDB, err := rond.Open(archive, tmp)
|
||||
must(err)
|
||||
b, err := rond.Parse(sqlDB)
|
||||
sqlDB.Close()
|
||||
must(err)
|
||||
|
||||
pg, err := sql.Open("pgx", dsn)
|
||||
must(err)
|
||||
defer pg.Close()
|
||||
ctx := context.Background()
|
||||
must(pg.PingContext(ctx))
|
||||
|
||||
fmt.Printf("=== 备份 %s ===\n", archive)
|
||||
fmt.Printf("备份: 到访 %d / 地点 %d / 位移 %d / 天气 %d / 活动 %d / 标签 %d / 原始定位 %d\n",
|
||||
len(b.Visits), len(b.Locations), len(b.Movements), len(b.Weather),
|
||||
len(b.Activities), len(b.Tags), b.RawVisitNum)
|
||||
var bFirst, bLast sql.NullTime
|
||||
for _, v := range b.Visits {
|
||||
if v.Arrival == nil {
|
||||
continue
|
||||
}
|
||||
if !bFirst.Valid || v.Arrival.Before(bFirst.Time) {
|
||||
bFirst = sql.NullTime{Time: *v.Arrival, Valid: true}
|
||||
}
|
||||
if !bLast.Valid || v.Arrival.After(bLast.Time) {
|
||||
bLast = sql.NullTime{Time: *v.Arrival, Valid: true}
|
||||
}
|
||||
}
|
||||
fmt.Printf("备份时间跨度: %s ~ %s\n", ts(bFirst), ts(bLast))
|
||||
|
||||
fmt.Printf("\n=== 库内 dataset #%d ===\n", datasetID)
|
||||
var name string
|
||||
var cnt, pcnt, raw int
|
||||
var gFirst, gLast sql.NullTime
|
||||
must(pg.QueryRowContext(ctx, `SELECT original_name, visit_count, place_count, raw_visit_count,
|
||||
first_visit_at, last_visit_at FROM datasets WHERE id=$1`, datasetID).
|
||||
Scan(&name, &cnt, &pcnt, &raw, &gFirst, &gLast))
|
||||
fmt.Printf("%s: 到访 %d / 地点 %d / 原始定位 %d,跨度 %s ~ %s\n", name, cnt, pcnt, raw, ts(gFirst), ts(gLast))
|
||||
|
||||
diffVisits(ctx, pg, datasetID, b)
|
||||
diffPlaces(ctx, pg, datasetID, b)
|
||||
diffMovements(ctx, pg, datasetID, b)
|
||||
diffWeather(ctx, pg, datasetID, b)
|
||||
diffActivities(ctx, pg, datasetID, b)
|
||||
diffTags(ctx, pg, datasetID, b)
|
||||
}
|
||||
|
||||
func ts(t sql.NullTime) string {
|
||||
if !t.Valid {
|
||||
return "-"
|
||||
}
|
||||
return t.Time.Local().Format("2006-01-02 15:04")
|
||||
}
|
||||
|
||||
type visitRow struct {
|
||||
pk, placeSrc, actSrc int
|
||||
arrival, departure *time.Time
|
||||
remark, emoji string
|
||||
bookmarked, userAdd bool
|
||||
}
|
||||
|
||||
func diffVisits(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
|
||||
fmt.Printf("\n---------- 到访 ----------\n")
|
||||
rows, err := pg.QueryContext(ctx, `SELECT v.src_pk, COALESCE(p.src_pk,0), COALESCE(a.src_pk,0),
|
||||
v.arrival, v.departure, COALESCE(v.remark,''), COALESCE(v.emoji,''), v.bookmarked, v.is_user_added
|
||||
FROM visits v LEFT JOIN places p ON p.id=v.place_id LEFT JOIN activities a ON a.id=v.activity_id
|
||||
WHERE v.dataset_id=$1`, dsID)
|
||||
must(err)
|
||||
db := map[int]visitRow{}
|
||||
for rows.Next() {
|
||||
var r visitRow
|
||||
var arr time.Time
|
||||
var dep sql.NullTime
|
||||
must(rows.Scan(&r.pk, &r.placeSrc, &r.actSrc, &arr, &dep, &r.remark, &r.emoji, &r.bookmarked, &r.userAdd))
|
||||
r.arrival = &arr
|
||||
if dep.Valid {
|
||||
r.departure = &dep.Time
|
||||
}
|
||||
db[r.pk] = r
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
bk := map[int]rond.Visit{}
|
||||
for _, v := range b.Visits {
|
||||
bk[v.SrcPK] = v
|
||||
}
|
||||
|
||||
var onlyBackup, onlyDB, bothDiff, same []int
|
||||
for pk, v := range bk {
|
||||
r, ok := db[pk]
|
||||
if !ok {
|
||||
onlyBackup = append(onlyBackup, pk)
|
||||
continue
|
||||
}
|
||||
if vdiff(v, r) != "" {
|
||||
bothDiff = append(bothDiff, pk)
|
||||
continue
|
||||
}
|
||||
same = append(same, pk)
|
||||
}
|
||||
for pk := range db {
|
||||
if _, ok := bk[pk]; !ok {
|
||||
onlyDB = append(onlyDB, pk)
|
||||
}
|
||||
}
|
||||
sort.Ints(onlyBackup)
|
||||
sort.Ints(onlyDB)
|
||||
sort.Ints(bothDiff)
|
||||
|
||||
fmt.Printf("同号且一致: %d\n", len(same))
|
||||
fmt.Printf("仅备份有(实机新增,库里没有): %d\n", len(onlyBackup))
|
||||
fmt.Printf("仅库里有(站点侧新增/实机已删除): %d\n", len(onlyDB))
|
||||
fmt.Printf("同号但内容不同(冲突): %d\n", len(bothDiff))
|
||||
|
||||
// 按到达时间做多重集匹配:判断「同号冲突」到底是同批数据换了主键,还是两批无关数据
|
||||
byTime := map[string][]int{}
|
||||
for pk, r := range db {
|
||||
if r.arrival != nil {
|
||||
byTime[r.arrival.UTC().Format(time.RFC3339)] = append(byTime[r.arrival.UTC().Format(time.RFC3339)], pk)
|
||||
}
|
||||
}
|
||||
matched, miss := 0, 0
|
||||
pair := [][2]int{}
|
||||
for pk, v := range bk {
|
||||
if v.Arrival == nil {
|
||||
continue
|
||||
}
|
||||
k := v.Arrival.UTC().Format(time.RFC3339)
|
||||
if list := byTime[k]; len(list) > 0 {
|
||||
matched++
|
||||
pair = append(pair, [2]int{pk, list[0]})
|
||||
byTime[k] = list[1:]
|
||||
} else {
|
||||
miss++
|
||||
}
|
||||
}
|
||||
fmt.Printf("\n[时间对齐] 备份到访按「到达时刻」能在库内找到同刻记录: %d 条,找不到: %d 条\n", matched, miss)
|
||||
sort.Slice(pair, func(i, j int) bool { return pair[i][0] < pair[j][0] })
|
||||
fmt.Printf("[时间对齐] 同刻对的 (备份pk→库pk) 前 12 组: ")
|
||||
for i, p := range pair {
|
||||
if i >= 12 {
|
||||
break
|
||||
}
|
||||
fmt.Printf("%d→%d ", p[0], p[1])
|
||||
}
|
||||
fmt.Println()
|
||||
nDiff := 0
|
||||
for _, p := range pair {
|
||||
if p[0] != p[1] {
|
||||
nDiff++
|
||||
}
|
||||
}
|
||||
fmt.Printf("[时间对齐] 同刻但主键不同: %d / %d\n", nDiff, len(pair))
|
||||
|
||||
show := func(label string, pks []int, f func(pk int) string) {
|
||||
if len(pks) == 0 {
|
||||
return
|
||||
}
|
||||
fmt.Printf("\n[%s] 共 %d 条,前 8 条:\n", label, len(pks))
|
||||
for i, pk := range pks {
|
||||
if i >= 8 {
|
||||
break
|
||||
}
|
||||
fmt.Printf(" #%d %s\n", pk, f(pk))
|
||||
}
|
||||
}
|
||||
show("仅备份有", onlyBackup, func(pk int) string {
|
||||
v := bk[pk]
|
||||
return fmt.Sprintf("到访 %s 离开 %s 地点src=%s", ft(v.Arrival), ft(v.Departure), ps(v.LocationSrcPK))
|
||||
})
|
||||
show("仅库里有", onlyDB, func(pk int) string {
|
||||
r := db[pk]
|
||||
return fmt.Sprintf("到访 %s 离开 %s 地点src=%d 站点新增=%v 备注=%q",
|
||||
ft(r.arrival), ftp(r.departure), r.placeSrc, r.userAdd, r.remark)
|
||||
})
|
||||
show("冲突", bothDiff, func(pk int) string {
|
||||
return vdiff(bk[pk], db[pk])
|
||||
})
|
||||
}
|
||||
|
||||
func vdiff(v rond.Visit, r visitRow) string {
|
||||
var d string
|
||||
if !eq(v.Arrival, r.arrival) {
|
||||
d += fmt.Sprintf("到达 %s→%s ", ft(v.Arrival), ft(r.arrival))
|
||||
}
|
||||
if !eq(v.Departure, r.departure) {
|
||||
d += fmt.Sprintf("离开 %s→%s ", ft(v.Departure), ft(r.departure))
|
||||
}
|
||||
if pv(v.LocationSrcPK) != r.placeSrc {
|
||||
d += fmt.Sprintf("地点 %s→%d ", ps(v.LocationSrcPK), r.placeSrc)
|
||||
}
|
||||
if av(v.ActivitySrcPK) != r.actSrc {
|
||||
d += fmt.Sprintf("活动 %s→%d ", ps(v.ActivitySrcPK), r.actSrc)
|
||||
}
|
||||
if v.Remark != r.remark {
|
||||
d += fmt.Sprintf("备注 %q→%q ", v.Remark, r.remark)
|
||||
}
|
||||
if v.Emoji != r.emoji {
|
||||
d += fmt.Sprintf("表情 %q→%q ", v.Emoji, r.emoji)
|
||||
}
|
||||
if v.Bookmarked != r.bookmarked {
|
||||
d += fmt.Sprintf("收藏 %v→%v ", v.Bookmarked, r.bookmarked)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func diffPlaces(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
|
||||
fmt.Printf("\n---------- 地点 ----------\n")
|
||||
type prow struct {
|
||||
name, poi, cc, prov, city, dist, sub, thor, tz string
|
||||
lat, lon float64
|
||||
}
|
||||
rows, err := pg.QueryContext(ctx, `SELECT src_pk, COALESCE(name,''), COALESCE(poi_category,''),
|
||||
COALESCE(country_code,''), COALESCE(province,''), COALESCE(city,''), COALESCE(district,''),
|
||||
COALESCE(sublocality,''), COALESCE(thoroughfare,''), COALESCE(timezone,''),
|
||||
COALESCE(lat,0), COALESCE(lon,0) FROM places WHERE dataset_id=$1`, dsID)
|
||||
must(err)
|
||||
db := map[int]prow{}
|
||||
for rows.Next() {
|
||||
var pk int
|
||||
var r prow
|
||||
must(rows.Scan(&pk, &r.name, &r.poi, &r.cc, &r.prov, &r.city, &r.dist, &r.sub, &r.thor, &r.tz, &r.lat, &r.lon))
|
||||
db[pk] = r
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
bk := map[int]rond.Location{}
|
||||
for _, l := range b.Locations {
|
||||
bk[l.SrcPK] = l
|
||||
}
|
||||
var onlyB, onlyD, diff []int
|
||||
for pk, l := range bk {
|
||||
r, ok := db[pk]
|
||||
if !ok {
|
||||
onlyB = append(onlyB, pk)
|
||||
continue
|
||||
}
|
||||
if l.Name != r.name || l.POICategory != r.poi || l.Province != r.prov || l.City != r.city ||
|
||||
l.District != r.dist || l.Sublocality != r.sub || l.Thoroughfare != r.thor || l.Timezone != r.tz ||
|
||||
l.CountryCode != r.cc || !near(l.Lat, r.lat) || !near(l.Lon, r.lon) {
|
||||
diff = append(diff, pk)
|
||||
}
|
||||
}
|
||||
for pk := range db {
|
||||
if _, ok := bk[pk]; !ok {
|
||||
onlyD = append(onlyD, pk)
|
||||
}
|
||||
}
|
||||
sort.Ints(onlyB)
|
||||
sort.Ints(onlyD)
|
||||
sort.Ints(diff)
|
||||
fmt.Printf("仅备份有: %d,仅库里有: %d,同号内容不同: %d\n", len(onlyB), len(onlyD), len(diff))
|
||||
for i, pk := range onlyB {
|
||||
if i >= 6 {
|
||||
break
|
||||
}
|
||||
fmt.Printf(" 仅备份 #%d %s (%s %s %s)\n", pk, bk[pk].Name, bk[pk].Province, bk[pk].City, bk[pk].POICategory)
|
||||
}
|
||||
for i, pk := range onlyD {
|
||||
if i >= 6 {
|
||||
break
|
||||
}
|
||||
fmt.Printf(" 仅库内 #%d %s (%s %s)\n", pk, db[pk].name, db[pk].prov, db[pk].city)
|
||||
}
|
||||
for i, pk := range diff {
|
||||
if i >= 6 {
|
||||
break
|
||||
}
|
||||
l, r := bk[pk], db[pk]
|
||||
fmt.Printf(" 冲突 #%d 备份[name=%q city=%q %.6f,%.6f] 库[name=%q city=%q %.6f,%.6f]\n",
|
||||
pk, l.Name, l.City, l.Lat, l.Lon, r.name, r.city, r.lat, r.lon)
|
||||
}
|
||||
}
|
||||
|
||||
func diffMovements(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
|
||||
fmt.Printf("\n---------- 位移 ----------\n")
|
||||
rows, err := pg.QueryContext(ctx, `SELECT src_pk, COALESCE(transport_src,0),
|
||||
COALESCE(from_visit_src,0), COALESCE(to_visit_src,0), started_at, ended_at
|
||||
FROM movements WHERE dataset_id=$1`, dsID)
|
||||
must(err)
|
||||
type mrow struct {
|
||||
t, f, to int
|
||||
s, e *time.Time
|
||||
}
|
||||
db := map[int]mrow{}
|
||||
for rows.Next() {
|
||||
var pk int
|
||||
var r mrow
|
||||
var s, e sql.NullTime
|
||||
must(rows.Scan(&pk, &r.t, &r.f, &r.to, &s, &e))
|
||||
if s.Valid {
|
||||
r.s = &s.Time
|
||||
}
|
||||
if e.Valid {
|
||||
r.e = &e.Time
|
||||
}
|
||||
db[pk] = r
|
||||
}
|
||||
rows.Close()
|
||||
var onlyB, onlyD int
|
||||
for _, m := range b.Movements {
|
||||
if _, ok := db[m.SrcPK]; !ok {
|
||||
onlyB++
|
||||
}
|
||||
}
|
||||
for pk := range db {
|
||||
found := false
|
||||
for _, m := range b.Movements {
|
||||
if m.SrcPK == pk {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
onlyD++
|
||||
}
|
||||
}
|
||||
fmt.Printf("备份 %d 条 / 库内 %d 条;仅备份有 %d,仅库里有 %d\n", len(b.Movements), len(db), onlyB, onlyD)
|
||||
}
|
||||
|
||||
func diffWeather(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
|
||||
var n int
|
||||
must(pg.QueryRowContext(ctx, `SELECT count(*) FROM weather WHERE dataset_id=$1`, dsID).Scan(&n))
|
||||
fmt.Printf("\n---------- 天气 ----------\n备份 %d 条 / 库内 %d 条\n", len(b.Weather), n)
|
||||
}
|
||||
|
||||
func diffActivities(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
|
||||
fmt.Printf("\n---------- 活动类型 ----------\n")
|
||||
rows, err := pg.QueryContext(ctx, `SELECT src_pk, name FROM activities WHERE dataset_id=$1`, dsID)
|
||||
must(err)
|
||||
db := map[int]string{}
|
||||
for rows.Next() {
|
||||
var pk int
|
||||
var n string
|
||||
must(rows.Scan(&pk, &n))
|
||||
db[pk] = n
|
||||
}
|
||||
rows.Close()
|
||||
for _, a := range b.Activities {
|
||||
n, ok := db[a.SrcPK]
|
||||
if !ok {
|
||||
fmt.Printf(" 仅备份 #%d %q\n", a.SrcPK, a.Name)
|
||||
} else if n != a.Name {
|
||||
fmt.Printf(" 冲突 #%d 备份=%q 库=%q\n", a.SrcPK, a.Name, n)
|
||||
}
|
||||
}
|
||||
for pk, n := range db {
|
||||
found := false
|
||||
for _, a := range b.Activities {
|
||||
if a.SrcPK == pk {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
fmt.Printf(" 仅库内 #%d %q\n", pk, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func diffTags(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
|
||||
var n int
|
||||
must(pg.QueryRowContext(ctx, `SELECT count(*) FROM tags WHERE dataset_id=$1`, dsID).Scan(&n))
|
||||
fmt.Printf("\n---------- 标签 ----------\n备份 %d 条 / 库内 %d 条\n", len(b.Tags), n)
|
||||
}
|
||||
|
||||
func eq(a, b *time.Time) bool {
|
||||
if a == nil || b == nil {
|
||||
return a == nil && b == nil
|
||||
}
|
||||
return a.Equal(*b)
|
||||
}
|
||||
|
||||
func pv(p *int) int {
|
||||
if p == nil {
|
||||
return 0
|
||||
}
|
||||
return *p
|
||||
}
|
||||
|
||||
func av(p *int) int { return pv(p) }
|
||||
|
||||
func ps(p *int) string {
|
||||
if p == nil {
|
||||
return "-"
|
||||
}
|
||||
return fmt.Sprint(*p)
|
||||
}
|
||||
|
||||
func ft(t *time.Time) string {
|
||||
if t == nil {
|
||||
return "-"
|
||||
}
|
||||
return t.Local().Format("01-02 15:04")
|
||||
}
|
||||
|
||||
func ftp(t *time.Time) string { return ft(t) }
|
||||
|
||||
func near(a, b float64) bool {
|
||||
d := a - b
|
||||
return d < 1e-9 && d > -1e-9
|
||||
}
|
||||
|
||||
func must(err error) {
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "错误:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
// 一次性探针:调用站点的导出逻辑产出一份 .rondbackup,用于核对
|
||||
// 「站点导出 → 导入 rond → rond 再导出」这条链路上 UUID 与 Z_PK 是否还连贯。
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
_ "github.com/jackc/pgx/v5/stdlib"
|
||||
|
||||
"rond-with-you/internal/rond"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 5 {
|
||||
fmt.Println("用法: exportprobe <dsn> <datasetID> <baseline.sqlite> <out.rondbackup>")
|
||||
os.Exit(2)
|
||||
}
|
||||
dsID, err := strconv.ParseInt(os.Args[2], 10, 64)
|
||||
must(err)
|
||||
db, err := sql.Open("pgx", os.Args[1])
|
||||
must(err)
|
||||
defer db.Close()
|
||||
|
||||
f, err := os.Create(os.Args[4])
|
||||
must(err)
|
||||
defer f.Close()
|
||||
must(rond.ExportRondbackup(context.Background(), db, dsID, os.Args[3], f))
|
||||
fmt.Printf("已导出到 %s\n", os.Args[4])
|
||||
}
|
||||
|
||||
func must(err error) {
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "错误:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
// 一次性探针:把一份 .fwss 与 PG 里的 fog_blocks 逐块比对,量化「叠加 / 替换」两种模式的影响。
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
_ "github.com/jackc/pgx/v5/stdlib"
|
||||
|
||||
"rond-with-you/internal/fog"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 3 {
|
||||
fmt.Println("用法: fogprobe <snapshot.fwss> <dsn> [userID]")
|
||||
os.Exit(2)
|
||||
}
|
||||
uid := int64(1)
|
||||
if len(os.Args) > 3 {
|
||||
uid, _ = strconv.ParseInt(os.Args[3], 10, 64)
|
||||
}
|
||||
data, err := os.ReadFile(os.Args[1])
|
||||
must(err)
|
||||
inc, incCells, err := fog.Decode(data)
|
||||
must(err)
|
||||
fmt.Printf("快照 %s: %d 块 / %d 格\n", os.Args[1], len(inc), incCells)
|
||||
|
||||
pg, err := sql.Open("pgx", os.Args[2])
|
||||
must(err)
|
||||
defer pg.Close()
|
||||
rows, err := pg.Query(`SELECT gbx, gby, bitmap FROM fog_blocks WHERE user_id=$1`, uid)
|
||||
must(err)
|
||||
local := map[[2]int][]byte{}
|
||||
localCells := 0
|
||||
for rows.Next() {
|
||||
var gx, gy int
|
||||
var bm []byte
|
||||
must(rows.Scan(&gx, &gy, &bm))
|
||||
local[[2]int{gx, gy}] = bm
|
||||
localCells += popcount(bm)
|
||||
}
|
||||
rows.Close()
|
||||
fmt.Printf("库内: %d 块 / %d 格\n", len(local), localCells)
|
||||
|
||||
var newBlocks, newCells int
|
||||
var novBlock, overlap int
|
||||
var overlapGain int
|
||||
for _, b := range inc {
|
||||
k := [2]int{b.GX, b.GY}
|
||||
old, ok := local[k]
|
||||
if !ok {
|
||||
newBlocks++
|
||||
newCells += popcount(b.Bitmap)
|
||||
continue
|
||||
}
|
||||
novBlock++
|
||||
gain := 0
|
||||
for i := range b.Bitmap {
|
||||
gain += poptab[b.Bitmap[i]&^old[i]]
|
||||
}
|
||||
if gain > 0 {
|
||||
overlap++
|
||||
overlapGain += gain
|
||||
}
|
||||
}
|
||||
var lostBlocks, lostCells, lostFull int
|
||||
for k, bm := range local {
|
||||
found := false
|
||||
for _, b := range inc {
|
||||
if b.GX == k[0] && b.GY == k[1] {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
lostBlocks++
|
||||
lostCells += popcount(bm)
|
||||
lostFull++
|
||||
}
|
||||
}
|
||||
fmt.Printf("\n【叠加合并 overlay】\n")
|
||||
fmt.Printf(" 新增整块: %d 块(+%d 格)\n", newBlocks, newCells)
|
||||
fmt.Printf(" 与已有块重叠: %d 块,其中 %d 块含新格(+%d 格)\n", novBlock, overlap, overlapGain)
|
||||
fmt.Printf(" 合计新增格: %d,合并后总量: %d 块 / %d 格\n",
|
||||
newCells+overlapGain, len(local)+newBlocks, localCells+newCells+overlapGain)
|
||||
|
||||
fmt.Printf("\n【整体替换 cover】\n")
|
||||
fmt.Printf(" 库里现有 %d 块会被丢弃(快照里完全没有这些块),损失 %d 格\n", lostFull, lostCells)
|
||||
fmt.Printf(" 替换后总量: %d 块 / %d 格\n", len(inc), incCells)
|
||||
fmt.Printf(" 净变化: %+d 块 / %+d 格\n", len(inc)-len(local), incCells-localCells)
|
||||
|
||||
fmt.Printf("\n【其它图层】\n")
|
||||
layers, err := fog.RawLayers(data)
|
||||
if err == nil {
|
||||
for k, v := range layers {
|
||||
fmt.Printf(" %s (%d 字节, %s)\n", k, len(v), hex.EncodeToString(v[:min(8, len(v))]))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
var poptab = func() [256]int {
|
||||
var t [256]int
|
||||
for i := range t {
|
||||
t[i] = strings.Count(strconv.FormatUint(uint64(i), 2), "1")
|
||||
}
|
||||
return t
|
||||
}()
|
||||
|
||||
func popcount(b []byte) int {
|
||||
n := 0
|
||||
for _, v := range b {
|
||||
n += poptab[v]
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func must(err error) {
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "错误:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
// 一次性探针:检查 rond 备份里是否存在「跨备份稳定」的对齐键。
|
||||
// 用完即可删。
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"rond-with-you/internal/rond"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 3 {
|
||||
fmt.Println("用法: idprobe <a.rondbackup> <b.rondbackup>")
|
||||
os.Exit(2)
|
||||
}
|
||||
a := load(os.Args[1])
|
||||
b := load(os.Args[2])
|
||||
fmt.Printf("\n=== A=%s ===\n=== B=%s ===\n", os.Args[1], os.Args[2])
|
||||
|
||||
fmt.Printf("\n---------- ZVISIT.ZIDENTIFIER_ 稳定性 ----------\n")
|
||||
fmt.Printf("A 有 UUID %d / %d,B 有 UUID %d / %d\n", len(a.visitUUID), len(a.visitPK), len(b.visitUUID), len(b.visitPK))
|
||||
inter := 0
|
||||
for u := range a.visitUUID {
|
||||
if _, ok := b.visitUUID[u]; ok {
|
||||
inter++
|
||||
}
|
||||
}
|
||||
fmt.Printf("两次备份 UUID 交集: %d(A 独有 %d,B 独有 %d)\n",
|
||||
inter, len(a.visitUUID)-inter, len(b.visitUUID)-inter)
|
||||
// UUID 相同则对比 pk 是否相同
|
||||
samePK, diffPK := 0, 0
|
||||
for u, pa := range a.visitUUID {
|
||||
if pb, ok := b.visitUUID[u]; ok {
|
||||
if pa == pb {
|
||||
samePK++
|
||||
} else {
|
||||
diffPK++
|
||||
}
|
||||
}
|
||||
}
|
||||
fmt.Printf("UUID 相同者中:Z_PK 也相同 %d,Z_PK 已变 %d\n", samePK, diffPK)
|
||||
|
||||
fmt.Printf("\n---------- ZACTIVITY.ZUID_ ----------\n")
|
||||
fmt.Printf("A UUID %d / %d,B UUID %d / %d\n", len(a.actUUID), len(a.actPK), len(b.actUUID), len(b.actPK))
|
||||
same, dif := 0, 0
|
||||
for u, pa := range a.actUUID {
|
||||
if pb, ok := b.actUUID[u]; ok {
|
||||
if pa == pb {
|
||||
same++
|
||||
} else {
|
||||
dif++
|
||||
}
|
||||
}
|
||||
}
|
||||
fmt.Printf("交集配对:Z_PK 相同 %d,Z_PK 已变 %d\n", same, dif)
|
||||
|
||||
fmt.Printf("\n---------- ZLOCATION 可用稳定列 ----------\n")
|
||||
fmt.Printf("A 列: %s\n", strings.Join(a.locCols, ", "))
|
||||
fmt.Printf("A 地点 %d,B 地点 %d\n", len(a.locPK), len(b.locPK))
|
||||
// 名称+粗坐标是否可作键
|
||||
type lk struct {
|
||||
name string
|
||||
lat float64
|
||||
lon float64
|
||||
}
|
||||
keyA := map[string][]int{}
|
||||
for pk, l := range a.locPK {
|
||||
k := fmt.Sprintf("%s|%.5f|%.5f", l.name, l.lat, l.lon)
|
||||
keyA[k] = append(keyA[k], pk)
|
||||
}
|
||||
hit, miss := 0, 0
|
||||
for pk, l := range b.locPK {
|
||||
k := fmt.Sprintf("%s|%.5f|%.5f", l.name, l.lat, l.lon)
|
||||
if _, ok := keyA[k]; ok {
|
||||
hit++
|
||||
} else {
|
||||
miss++
|
||||
if miss <= 5 {
|
||||
fmt.Printf(" B 独有地点 #%d %q %.6f,%.6f\n", pk, l.name, l.lat, l.lon)
|
||||
}
|
||||
}
|
||||
}
|
||||
fmt.Printf("按「名称+坐标(1e-5)」键:B 命中 A %d,未命中 %d\n", hit, miss)
|
||||
}
|
||||
|
||||
type loc struct {
|
||||
name string
|
||||
lat, lon float64
|
||||
}
|
||||
|
||||
type snap struct {
|
||||
visitUUID map[string]int
|
||||
visitPK map[int]bool
|
||||
actUUID map[string]int
|
||||
actPK map[int]bool
|
||||
locPK map[int]loc
|
||||
locCols []string
|
||||
}
|
||||
|
||||
func load(path string) *snap {
|
||||
dir, err := os.MkdirTemp("", "idprobe")
|
||||
must(err)
|
||||
defer os.RemoveAll(dir)
|
||||
db, err := rond.Open(path, dir)
|
||||
must(err)
|
||||
defer db.Close()
|
||||
|
||||
s := &snap{visitUUID: map[string]int{}, visitPK: map[int]bool{},
|
||||
actUUID: map[string]int{}, actPK: map[int]bool{}, locPK: map[int]loc{}}
|
||||
|
||||
rows, err := db.Query(`SELECT Z_PK, hex(ZIDENTIFIER_) FROM ZVISIT`)
|
||||
must(err)
|
||||
for rows.Next() {
|
||||
var pk int
|
||||
var u sql.NullString
|
||||
must(rows.Scan(&pk, &u))
|
||||
s.visitPK[pk] = true
|
||||
if u.Valid && u.String != "" && strings.Trim(u.String, "0") != "" {
|
||||
s.visitUUID[u.String] = pk
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
rows, err = db.Query(`SELECT Z_PK, hex(ZUID_), COALESCE(ZNAME_,'') FROM ZACTIVITY`)
|
||||
must(err)
|
||||
for rows.Next() {
|
||||
var pk int
|
||||
var u sql.NullString
|
||||
var n string
|
||||
must(rows.Scan(&pk, &u, &n))
|
||||
s.actPK[pk] = true
|
||||
if u.Valid && u.String != "" && strings.Trim(u.String, "0") != "" {
|
||||
s.actUUID[u.String] = pk
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
rows, err = db.Query(`PRAGMA table_info(ZLOCATION)`)
|
||||
must(err)
|
||||
for rows.Next() {
|
||||
var cid int
|
||||
var name, typ string
|
||||
var notnull int
|
||||
var dflt sql.NullString
|
||||
var pk int
|
||||
must(rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk))
|
||||
s.locCols = append(s.locCols, name)
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
rows, err = db.Query(`SELECT Z_PK, COALESCE(ZNAME_,''), COALESCE(ZLATITUDE,0), COALESCE(ZLONGITUDE,0) FROM ZLOCATION`)
|
||||
must(err)
|
||||
for rows.Next() {
|
||||
var pk int
|
||||
var l loc
|
||||
must(rows.Scan(&pk, &l.name, &l.lat, &l.lon))
|
||||
s.locPK[pk] = l
|
||||
}
|
||||
rows.Close()
|
||||
return s
|
||||
}
|
||||
|
||||
func must(err error) {
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "错误:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
// 一次性探针:在 rond 备份的 SQLite 上跑一条查询并打印结果(BLOB 打十六进制)。
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"rond-with-you/internal/rond"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 3 {
|
||||
fmt.Println(`用法: sqlprobe <backup> "SELECT ..."`)
|
||||
os.Exit(2)
|
||||
}
|
||||
dir, err := os.MkdirTemp("", "sqlprobe")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
db, err := rond.Open(os.Args[1], dir)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
rows, err := db.Query(os.Args[2])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
cols, _ := rows.Columns()
|
||||
fmt.Println(strings.Join(cols, " | "))
|
||||
for rows.Next() {
|
||||
vals := make([]any, len(cols))
|
||||
ptrs := make([]any, len(cols))
|
||||
for i := range vals {
|
||||
ptrs[i] = &vals[i]
|
||||
}
|
||||
if err := rows.Scan(ptrs...); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
parts := make([]string, len(cols))
|
||||
for i, v := range vals {
|
||||
switch t := v.(type) {
|
||||
case nil:
|
||||
parts[i] = "-"
|
||||
case []byte:
|
||||
parts[i] = fmt.Sprintf("%x", t)
|
||||
default:
|
||||
parts[i] = fmt.Sprint(t)
|
||||
}
|
||||
}
|
||||
fmt.Println(strings.Join(parts, " | "))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user