备份与迷雾的差异分析、合并执行,附属数据改挂坐标键

备份(.rondbackup)与迷雾(.fwss)各加一层「先看差异再动手」:

- 迷雾:上传先出差异页,说清叠加能补多少、整体替换会丢多少,并在真实底图上
  用绿/灰/红标出会新增 / 不变 / 会丢的区域
- 备份:与当前数据集按 UUID 逐条比对(rond 每次导出都会重排 Z_PK,编号认不出人),
  新增合入、冲突逐条选、仅站内有逐条勾选删除,全在一个事务里执行;
  合入的行写回 entity_raw,保证再传同一份备份时差异归零
- 地点按坐标对齐:站点的「补全地址」会把地点改名,拿名称当键会让一整个地点的
  到访全变成假冲突
- 附属数据(备注/结论/访客票/图片)从 dataset_id+src_pk 换成坐标键,换数据集不再失联;
  访客票没有 dataset_id,回填按活跃数据集认领
- 修正 contentHash 拿 Z_PK 当指纹的隐患,以及两处「src_pk 跨快照稳定」的错误注释

导出格式(.rondbackup / .fwss)保持原样,往返测试通过。
This commit is contained in:
tamakyi
2026-09-19 16:04:00 +08:00
parent 9e01923244
commit 30c502ce7b
43 changed files with 5968 additions and 170 deletions
+150 -13
View File
@@ -171,38 +171,50 @@ CREATE TABLE IF NOT EXISTS fog_blocks (
PRIMARY KEY (user_id, gbx, gby)
);
-- 地点备注与别名。按 src_pk(rond 内部地点 ID,跨快照稳定)而不是 places.id 存,
-- 这样重新上传备份、数据集换了新 id 之后备注依然挂着。
-- dataset_id 是后加的:src_pk 只在单个数据集内唯一,缺了它切数据集时同编号的
-- 地点会把别人的备注/别名认领过去。
-- 地点备注与别名。挂**坐标键**(place_key)而不是 src_pk:
--
-- src_pk(rond 的 Z_PK)只在单个数据集内有效——rond 每次导出/同步都会重排编号
-- (实测两份备份里 822 条到访的 UUID 完全一致、Z_PK 100% 不同),所以换一份备份、
-- 换个数据集之后,同编号的完全是另一个地点,备注就再也挂不回去了。
-- 坐标是地点的地理事实,跨备份稳定:place_key = 经纬度各取 6 位小数拼起来
-- (约 0.1m 精度),与 Go 侧的 rond.PlaceCoordKey 必须逐位一致。
-- dataset_id / src_pk 保留下来只作「最后一次写入时挂在哪儿」的排查线索。
CREATE TABLE IF NOT EXISTS place_notes (
user_id BIGINT NOT NULL,
place_key TEXT NOT NULL,
dataset_id BIGINT NOT NULL DEFAULT 0,
src_pk INT NOT NULL,
src_pk INT NOT NULL DEFAULT 0,
alias TEXT NOT NULL DEFAULT '',
note TEXT NOT NULL DEFAULT '',
rond_note TEXT NOT NULL DEFAULT '',
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, dataset_id, src_pk)
PRIMARY KEY (user_id, place_key)
);
-- 站长对地点的「官方结论」:推荐(1) / 踩雷(-1),与自由文本备注分开存,同样按 src_pk。
-- 站长对地点的「官方结论」:推荐(1) / 踩雷(-1),与备注一样按坐标键存。
CREATE TABLE IF NOT EXISTS place_votes (
user_id BIGINT NOT NULL,
place_key TEXT NOT NULL,
dataset_id BIGINT NOT NULL DEFAULT 0,
src_pk INT NOT NULL,
src_pk INT NOT NULL DEFAULT 0,
verdict SMALLINT NOT NULL,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, dataset_id, src_pk)
PRIMARY KEY (user_id, place_key)
);
-- 访客对地点的投票:一人一票、可改可撤。voter_key 由匿名 cookie + IP 派生,
-- 与数据集无关(换快照不丢),故不挂 dataset_id。
-- 访客对地点的投票:一人一票、可改可撤。voter_key 由匿名 cookie + IP 派生。
--
-- 这是站点自己的数据(rondbackup 里没有这一项),但同样要认得出「是哪个地点」:
-- 挂 place_key 而不是 src_pk,理由与备注、结论一样。src_pk 只留作排查线索。
-- 迁移时这张表没有 dataset_id 可依,只能按**活跃数据集**认领那个编号属于哪个地点
-- (见文件末尾的迁移块)。
CREATE TABLE IF NOT EXISTS place_visitor_votes (
src_pk INT NOT NULL,
place_key TEXT NOT NULL,
voter_key TEXT NOT NULL,
vote SMALLINT NOT NULL,
src_pk INT NOT NULL DEFAULT 0,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (src_pk, voter_key)
PRIMARY KEY (place_key, voter_key)
);
CREATE INDEX IF NOT EXISTS idx_pvv_src ON place_visitor_votes(src_pk);
@@ -309,9 +321,17 @@ UPDATE place_notes SET dataset_id = COALESCE((SELECT id FROM datasets WHERE is_a
UPDATE place_votes SET dataset_id = COALESCE((SELECT id FROM datasets WHERE is_active ORDER BY id DESC LIMIT 1), 0)
WHERE dataset_id = 0;
-- ⚠️ 这一段只在「还没搬到坐标键」的库上跑:文件末尾那次迁移会把主键换成
-- (user_id, place_key),这里要是也跟着改回去,两张表就会在每次启动时来回翻,
-- 永远停在最后一个执行的块上。
DO $$
DECLARE cols text;
BEGIN
IF EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_name = 'place_notes' AND column_name = 'place_key') THEN
RETURN;
END IF;
SELECT string_agg(a.attname, ',' ORDER BY array_position(c.conkey, a.attnum))
INTO cols
FROM pg_constraint c
@@ -395,6 +415,7 @@ ALTER TABLE places ADD COLUMN IF NOT EXISTS src_lon DOUBLE PRECISION;
-- SQL 里写不出来。启动时由 ingest.BackfillAllCoordSource 统一处理,那里同时能识别出
-- 「站点导出的备份给新建地点合成的 ZLOCATION 行」,不会把它误当成真机的原始 GPS。
-- 地点图片:站长给地点配的图。字节由**浏览器**转成 WebP 后再上传(服务端只负责收下与转存),
-- 1 核机器不用扛解码。storage 记这一张实际放在哪、object_key 是本地相对路径或对象存储 key:
-- 换了存储后端之后旧图仍按各自那一列去取,不会因为改了设置就全变破图。
@@ -402,6 +423,9 @@ CREATE TABLE IF NOT EXISTS place_images (
id BIGSERIAL PRIMARY KEY,
dataset_id BIGINT NOT NULL REFERENCES datasets(id) ON DELETE CASCADE,
src_pk INT NOT NULL,
-- place_key 与备注、结论共用同一套坐标键:换数据集后图片还挂得回原地点。
-- 对象键里仍然带 dataset_id(那只是存储路径,与「认不认得出是同一个地点」无关)。
place_key TEXT NOT NULL DEFAULT '',
storage TEXT NOT NULL DEFAULT 'local',
object_key TEXT NOT NULL,
width INT NOT NULL DEFAULT 0,
@@ -412,3 +436,116 @@ CREATE TABLE IF NOT EXISTS place_images (
UNIQUE (dataset_id, src_pk, object_key)
);
CREATE INDEX IF NOT EXISTS idx_place_images_place ON place_images(dataset_id, src_pk, position);
-- ---------- 附属数据改挂坐标键(一次性迁移)----------
-- 备注 / 结论 / 图片原本挂 (dataset_id, src_pk),换个数据集就全看不见了。
-- 这里把它们搬到坐标键上;只在「还没有 place_key 列」时执行,所以新库直接走上面的
-- CREATE TABLE,老库走这一段。
--
-- 坐标键的算法必须与 Go 侧 rond.PlaceCoordKey 逐位一致:
-- fmt.Sprintf("%.6f|%.6f", lat, lon) ↔ to_char(lat,'FM999990.000000')||'|'||to_char(lon,'FM999990.000000')
-- 已在真实数据上逐条比对过(368/368 相同)。改任何一边都要重新比。
DO $$
DECLARE
cols text;
BEGIN
-- 判据是**主键**而不是「列在不在」:半迁移过的库(列补上了、主键还是老的三列)
-- 也得能修好,否则这张表会一直卡在中间状态。
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_name = 'place_notes' AND column_name = 'place_key') THEN
ALTER TABLE place_notes ADD COLUMN place_key TEXT;
END IF;
SELECT string_agg(a.attname, ',' ORDER BY array_position(c.conkey, a.attnum))
INTO cols FROM pg_constraint c
JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = ANY(c.conkey)
WHERE c.conname = 'place_notes_pkey';
IF cols IS DISTINCT FROM 'user_id,place_key' THEN
UPDATE place_notes n SET place_key = (
SELECT to_char(p.lat, 'FM999990.000000') || '|' || to_char(p.lon, 'FM999990.000000')
FROM places p
WHERE p.dataset_id = n.dataset_id AND p.src_pk = n.src_pk
AND p.lat IS NOT NULL AND p.lon IS NOT NULL)
WHERE n.place_key IS NULL;
-- 地点已经不在库里的(被删过),留一个不会撞的孤儿键,宁可留着也别丢
UPDATE place_notes SET place_key = 'orphan:' || dataset_id || ':' || src_pk
WHERE place_key IS NULL;
-- 同一坐标上有多条时只留最新的一条,否则加不上主键
DELETE FROM place_notes a USING place_notes b
WHERE a.user_id = b.user_id AND a.place_key = b.place_key
AND (a.updated_at, a.ctid) < (b.updated_at, b.ctid);
ALTER TABLE place_notes ALTER COLUMN place_key SET NOT NULL;
ALTER TABLE place_notes DROP CONSTRAINT IF EXISTS place_notes_pkey;
ALTER TABLE place_notes ADD PRIMARY KEY (user_id, place_key);
END IF;
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_name = 'place_votes' AND column_name = 'place_key') THEN
ALTER TABLE place_votes ADD COLUMN place_key TEXT;
END IF;
SELECT string_agg(a.attname, ',' ORDER BY array_position(c.conkey, a.attnum))
INTO cols FROM pg_constraint c
JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = ANY(c.conkey)
WHERE c.conname = 'place_votes_pkey';
IF cols IS DISTINCT FROM 'user_id,place_key' THEN
UPDATE place_votes n SET place_key = (
SELECT to_char(p.lat, 'FM999990.000000') || '|' || to_char(p.lon, 'FM999990.000000')
FROM places p
WHERE p.dataset_id = n.dataset_id AND p.src_pk = n.src_pk
AND p.lat IS NOT NULL AND p.lon IS NOT NULL)
WHERE n.place_key IS NULL;
UPDATE place_votes SET place_key = 'orphan:' || dataset_id || ':' || src_pk
WHERE place_key IS NULL;
DELETE FROM place_votes a USING place_votes b
WHERE a.user_id = b.user_id AND a.place_key = b.place_key
AND (a.updated_at, a.ctid) < (b.updated_at, b.ctid);
ALTER TABLE place_votes ALTER COLUMN place_key SET NOT NULL;
ALTER TABLE place_votes DROP CONSTRAINT IF EXISTS place_votes_pkey;
ALTER TABLE place_votes ADD PRIMARY KEY (user_id, place_key);
END IF;
-- 图片自己有 id 主键,只需补列 + 回填;查询走文件末尾那个索引
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_name = 'place_images' AND column_name = 'place_key') THEN
ALTER TABLE place_images ADD COLUMN place_key TEXT NOT NULL DEFAULT '';
END IF;
UPDATE place_images i SET place_key = COALESCE((
SELECT to_char(p.lat, 'FM999990.000000') || '|' || to_char(p.lon, 'FM999990.000000')
FROM places p
WHERE p.dataset_id = i.dataset_id AND p.src_pk = i.src_pk
AND p.lat IS NOT NULL AND p.lon IS NOT NULL),
'orphan:' || i.dataset_id || ':' || i.src_pk)
WHERE i.place_key = '';
-- 访客票:这张表没有 dataset_id,回填时只能按**活跃数据集**认领那个编号属于哪个地点。
-- 两个数据集里都有同一编号时也只认活跃的那个——这是唯一说得通的猜法,
-- 认不出来的留孤儿键,票不丢。
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_name = 'place_visitor_votes' AND column_name = 'place_key') THEN
ALTER TABLE place_visitor_votes ADD COLUMN place_key TEXT;
END IF;
SELECT string_agg(a.attname, ',' ORDER BY array_position(c.conkey, a.attnum))
INTO cols FROM pg_constraint c
JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = ANY(c.conkey)
WHERE c.conname = 'place_visitor_votes_pkey';
IF cols IS DISTINCT FROM 'place_key,voter_key' THEN
UPDATE place_visitor_votes vv SET place_key = COALESCE((
SELECT to_char(p.lat, 'FM999990.000000') || '|' || to_char(p.lon, 'FM999990.000000')
FROM places p JOIN datasets d ON d.id = p.dataset_id
WHERE d.is_active AND p.src_pk = vv.src_pk
AND p.lat IS NOT NULL AND p.lon IS NOT NULL
ORDER BY p.id LIMIT 1),
'orphan:' || vv.src_pk)
WHERE vv.place_key IS NULL;
-- 同一个访客对同一坐标只剩一条:留最新的
DELETE FROM place_visitor_votes a USING place_visitor_votes b
WHERE a.voter_key = b.voter_key AND a.place_key = b.place_key
AND (a.updated_at, a.ctid) < (b.updated_at, b.ctid);
ALTER TABLE place_visitor_votes ALTER COLUMN place_key SET NOT NULL;
ALTER TABLE place_visitor_votes DROP CONSTRAINT IF EXISTS place_visitor_votes_pkey;
ALTER TABLE place_visitor_votes ADD PRIMARY KEY (place_key, voter_key);
END IF;
END $$;
CREATE INDEX IF NOT EXISTS idx_place_images_key ON place_images(place_key, position);
CREATE INDEX IF NOT EXISTS idx_pvv_key ON place_visitor_votes(place_key);
+90
View File
@@ -0,0 +1,90 @@
package fog
// Diff 是一份待导入快照与当前迷雾之间的差异画像。
//
// 导入有两种语义完全不同的模式:叠加只做按位或、不可能丢数据;整体替换会先清空,
// 快照里没有的块一律消失。以前两种模式在界面上只是一个下拉框,站长在下手之前
// 看不到「这次能补多少」和「这次会丢多少」,实测里整体替换静默丢掉过 14% 的已探索区域。
type Diff struct {
CurBlocks, CurCells int // 库里现有
IncBlocks, IncCells int // 快照本身
// GainBlocks/GainCells 是叠加模式真正新增的部分:快照里当前完全没有的整块,
// 加上重叠块里补亮的格。
GainBlocks int
GainCells int
// OverlapBlocks 是与现有块坐标相同的块数,其中 SameBlocks 是一格都不会变的。
OverlapBlocks int
SameBlocks int
// LostBlocks/LostCells 是快照里完全没有、只有整体替换才会消失的部分。
LostBlocks int
LostCells int
curAreaM2, incAreaM2, gainAreaM2, lostAreaM2 float64
}
// DiffBlocks 比对当前块集与一份待导入快照。入参块集不要求有序。
func DiffBlocks(cur, inc []Block) Diff {
var d Diff
d.CurBlocks, d.IncBlocks = len(cur), len(inc)
curMap := make(map[[2]int][]byte, len(cur))
for _, b := range cur {
curMap[[2]int{b.GX, b.GY}] = b.Bitmap
n := popcountSum(b.Bitmap)
d.CurCells += n
d.curAreaM2 += float64(n) * cellAreaM2(b.GX, b.GY)
}
incSet := make(map[[2]int]struct{}, len(inc))
for _, b := range inc {
incSet[[2]int{b.GX, b.GY}] = struct{}{}
n := popcountSum(b.Bitmap)
d.IncCells += n
d.incAreaM2 += float64(n) * cellAreaM2(b.GX, b.GY)
old, ok := curMap[[2]int{b.GX, b.GY}]
if !ok {
d.GainBlocks++
d.GainCells += n
d.gainAreaM2 += float64(n) * cellAreaM2(b.GX, b.GY)
continue
}
d.OverlapBlocks++
gain := 0
for i := range b.Bitmap {
gain += int(popcount[b.Bitmap[i]&^old[i]])
}
if gain == 0 {
d.SameBlocks++
continue
}
d.GainCells += gain
d.gainAreaM2 += float64(gain) * cellAreaM2(b.GX, b.GY)
}
for _, b := range cur {
if _, ok := incSet[[2]int{b.GX, b.GY}]; ok {
continue
}
n := popcountSum(b.Bitmap)
d.LostBlocks++
d.LostCells += n
d.lostAreaM2 += float64(n) * cellAreaM2(b.GX, b.GY)
}
return d
}
func (d Diff) CurAreaKm2() float64 { return d.curAreaM2 / 1e6 }
func (d Diff) IncAreaKm2() float64 { return d.incAreaM2 / 1e6 }
func (d Diff) GainAreaKm2() float64 { return d.gainAreaM2 / 1e6 }
func (d Diff) LostAreaKm2() float64 { return d.lostAreaM2 / 1e6 }
// MergeAreaKm2 是叠加后的总量,CoverAreaKm2 是整体替换后的总量。
func (d Diff) MergeAreaKm2() float64 { return (d.curAreaM2 + d.gainAreaM2) / 1e6 }
func (d Diff) CoverAreaKm2() float64 { return d.incAreaM2 / 1e6 }
// Changed 报告叠加是否会让地图有任何变化——为假说明这份快照已经被完全覆盖过。
func (d Diff) Changed() bool { return d.GainCells > 0 }
+88
View File
@@ -0,0 +1,88 @@
package fog
import "testing"
// blk 造一个块,cells 是块内的 (列, 行) 坐标。
func blk(gx, gy int, cells ...[2]int) Block {
bm := make([]byte, bitmapSize)
for _, c := range cells {
bm[c[1]*8+c[0]/8] |= 0x80 >> (c[0] % 8)
}
return Block{GX: gx, GY: gy, Bitmap: bm}
}
func TestDiffEmptyBase(t *testing.T) {
d := DiffBlocks(nil, []Block{blk(10, 20, [2]int{0, 0}, [2]int{1, 0})})
if d.CurBlocks != 0 || d.CurCells != 0 {
t.Fatalf("空库不该有现有数据: %+v", d)
}
if d.GainBlocks != 1 || d.GainCells != 2 || d.IncCells != 2 {
t.Fatalf("空库叠加应为全部新增: %+v", d)
}
if d.LostBlocks != 0 || d.LostCells != 0 || !d.Changed() {
t.Fatalf("空库不该有丢失: %+v", d)
}
}
func TestDiffIdentical(t *testing.T) {
cur := []Block{blk(10, 20, [2]int{0, 0}, [2]int{1, 0}), blk(11, 20, [2]int{5, 5})}
d := DiffBlocks(cur, cur)
if d.GainCells != 0 || d.Changed() {
t.Fatalf("同一份快照再叠一次不该有任何新增: %+v", d)
}
if d.LostBlocks != 0 || d.LostCells != 0 {
t.Fatalf("同一份快照不该有可丢的部分: %+v", d)
}
if d.OverlapBlocks != 2 || d.SameBlocks != 2 {
t.Fatalf("两个块都应算作「重叠且无变化」: %+v", d)
}
}
func TestDiffOverlapGain(t *testing.T) {
d := DiffBlocks(
[]Block{blk(10, 20, [2]int{0, 0})},
[]Block{blk(10, 20, [2]int{0, 0}, [2]int{3, 4})})
if d.GainBlocks != 0 || d.GainCells != 1 {
t.Fatalf("重叠块应只补亮多出的那一格: %+v", d)
}
if d.OverlapBlocks != 1 || d.SameBlocks != 0 {
t.Fatalf("重叠但补亮了,不该记为无变化: %+v", d)
}
if d.LostBlocks != 0 {
t.Fatalf("同一个块不该被判为会丢失: %+v", d)
}
}
func TestDiffDisjoint(t *testing.T) {
d := DiffBlocks(
[]Block{blk(10, 20, [2]int{0, 0})},
[]Block{blk(30, 40, [2]int{1, 1})})
if d.GainBlocks != 1 || d.GainCells != 1 {
t.Fatalf("另一块应是新增: %+v", d)
}
if d.LostBlocks != 1 || d.LostCells != 1 {
t.Fatalf("库里那块在快照里没有,整体替换会丢掉: %+v", d)
}
}
// 面积按块中心纬度做 cos 修正,越靠近两极越小;总量关系也要自洽。
func TestDiffAreas(t *testing.T) {
equator := DiffBlocks(nil, []Block{blk(100, WorldCells/BlockCells/2, [2]int{1, 1})})
high := DiffBlocks(nil, []Block{blk(100, 20000, [2]int{1, 1})})
if high.GainAreaKm2() <= 0 {
t.Fatalf("高纬单格面积应为正: %v", high.GainAreaKm2())
}
if !(equator.GainAreaKm2() > high.GainAreaKm2()) {
t.Fatalf("赤道单格应大于高纬: %v vs %v", equator.GainAreaKm2(), high.GainAreaKm2())
}
m := DiffBlocks(
[]Block{blk(10, 20, [2]int{0, 0})},
[]Block{blk(10, 20, [2]int{0, 0}, [2]int{9, 9})})
if diff := m.MergeAreaKm2() - (m.CurAreaKm2() + m.GainAreaKm2()); diff > 1e-12 || diff < -1e-12 {
t.Fatalf("叠加后总量应等于现有加新增: %v", diff)
}
if m.CoverAreaKm2() != m.IncAreaKm2() {
t.Fatalf("整体替换后总量应等于快照本身: %v vs %v", m.CoverAreaKm2(), m.IncAreaKm2())
}
}
+6
View File
@@ -180,6 +180,12 @@ func mercY(lat float64) float64 {
return (math.Pi - math.Asinh(math.Tan(rad))) / (2 * math.Pi) * WorldCells
}
// BlockLonLat 返回某块中心的 WGS-84 经纬度,供差异地图定位视野。
func BlockLonLat(gbx, gby int) (lat, lon float64) {
lon, lat = cellLonLat(gbx*BlockCells+BlockCells/2, gby*BlockCells+BlockCells/2)
return
}
func cellLonLat(cx, cy int) (float64, float64) {
lon := (float64(cx)+0.5)/WorldCells*360.0 - 180.0
n := 2 * math.Pi * (float64(cy) + 0.5) / WorldCells
+200
View File
@@ -0,0 +1,200 @@
package fog
import (
"image"
"math"
)
// 差异图的三种归属。改这里的色值要同步 templates/admin/fog-preview.html 的图例。
var (
DiffBothColor = [4]byte{126, 134, 146, 148} // 两边都有:导入不影响
DiffGainColor = [4]byte{36, 164, 90, 205} // 只有快照有:叠加会新增
DiffLostColor = [4]byte{214, 56, 56, 210} // 只有库里有:整体替换会丢
)
const (
diffNone uint8 = iota
diffBoth
diffGain
diffLost
)
// RenderDiffTile 渲染一张 256x256 的差异瓦片,供站长在真实底图上核对
// 「这份快照能补上哪里」与「整体替换会丢掉哪里」。
//
// 与 RenderTile 一样按底图坐标系做 WGS84→GCJ02 纠偏;不做描边——三种颜色
// 已经够区分,再描边在粗缩放级下会糊成一片。
//
// 逐格判定只在放大到「一格不小于一像素」(z>=14)时做;更粗的级别下一个块还不到
// 一个像素,按块内占多数的类别整块涂色。绘制顺序 both→gain→lost,让「会新增」
// 和「会丢」压住底色,重叠处不至于被灰吃掉。
func RenderDiffTile(cur, inc map[[2]int][]byte, z, x, y int, crs string) *image.NRGBA {
if z < 0 || z > 22 {
return nil
}
step := WorldCells >> z
scale := 256.0 / float64(step)
ox, oy := float64(x*step), float64(y*step)
layer := make([]uint8, 256*256)
mark := func(px, py, w, h int, kind uint8) {
for r := 0; r < h; r++ {
yy := py + r
if yy < 0 || yy > 255 {
continue
}
row := yy * 256
for c := 0; c < w; c++ {
xx := px + c
if xx < 0 || xx > 255 {
continue
}
layer[row+xx] = kind
}
}
}
// 直接遍历块表而不是全球块范围:低缩放级下范围循环会膨胀到数十亿次空查询。
// GCJ 偏移最多约 700m(74 格),粗裁剪时两侧各放宽两块保证不漏。
const margin = 2 * BlockCells
lo, hi := float64(x*step-margin), float64((x+1)*step+margin)
rlo, rhi := float64(y*step-margin), float64((y+1)*step+margin)
shift := crs == "gcj02"
draw := func(pass uint8, gbx, gby int, cb, ib []byte) {
bx, by := float64(gbx*BlockCells), float64(gby*BlockCells)
if bx+BlockCells <= lo || bx >= hi || by+BlockCells <= rlo || by >= rhi {
return
}
ddx, ddy := 0.0, 0.0
if shift {
ddx, ddy = blockShift(gbx, gby)
}
u0 := (bx + ddx - ox) * scale
v0 := (by + ddy - oy) * scale
if scale >= 1 {
w := int(math.Ceil(scale))
for row := 0; row < BlockCells; row++ {
base := row * 8
for colB := 0; colB < 8; colB++ {
var cbv, ibv byte
if cb != nil {
cbv = cb[base+colB]
}
if ib != nil {
ibv = ib[base+colB]
}
if cbv|ibv == 0 {
continue
}
py := int(math.Floor(v0 + float64(row)*scale))
for bit := 0; bit < 8; bit++ {
m := byte(0x80 >> bit)
kind := cellKind(cbv&m != 0, ibv&m != 0)
if kind != pass {
continue
}
px := int(math.Floor(u0 + float64(colB*8+bit)*scale))
mark(px, py, w, w, kind)
}
}
}
return
}
// 一格不足一像素:整块按多数类涂,三种混在一起的块以数量最多的为准
if dominantKind(blockDiffCounts(cb, ib)) != pass {
return
}
u1 := (bx + BlockCells + ddx - ox) * scale
v1 := (by + BlockCells + ddy - oy) * scale
px0, py0 := int(math.Floor(u0)), int(math.Floor(v0))
px1, py1 := int(math.Ceil(u1)), int(math.Ceil(v1))
if px1-px0 < 1 {
px1 = px0 + 1
}
if py1-py0 < 1 {
py1 = py0 + 1
}
mark(px0, py0, px1-px0, py1-py0, pass)
}
for _, pass := range []uint8{diffBoth, diffGain, diffLost} {
for key, cb := range cur {
draw(pass, key[0], key[1], cb, inc[key])
}
// 只在快照里的块单独走一遍,两边都有的块不再重复画
for key, ib := range inc {
if _, dup := cur[key]; dup {
continue
}
draw(pass, key[0], key[1], nil, ib)
}
}
img := image.NewNRGBA(image.Rect(0, 0, 256, 256))
for i, kind := range layer {
if kind == diffNone {
continue
}
var c [4]byte
switch kind {
case diffBoth:
c = DiffBothColor
case diffGain:
c = DiffGainColor
default:
c = DiffLostColor
}
o := i * 4
img.Pix[o], img.Pix[o+1], img.Pix[o+2], img.Pix[o+3] = c[0], c[1], c[2], c[3]
}
return img
}
func cellKind(inCur, inInc bool) uint8 {
switch {
case inCur && inInc:
return diffBoth
case inInc:
return diffGain
case inCur:
return diffLost
}
return diffNone
}
// blockDiffCounts 数一个块里三类各占多少格。两个位图都可能缺失(只有一边有这个块)。
func blockDiffCounts(cb, ib []byte) (both, gain, lost int) {
n := len(cb)
if len(ib) > n {
n = len(ib)
}
for i := 0; i < n; i++ {
var c, in byte
if i < len(cb) {
c = cb[i]
}
if i < len(ib) {
in = ib[i]
}
both += int(popcount[c&in])
gain += int(popcount[in&^c])
lost += int(popcount[c&^in])
}
return
}
// dominantKind 取三类里占得最多的一类,用来在粗缩放级下决定整块的着色。
func dominantKind(both, gain, lost int) uint8 {
switch {
case gain > 0 && gain >= both && gain >= lost:
return diffGain
case lost > 0 && lost >= both:
return diffLost
case both > 0:
return diffBoth
}
return diffNone
}
+114
View File
@@ -0,0 +1,114 @@
package fog
import (
"image"
"math"
"testing"
)
func bmOf(cells ...[2]int) []byte {
bm := make([]byte, bitmapSize)
for _, c := range cells {
bm[c[1]*8+c[0]/8] |= 0x80 >> (c[0] % 8)
}
return bm
}
func pixAt(img *image.NRGBA, px, py int) [4]byte {
i := (py*256 + px) * 4
return [4]byte{img.Pix[i], img.Pix[i+1], img.Pix[i+2], img.Pix[i+3]}
}
// z=18 时一格正好 16 像素,块 (100,200) 完整落在瓦片 (400,800) 里,
// 格 (col,row) 的中心像素是 (col*16+8, row*16+8)。
func TestRenderDiffTilePerCell(t *testing.T) {
key := [2]int{100, 200}
cur := map[[2]int][]byte{key: bmOf([2]int{0, 0}, [2]int{4, 0})}
inc := map[[2]int][]byte{key: bmOf([2]int{0, 0}, [2]int{1, 0})}
img := RenderDiffTile(cur, inc, 18, 400, 800, "wgs84")
if img == nil {
t.Fatal("瓦片不该为空")
}
if got := pixAt(img, 8, 8); got != DiffBothColor {
t.Errorf("两边都有的格应为灰色,实际 %v", got)
}
if got := pixAt(img, 24, 8); got != DiffGainColor {
t.Errorf("只有快照有的格应为绿色,实际 %v", got)
}
if got := pixAt(img, 72, 8); got != DiffLostColor {
t.Errorf("只有库里有、整体替换会丢的格应为红色,实际 %v", got)
}
if got := pixAt(img, 200, 200); got[3] != 0 {
t.Errorf("没被探索的格应保持透明,实际 %v", got)
}
}
func TestRenderDiffTileSingleSided(t *testing.T) {
key := [2]int{100, 200}
onlyCur := RenderDiffTile(map[[2]int][]byte{key: bmOf([2]int{3, 3})}, nil, 18, 400, 800, "wgs84")
if got := pixAt(onlyCur, 56, 56); got != DiffLostColor {
t.Errorf("快照里完全没有的块应整块为红,实际 %v", got)
}
onlyInc := RenderDiffTile(nil, map[[2]int][]byte{key: bmOf([2]int{3, 3})}, 18, 400, 800, "wgs84")
if got := pixAt(onlyInc, 56, 56); got != DiffGainColor {
t.Errorf("库里完全没有的块应整块为绿,实际 %v", got)
}
}
// z=10 时一格不足一像素,块只占 4x4 像素,按块内多数类涂色。
func TestRenderDiffTileCoarseZoomUsesDominantKind(t *testing.T) {
key := [2]int{100, 200}
cur := map[[2]int][]byte{key: bmOf([2]int{0, 0})}
inc := map[[2]int][]byte{key: bmOf([2]int{1, 1}, [2]int{2, 2}, [2]int{3, 3})}
img := RenderDiffTile(cur, inc, 10, 1, 3, "wgs84")
const step, scale = WorldCells >> 10, 256.0 / (WorldCells >> 10)
bx, by := float64(key[0]*BlockCells), float64(key[1]*BlockCells)
px := int(math.Floor((bx - 1*step) * scale))
py := int(math.Floor((by - 3*step) * scale))
if got := pixAt(img, px+1, py+1); got != DiffGainColor {
t.Errorf("三格新增、一格不变的块应以绿色为主,实际 %v", got)
}
if w := int(math.Ceil(float64(BlockCells) * scale)); w != 4 {
t.Fatalf("z=10 下块应占 4 像素,实际 %d", w)
}
}
func TestRenderDiffTileEdgeCases(t *testing.T) {
if RenderDiffTile(nil, nil, 18, 400, 800, "wgs84") == nil {
t.Error("两边都空也应返回一张空瓦片")
}
if RenderDiffTile(nil, nil, 23, 0, 0, "wgs84") != nil {
t.Error("超出缩放范围应返回 nil")
}
}
func TestDominantKind(t *testing.T) {
cases := []struct {
both, gain, lost int
want uint8
}{
{0, 0, 0, diffNone},
{5, 1, 0, diffBoth},
{1, 5, 0, diffGain},
{1, 0, 5, diffLost},
{2, 2, 0, diffGain}, // 平局让「会新增」胜出
{3, 1, 1, diffBoth},
}
for _, c := range cases {
if got := dominantKind(c.both, c.gain, c.lost); got != c.want {
t.Errorf("dominantKind(%d,%d,%d)=%d,期望 %d", c.both, c.gain, c.lost, got, c.want)
}
}
}
func TestBlockDiffCounts(t *testing.T) {
both, gain, lost := blockDiffCounts(bmOf([2]int{0, 0}, [2]int{1, 0}), bmOf([2]int{0, 0}, [2]int{2, 0}))
if both != 1 || gain != 1 || lost != 1 {
t.Errorf("一块里应各有一格:both=%d gain=%d lost=%d", both, gain, lost)
}
both, gain, lost = blockDiffCounts(nil, bmOf([2]int{5, 5}))
if both != 0 || gain != 1 || lost != 0 {
t.Errorf("只有快照有该块:both=%d gain=%d lost=%d", both, gain, lost)
}
}
+30 -4
View File
@@ -67,15 +67,41 @@ func insertEntityRaw(ctx context.Context, tx *sql.Tx, datasetID int64, rows map[
}
// contentHash 给备份内容算指纹,用来识别「内容没变、只是重新导出了一次」的上传。
// rond 每次导出连 WAL 一起打包,字节必然不同,文件摘要挡不住这种重复;
// 但同一批到访的 Z_PK 是稳定的,所以按到访自然键排序后取摘要即可。
// rond 每次导出连 WAL 一起打包,字节必然不同,文件摘要挡不住这种重复。
//
// 指纹按**到访/活动/地点的稳定标识**算,绝不能掺 Z_PK:rond 每次导出/同步都会重排
// 编号(实测两份备份里 822 条到访的 UUID 完全一致、Z_PK 100% 不同),拿编号当指纹会把
// 「同一批数据重导一次」判成新内容 —— 于是新建数据集,而备注与投票是按 dataset_id
// 隔离的,换了数据集就再也看不见。
//
// 没有 UUID 的老备份退回「到达时间 + 地点坐标」,仍比编号可靠。
func contentHash(b *rond.Backup) string {
keys := make([]string, 0, len(b.Visits))
locCoord := make(map[int]string, len(b.Locations))
keys := make([]string, 0, len(b.Visits)+len(b.Locations)+len(b.Activities))
for _, l := range b.Locations {
locCoord[l.SrcPK] = fmt.Sprintf("%.6f/%.6f", l.Lat, l.Lon)
keys = append(keys, "L"+l.Name+"/"+locCoord[l.SrcPK])
}
for _, v := range b.Visits {
if v.Arrival == nil {
continue // 导入时就被跳过的行,不该影响指纹
}
if u := b.VisitUUID[v.SrcPK]; u != "" {
keys = append(keys, "V"+u)
continue
}
keys = append(keys, fmt.Sprintf("%d|%d", v.SrcPK, v.Arrival.Unix()))
loc := ""
if v.LocationSrcPK != nil {
loc = locCoord[*v.LocationSrcPK]
}
keys = append(keys, fmt.Sprintf("T%d/%s", v.Arrival.Unix(), loc))
}
for _, a := range b.Activities {
if u := b.ActivityUUID[a.SrcPK]; u != "" {
keys = append(keys, "A"+u)
continue
}
keys = append(keys, "A"+a.Name)
}
sort.Strings(keys)
h := sha256.New()
+107
View File
@@ -0,0 +1,107 @@
package ingest
import (
"fmt"
"testing"
"time"
"rond-with-you/internal/rond"
)
// sampleBackup 造一份最小的备份:到访 + 地点 + 活动,编号与 UUID 分别给定。
func sampleBackup(pks []int, uuids []string) *rond.Backup {
b := &rond.Backup{VisitUUID: map[int]string{}, ActivityUUID: map[int]string{}}
for i, pk := range pks {
at := time.Unix(int64(1700000000+i*3600), 0)
b.Visits = append(b.Visits, rond.Visit{SrcPK: pk, Arrival: &at, LocationSrcPK: intp(1)})
b.VisitUUID[pk] = uuids[i]
}
b.Locations = []rond.Location{{SrcPK: 1, Name: "家", Lat: 24.1, Lon: 109.1}}
b.Activities = []rond.Activity{{SrcPK: 9, Name: "工作"}}
b.ActivityUUID[9] = "cccccccccccccccccccccccccccccccc"
return b
}
func intp(v int) *int { return &v }
// 同一批数据重新导出一次:UUID 一个没变、Z_PK 全被打乱(顺序也变了),
// 指纹必须一样。否则会被判成「新内容」→ 新建数据集 → 按 dataset_id 隔离的
// 备注与投票全部失联。
func TestContentHashIgnoresPKReorder(t *testing.T) {
a := sampleBackup([]int{1, 2, 3}, []string{
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
"dddddddddddddddddddddddddddddddd",
})
b := sampleBackup([]int{555, 276, 88}, []string{
"dddddddddddddddddddddddddddddddd",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
})
if contentHash(a) != contentHash(b) {
t.Fatal("编号重排不该改变指纹")
}
}
// 内容真的变了(多一条到访)就得算不同的指纹,否则新数据会被当成重复上传而丢掉。
func TestContentHashDetectsNewVisit(t *testing.T) {
a := sampleBackup([]int{1, 2}, []string{
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
})
b := sampleBackup([]int{1, 2, 3}, []string{
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
"dddddddddddddddddddddddddddddddd",
})
if contentHash(a) == contentHash(b) {
t.Fatal("多了一条到访就是不同内容")
}
}
// 只改了地点名(到访一条没动)也算内容变了:站点是靠地名认地点的。
func TestContentHashDetectsPlaceRename(t *testing.T) {
a := sampleBackup([]int{1}, []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"})
b := sampleBackup([]int{1}, []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"})
b.Locations[0].Name = "老家"
if contentHash(a) == contentHash(b) {
t.Fatal("地点改名应改变指纹")
}
}
// 没有 UUID 的老备份退回「到达时间 + 地点坐标」,同一批数据重导仍要认出是同一份。
func TestContentHashFallsBackWithoutUUID(t *testing.T) {
a := sampleBackup([]int{1, 2}, []string{"", ""})
b := sampleBackup([]int{700, 900}, []string{"", ""})
if contentHash(a) != contentHash(b) {
t.Fatal("没有 UUID 时应按到达时间与地点兜底,不该受编号影响")
}
c := sampleBackup([]int{1, 2}, []string{"", ""})
c.Visits[1].Arrival = ptrTime(time.Unix(1700000000, 0)) // 改掉第二条的时间
if contentHash(a) == contentHash(c) {
t.Fatal("时间不同应算不同内容")
}
}
// 没有到达时间的行导入时就被跳过,不该参与指纹。
func TestContentHashSkipsRowsWithoutArrival(t *testing.T) {
a := sampleBackup([]int{1}, []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"})
b := sampleBackup([]int{1}, []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"})
b.Visits = append(b.Visits, rond.Visit{SrcPK: 42})
if contentHash(a) != contentHash(b) {
t.Fatal("缺到达时间的行不该影响指纹")
}
}
// 指纹是稳定的十六进制摘要,不是随手拼的字符串。
func TestContentHashShape(t *testing.T) {
h := contentHash(sampleBackup([]int{1}, []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}))
if len(h) != 64 {
t.Fatalf("应是 64 位 sha256 十六进制,实际 %d 位: %s", len(h), h)
}
if _, err := fmt.Sscanf(h, "%x", new(string)); err != nil {
t.Fatalf("不是纯十六进制: %v", err)
}
}
func ptrTime(t time.Time) *time.Time { return &t }
+62 -37
View File
@@ -1,11 +1,14 @@
// Package notes 存放站长为地点加的别名与备注。
// 备注按 rond 的地点 ID(src_pk)而非数据库自增 id 存储,
// 因此重新上传备份、地点换了新 id 之后内容依然挂在原地。
//
// 备注挂的是**坐标键**(place_key)而不是 rond 的地点编号:src_pk 只在单个数据集内有效
// (rond 每次导出都会重排 Z_PK,实测两份备份里 822 条到访的 UUID 一致而 Z_PK 100% 不同),
// 换个数据集同编号就是另一个地点,备注会全部失联。坐标是地理事实,跨数据集稳定。
package notes
import (
"context"
"database/sql"
"fmt"
"strings"
"time"
)
@@ -20,15 +23,26 @@ type Note struct {
type Store struct{ DB *sql.DB }
// All 返回某个数据集的全站备注(站点只有一个站长账号,读的时候不按 user_id 过滤,
// 这样匿名访客也能看到别名;写的时候才需要登录用户的 id)。
// 后写覆盖先写,保证同一地点只保留一条。必须按 dataset_id 过滤:
// src_pk 只在单个数据集内唯一,跨数据集同编号是完全不同的地点。
// placeKeyExpr 把某个 places 别名下的坐标换算成 place_key。
// **必须与 Go 侧的 rond.PlaceCoordKey 逐位一致**:
//
// fmt.Sprintf("%.6f|%.6f") ↔ to_char(x,'FM999990.000000')||'|'||to_char(y,'FM999990.000000')
//
// 已在真实数据上逐条比对(368/368 相同)。改任何一边都要重新比。
func placeKeyExpr(alias string) string {
return fmt.Sprintf(
"to_char(%[1]s.lat,'FM999990.000000')||'|'||to_char(%[1]s.lon,'FM999990.000000')", alias)
}
// All 返回某个数据集里「有备注的地点」,键是地点在该数据集内的编号。
// 站点只有一个站长账号,读的时候不按 user_id 过滤(匿名访客也要看到别名)。
func (s *Store) All(ctx context.Context, datasetID int64) (map[int]Note, error) {
out := map[int]Note{}
rows, err := s.DB.QueryContext(ctx,
`SELECT src_pk, alias, note, rond_note, updated_at FROM place_notes
WHERE dataset_id=$1 ORDER BY updated_at DESC`, datasetID)
rows, err := s.DB.QueryContext(ctx, `
SELECT p.src_pk, n.alias, n.note, n.rond_note, n.updated_at
FROM places p
JOIN place_notes n ON n.place_key = `+placeKeyExpr("p")+`
WHERE p.dataset_id = $1 AND p.lat IS NOT NULL AND p.lon IS NOT NULL`, datasetID)
if err != nil {
return out, err
}
@@ -46,9 +60,9 @@ func (s *Store) All(ctx context.Context, datasetID int64) (map[int]Note, error)
// SetNote 只改备注、保留别名(批量「清空备注」用);两者都为空则删除记录。
func (s *Store) SetNote(ctx context.Context, userID, datasetID int64, srcPK int, content string) error {
var alias string
err := s.DB.QueryRowContext(ctx,
`SELECT alias FROM place_notes WHERE user_id=$1 AND dataset_id=$2 AND src_pk=$3`,
userID, datasetID, srcPK).Scan(&alias)
err := s.DB.QueryRowContext(ctx, `SELECT n.alias FROM place_notes n
JOIN places p ON n.place_key = `+placeKeyExpr("p")+`
WHERE n.user_id=$1 AND p.dataset_id=$2 AND p.src_pk=$3`, userID, datasetID, srcPK).Scan(&alias)
if err != nil && err != sql.ErrNoRows {
return err
}
@@ -57,8 +71,9 @@ func (s *Store) SetNote(ctx context.Context, userID, datasetID int64, srcPK int,
// Delete 删除某地点的整条备注与别名。
func (s *Store) Delete(ctx context.Context, userID, datasetID int64, srcPK int) error {
_, err := s.DB.ExecContext(ctx,
`DELETE FROM place_notes WHERE user_id=$1 AND dataset_id=$2 AND src_pk=$3`, userID, datasetID, srcPK)
_, err := s.DB.ExecContext(ctx, `DELETE FROM place_notes n USING places p
WHERE n.user_id=$1 AND p.dataset_id=$2 AND p.src_pk=$3
AND n.place_key = `+placeKeyExpr("p"), userID, datasetID, srcPK)
return err
}
@@ -68,21 +83,28 @@ func (s *Store) Save(ctx context.Context, userID, datasetID int64, srcPK int, al
content = strings.TrimSpace(content)
if alias == "" && content == "" {
// 没有 rond 备注时直接删行;有则只清空手写部分,别把回填结果一起删掉
_, err := s.DB.ExecContext(ctx,
`DELETE FROM place_notes WHERE user_id=$1 AND dataset_id=$2 AND src_pk=$3 AND rond_note=''`,
userID, datasetID, srcPK)
if err != nil {
if _, err := s.DB.ExecContext(ctx, `DELETE FROM place_notes n USING places p
WHERE n.user_id=$1 AND p.dataset_id=$2 AND p.src_pk=$3
AND n.place_key = `+placeKeyExpr("p")+` AND n.rond_note=''`,
userID, datasetID, srcPK); err != nil {
return err
}
_, err = s.DB.ExecContext(ctx,
`UPDATE place_notes SET alias='', note='', updated_at=now()
WHERE user_id=$1 AND dataset_id=$2 AND src_pk=$3`, userID, datasetID, srcPK)
_, err := s.DB.ExecContext(ctx, `UPDATE place_notes n
SET alias='', note='', updated_at=now()
FROM places p
WHERE n.user_id=$1 AND p.dataset_id=$2 AND p.src_pk=$3
AND n.place_key = `+placeKeyExpr("p"), userID, datasetID, srcPK)
return err
}
_, err := s.DB.ExecContext(ctx, `INSERT INTO place_notes (user_id, dataset_id, src_pk, alias, note, updated_at)
VALUES ($1,$2,$3,$4,$5,now())
ON CONFLICT (user_id, dataset_id, src_pk)
DO UPDATE SET alias=EXCLUDED.alias, note=EXCLUDED.note, updated_at=now()`,
// 先按坐标键算出这条地点该挂在哪,找不到该地点就什么也不写
_, err := s.DB.ExecContext(ctx, `INSERT INTO place_notes
(user_id, place_key, dataset_id, src_pk, alias, note, updated_at)
SELECT $1, `+placeKeyExpr("p")+`, $2, $3, $4, $5, now()
FROM places p
WHERE p.dataset_id=$2 AND p.src_pk=$3 AND p.lat IS NOT NULL AND p.lon IS NOT NULL
ON CONFLICT (user_id, place_key) DO UPDATE SET
alias=EXCLUDED.alias, note=EXCLUDED.note,
dataset_id=EXCLUDED.dataset_id, src_pk=EXCLUDED.src_pk, updated_at=now()`,
userID, datasetID, srcPK, alias, content)
return err
}
@@ -92,20 +114,22 @@ func (s *Store) Save(ctx context.Context, userID, datasetID int64, srcPK int, al
// 只重写 rond_note 一列,站长手写的 note 一概不动。返回写了多少行。
func (s *Store) BackfillRond(ctx context.Context, userID, datasetID int64) (int, error) {
// DISTINCT 在「日期 + 内容」这一层做,同一天重复写的同一句只留一条
const agg = `
agg := `
WITH raw AS (
SELECT p.src_pk, to_char(v.arrival, 'YYYY-MM-DD') || ' ' || btrim(v.remark) AS line
SELECT ` + placeKeyExpr("p") + ` AS key,
to_char(v.arrival, 'YYYY-MM-DD') || ' ' || btrim(v.remark) AS line
FROM visits v JOIN places p ON p.id = v.place_id
WHERE v.dataset_id = $1 AND btrim(COALESCE(v.remark, '')) <> ''
AND p.lat IS NOT NULL AND p.lon IS NOT NULL
), uniq AS (
SELECT DISTINCT src_pk, line, left(line, 10) AS day FROM raw
SELECT DISTINCT key, line, left(line, 10) AS day FROM raw
), agg AS (
SELECT src_pk, string_agg(line, E'\n' ORDER BY day, line) AS txt FROM uniq GROUP BY src_pk
SELECT key, string_agg(line, E'\n' ORDER BY day, line) AS txt FROM uniq GROUP BY key
)`
res, err := s.DB.ExecContext(ctx, agg+`
INSERT INTO place_notes (user_id, dataset_id, src_pk, alias, note, rond_note, updated_at)
SELECT $2, $1, src_pk, '', '', txt, now() FROM agg
ON CONFLICT (user_id, dataset_id, src_pk) DO UPDATE SET rond_note = EXCLUDED.rond_note`,
INSERT INTO place_notes (user_id, place_key, dataset_id, src_pk, alias, note, rond_note, updated_at)
SELECT $2, key, $1, 0, '', '', txt, now() FROM agg
ON CONFLICT (user_id, place_key) DO UPDATE SET rond_note = EXCLUDED.rond_note`,
datasetID, userID)
if err != nil {
return 0, err
@@ -113,15 +137,16 @@ func (s *Store) BackfillRond(ctx context.Context, userID, datasetID int64) (int,
affected, _ := res.RowsAffected()
// 备注被删掉的地点要清掉旧值,否则会一直留着过期的 rond 内容。
// 限定在本数据集的地点里:datasetID 不合法时子查询为空,一行都不会动,
// 不会把别的数据集的回填结果误清掉。
// 限定在本数据集的地点里:datasetID 不合法时子查询为空,一行都不会动。
if _, err := s.DB.ExecContext(ctx, `
UPDATE place_notes n SET rond_note = ''
WHERE n.user_id = $2 AND n.dataset_id = $1 AND n.rond_note <> ''
AND n.src_pk IN (SELECT src_pk FROM places WHERE dataset_id = $1)
WHERE n.user_id = $2 AND n.rond_note <> ''
AND n.place_key IN (SELECT `+placeKeyExpr("p")+`
FROM places p WHERE p.dataset_id = $1 AND p.lat IS NOT NULL AND p.lon IS NOT NULL)
AND NOT EXISTS (
SELECT 1 FROM visits v JOIN places p ON p.id = v.place_id
WHERE v.dataset_id = $1 AND p.src_pk = n.src_pk AND btrim(COALESCE(v.remark, '')) <> ''
WHERE v.dataset_id = $1 AND `+placeKeyExpr("p")+` = n.place_key
AND btrim(COALESCE(v.remark, '')) <> ''
)`, datasetID, userID); err != nil {
return 0, err
}
+390
View File
@@ -0,0 +1,390 @@
package rond
import (
"fmt"
"sort"
"time"
)
// 两份数据的比对。
//
// 对齐键的选择是这个文件的全部要点:rond 每次导出/同步都会重排 Z_PK(实测两份备份里
// 822 条到访的 UUID 完全一致、Z_PK 100% 不同),所以 src_pk 只能在一个数据集内部使用。
// 跨备份对齐一律走真机写的 UUID;地点/标签/交通方式没有 UUID,退到「名称 + 坐标」。
//
// 这里只做判定,不碰数据库、也不写任何东西。
// StoredVisit 是库里的一条到访,外加从 entity_raw 留档取出的对齐键。
type StoredVisit struct {
SrcPK int
UUID string // 空 = 站点新建的行(真机不会给它 UUID)
Arrival *time.Time
Departure *time.Time
Remark string
Emoji string
Bookmarked bool
UserAdded bool
PlaceKey string // 地点对齐键,空 = 该到访没有地点
PlaceName string // 仅用于展示
ActivityName string
}
// StoredPlace 是库里的一个地点。
type StoredPlace struct {
SrcPK int
Key string
Name string
Lat, Lon float64
Province string
City string
POICategory string
VisitCount int
}
// StoredActivity 是库里的一个活动类型。
type StoredActivity struct {
SrcPK int
Name string
UUID string
}
// FieldChange 是冲突里的一项字段差异。From 是库里当前的值,To 是备份里的值。
type FieldChange struct {
Label string
From string
To string
}
// VisitConflict 是同一条到访(UUID 相同)两边字段不一致。
type VisitConflict struct {
Backup Visit
DB StoredVisit
Fields []FieldChange
}
// VisitDiff 是到访的差异画像。
type VisitDiff struct {
Added []Visit // 备份有、库里没有
Removed []StoredVisit // 库里有、备份里没有
Changed []VisitConflict
Same int // 对齐上且完全一致
}
// PlaceDiff 是地点的差异画像。
type PlaceDiff struct {
Added []Location
Removed []StoredPlace
Changed []PlaceConflict
Same int
}
type PlaceConflict struct {
Backup Location
DB StoredPlace
Fields []FieldChange
}
// ActivityDiff 是活动类型的差异画像(按 UUID 对齐,退不到就按名称)。
type ActivityDiff struct {
Added []Activity
Removed []StoredActivity
Changed []ActivityConflict
Same int
}
type ActivityConflict struct {
Backup Activity
DB StoredActivity
Fields []FieldChange
}
// PlaceCoordKey 是地点的跨备份对齐键:**只认坐标,不认名称**。
//
// 地点没有 UUID,名称又会被站点自己的「补全缺失地址」改掉(实测:站内把
// 「体育馆美食街」反查成了「河池市国民体质监测中心」,坐标一字未动,于是 17 条
// 到访全被误报成「换了地点」)。坐标取 6 位小数(约 0.1m)——足以吸收浮点往返误差,
// 又不会把挨着的两个地点并成一个。
func PlaceCoordKey(lat, lon float64) string {
return fmt.Sprintf("%.6f|%.6f", lat, lon)
}
func coordStr(v float64) string { return fmt.Sprintf("%.6f", v) }
// DiffVisits 按 UUID 对齐两边的到访。
//
// 真机偶尔有条目没写 UUID(实测 824 行到访里有 2 行),只按 UUID 配对会让同一条既算
// 「新增」又算「仅站内有」;所以没有 UUID 的行退到「到达时间 + 地点」兜底配对。
// 库里那些连兜底也配不上的(站点新建的行)一律算「库里有、备份没有」。
func DiffVisits(stored []StoredVisit, b *Backup) VisitDiff {
placeKey, placeName := map[int]string{}, map[int]string{}
for _, l := range b.Locations {
// 没有坐标的地点不参与坐标对齐,否则它们会共用 "0.000000|0.000000" 互相认领
if l.Lat != 0 || l.Lon != 0 {
placeKey[l.SrcPK] = PlaceCoordKey(l.Lat, l.Lon)
}
placeName[l.SrcPK] = placeLabel(l.Name, l.Lat, l.Lon)
}
actName := map[int]string{}
for _, a := range b.Activities {
actName[a.SrcPK] = a.Name
}
used := make([]bool, len(stored))
byUUID := make(map[string]int, len(stored))
byTime := map[string][]int{}
for i, sv := range stored {
if sv.UUID != "" {
byUUID[sv.UUID] = i
continue
}
if sv.Arrival != nil {
k := timeKey(*sv.Arrival, sv.PlaceKey)
byTime[k] = append(byTime[k], i)
}
}
var d VisitDiff
classify := func(i int, v Visit) {
used[i] = true
sv := stored[i]
if fields := visitFieldChanges(sv, v, placeKey, placeName, actName); len(fields) > 0 {
d.Changed = append(d.Changed, VisitConflict{Backup: v, DB: sv, Fields: fields})
} else {
d.Same++
}
}
for _, v := range b.Visits {
if v.Arrival == nil {
continue // 没有到达时间的行导入时就被跳过了,不该被算成「新增」
}
if uuid := b.VisitUUID[v.SrcPK]; uuid != "" {
if i, ok := byUUID[uuid]; ok {
classify(i, v)
continue
}
d.Added = append(d.Added, v)
continue
}
bpk := ""
if v.LocationSrcPK != nil {
bpk = placeKey[*v.LocationSrcPK]
}
if list := byTime[timeKey(*v.Arrival, bpk)]; len(list) > 0 {
byTime[timeKey(*v.Arrival, bpk)] = list[1:]
classify(list[0], v)
continue
}
d.Added = append(d.Added, v)
}
for i, sv := range stored {
if !used[i] {
d.Removed = append(d.Removed, sv)
}
}
sortVisits(d)
return d
}
// timeKey 是没有 UUID 时的兜底配对键。只到秒:苹果时间戳转过来是浮点秒,
// 而 PG 存的是微秒,逐位比会永远配不上。
func timeKey(t time.Time, place string) string {
return fmt.Sprintf("%d|%s", t.Unix(), place)
}
func visitFieldChanges(sv StoredVisit, v Visit, placeKey, placeName, actName map[int]string) []FieldChange {
var out []FieldChange
add := func(label, from, to string) {
if from != to {
out = append(out, FieldChange{Label: label, From: from, To: to})
}
}
if !sameTime(sv.Arrival, v.Arrival) {
add("到达时间", fmtTime(sv.Arrival), fmtTime(v.Arrival))
}
if !sameTime(sv.Departure, v.Departure) {
add("离开时间", fmtTime(sv.Departure), fmtTime(v.Departure))
}
// 地点按坐标判、显示名称;活动按名称判。都不能比 src_pk——两边编号空间不同。
bp, bn := "", ""
if v.LocationSrcPK != nil {
bp, bn = placeKey[*v.LocationSrcPK], placeName[*v.LocationSrcPK]
}
// 地点按坐标判、显示名称:名称被「补全地址」反查改过,不算换了地点
if sv.PlaceKey != bp {
out = append(out, FieldChange{Label: "地点", From: sv.PlaceName, To: bn})
}
bl := ""
if v.ActivitySrcPK != nil {
bl = actName[*v.ActivitySrcPK]
}
add("活动", sv.ActivityName, bl)
add("备注", sv.Remark, v.Remark)
add("表情", sv.Emoji, v.Emoji)
add("收藏", fmtBool(sv.Bookmarked), fmtBool(v.Bookmarked))
return out
}
// placeLabel 给没有名字的地点一个可读的占位。
func placeLabel(name string, lat, lon float64) string {
if name != "" {
return name
}
return fmt.Sprintf("%.5f, %.5f", lat, lon)
}
// DiffPlaces 对齐地点。坐标优先(改名不影响配对),配不上再按名称找一遍
// (覆盖「站点把坐标纠偏过」这种),两样都对不上才算新增。
func DiffPlaces(stored []StoredPlace, b *Backup) PlaceDiff {
byCoord := make(map[string]StoredPlace, len(stored))
byName := map[string]StoredPlace{}
for _, sp := range stored {
if sp.Lat != 0 || sp.Lon != 0 {
byCoord[PlaceCoordKey(sp.Lat, sp.Lon)] = sp
}
if _, dup := byName[sp.Name]; !dup && sp.Name != "" {
byName[sp.Name] = sp
}
}
used := map[int]bool{}
var d PlaceDiff
classify := func(sp StoredPlace, l Location) {
used[sp.SrcPK] = true
if fields := placeFieldChanges(sp, l); len(fields) > 0 {
d.Changed = append(d.Changed, PlaceConflict{Backup: l, DB: sp, Fields: fields})
} else {
d.Same++
}
}
for _, l := range b.Locations {
if sp, ok := byCoord[PlaceCoordKey(l.Lat, l.Lon)]; ok && !used[sp.SrcPK] {
classify(sp, l)
continue
}
if sp, ok := byName[l.Name]; ok && !used[sp.SrcPK] {
classify(sp, l)
continue
}
d.Added = append(d.Added, l)
}
for _, sp := range stored {
if !used[sp.SrcPK] {
d.Removed = append(d.Removed, sp)
}
}
sort.Slice(d.Added, func(i, j int) bool { return d.Added[i].Name < d.Added[j].Name })
sort.Slice(d.Removed, func(i, j int) bool { return d.Removed[i].Name < d.Removed[j].Name })
sort.Slice(d.Changed, func(i, j int) bool { return d.Changed[i].DB.Name < d.Changed[j].DB.Name })
return d
}
// placeFieldChanges 比地点上除对齐键之外的字段。名称也算——按坐标配上之后,
// 「名字不一样」正是站长最想看到的那类差异。
func placeFieldChanges(sp StoredPlace, l Location) []FieldChange {
var out []FieldChange
add := func(label, from, to string) {
if from != to {
out = append(out, FieldChange{Label: label, From: from, To: to})
}
}
add("名称", sp.Name, l.Name)
add("纬度", coordStr(sp.Lat), coordStr(l.Lat))
add("经度", coordStr(sp.Lon), coordStr(l.Lon))
add("省份", sp.Province, l.Province)
add("城市", sp.City, l.City)
add("分类", sp.POICategory, l.POICategory)
return out
}
// DiffActivities 优先按 UUID 对齐;库里没有 UUID 的(站点新建)退回名称匹配,
// 匹配不上就算「库里有、备份没有」。
func DiffActivities(stored []StoredActivity, b *Backup) ActivityDiff {
byUUID := map[string]StoredActivity{}
byName := map[string]StoredActivity{}
for _, sa := range stored {
if sa.UUID != "" {
byUUID[sa.UUID] = sa
}
if _, dup := byName[sa.Name]; !dup {
byName[sa.Name] = sa
}
}
var d ActivityDiff
matched := map[int]bool{}
for _, a := range b.Activities {
sa, ok := byUUID[b.ActivityUUID[a.SrcPK]]
if !ok {
if sa, ok = byName[a.Name]; !ok {
d.Added = append(d.Added, a)
continue
}
}
matched[sa.SrcPK] = true
if sa.Name != a.Name {
d.Changed = append(d.Changed, ActivityConflict{Backup: a, DB: sa,
Fields: []FieldChange{{Label: "名称", From: sa.Name, To: a.Name}}})
} else {
d.Same++
}
}
for _, sa := range stored {
if !matched[sa.SrcPK] {
d.Removed = append(d.Removed, sa)
}
}
sort.Slice(d.Added, func(i, j int) bool { return d.Added[i].Name < d.Added[j].Name })
sort.Slice(d.Removed, func(i, j int) bool { return d.Removed[i].Name < d.Removed[j].Name })
return d
}
func sortVisits(d VisitDiff) {
tv := func(t *time.Time) int64 {
if t == nil {
return 0
}
return t.Unix()
}
sort.Slice(d.Added, func(i, j int) bool { return tv(d.Added[i].Arrival) < tv(d.Added[j].Arrival) })
sort.Slice(d.Removed, func(i, j int) bool { return tv(d.Removed[i].Arrival) < tv(d.Removed[j].Arrival) })
sort.Slice(d.Changed, func(i, j int) bool { return tv(d.Changed[i].Backup.Arrival) < tv(d.Changed[j].Backup.Arrival) })
}
func sameTime(a, b *time.Time) bool {
if a == nil || b == nil {
return a == nil && b == nil
}
return a.Equal(*b)
}
func fmtTime(t *time.Time) string {
if t == nil {
return "—"
}
return t.Local().Format("2006-01-02 15:04")
}
func fmtBool(v bool) string {
if v {
return "是"
}
return "否"
}
// Counts 汇总各实体的三类条数,供概览展示。
type Counts struct {
Added, Removed, Changed, Same int
}
func (c Counts) Total() int { return c.Added + c.Removed + c.Changed + c.Same }
func VisitCounts(d VisitDiff) Counts {
return Counts{len(d.Added), len(d.Removed), len(d.Changed), d.Same}
}
func PlaceCounts(d PlaceDiff) Counts {
return Counts{len(d.Added), len(d.Removed), len(d.Changed), d.Same}
}
func ActivityCounts(d ActivityDiff) Counts {
return Counts{len(d.Added), len(d.Removed), len(d.Changed), d.Same}
}
+293
View File
@@ -0,0 +1,293 @@
package rond
import (
"testing"
"time"
)
func tp(s string) *time.Time {
t, err := time.Parse("2006-01-02 15:04", s)
if err != nil {
panic(err)
}
return &t
}
func ip(v int) *int { return &v }
// 两组编号刻意完全不同:rond 每次导出都会重排 Z_PK,对齐只能靠 UUID。
const (
uuidA = "11111111111111111111111111111111"
uuidB = "22222222222222222222222222222222"
uuidC = "33333333333333333333333333333333"
)
func sampleBackup() *Backup {
return &Backup{
Visits: []Visit{
{SrcPK: 555, Arrival: tp("2026-09-01 10:00"), Departure: tp("2026-09-01 11:00"),
LocationSrcPK: ip(3), ActivitySrcPK: ip(7)},
{SrcPK: 556, Arrival: tp("2026-09-02 10:00"), LocationSrcPK: ip(3)},
},
Locations: []Location{
{SrcPK: 3, Name: "家", Lat: 24.1, Lon: 109.1, City: "柳州市"},
{SrcPK: 4, Name: "公司", Lat: 24.2, Lon: 109.2, City: "柳州市"},
},
Activities: []Activity{
{SrcPK: 7, Name: "工作"},
{SrcPK: 8, Name: "散步"},
},
VisitUUID: map[int]string{555: uuidA, 556: uuidB},
ActivityUUID: map[int]string{7: uuidC},
}
}
func TestDiffVisitsAlignsByUUIDNotPK(t *testing.T) {
stored := []StoredVisit{
{SrcPK: 1, UUID: uuidA, Arrival: tp("2026-09-01 10:00"), Departure: tp("2026-09-01 11:00"),
PlaceKey: PlaceCoordKey(24.1, 109.1), ActivityName: "工作"},
{SrcPK: 2, UUID: uuidB, Arrival: tp("2026-09-02 10:00"),
PlaceKey: PlaceCoordKey(24.1, 109.1)},
}
d := DiffVisits(stored, sampleBackup())
if len(d.Added) != 0 || len(d.Removed) != 0 || len(d.Changed) != 0 {
t.Fatalf("两边一致时不该有差异: +%d -%d ~%d", len(d.Added), len(d.Removed), len(d.Changed))
}
if d.Same != 2 {
t.Fatalf("应配上 2 条,实际 %d", d.Same)
}
}
func TestDiffVisitsFieldChanges(t *testing.T) {
stored := []StoredVisit{{
SrcPK: 1, UUID: uuidA, Arrival: tp("2026-09-01 09:00"), Departure: tp("2026-09-01 11:00"),
PlaceKey: PlaceCoordKey(24.1, 109.1), ActivityName: "工作", Remark: "站点备注",
}}
d := DiffVisits(stored, sampleBackup())
if len(d.Changed) != 1 {
t.Fatalf("到达时间不同应算冲突,实际 %d 条", len(d.Changed))
}
f := d.Changed[0].Fields
if len(f) != 2 {
t.Fatalf("应报出到达时间与备注两项,实际 %v", f)
}
if f[0].Label != "到达时间" || f[1].Label != "备注" {
t.Fatalf("字段名不对: %v", f)
}
if f[1].From != "站点备注" || f[1].To != "" {
t.Fatalf("方向搞反了(From 应是库里的值): %+v", f[1])
}
}
// 站点新建的到访没有 UUID,而且它的 src_pk 很可能与真机新行的编号撞上——
// 绝不能因为编号相同就当成同一条。
func TestDiffVisitsIgnoresPKCollision(t *testing.T) {
stored := []StoredVisit{{
SrcPK: 555, UUID: "", Arrival: tp("2026-08-01 08:00"), UserAdded: true,
PlaceKey: PlaceCoordKey(30.0, 120.0),
}}
d := DiffVisits(stored, sampleBackup())
if len(d.Added) != 2 {
t.Fatalf("备份里两条都是新增,实际 %d", len(d.Added))
}
if len(d.Removed) != 1 {
t.Fatalf("站点补录那条应算「仅库里有」,实际 %d", len(d.Removed))
}
if !d.Removed[0].UserAdded {
t.Error("站点补录的标记要带上,页面上要区分开")
}
}
// 真机偶尔有不带 UUID 的到访,只按 UUID 配对会让同一条既算新增又算仅站内有。
func TestDiffVisitsFallsBackWhenUUIDMissing(t *testing.T) {
b := &Backup{
Visits: []Visit{
{SrcPK: 900, Arrival: tp("2026-09-01 10:00"), LocationSrcPK: ip(3)},
{SrcPK: 901, Arrival: tp("2026-09-02 10:00"), LocationSrcPK: ip(3)},
},
Locations: []Location{{SrcPK: 3, Name: "家", Lat: 24.1, Lon: 109.1}},
VisitUUID: map[int]string{}, // 一条都没有 UUID
ActivityUUID: map[int]string{},
}
stored := []StoredVisit{
{SrcPK: 1, Arrival: tp("2026-09-01 10:00"), PlaceKey: PlaceCoordKey(24.1, 109.1)},
{SrcPK: 2, Arrival: tp("2026-09-02 10:00"), PlaceKey: PlaceCoordKey(24.1, 109.1)},
}
d := DiffVisits(stored, b)
if len(d.Added) != 0 || len(d.Removed) != 0 {
t.Fatalf("没有 UUID 时应按「到达时间 + 地点」兜底配对: +%d -%d", len(d.Added), len(d.Removed))
}
if d.Same != 2 {
t.Fatalf("两条都该配上,实际 %d", d.Same)
}
}
// 兜底配对要认得出「同一条但时间被改过」——时间不同就配不上,此时按新增 + 仅站内有报出,
// 总比静默当成同一条好。
func TestDiffVisitsFallbackNeedsSameTime(t *testing.T) {
b := &Backup{
Visits: []Visit{{SrcPK: 900, Arrival: tp("2026-09-01 10:00"), LocationSrcPK: ip(3)}},
Locations: []Location{{SrcPK: 3, Name: "家", Lat: 24.1, Lon: 109.1}},
VisitUUID: map[int]string{},
}
stored := []StoredVisit{{SrcPK: 1, Arrival: tp("2026-09-01 11:00"), PlaceKey: PlaceCoordKey(24.1, 109.1)}}
d := DiffVisits(stored, b)
if len(d.Added) != 1 || len(d.Removed) != 1 {
t.Fatalf("时间对不上时该分别报出: +%d -%d", len(d.Added), len(d.Removed))
}
}
func TestDiffVisitsSkipsRowsWithoutArrival(t *testing.T) {
b := sampleBackup()
b.Visits = append(b.Visits, Visit{SrcPK: 557, LocationSrcPK: ip(3)})
d := DiffVisits(nil, b)
if len(d.Added) != 2 {
t.Fatalf("没有到达时间的行导入时就被跳过,不该算新增,实际 %d", len(d.Added))
}
}
func TestDiffPlacesByKey(t *testing.T) {
b := sampleBackup()
stored := []StoredPlace{
{SrcPK: 10, Key: PlaceCoordKey(24.1, 109.1), Name: "家", Lat: 24.1, Lon: 109.1, City: "柳州市"},
}
d := DiffPlaces(stored, b)
if len(d.Added) != 1 || d.Added[0].Name != "公司" {
t.Fatalf("应只有「公司」是新增: %+v", d.Added)
}
if len(d.Removed) != 0 || len(d.Changed) != 0 || d.Same != 1 {
t.Fatalf("「家」应配上且无差异: same=%d changed=%d removed=%d", d.Same, len(d.Changed), len(d.Removed))
}
}
func TestDiffPlacesReportsFieldChanges(t *testing.T) {
b := sampleBackup()
stored := []StoredPlace{
{SrcPK: 10, Key: PlaceCoordKey(24.1, 109.1), Name: "家", Lat: 24.1, Lon: 109.1, City: "桂林市"},
}
d := DiffPlaces(stored, b)
if len(d.Changed) != 1 {
t.Fatalf("城市不同应算冲突,实际 %d", len(d.Changed))
}
if f := d.Changed[0].Fields; len(f) != 1 || f[0].Label != "城市" || f[0].From != "桂林市" || f[0].To != "柳州市" {
t.Fatalf("字段差异不对: %+v", f)
}
}
func TestDiffActivities(t *testing.T) {
b := sampleBackup()
stored := []StoredActivity{
{SrcPK: 20, Name: "工作", UUID: uuidC},
{SrcPK: 21, Name: "站点自建"},
}
d := DiffActivities(stored, b)
if len(d.Added) != 1 || d.Added[0].Name != "散步" {
t.Fatalf("「散步」应是新增: %+v", d.Added)
}
if len(d.Removed) != 1 || d.Removed[0].Name != "站点自建" {
t.Fatalf("站点自建的活动应算仅库里有: %+v", d.Removed)
}
if d.Same != 1 {
t.Fatalf("「工作」应配上,实际 same=%d", d.Same)
}
}
// 活动改了名:UUID 还在,就该认出来是同一个,只报名称变化,而不是「删一个加一个」。
func TestDiffActivitiesRenameByUUID(t *testing.T) {
b := sampleBackup()
stored := []StoredActivity{{SrcPK: 20, Name: "上班", UUID: uuidC}}
d := DiffActivities(stored, b)
if len(d.Added) != 1 || len(d.Removed) != 0 {
t.Fatalf("只该多出「散步」,「上班」应认出是同一个: +%d -%d", len(d.Added), len(d.Removed))
}
if len(d.Changed) != 1 || d.Changed[0].Fields[0].From != "上班" || d.Changed[0].Fields[0].To != "工作" {
t.Fatalf("应按 UUID 认出重命名: %+v", d.Changed)
}
}
func TestPlaceCoordKeyStable(t *testing.T) {
a := PlaceCoordKey(24.1, 109.1)
if a != PlaceCoordKey(24.1, 109.1) {
t.Fatal("同样的输入应得到同样的键")
}
if a == PlaceCoordKey(24.1, 109.2) {
t.Fatal("坐标不同不该算同一个地点")
}
}
// 站点把地点反查改名(坐标没动)时,到访层面不该报「换了地点」——
// 实测里这一条会让 17 条到访全变成假冲突。
func TestDiffVisitsIgnoresPlaceRename(t *testing.T) {
b := &Backup{
Visits: []Visit{{SrcPK: 5, Arrival: tp("2026-09-01 10:00"), LocationSrcPK: ip(3)}},
Locations: []Location{{SrcPK: 3, Name: "河池市国民体质监测中心", Lat: 24.691827, Lon: 108.035268}},
VisitUUID: map[int]string{5: uuidA},
}
stored := []StoredVisit{{
SrcPK: 1, UUID: uuidA, Arrival: tp("2026-09-01 10:00"),
PlaceKey: PlaceCoordKey(24.691827, 108.035268), PlaceName: "体育馆美食街",
}}
d := DiffVisits(stored, b)
if len(d.Changed) != 0 {
t.Fatalf("只改了地点名不该算到访冲突: %+v", d.Changed[0].Fields)
}
if d.Same != 1 {
t.Fatalf("应算作一致,实际 same=%d", d.Same)
}
}
// 反之,地点层面要把改名报出来。
func TestDiffPlacesReportsRename(t *testing.T) {
b := &Backup{Locations: []Location{
{SrcPK: 3, Name: "河池市国民体质监测中心", Lat: 24.691827, Lon: 108.035268, City: "河池市"},
}}
stored := []StoredPlace{{
SrcPK: 10, Key: PlaceCoordKey(24.691827, 108.035268), Name: "体育馆美食街",
Lat: 24.691827, Lon: 108.035268, City: "河池市",
}}
d := DiffPlaces(stored, b)
if len(d.Added) != 0 || len(d.Removed) != 0 {
t.Fatalf("坐标一样就是同一个地点: +%d -%d", len(d.Added), len(d.Removed))
}
if len(d.Changed) != 1 {
t.Fatalf("改名应报成一条冲突,实际 %d", len(d.Changed))
}
f := d.Changed[0].Fields
if len(f) != 1 || f[0].Label != "名称" || f[0].From != "体育馆美食街" || f[0].To != "河池市国民体质监测中心" {
t.Fatalf("字段差异不对: %+v", f)
}
}
// 没有坐标的地点不能都落到同一个键上,否则两个不同地点会互相认领。
func TestDiffPlacesWithoutCoordDoesNotCollide(t *testing.T) {
b := &Backup{Locations: []Location{{SrcPK: 3, Name: "甲地"}}}
stored := []StoredPlace{
{SrcPK: 10, Name: "甲地"},
{SrcPK: 11, Name: "乙地"},
}
d := DiffPlaces(stored, b)
if len(d.Added) != 0 || len(d.Removed) != 1 {
t.Fatalf("应只配上同名的「甲地」: +%d -%d", len(d.Added), len(d.Removed))
}
if d.Removed[0].Name != "乙地" {
t.Fatalf("没配上的应是「乙地」,实际 %q", d.Removed[0].Name)
}
if d.Same != 1 {
t.Fatalf("「甲地」应算一致,实际 %d", d.Same)
}
}
// 站点把坐标纠偏过(名称没变)也要认得出是同一个地点。
func TestDiffPlacesMatchesByNameWhenCoordMoved(t *testing.T) {
b := &Backup{Locations: []Location{{SrcPK: 3, Name: "家", Lat: 24.100001, Lon: 109.100001}}}
stored := []StoredPlace{{
SrcPK: 10, Key: PlaceCoordKey(24.1, 109.1), Name: "家", Lat: 24.1, Lon: 109.1,
}}
d := DiffPlaces(stored, b)
if len(d.Added) != 0 || len(d.Removed) != 0 {
t.Fatalf("名称一样就该配上: +%d -%d", len(d.Added), len(d.Removed))
}
if len(d.Changed) != 1 || d.Changed[0].Fields[0].Label != "纬度" {
t.Fatalf("应报出坐标差异: %+v", d.Changed)
}
}
+6 -3
View File
@@ -1332,9 +1332,12 @@ var uuidNamespace = [16]byte{0x6b, 0x1c, 0x9a, 0x2e, 0x42, 0x7d, 0x4f, 0x16,
// uuidV5 由命名空间与名字生成 RFC 4122 版本 5 的 UUID(16 字节原始形式)。
//
// rond 给每条到访、每个活动都存一个 16 字节 UUID(ZIDENTIFIER_ / ZUID_,真机里全是 v4),
// 站点这边没有对应字段。这里用确定性算法补:名字只取表名 + src_pk(src_pk 跨快照稳定),
// 于是同一条记录每次导出拿到的都是同一个 UUID——若改成随机,每次导出的包都会让 rond
// 把同一批记录当成新的。
// 站点新建的行没有真机 UUID 可继承。这里用确定性算法补:名字取表名 + src_pk
// (src_pk 在**同一个数据集内**稳定,够用了),于是同一条记录每次导出拿到的都是同一个
// UUID——若改成随机,每次导出的包都会让 rond 把同一批记录当成新的。
//
// 注意这只保证「同一数据集内幂等」,不是跨备份的稳定标识:rond 那边每次导入都会重排
// Z_PK,真正的跨备份对齐键是它原生的 ZIDENTIFIER_(见 diff.go)。
func uuidV5(name string) []byte {
h := sha1.New()
h.Write(uuidNamespace[:])
+170
View File
@@ -0,0 +1,170 @@
package rond
// 合并计划:把「差异画像 + 站长在页面上做的选择」翻译成一份可执行的改动清单。
// 这里只做判定,不碰数据库、也不写任何东西。
// 站长对一条冲突的处理方式。
const (
PickBackup = "backup" // 用备份(实机)的值覆盖站内
PickDB = "db" // 保留站内的值
PickSkip = "skip" // 这条不动
)
// Choices 是差异页提交上来的选择。冲突是逐条选的,键分别是到访 UUID、
// 地点坐标键、活动 UUID。
type Choices struct {
AddVisits bool // 把备份里多出来的到访并进来
AddPlaces bool
AddActs bool
VisitClash map[string]string
PlaceClash map[string]string
ActClash map[string]string
// 删除:「仅站内有」的记录按库里编号勾选,默认一个都不删。
// 键是 src_pk 而不是 UUID——站点补录的行压根没有 UUID。
DelVisits map[int]bool
DelPlaces map[int]bool
DelActs map[int]bool
}
func (c Choices) visitPick(key string) string { return pickOf(c.VisitClash, key) }
func (c Choices) placePick(key string) string { return pickOf(c.PlaceClash, key) }
func (c Choices) activityPick(key string) string { return pickOf(c.ActClash, key) }
func pickOf(m map[string]string, key string) string {
switch m[key] {
case PickBackup, PickDB, PickSkip:
return m[key]
}
return PickDB // 没提交或提交了怪值:默认保留站内的,不误伤
}
// VisitPatch 表示用备份侧的值覆盖库里的那一条。
type VisitPatch struct {
UUID string // 对齐键
V Visit
}
// PlacePatch 同理,键是坐标键。
type PlacePatch struct {
CoordKey string
L Location
}
// ActivityPatch 键是活动 UUID(没有 UUID 时退回名称)。
type ActivityPatch struct {
Key string
A Activity
}
// MergePlan 是一次合并要做的全部改动。
type MergePlan struct {
AddPlaces []Location
UpdPlaces []PlacePatch
DelPlaces []StoredPlace
AddActs []Activity
UpdActs []ActivityPatch
DelActs []StoredActivity
AddVisits []Visit
UpdVisits []VisitPatch
DelVisits []StoredVisit
}
// Empty 报告这份计划什么事都不用做。
func (p MergePlan) Empty() bool {
return len(p.AddPlaces)+len(p.UpdPlaces)+len(p.DelPlaces)+
len(p.AddActs)+len(p.UpdActs)+len(p.DelActs)+
len(p.AddVisits)+len(p.UpdVisits)+len(p.DelVisits) == 0
}
// BuildMergePlan 按选择把差异筛成改动清单。
func BuildMergePlan(vd VisitDiff, pd PlaceDiff, ad ActivityDiff, c Choices) MergePlan {
var p MergePlan
if c.AddPlaces {
p.AddPlaces = pd.Added
}
for _, cf := range pd.Changed {
if c.placePick(cf.DB.Key) == PickBackup {
p.UpdPlaces = append(p.UpdPlaces, PlacePatch{CoordKey: cf.DB.Key, L: cf.Backup})
}
}
for _, sp := range pd.Removed {
if c.DelPlaces[sp.SrcPK] {
p.DelPlaces = append(p.DelPlaces, sp)
}
}
if c.AddActs {
p.AddActs = ad.Added
}
for _, cf := range ad.Changed {
if c.activityPick(activityKey(cf.DB)) == PickBackup {
p.UpdActs = append(p.UpdActs, ActivityPatch{Key: activityKey(cf.DB), A: cf.Backup})
}
}
for _, sa := range ad.Removed {
if c.DelActs[sa.SrcPK] {
p.DelActs = append(p.DelActs, sa)
}
}
if c.AddVisits {
p.AddVisits = vd.Added
// 新到访可能挂在一个「备份里新出现、库里还没有」的地点/活动上:
// 不把它们一起建出来,到访的 place_id / activity_id 就无从映射,只能整条丢掉。
p.AddPlaces = appendMissingPlaces(p.AddPlaces, pd.Added)
p.AddActs = appendMissingActs(p.AddActs, ad.Added)
}
for _, cf := range vd.Changed {
if c.visitPick(cf.DB.UUID) == PickBackup {
p.UpdVisits = append(p.UpdVisits, VisitPatch{UUID: cf.DB.UUID, V: cf.Backup})
}
}
for _, sv := range vd.Removed {
if c.DelVisits[sv.SrcPK] {
p.DelVisits = append(p.DelVisits, sv)
}
}
return p
}
func appendMissingPlaces(have, more []Location) []Location {
seen := make(map[int]bool, len(have))
for _, l := range have {
seen[l.SrcPK] = true
}
for _, l := range more {
if !seen[l.SrcPK] {
have = append(have, l)
seen[l.SrcPK] = true
}
}
return have
}
func appendMissingActs(have, more []Activity) []Activity {
seen := make(map[int]bool, len(have))
for _, a := range have {
seen[a.SrcPK] = true
}
for _, a := range more {
if !seen[a.SrcPK] {
have = append(have, a)
seen[a.SrcPK] = true
}
}
return have
}
// activityKey 是活动冲突在页面上的标识:优先 UUID(改名也能对上),没有才用名称。
func activityKey(sa StoredActivity) string {
if sa.UUID != "" {
return sa.UUID
}
return "n:" + sa.Name
}
// ActivityKey 供页面与处理函数共用同一套键。
func ActivityKey(sa StoredActivity) string { return activityKey(sa) }
+76
View File
@@ -0,0 +1,76 @@
package rond
import "testing"
func TestBuildMergePlanPicks(t *testing.T) {
vd := VisitDiff{
Added: []Visit{{SrcPK: 900, Arrival: tp("2026-09-09 10:00")}},
Removed: []StoredVisit{{SrcPK: 5, UUID: uuidA}},
Changed: []VisitConflict{
{Backup: Visit{SrcPK: 901, Arrival: tp("2026-09-01 12:00")}, DB: StoredVisit{SrcPK: 1, UUID: uuidA}},
{Backup: Visit{SrcPK: 902, Arrival: tp("2026-09-02 12:00")}, DB: StoredVisit{SrcPK: 2, UUID: uuidB}},
},
}
pd := PlaceDiff{Added: []Location{{SrcPK: 3, Name: "新地点"}}}
ad := ActivityDiff{Added: []Activity{{SrcPK: 8, Name: "散步"}}}
p := BuildMergePlan(vd, pd, ad, Choices{
AddVisits: true,
AddPlaces: true,
AddActs: false,
VisitClash: map[string]string{uuidA: PickBackup, uuidB: PickDB},
})
if len(p.AddVisits) != 1 || p.AddVisits[0].SrcPK != 900 {
t.Fatalf("新增到访应合入一条: %+v", p.AddVisits)
}
if len(p.UpdVisits) != 1 || p.UpdVisits[0].UUID != uuidA {
t.Fatalf("只有选了「用实机」的那条才该更新: %+v", p.UpdVisits)
}
if len(p.AddPlaces) != 1 {
t.Fatalf("勾了「并进来」就该带上新增地点: %d", len(p.AddPlaces))
}
// 即使没单独勾活动,合入到访时也要把它可能挂着的「库里还没有的活动」带上,
// 否则那条到访的 activity_id 无从映射,只能整条丢掉。
if len(p.AddActs) != 1 || p.AddActs[0].Name != "散步" {
t.Fatalf("合入到访应连带它需要的活动: %+v", p.AddActs)
}
// 「仅站内有」永远不进计划:删除是单独的事,绝不顺手做
if len(p.UpdPlaces)+len(p.UpdActs) != 0 {
t.Fatal("不该有地点/活动更新")
}
}
// 没提交的键、或提交了怪值,一律当成「保留站内」——不误伤。
func TestBuildMergePlanDefaultsToKeepDB(t *testing.T) {
vd := VisitDiff{Changed: []VisitConflict{
{Backup: Visit{SrcPK: 901}, DB: StoredVisit{SrcPK: 1, UUID: uuidA}},
}}
p := BuildMergePlan(vd, PlaceDiff{}, ActivityDiff{}, Choices{VisitClash: map[string]string{uuidA: "wat"}})
if len(p.UpdVisits) != 0 {
t.Fatalf("怪值不该被当成「用实机」: %+v", p.UpdVisits)
}
if !p.Empty() {
t.Fatal("什么都没选,计划应当是空的")
}
}
// 冲突选了「跳过」也不动。
func TestBuildMergePlanSkip(t *testing.T) {
vd := VisitDiff{Changed: []VisitConflict{
{Backup: Visit{SrcPK: 901}, DB: StoredVisit{SrcPK: 1, UUID: uuidA}},
}}
p := BuildMergePlan(vd, PlaceDiff{}, ActivityDiff{}, Choices{VisitClash: map[string]string{uuidA: PickSkip}})
if !p.Empty() {
t.Fatalf("跳过就不该产生改动: %+v", p)
}
}
// 活动改了名也要能用 UUID 当键对得上。
func TestActivityKeyPrefersUUID(t *testing.T) {
if got := ActivityKey(StoredActivity{SrcPK: 1, Name: "上班", UUID: uuidC}); got != uuidC {
t.Fatalf("有 UUID 就用 UUID,实际 %q", got)
}
if got := ActivityKey(StoredActivity{SrcPK: 1, Name: "上班"}); got != "n:上班" {
t.Fatalf("没有 UUID 退回名称,实际 %q", got)
}
}
+36
View File
@@ -156,6 +156,13 @@ type Backup struct {
// RawRows 是各实体在 Core Data 里的完整原始行:「表名 -> Z_PK -> 列名 -> 值」。
// 站点只映射了其中一部分字段,留档让导出能逐列还原,不必依赖基线库补剩下的列。
RawRows map[string]map[int]map[string]any
// VisitUUID / ActivityUUID 是真机给每条到访、每个活动存的 16 字节 UUID(小写十六进制)。
// **这是跨备份唯一稳定的对齐键**:rond 每次导出/同步都会重排 Z_PK——实测两份备份里
// 822 条到访的 UUID 完全一致、Z_PK 100% 不同——所以「这份备份」与「库里的那一份」
// 之间只能按它对齐。地点/标签/交通方式没有 UUID,只能退到名称与坐标。
VisitUUID map[int]string
ActivityUUID map[int]string
}
// Open 解压备份包到 dir,返回可用的 SQLite 连接。
@@ -465,9 +472,38 @@ func Parse(db *sql.DB) (*Backup, error) {
} else {
log.Printf("抓取原始行留档失败(导出将退回基线补列): %v", rerr)
}
// 对齐键:导出不依赖它,但「两份数据比对」全靠它,缺了就只能放弃逐条对齐
if b.VisitUUID, err = uuidColumn(db, "ZVISIT", "ZIDENTIFIER_"); err != nil {
return nil, fmt.Errorf("读取到访 UUID: %w", err)
}
if b.ActivityUUID, err = uuidColumn(db, "ZACTIVITY", "ZUID_"); err != nil {
return nil, fmt.Errorf("读取活动 UUID: %w", err)
}
return b, nil
}
// uuidColumn 读出某表某一列的 UUID,返回 Z_PK -> 小写十六进制。
// 列不存在(很老的备份)返回空表而不是错误;全零或空的当作没有。
func uuidColumn(db *sql.DB, table, col string) (map[int]string, error) {
rows, err := db.Query(fmt.Sprintf(`SELECT Z_PK, hex(%s) FROM %s`, col, table))
if err != nil {
return nil, nil
}
defer rows.Close()
out := map[int]string{}
for rows.Next() {
var pk int
var h sql.NullString
if err := rows.Scan(&pk, &h); err != nil {
return nil, err
}
if h.Valid && strings.Trim(h.String, "0") != "" {
out[pk] = strings.ToLower(h.String)
}
}
return out, rows.Err()
}
// rawTables 是需要整行留档的 Core Data 表(站点管理的实体 + 标签分组)。
var rawTables = []string{"ZLOCATION", "ZVISIT", "ZMOVEMENT", "ZHOURLYWEATHER", "ZACTIVITY", "ZTAG", "ZTAGGROUP", "ZTRANSPORT"}
+7 -5
View File
@@ -666,7 +666,8 @@ const placeCols = `p.id, COALESCE(p.name,''), COALESCE(p.poi_category,''), COALE
EXISTS (SELECT 1 FROM visits v3 WHERE v3.place_id=p.id AND v3.is_home),
EXISTS (SELECT 1 FROM visits v4 WHERE v4.place_id=p.id AND v4.is_work),
p.src_pk,
EXISTS (SELECT 1 FROM place_images pi WHERE pi.dataset_id=p.dataset_id AND pi.src_pk=p.src_pk)`
EXISTS (SELECT 1 FROM place_images pi WHERE pi.place_key =
to_char(p.lat,'FM999990.000000')||'|'||to_char(p.lon,'FM999990.000000'))`
// PlaceCols 与 ScanPlace 配对,供包外构造只取单个地点的查询。
const PlaceCols = placeCols
@@ -708,11 +709,12 @@ func (d DB) Places(ctx context.Context, f Filter, sortBy string, limit, offset i
args := append([]any{f.DatasetID}, inner.args...)
base := `FROM places p WHERE p.dataset_id = $1 AND EXISTS (SELECT 1 FROM visits v WHERE ` + inner.expr() + `)`
if f.Verdict != 0 {
// 结论存在 place_votes,按 src_pk 关联;占位符接在内层条件之后。
// dataset_id 也要对上:src_pk 只在单个数据集内唯一。
// 结论挂坐标键(换数据集也要跟得住),按本数据集的地点圈定范围;
// 占位符接在内层条件之后。
args = append(args, f.Verdict)
base += fmt.Sprintf(` AND EXISTS (SELECT 1 FROM place_votes pv
WHERE pv.src_pk = p.src_pk AND pv.dataset_id = p.dataset_id AND pv.verdict = $%d)`, len(args))
base += fmt.Sprintf(` AND EXISTS (SELECT 1 FROM place_votes pv WHERE pv.place_key =
to_char(p.lat,'FM999990.000000')||'|'||to_char(p.lon,'FM999990.000000')
AND pv.verdict = $%d)`, len(args))
}
q := `SELECT ` + placeCols + ` ` + base + ` ORDER BY ` + order + fmt.Sprintf(` LIMIT %d OFFSET %d`, limit, offset)
+78 -34
View File
@@ -2,14 +2,23 @@
// - place_votes 站长对地点的官方结论(单值)
// - place_visitor_votes 访客点赞 / 点踩(一人一票,可改可撤)
//
// 两者都按 rond 的地点 ID(src_pk)存储,重新上传备份、地点换了新 id 也不丢。
// 站长的「官方结论」按**坐标键**(place_key)存储:src_pk 只在单个数据集内有效,
// 换个数据集同编号就是另一个地点,结论会挂错人。
// 访客票仍按 src_pk 存——那张表没有 dataset_id,无从判断票是投给哪个数据集的,
// 迁移有歧义,留待单独处理。
package votes
import (
"context"
"database/sql"
"fmt"
)
// placeKeyExpr 把 places p 这一行的坐标换算成 place_key。
// **必须与 Go 侧的 rond.PlaceCoordKey 以及 notes 包里的同款表达式逐位一致**:
// fmt.Sprintf("%.6f|%.6f") ↔ to_char(x,'FM999990.000000')||'|'||to_char(y,'FM999990.000000')
const placeKeyExpr = `to_char(p.lat,'FM999990.000000')||'|'||to_char(p.lon,'FM999990.000000')`
// VerdictRecommend / VerdictAvoid 是官方结论的取值。
const (
VerdictAvoid = -1
@@ -24,11 +33,13 @@ type Tally struct {
Down int `json:"down"`
}
// Verdicts 返回某个数据集的站长结论:src_pk -> verdict(1/-1)。
// 必须按 dataset_id 过滤:src_pk 只在单个数据集内唯一。
// Verdicts 返回某个数据集的站长结论:该数据集里的 src_pk -> verdict(1/-1)。
func (s *Store) Verdicts(ctx context.Context, datasetID int64) (map[int]int, error) {
out := map[int]int{}
rows, err := s.DB.QueryContext(ctx, `SELECT src_pk, verdict FROM place_votes WHERE dataset_id=$1`, datasetID)
rows, err := s.DB.QueryContext(ctx, `
SELECT p.src_pk, v.verdict FROM places p
JOIN place_votes v ON v.place_key = `+placeKeyExpr+`
WHERE p.dataset_id=$1 AND p.lat IS NOT NULL AND p.lon IS NOT NULL`, datasetID)
if err != nil {
return out, err
}
@@ -46,8 +57,9 @@ func (s *Store) Verdicts(ctx context.Context, datasetID int64) (map[int]int, err
// Verdict 取单个地点的站长结论;未标注返回 0。
func (s *Store) Verdict(ctx context.Context, datasetID int64, srcPK int) (int, error) {
var v int
err := s.DB.QueryRowContext(ctx,
`SELECT verdict FROM place_votes WHERE dataset_id=$1 AND src_pk=$2`, datasetID, srcPK).Scan(&v)
err := s.DB.QueryRowContext(ctx, `SELECT v.verdict FROM place_votes v
JOIN places p ON v.place_key = `+placeKeyExpr+`
WHERE p.dataset_id=$1 AND p.src_pk=$2`, datasetID, srcPK).Scan(&v)
if err == sql.ErrNoRows {
return 0, nil
}
@@ -57,24 +69,46 @@ func (s *Store) Verdict(ctx context.Context, datasetID int64, srcPK int) (int, e
// SaveVerdict 设置站长结论;verdict 为 0 表示取消结论。
func (s *Store) SaveVerdict(ctx context.Context, userID, datasetID int64, srcPK, verdict int) error {
if verdict == 0 {
_, err := s.DB.ExecContext(ctx,
`DELETE FROM place_votes WHERE user_id=$1 AND dataset_id=$2 AND src_pk=$3`, userID, datasetID, srcPK)
_, err := s.DB.ExecContext(ctx, `DELETE FROM place_votes v USING places p
WHERE v.user_id=$1 AND p.dataset_id=$2 AND p.src_pk=$3
AND v.place_key = `+placeKeyExpr, userID, datasetID, srcPK)
return err
}
_, err := s.DB.ExecContext(ctx, `INSERT INTO place_votes (user_id, dataset_id, src_pk, verdict, updated_at)
VALUES ($1,$2,$3,$4,now())
ON CONFLICT (user_id, dataset_id, src_pk) DO UPDATE SET verdict=EXCLUDED.verdict, updated_at=now()`,
// 先按坐标键算出这条地点该挂在哪,找不到该地点就什么也不写
_, err := s.DB.ExecContext(ctx, `INSERT INTO place_votes
(user_id, place_key, dataset_id, src_pk, verdict, updated_at)
SELECT $1, `+placeKeyExpr+`, $2, $3, $4, now()
FROM places p
WHERE p.dataset_id=$2 AND p.src_pk=$3 AND p.lat IS NOT NULL AND p.lon IS NOT NULL
ON CONFLICT (user_id, place_key) DO UPDATE SET
verdict=EXCLUDED.verdict, dataset_id=EXCLUDED.dataset_id,
src_pk=EXCLUDED.src_pk, updated_at=now()`,
userID, datasetID, srcPK, verdict)
return err
}
// Tallies 返回全部地点的访客票数。票表规模很小(个人站点),一次读全更省事,
// 也避免给 pgx 传数组参数带来的类型歧义。
func (s *Store) Tallies(ctx context.Context) (map[int]Tally, error) {
// placeKeyOf 把某个数据集里的地点编号换算成坐标键——访客票也是挂在坐标上的,
// 换数据集之后票仍然跟着同一个地点。
func (s *Store) placeKeyOf(ctx context.Context, datasetID int64, srcPK int) (string, error) {
var key string
err := s.DB.QueryRowContext(ctx, `SELECT `+placeKeyExpr+` FROM places p
WHERE p.dataset_id=$1 AND p.src_pk=$2 AND p.lat IS NOT NULL AND p.lon IS NOT NULL`,
datasetID, srcPK).Scan(&key)
if err == sql.ErrNoRows {
return "", fmt.Errorf("地点不存在或没有坐标")
}
return key, err
}
// Tallies 返回某个数据集里各地点的访客票数:src_pk -> Tally。
// 票表规模很小(个人站点),一次读全更省事,也避免给 pgx 传数组参数带来的类型歧义。
func (s *Store) Tallies(ctx context.Context, datasetID int64) (map[int]Tally, error) {
out := map[int]Tally{}
rows, err := s.DB.QueryContext(ctx, `SELECT src_pk,
count(*) FILTER (WHERE vote=1), count(*) FILTER (WHERE vote=-1)
FROM place_visitor_votes GROUP BY src_pk`)
rows, err := s.DB.QueryContext(ctx, `SELECT p.src_pk,
count(*) FILTER (WHERE vv.vote=1), count(*) FILTER (WHERE vv.vote=-1)
FROM places p JOIN place_visitor_votes vv ON vv.place_key = `+placeKeyExpr+`
WHERE p.dataset_id=$1 AND p.lat IS NOT NULL AND p.lon IS NOT NULL
GROUP BY p.src_pk`, datasetID)
if err != nil {
return out, err
}
@@ -91,21 +125,24 @@ func (s *Store) Tallies(ctx context.Context) (map[int]Tally, error) {
}
// Tally 取单个地点的访客票数。
func (s *Store) Tally(ctx context.Context, srcPK int) (Tally, error) {
func (s *Store) Tally(ctx context.Context, datasetID int64, srcPK int) (Tally, error) {
var t Tally
err := s.DB.QueryRowContext(ctx, `SELECT count(*) FILTER (WHERE vote=1), count(*) FILTER (WHERE vote=-1)
FROM place_visitor_votes WHERE src_pk=$1`, srcPK).Scan(&t.Up, &t.Down)
err := s.DB.QueryRowContext(ctx, `SELECT
count(*) FILTER (WHERE vv.vote=1), count(*) FILTER (WHERE vv.vote=-1)
FROM place_visitor_votes vv JOIN places p ON vv.place_key = `+placeKeyExpr+`
WHERE p.dataset_id=$1 AND p.src_pk=$2`, datasetID, srcPK).Scan(&t.Up, &t.Down)
return t, err
}
// VoterAll 返回该访客投过的全部票:src_pk -> vote,供地图一次性标注「我投的」。
func (s *Store) VoterAll(ctx context.Context, voterKey string) (map[int]int, error) {
func (s *Store) VoterAll(ctx context.Context, datasetID int64, voterKey string) (map[int]int, error) {
out := map[int]int{}
if voterKey == "" {
return out, nil
}
rows, err := s.DB.QueryContext(ctx,
`SELECT src_pk, vote FROM place_visitor_votes WHERE voter_key=$1`, voterKey)
rows, err := s.DB.QueryContext(ctx, `SELECT p.src_pk, vv.vote
FROM place_visitor_votes vv JOIN places p ON vv.place_key = `+placeKeyExpr+`
WHERE p.dataset_id=$1 AND vv.voter_key=$2`, datasetID, voterKey)
if err != nil {
return out, err
}
@@ -121,13 +158,15 @@ func (s *Store) VoterAll(ctx context.Context, voterKey string) (map[int]int, err
}
// Voter 返回该访客对某地点的既有投票(1/-1/0)。voterKey 为空表示拿不到身份。
func (s *Store) Voter(ctx context.Context, srcPK int, voterKey string) int {
func (s *Store) Voter(ctx context.Context, datasetID int64, srcPK int, voterKey string) int {
if voterKey == "" {
return 0
}
var v int
if err := s.DB.QueryRowContext(ctx,
`SELECT vote FROM place_visitor_votes WHERE src_pk=$1 AND voter_key=$2`, srcPK, voterKey).Scan(&v); err != nil {
if err := s.DB.QueryRowContext(ctx, `SELECT vv.vote FROM place_visitor_votes vv
JOIN places p ON vv.place_key = `+placeKeyExpr+`
WHERE p.dataset_id=$1 AND p.src_pk=$2 AND vv.voter_key=$3`,
datasetID, srcPK, voterKey).Scan(&v); err != nil {
return 0
}
return v
@@ -135,30 +174,35 @@ func (s *Store) Voter(ctx context.Context, srcPK int, voterKey string) int {
// Cast 记录访客投票:未投过则新增;投同一票则撤销(再点一次取消);投另一票则改为该票。
// 返回更新后的票数与该访客最终的选择。
func (s *Store) Cast(ctx context.Context, srcPK int, voterKey string, vote int) (Tally, int, error) {
func (s *Store) Cast(ctx context.Context, datasetID int64, srcPK int, voterKey string, vote int) (Tally, int, error) {
key, err := s.placeKeyOf(ctx, datasetID, srcPK)
if err != nil {
return Tally{}, 0, err
}
var cur int
err := s.DB.QueryRowContext(ctx,
`SELECT vote FROM place_visitor_votes WHERE src_pk=$1 AND voter_key=$2`, srcPK, voterKey).Scan(&cur)
err = s.DB.QueryRowContext(ctx,
`SELECT vote FROM place_visitor_votes WHERE place_key=$1 AND voter_key=$2`, key, voterKey).Scan(&cur)
switch {
case err == sql.ErrNoRows:
_, err = s.DB.ExecContext(ctx,
`INSERT INTO place_visitor_votes (src_pk, voter_key, vote) VALUES ($1,$2,$3)`, srcPK, voterKey, vote)
`INSERT INTO place_visitor_votes (place_key, voter_key, vote, src_pk) VALUES ($1,$2,$3,$4)`,
key, voterKey, vote, srcPK)
cur = vote
case err != nil:
return Tally{}, 0, err
case cur == vote:
_, err = s.DB.ExecContext(ctx,
`DELETE FROM place_visitor_votes WHERE src_pk=$1 AND voter_key=$2`, srcPK, voterKey)
`DELETE FROM place_visitor_votes WHERE place_key=$1 AND voter_key=$2`, key, voterKey)
cur = 0
default:
_, err = s.DB.ExecContext(ctx,
`UPDATE place_visitor_votes SET vote=$3, updated_at=now() WHERE src_pk=$1 AND voter_key=$2`,
srcPK, voterKey, vote)
`UPDATE place_visitor_votes SET vote=$3, src_pk=$4, updated_at=now()
WHERE place_key=$1 AND voter_key=$2`, key, voterKey, vote, srcPK)
cur = vote
}
if err != nil {
return Tally{}, 0, err
}
t, err := s.Tally(ctx, srcPK)
t, err := s.Tally(ctx, datasetID, srcPK)
return t, cur, err
}
+346
View File
@@ -0,0 +1,346 @@
package web
import (
"context"
"database/sql"
"encoding/base64"
"encoding/hex"
"fmt"
"io"
"log"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"rond-with-you/internal/ingest"
"rond-with-you/internal/rond"
)
// rondbackup 的差异预演:把上传的备份与当前展示的数据集按稳定键对齐,只读地摆出
// 新增 / 仅库里有 / 冲突三类,**不写任何数据**。合并执行是下一步的事。
const backupStashExt = ".rondbackup"
// diffListLimit 是每类差异在页面上最多列出的条数。867 条全铺出来既慢又没人看,
// 超过就只给前若干条并注明。
const diffListLimit = 200
func (s *Server) backupStashDir() string {
return filepath.Join(s.cfg.DataDir, "backup-staging")
}
// adminBackupPreview 收下备份、暂存,跳到差异页。这里不校验备份内容——
// 差异页反正是要解析一遍的,失败时让它报错即可,免得同一份包解两次。
func (s *Server) adminBackupPreview(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxUploadBytes+1<<20)
if err := r.ParseMultipartForm(32 << 20); err != nil {
s.adminFlash(w, r, "err", "读取上传内容失败:"+err.Error())
return
}
file, hdr, err := r.FormFile("backup")
if err != nil {
s.adminFlash(w, r, "err", "请选择要分析的 .rondbackup 文件")
return
}
defer file.Close()
name := filepath.Base(hdr.Filename)
lower := strings.ToLower(name)
if !strings.HasSuffix(lower, ".rondbackup") && !strings.HasSuffix(lower, ".zip") {
s.adminFlash(w, r, "err", "文件类型不正确,请上传 rond 导出的 .rondbackup 备份")
return
}
data, err := io.ReadAll(io.LimitReader(file, maxUploadBytes+1))
if err != nil {
s.adminFlash(w, r, "err", "读取文件失败:"+err.Error())
return
}
if int64(len(data)) > maxUploadBytes {
s.adminFlash(w, r, "err", "备份文件过大,疑似不是 rond 备份")
return
}
token, err := stashPut(s.backupStashDir(), backupStashExt, data)
if err != nil {
s.serverError(w, r, err)
return
}
http.Redirect(w, r, "/admin/preview/"+token, http.StatusSeeOther)
}
// adminBackupPreviewPage 渲染差异页。每次都重新算:库里的数据随时可能被别处改掉。
func (s *Server) adminBackupPreviewPage(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
p, err := s.pageBase(r, "admin", "备份差异预览")
if err != nil {
s.serverError(w, r, err)
return
}
token := r.PathValue("token")
path, ok := stashPath(s.backupStashDir(), backupStashExt, token)
if !ok {
s.adminFlash(w, r, "err", "预览链接无效,请重新上传备份")
return
}
if _, err := os.Stat(path); err != nil {
s.adminFlash(w, r, "err", "这次预览已过期,请重新上传备份")
return
}
ds, err := s.datasetFor(ctx)
if err != nil {
s.serverError(w, r, err)
return
}
if ds == nil {
s.adminFlash(w, r, "err", "还没有可比较的数据集,请先直接导入一份备份")
return
}
tmp, err := ingest.TempDir(s.cfg.DataDir, "preview")
if err != nil {
s.serverError(w, r, err)
return
}
defer os.RemoveAll(tmp)
sqlDB, err := rond.Open(path, tmp)
if err != nil {
s.adminFlash(w, r, "err", "解析备份失败:"+err.Error())
return
}
backup, perr := rond.Parse(sqlDB)
sqlDB.Close()
if perr != nil {
s.adminFlash(w, r, "err", "读取备份数据失败:"+perr.Error())
return
}
storedVisits, err := s.loadStoredVisits(ctx, ds.ID)
if err != nil {
s.serverError(w, r, err)
return
}
storedPlaces, err := s.loadStoredPlaces(ctx, ds.ID)
if err != nil {
s.serverError(w, r, err)
return
}
storedActs, err := s.loadStoredActivities(ctx, ds.ID)
if err != nil {
s.serverError(w, r, err)
return
}
vd := rond.DiffVisits(storedVisits, backup)
pd := rond.DiffPlaces(storedPlaces, backup)
ad := rond.DiffActivities(storedActs, backup)
d := BackupDiffData{
Page: p, Token: token,
DatasetName: ds.Name,
VisitCounts: rond.VisitCounts(vd),
PlaceCounts: rond.PlaceCounts(pd),
ActCounts: rond.ActivityCounts(ad),
PlaceAdded: limitSlice(pd.Added, diffListLimit),
PlaceGone: limitSlice(pd.Removed, diffListLimit),
PlaceClash: limitSlice(pd.Changed, diffListLimit),
ActAdded: limitSlice(ad.Added, diffListLimit),
ActGone: limitSlice(ad.Removed, diffListLimit),
ActClash: limitSlice(ad.Changed, diffListLimit),
BackupVisits: len(backup.Visits), BackupPlaces: len(backup.Locations),
BackupActs: len(backup.Activities), BackupMovements: len(backup.Movements),
BackupWeather: len(backup.Weather), BackupRaw: backup.RawVisitNum,
StoredVisits: len(storedVisits), StoredPlaces: len(storedPlaces),
StoredActs: len(storedActs),
}
d.VisitAdded, d.VisitGone, d.VisitClash = visitRows(vd, backup, diffListLimit)
d.Truncated = anyTruncated(vd, pd, ad)
d.StoredMovements, d.StoredWeather, err = s.countOther(ctx, ds.ID)
if err != nil {
s.serverError(w, r, err)
return
}
d.Identical = d.VisitCounts.Added+d.VisitCounts.Removed+d.VisitCounts.Changed+
d.PlaceCounts.Added+d.PlaceCounts.Removed+d.PlaceCounts.Changed+
d.ActCounts.Added+d.ActCounts.Removed+d.ActCounts.Changed == 0
s.render(w, "admin/backup-preview", d)
}
// visitRows 把三类到访差异摊平成展示行。备份里的地点/活动只有编号,要借备份自己的
// 表翻成名称;库里那侧的名称在加载时就一并取出来了。
func visitRows(vd rond.VisitDiff, b *rond.Backup, limit int) (added, gone, clash []VisitRow) {
place, act := map[int]string{}, map[int]string{}
for _, l := range b.Locations {
place[l.SrcPK] = placeLabel(l.Name, l.Lat, l.Lon)
}
for _, a := range b.Activities {
act[a.SrcPK] = a.Name
}
fromBackup := func(v rond.Visit) VisitRow {
r := VisitRow{Arrival: v.Arrival, Departure: v.Departure, Remark: v.Remark, Emoji: v.Emoji}
if v.LocationSrcPK != nil {
r.Place = place[*v.LocationSrcPK]
}
if v.ActivitySrcPK != nil {
r.Activity = act[*v.ActivitySrcPK]
}
return r
}
for _, v := range limitSlice(vd.Added, limit) {
added = append(added, fromBackup(v))
}
for _, sv := range limitSlice(vd.Removed, limit) {
gone = append(gone, VisitRow{SrcPK: sv.SrcPK, Arrival: sv.Arrival, Departure: sv.Departure,
Place: sv.PlaceName, Activity: sv.ActivityName, Remark: sv.Remark, Emoji: sv.Emoji,
UserAdded: sv.UserAdded})
}
for _, c := range limitSlice(vd.Changed, limit) {
r := fromBackup(c.Backup)
r.Key = c.DB.UUID
r.UserAdded = c.DB.UserAdded
r.Fields = c.Fields
clash = append(clash, r)
}
return
}
func placeLabel(name string, lat, lon float64) string {
if name != "" {
return name
}
return fmt.Sprintf("%.5f, %.5f", lat, lon)
}
// anyTruncated 报告是否有哪一类差异多到被截断了。
func anyTruncated(vd rond.VisitDiff, pd rond.PlaceDiff, ad rond.ActivityDiff) bool {
for _, n := range []int{len(vd.Added), len(vd.Removed), len(vd.Changed),
len(pd.Added), len(pd.Removed), len(pd.Changed),
len(ad.Added), len(ad.Removed), len(ad.Changed)} {
if n > diffListLimit {
return true
}
}
return false
}
func limitSlice[T any](in []T, n int) []T {
if len(in) <= n {
return in
}
return in[:n]
}
func (s *Server) countOther(ctx context.Context, datasetID int64) (movements, weather int, err error) {
err = s.db.QueryRowContext(ctx, `SELECT
(SELECT count(*) FROM movements WHERE dataset_id=$1),
(SELECT count(*) FROM weather WHERE dataset_id=$1)`, datasetID).Scan(&movements, &weather)
return
}
// loadStoredVisits 读出该数据集的全部到访,并把原始行留档里的 UUID 解出来当对齐键。
// 没有留档的行就是站点新建的(真机不会给它 UUID),对齐键留空。
func (s *Server) loadStoredVisits(ctx context.Context, datasetID int64) ([]rond.StoredVisit, error) {
rows, err := s.db.QueryContext(ctx, `
SELECT v.src_pk, v.arrival, v.departure, COALESCE(v.remark,''), COALESCE(v.emoji,''),
v.bookmarked, v.is_user_added, COALESCE(p.src_pk,0),
COALESCE(p.name,''), COALESCE(p.lat,0), COALESCE(p.lon,0), COALESCE(a.name,''),
COALESCE(er.raw #>> '{ZIDENTIFIER_,$blob}','')
FROM visits v
LEFT JOIN places p ON p.id = v.place_id
LEFT JOIN activities a ON a.id = v.activity_id
LEFT JOIN entity_raw er ON er.dataset_id = v.dataset_id
AND er.entity = 'ZVISIT' AND er.src_pk = v.src_pk
WHERE v.dataset_id = $1`, datasetID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []rond.StoredVisit
for rows.Next() {
var sv rond.StoredVisit
var arr time.Time
var dep sql.NullTime
var placePK int
var placeName string
var lat, lon float64
var b64 string
if err := rows.Scan(&sv.SrcPK, &arr, &dep, &sv.Remark, &sv.Emoji, &sv.Bookmarked,
&sv.UserAdded, &placePK, &placeName, &lat, &lon, &sv.ActivityName, &b64); err != nil {
return nil, err
}
sv.Arrival = &arr
if dep.Valid {
sv.Departure = &dep.Time
}
sv.PlaceName = placeName
// 没有坐标的地点不能都落到 "0.000000|0.000000" 上,否则两个地点会互相认领
if placePK > 0 && (lat != 0 || lon != 0) {
sv.PlaceKey = rond.PlaceCoordKey(lat, lon)
}
sv.UUID = uuidFromBase64(b64)
out = append(out, sv)
}
return out, rows.Err()
}
func (s *Server) loadStoredPlaces(ctx context.Context, datasetID int64) ([]rond.StoredPlace, error) {
rows, err := s.db.QueryContext(ctx, `SELECT src_pk, COALESCE(name,''), COALESCE(lat,0), COALESCE(lon,0),
COALESCE(province,''), COALESCE(city,''), COALESCE(poi_category,''), visit_count
FROM places WHERE dataset_id=$1`, datasetID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []rond.StoredPlace
for rows.Next() {
var sp rond.StoredPlace
if err := rows.Scan(&sp.SrcPK, &sp.Name, &sp.Lat, &sp.Lon,
&sp.Province, &sp.City, &sp.POICategory, &sp.VisitCount); err != nil {
return nil, err
}
if sp.Lat != 0 || sp.Lon != 0 {
sp.Key = rond.PlaceCoordKey(sp.Lat, sp.Lon)
}
out = append(out, sp)
}
return out, rows.Err()
}
// loadStoredActivities 读活动类型;UUID 同样从留档里取,用于识别「改了名但其实是同一个」。
func (s *Server) loadStoredActivities(ctx context.Context, datasetID int64) ([]rond.StoredActivity, error) {
rows, err := s.db.QueryContext(ctx, `SELECT a.src_pk, a.name,
COALESCE(er.raw #>> '{ZUID_,$blob}','')
FROM activities a
LEFT JOIN entity_raw er ON er.dataset_id = a.dataset_id
AND er.entity = 'ZACTIVITY' AND er.src_pk = a.src_pk
WHERE a.dataset_id=$1`, datasetID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []rond.StoredActivity
for rows.Next() {
var sa rond.StoredActivity
var b64 string
if err := rows.Scan(&sa.SrcPK, &sa.Name, &b64); err != nil {
return nil, err
}
sa.UUID = uuidFromBase64(b64)
out = append(out, sa)
}
return out, rows.Err()
}
// uuidFromBase64 把留档里的 $blob(base64 的 16 字节 UUID)转成小写十六进制,
// 好跟备份侧 hex(ZIDENTIFIER_) 的写法对上。
func uuidFromBase64(s string) string {
if s == "" {
return ""
}
b, err := base64.StdEncoding.DecodeString(s)
if err != nil || len(b) != 16 {
log.Printf("留档里的 UUID 不是 16 字节,忽略: %q", s)
return ""
}
return hex.EncodeToString(b)
}
+164
View File
@@ -0,0 +1,164 @@
package web
import (
"bytes"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"testing"
)
// previewBackup 走真实的上传处理函数:POST → 303 → GET 差异页,返回渲染结果。
func previewBackup(t *testing.T, f *movementFixture, path string) string {
t.Helper()
_, page := previewBackupToken(t, f, path)
return page
}
// previewBackupToken 同上,但把令牌也交出来,供后续的合并步骤使用。
func previewBackupToken(t *testing.T, f *movementFixture, path string) (string, string) {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("读取 %s 失败: %v", path, err)
}
var buf bytes.Buffer
mw := multipart.NewWriter(&buf)
fw, err := mw.CreateFormFile("backup", filepath.Base(path))
if err != nil {
t.Fatalf("构造上传表单失败: %v", err)
}
if _, err := fw.Write(data); err != nil {
t.Fatalf("写入上传内容失败: %v", err)
}
mw.Close()
rec := httptest.NewRecorder()
req := httptest.NewRequest("POST", "/admin/preview", &buf)
req.Header.Set("Content-Type", mw.FormDataContentType())
f.srv.adminBackupPreview(rec, req.WithContext(f.ctx))
if rec.Code != http.StatusSeeOther {
t.Fatalf("上传应跳转到差异页: code=%d body=%s", rec.Code, rec.Body.String())
}
loc := rec.Header().Get("Location")
token := strings.TrimPrefix(loc, "/admin/preview/")
if token == loc || len(token) != stashTokenLen {
t.Fatalf("差异页地址不对: %q", loc)
}
rec = httptest.NewRecorder()
req = httptest.NewRequest("GET", loc, nil).WithContext(f.ctx)
req.SetPathValue("token", token)
f.srv.adminBackupPreviewPage(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("差异页渲染失败: code=%d body=%s", rec.Code, rec.Body.String())
}
return token, rec.Body.String()
}
// visitCounts 从概览表里取出「到访」那一行的 (备份新增, 仅站内有, 冲突)。
// 不能全页 Contains 数字——地点行也有同样的 data-label,会张冠李戴。
func visitCounts(t *testing.T, page string) (added, gone, clash int) {
t.Helper()
row := regexp.MustCompile(`(?s)<td>到访</td>(.*?)</tr>`).FindStringSubmatch(page)
if row == nil {
t.Fatal("概览表里找不到「到访」行")
}
nums := regexp.MustCompile(`data-label="(备份新增|仅站内有|冲突)">(\d+)<`).FindAllStringSubmatch(row[1], -1)
if len(nums) != 3 {
t.Fatalf("到访行应有三个计数,实际 %d", len(nums))
}
for _, n := range nums {
v, _ := strconv.Atoi(n[2])
switch n[1] {
case "备份新增":
added = v
case "仅站内有":
gone = v
case "冲突":
clash = v
}
}
return
}
// 同一份备份再传一次:UUID 对得上、字段也没变,到访应当一条不差。
//
// 这条也顺带钉住「按 UUID 而不是编号对齐」:库内那些行的编号与备份里完全不同
// (rond 重排过),只按编号比会让 800 多条全变冲突。
//
// 只断言到访,不断言「完全一致」整句:导入后站点会反查补全地点(改名称/地址),
// 地点层面因此真的存在差异,那是预期行为,不该由这个用例来管。
func TestBackupDiffIdentical(t *testing.T) {
f := newMovementFixture(t)
page := previewBackup(t, f, f.src)
if a, g, c := visitCounts(t, page); a != 0 || g != 0 || c != 0 {
t.Fatalf("同一份备份的到访不该有差异: 新增 %d 仅站内有 %d 冲突 %d", a, g, c)
}
}
// 站内改过备注、又补录了一条到访,差异页要把这两件事分别认出来。
//
// 这里的关键是「按 UUID 对齐」:库里那些行的 Z_PK 与备份里完全不同
// (rond 重排过),只按编号比会把 800 多条全判成冲突。
func TestBackupDiffDetectsSiteEdits(t *testing.T) {
f := newMovementFixture(t)
var pk int
if err := f.pg.QueryRowContext(f.ctx, `UPDATE visits SET remark='站点改过的备注'
WHERE id = (SELECT id FROM visits WHERE dataset_id=$1 ORDER BY src_pk LIMIT 1)
RETURNING src_pk`, f.dsID).Scan(&pk); err != nil {
t.Fatalf("改备注失败: %v", err)
}
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO visits
(dataset_id, src_pk, arrival, departure, is_user_added, remark)
VALUES ($1, 999999, now(), now(), true, '站点补录的一条')`, f.dsID); err != nil {
t.Fatalf("插入补录到访失败: %v", err)
}
page := previewBackup(t, f, f.src)
// 「站点改过的备注」只可能从冲突行的字段差异里渲染出来;「站内补录」是那条补录行的来源标记
for _, want := range []string{"备份差异预览", "站点改过的备注", "站点补录的一条", ">站内补录<"} {
if !strings.Contains(page, want) {
t.Errorf("差异页缺少 %q", want)
}
}
a, g, c := visitCounts(t, page)
if c != 1 {
t.Errorf("改过一条备注,到访冲突数应为 1,实际 %d", c)
}
if g != 1 {
t.Errorf("补录了一条,到访「仅站内有」应为 1,实际 %d", g)
}
if a != 0 {
t.Errorf("备份里没有多出来的到访,实际 %d", a)
}
}
// 差异页是只读的:渲染前后库里的行数必须一模一样。
func TestBackupDiffPreviewWritesNothing(t *testing.T) {
f := newMovementFixture(t)
count := func() (visits, places, raw int) {
t.Helper()
if err := f.pg.QueryRowContext(f.ctx, `SELECT
(SELECT count(*) FROM visits WHERE dataset_id=$1),
(SELECT count(*) FROM places WHERE dataset_id=$1),
(SELECT count(*) FROM entity_raw WHERE dataset_id=$1)`, f.dsID).
Scan(&visits, &places, &raw); err != nil {
t.Fatalf("统计失败: %v", err)
}
return
}
v0, p0, r0 := count()
previewBackup(t, f, f.src)
v1, p1, r1 := count()
if v0 != v1 || p0 != p1 || r0 != r1 {
t.Fatalf("差异页不该写库: 到访 %d→%d 地点 %d→%d 留档 %d→%d", v0, v1, p0, p1, r0, r1)
}
}
+753
View File
@@ -0,0 +1,753 @@
package web
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"log"
"net/http"
"os"
"strconv"
"strings"
"rond-with-you/internal/geo"
"rond-with-you/internal/ingest"
"rond-with-you/internal/rond"
)
// 差异页确认后的合并执行。
//
// 全部改动在一个事务里完成:半合并的数据集比没合并更难收拾(比如地点建了、
// 到访没建,地图上就多一个永远没人去的点)。删除不在这里——删地点会级联删掉
// 它的全部到访,那件事单独再议。
//
// 编号是这里最容易出错的地方:**库里那侧的编号空间才是唯一有效的**。
// 新插入的行拿 nextSrcPK 发的新号,于是位移的 from_visit_src、天气的 visit_src
// 都必须写「库里的新号」,而不是备份里的原号。
type mergeStats struct {
Places, Activities, Tags int
Visits, Movements, Weather int
UpdPlaces, UpdActs, UpdVisits int
DelPlaces, DelActs, DelVisits int
}
func (s *Server) adminBackupMerge(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
ds, err := s.datasetFor(ctx)
if err != nil {
s.serverError(w, r, err)
return
}
if ds == nil {
s.adminFlash(w, r, "err", "没有可合并的数据集")
return
}
token := r.PathValue("token")
path, ok := stashPath(s.backupStashDir(), backupStashExt, token)
if !ok {
s.adminFlash(w, r, "err", "合并链接无效,请重新上传备份")
return
}
if _, err := os.Stat(path); err != nil {
s.adminFlash(w, r, "err", "这次预览已过期,请重新上传备份")
return
}
if err := r.ParseForm(); err != nil {
s.serverError(w, r, err)
return
}
backup, err := s.parseBackupFile(path)
if err != nil {
s.adminFlash(w, r, "err", err.Error())
return
}
visits, places, acts, err := s.loadStoredAll(ctx, ds.ID)
if err != nil {
s.serverError(w, r, err)
return
}
// 重新算一遍差异再执行:从渲染差异页到点下按钮之间,库可能已经被别处改过
vd := rond.DiffVisits(visits, backup)
pd := rond.DiffPlaces(places, backup)
ad := rond.DiffActivities(acts, backup)
addNew := r.PostFormValue("add_new") == "1"
plan := rond.BuildMergePlan(vd, pd, ad, rond.Choices{
AddVisits: addNew, AddPlaces: addNew, AddActs: addNew,
VisitClash: pickForm(r, "vc_"),
PlaceClash: pickForm(r, "pc_"),
ActClash: pickForm(r, "ac_"),
DelVisits: pickIntForm(r, "dv_"),
DelPlaces: pickIntForm(r, "dp_"),
DelActs: pickIntForm(r, "da_"),
})
if plan.Empty() {
s.adminFlash(w, r, "ok", "按当前的选择没有要写入的改动,数据未变")
return
}
st, err := s.applyMergePlan(ctx, ds, backup, plan, visits)
if err != nil {
log.Printf("合并失败: %v", err)
s.adminFlash(w, r, "err", "合并失败,已整体回滚、数据未变:"+err.Error())
return
}
stashDrop(s.backupStashDir(), backupStashExt, token)
msg := fmt.Sprintf("合并完成:新增 %d 个地点 / %d 个活动 / %d 条到访;更新 %d 个地点 / %d 个活动 / %d 条到访",
st.Places, st.Activities, st.Visits, st.UpdPlaces, st.UpdActs, st.UpdVisits)
if st.DelVisits+st.DelActs+st.DelPlaces > 0 {
msg += fmt.Sprintf(";删除 %d 条到访 / %d 个活动 / %d 个地点", st.DelVisits, st.DelActs, st.DelPlaces)
}
if st.Tags > 0 {
msg += fmt.Sprintf(";顺带建了 %d 个标签", st.Tags)
}
if st.Movements+st.Weather > 0 {
msg += fmt.Sprintf(";随到访带入 %d 条位移、%d 条天气", st.Movements, st.Weather)
}
s.adminFlash(w, r, "ok", msg)
}
// pickIntForm 收集「前缀 + 整数」形式的勾选,比如 dv_825=1(要删除的到访)。
func pickIntForm(r *http.Request, prefix string) map[int]bool {
out := map[int]bool{}
for k, vs := range r.PostForm {
if len(vs) == 0 || vs[0] == "" || !strings.HasPrefix(k, prefix) {
continue
}
if n, err := strconv.Atoi(strings.TrimPrefix(k, prefix)); err == nil {
out[n] = true
}
}
return out
}
// pickForm 收集「前缀 + 键」形式的逐条选择,比如 vc_<uuid>=backup。
func pickForm(r *http.Request, prefix string) map[string]string {
out := map[string]string{}
for k, vs := range r.PostForm {
if len(vs) == 0 || !strings.HasPrefix(k, prefix) {
continue
}
out[strings.TrimPrefix(k, prefix)] = vs[0]
}
return out
}
func (s *Server) loadStoredAll(ctx context.Context, dsID int64) ([]rond.StoredVisit, []rond.StoredPlace, []rond.StoredActivity, error) {
visits, err := s.loadStoredVisits(ctx, dsID)
if err != nil {
return nil, nil, nil, err
}
places, err := s.loadStoredPlaces(ctx, dsID)
if err != nil {
return nil, nil, nil, err
}
acts, err := s.loadStoredActivities(ctx, dsID)
if err != nil {
return nil, nil, nil, err
}
return visits, places, acts, nil
}
func (s *Server) parseBackupFile(path string) (*rond.Backup, error) {
tmp, err := ingest.TempDir(s.cfg.DataDir, "merge")
if err != nil {
return nil, err
}
defer os.RemoveAll(tmp)
db, err := rond.Open(path, tmp)
if err != nil {
return nil, fmt.Errorf("解析备份失败:%w", err)
}
defer db.Close()
b, err := rond.Parse(db)
if err != nil {
return nil, fmt.Errorf("读取备份数据失败:%w", err)
}
return b, nil
}
func (s *Server) applyMergePlan(ctx context.Context, ds *Dataset, b *rond.Backup, plan rond.MergePlan,
storedVisits []rond.StoredVisit) (*mergeStats, error) {
st := &mergeStats{}
err := s.withTx(ctx, func(ex execer) error {
placeID, err := s.mergePlaces(ctx, ex, ds.ID, b, plan, st)
if err != nil {
return err
}
actID, err := s.mergeActivities(ctx, ex, ds.ID, b, plan, st)
if err != nil {
return err
}
tagID, err := s.mergeTags(ctx, ex, ds.ID, b, st)
if err != nil {
return err
}
if err := s.mergeVisits(ctx, ex, ds.ID, b, plan, placeID, actID, tagID, storedVisits, st); err != nil {
return err
}
// 删除放在最后:先让新增/更新落定,再按最终状态删
if err := s.applyDeletions(ctx, ex, ds.ID, plan, st); err != nil {
return err
}
return s.recountPlaces(ctx, ex, ds.ID)
})
if err != nil {
return nil, err
}
return st, nil
}
// mergePlaces 先把备份地点映射到库里的 places.id(已存在的直接映射),
// 再处理冲突更新与新增,返回「备份地点编号 -> 库 id」。
func (s *Server) mergePlaces(ctx context.Context, ex execer, dsID int64, b *rond.Backup,
plan rond.MergePlan, st *mergeStats) (map[int]int64, error) {
byCoord, byName := map[string]int64{}, map[string]int64{}
rows, err := ex.QueryContext(ctx, `SELECT id, COALESCE(name,''), COALESCE(lat,0), COALESCE(lon,0)
FROM places WHERE dataset_id=$1`, dsID)
if err != nil {
return nil, err
}
for rows.Next() {
var id int64
var name string
var lat, lon float64
if err := rows.Scan(&id, &name, &lat, &lon); err != nil {
rows.Close()
return nil, err
}
if lat != 0 || lon != 0 {
byCoord[rond.PlaceCoordKey(lat, lon)] = id
}
if _, dup := byName[name]; !dup && name != "" {
byName[name] = id
}
}
rows.Close()
if err := rows.Err(); err != nil {
return nil, err
}
placeID := map[int]int64{}
for _, l := range b.Locations {
if l.Lat != 0 || l.Lon != 0 {
if id, ok := byCoord[rond.PlaceCoordKey(l.Lat, l.Lon)]; ok {
placeID[l.SrcPK] = id
continue
}
}
if l.Name != "" {
if id, ok := byName[l.Name]; ok {
placeID[l.SrcPK] = id
}
}
}
for _, up := range plan.UpdPlaces {
id := placeID[up.L.SrcPK]
if id == 0 {
id = byCoord[up.CoordKey]
}
if id == 0 {
id = byName[up.L.Name]
}
if id == 0 {
return nil, fmt.Errorf("找不到要更新的地点(%s)", up.L.Name)
}
if _, err := ex.ExecContext(ctx, `UPDATE places SET name=$1, poi_category=$2, lat=$3, lon=$4,
country_code=$5, province=$6, city=$7, district=$8, sublocality=$9, thoroughfare=$10, timezone=$11
WHERE id=$12`,
nullStr(up.L.Name), nullStr(up.L.POICategory), up.L.Lat, up.L.Lon,
nullStr(up.L.CountryCode), nullStr(up.L.Province), nullStr(up.L.City), nullStr(up.L.District),
nullStr(up.L.Sublocality), nullStr(up.L.Thoroughfare), nullStr(up.L.Timezone), id); err != nil {
return nil, fmt.Errorf("更新地点: %w", err)
}
placeID[up.L.SrcPK] = id
st.UpdPlaces++
}
for _, l := range plan.AddPlaces {
pk, err := s.nextSrcPK(ctx, ex, dsID, placeSeq)
if err != nil {
return nil, err
}
var id int64
if err := ex.QueryRowContext(ctx, `INSERT INTO places
(dataset_id, src_pk, name, poi_category, lat, lon, country_code, province, city,
district, sublocality, thoroughfare, timezone)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13) RETURNING id`,
dsID, pk, nullStr(l.Name), nullStr(l.POICategory), l.Lat, l.Lon, nullStr(l.CountryCode),
nullStr(l.Province), nullStr(l.City), nullStr(l.District), nullStr(l.Sublocality),
nullStr(l.Thoroughfare), nullStr(l.Timezone)).Scan(&id); err != nil {
return nil, fmt.Errorf("新增地点: %w", err)
}
placeID[l.SrcPK] = id
st.Places++
if err := writeRawRow(ctx, ex, dsID, "ZLOCATION", pk, b.RawRows["ZLOCATION"][l.SrcPK]); err != nil {
return nil, err
}
}
return placeID, nil
}
func (s *Server) mergeActivities(ctx context.Context, ex execer, dsID int64, b *rond.Backup,
plan rond.MergePlan, st *mergeStats) (map[int]int64, error) {
byUUID, byName := map[string]int64{}, map[string]int64{}
rows, err := ex.QueryContext(ctx, `SELECT a.id, a.name, COALESCE(er.raw #>> '{ZUID_,$blob}','')
FROM activities a
LEFT JOIN entity_raw er ON er.dataset_id = a.dataset_id
AND er.entity = 'ZACTIVITY' AND er.src_pk = a.src_pk
WHERE a.dataset_id=$1`, dsID)
if err != nil {
return nil, err
}
for rows.Next() {
var id int64
var name, b64 string
if err := rows.Scan(&id, &name, &b64); err != nil {
rows.Close()
return nil, err
}
if u := uuidFromBase64(b64); u != "" {
byUUID[u] = id
}
if _, dup := byName[name]; !dup && name != "" {
byName[name] = id
}
}
rows.Close()
if err := rows.Err(); err != nil {
return nil, err
}
actID := map[int]int64{}
for _, a := range b.Activities {
if u := b.ActivityUUID[a.SrcPK]; u != "" {
if id, ok := byUUID[u]; ok {
actID[a.SrcPK] = id
continue
}
}
if id, ok := byName[a.Name]; ok {
actID[a.SrcPK] = id
}
}
for _, up := range plan.UpdActs {
id := actID[up.A.SrcPK]
if id == 0 {
id = byName[up.A.Name]
}
if id == 0 {
return nil, fmt.Errorf("找不到要更新的活动(%s)", up.A.Name)
}
if _, err := ex.ExecContext(ctx, `UPDATE activities SET name=$1, color=$2, icon=$3,
is_home=$4, is_work=$5, is_excluded=$6, is_archived=$7 WHERE id=$8`,
up.A.Name, nullStr(up.A.Color), nullStr(up.A.Icon), up.A.IsHome, up.A.IsWork,
up.A.IsExcluded, up.A.IsArchived, id); err != nil {
return nil, fmt.Errorf("更新活动: %w", err)
}
st.UpdActs++
}
for _, a := range plan.AddActs {
pk, err := s.nextSrcPK(ctx, ex, dsID, activitySeq)
if err != nil {
return nil, err
}
var id int64
if err := ex.QueryRowContext(ctx, `INSERT INTO activities
(dataset_id, src_pk, name, color, icon, is_home, is_work, is_excluded, is_archived)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) RETURNING id`,
dsID, pk, a.Name, nullStr(a.Color), nullStr(a.Icon),
a.IsHome, a.IsWork, a.IsExcluded, a.IsArchived).Scan(&id); err != nil {
return nil, fmt.Errorf("新增活动: %w", err)
}
actID[a.SrcPK] = id
st.Activities++
if err := writeRawRow(ctx, ex, dsID, "ZACTIVITY", pk, b.RawRows["ZACTIVITY"][a.SrcPK]); err != nil {
return nil, err
}
}
return actID, nil
}
// mergeTags 只做「库里有同名的就复用、没有就建」——标签是纯名称实体,没有别的标识。
func (s *Server) mergeTags(ctx context.Context, ex execer, dsID int64, b *rond.Backup, st *mergeStats) (map[int]int64, error) {
byName := map[string]int64{}
rows, err := ex.QueryContext(ctx, `SELECT id, name FROM tags WHERE dataset_id=$1`, dsID)
if err != nil {
return nil, err
}
for rows.Next() {
var id int64
var name string
if err := rows.Scan(&id, &name); err != nil {
rows.Close()
return nil, err
}
byName[name] = id
}
rows.Close()
if err := rows.Err(); err != nil {
return nil, err
}
tagID := map[int]int64{}
for _, t := range b.Tags {
if id, ok := byName[t.Name]; ok {
tagID[t.SrcPK] = id
continue
}
pk, err := s.nextSrcPK(ctx, ex, dsID, tagSeq)
if err != nil {
return nil, err
}
var id int64
if err := ex.QueryRowContext(ctx, `INSERT INTO tags (dataset_id, src_pk, name, color, group_name)
VALUES ($1,$2,$3,$4,NULLIF($5,'')) RETURNING id`,
dsID, pk, t.Name, nullStr(t.Color), t.GroupName).Scan(&id); err != nil {
return nil, fmt.Errorf("新增标签: %w", err)
}
byName[t.Name] = id
tagID[t.SrcPK] = id
st.Tags++
if err := writeRawRow(ctx, ex, dsID, "ZTAG", pk, b.RawRows["ZTAG"][t.SrcPK]); err != nil {
return nil, err
}
}
return tagID, nil
}
func (s *Server) mergeVisits(ctx context.Context, ex execer, dsID int64, b *rond.Backup, plan rond.MergePlan,
placeID, actID, tagID map[int]int64, storedVisits []rond.StoredVisit, st *mergeStats) error {
// 备份到访编号 -> 库里的 visits.src_pk:位移的 from/to_visit_src 与天气的
// visit_src 写的都是**库里的编号**,端点在备份里的原号在这里换成库里的号。
visSrc := map[int]int{}
storedByUUID := map[string]rond.StoredVisit{}
for _, sv := range storedVisits {
if sv.UUID != "" {
storedByUUID[sv.UUID] = sv
}
}
for _, v := range b.Visits {
if u := b.VisitUUID[v.SrcPK]; u != "" {
if sv, ok := storedByUUID[u]; ok {
visSrc[v.SrcPK] = sv.SrcPK
}
}
}
activityByPK := map[int]rond.Activity{}
for _, a := range b.Activities {
activityByPK[a.SrcPK] = a
}
visitPlace := map[int]int{}
for _, v := range b.Visits {
if v.LocationSrcPK != nil {
visitPlace[v.SrcPK] = *v.LocationSrcPK
}
}
resolve := func(v rond.Visit) (pid, aid any, isHome, isWork bool) {
if v.LocationSrcPK != nil {
if x, ok := placeID[*v.LocationSrcPK]; ok {
pid = x
}
}
if v.ActivitySrcPK != nil {
if x, ok := actID[*v.ActivitySrcPK]; ok {
aid = x
}
if a, ok := activityByPK[*v.ActivitySrcPK]; ok {
isHome, isWork = a.IsHome, a.IsWork
}
}
return
}
for _, up := range plan.UpdVisits {
sv, ok := storedByUUID[up.UUID]
if !ok {
return fmt.Errorf("找不到要更新的到访(%s)", up.UUID)
}
pid, aid, isHome, isWork := resolve(up.V)
if _, err := ex.ExecContext(ctx, `UPDATE visits SET place_id=$1, activity_id=$2, arrival=$3, departure=$4,
duration_min=$5, is_home=$6, is_work=$7, bookmarked=$8, remark=NULLIF($9,''), emoji=NULLIF($10,'')
WHERE dataset_id=$11 AND src_pk=$12`,
pid, aid, up.V.Arrival, up.V.Departure, visitDuration(up.V), isHome, isWork,
up.V.Bookmarked, up.V.Remark, up.V.Emoji, dsID, sv.SrcPK); err != nil {
return fmt.Errorf("更新到访: %w", err)
}
st.UpdVisits++
}
addedVisit := map[int]bool{}
for _, v := range plan.AddVisits {
pid, aid, isHome, isWork := resolve(v)
pk, err := s.nextSrcPK(ctx, ex, dsID, visitSeq)
if err != nil {
return err
}
var id int64
if err := ex.QueryRowContext(ctx, `INSERT INTO visits
(dataset_id, src_pk, place_id, activity_id, arrival, departure, duration_min,
is_home, is_work, bookmarked, is_user_added, remark, emoji, weather_symbol)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,NULLIF($12,''),NULLIF($13,''),NULLIF($14,''))
ON CONFLICT DO NOTHING RETURNING id`,
dsID, pk, pid, aid, v.Arrival, v.Departure, visitDuration(v), isHome, isWork,
v.Bookmarked, v.UserAdded, v.Remark, v.Emoji, v.WeatherSymbol).Scan(&id); err != nil {
if err == sql.ErrNoRows {
continue
}
return fmt.Errorf("新增到访: %w", err)
}
visSrc[v.SrcPK] = pk
addedVisit[v.SrcPK] = true
st.Visits++
// 留档必须写:下次再比对时库侧就是靠这里的 UUID 认出这一条,
// 少了它只能退到「到达时间 + 坐标」兜底。
if err := writeRawRow(ctx, ex, dsID, "ZVISIT", pk, b.RawRows["ZVISIT"][v.SrcPK]); err != nil {
return err
}
for _, tpk := range v.TagSrcPKs {
if tid, ok := tagID[tpk]; ok {
if _, err := ex.ExecContext(ctx, `INSERT INTO visit_tags (visit_id, tag_id)
VALUES ($1,$2) ON CONFLICT DO NOTHING`, id, tid); err != nil {
return fmt.Errorf("写入到访标签: %w", err)
}
}
}
}
transName, transColor, transIcon := map[int]string{}, map[int]string{}, map[int]string{}
for _, t := range b.Transports {
transName[t.SrcPK], transColor[t.SrcPK], transIcon[t.SrcPK] = t.Name, t.Color, t.Icon
}
coordOf := map[int][2]float64{}
for _, l := range b.Locations {
coordOf[l.SrcPK] = [2]float64{l.Lat, l.Lon}
}
// 位移与天气跟着**新插入的到访**走:只插这一轮真正补进来的那些,
// 否则库里已有的位移会被整份复制一遍。另一端可以是老到访(库里已经有)。
for _, m := range b.Movements {
if m.FromVisitSrc == nil || m.ToVisitSrc == nil {
continue
}
if !addedVisit[*m.FromVisitSrc] && !addedVisit[*m.ToVisitSrc] {
continue
}
fsrc, ok1 := visSrc[*m.FromVisitSrc]
tsrc, ok2 := visSrc[*m.ToVisitSrc]
if !ok1 || !ok2 {
continue
}
var fromPID, toPID any
if fp, ok := visitPlace[*m.FromVisitSrc]; ok {
if x, ok := placeID[fp]; ok {
fromPID = x
}
}
if tp, ok := visitPlace[*m.ToVisitSrc]; ok {
if x, ok := placeID[tp]; ok {
toPID = x
}
}
var dist any
if fp, ok1 := visitPlace[*m.FromVisitSrc]; ok1 {
if tp, ok2 := visitPlace[*m.ToVisitSrc]; ok2 {
c1, c2 := coordOf[fp], coordOf[tp]
if c1[0] != 0 && c2[0] != 0 {
dist = geo.Haversine(c1[0], c1[1], c2[0], c2[1])
}
}
}
var tname, tcolor, ticon any
if m.TransportSrc != nil {
tname = nullStr(transName[*m.TransportSrc])
tcolor = nullStr(transColor[*m.TransportSrc])
ticon = nullStr(transIcon[*m.TransportSrc])
}
var dur any
if m.Start != nil && m.End != nil {
if d := int(m.End.Sub(*m.Start).Minutes()); d >= 0 {
dur = d
}
}
pk, err := s.nextSrcPK(ctx, ex, dsID, movementSeq)
if err != nil {
return err
}
if _, err := ex.ExecContext(ctx, `INSERT INTO movements
(dataset_id, src_pk, transport_src, transport_name, transport_color, transport_icon,
from_visit_src, to_visit_src, from_place_id, to_place_id, started_at, ended_at, duration_min, distance_km)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT DO NOTHING`,
dsID, pk, m.TransportSrc, tname, tcolor, ticon, fsrc, tsrc,
fromPID, toPID, m.Start, m.End, dur, dist); err != nil {
return fmt.Errorf("新增位移: %w", err)
}
st.Movements++
if err := writeRawRow(ctx, ex, dsID, "ZMOVEMENT", pk, b.RawRows["ZMOVEMENT"][m.SrcPK]); err != nil {
return err
}
}
for _, w := range b.Weather {
if w.VisitSrc == nil || w.At == nil {
continue
}
if !addedVisit[*w.VisitSrc] {
continue
}
vsrc, ok := visSrc[*w.VisitSrc]
if !ok {
continue
}
// 站内删到访时会把天气的归属置空而保留记录(见 editVisitDelete)。
// 这类「同一时刻、没有归属」的天气先认领回来,否则同一条到访找回来之后
// 会多出一份重复的天气。
var claimed int64
if err := ex.QueryRowContext(ctx, `UPDATE weather SET visit_src=$1
WHERE dataset_id=$2 AND at=$3 AND visit_src IS NULL RETURNING id`,
vsrc, dsID, w.At).Scan(&claimed); err == nil {
continue
}
pk, err := s.nextSrcPK(ctx, ex, dsID, weatherSeq)
if err != nil {
return err
}
if _, err := ex.ExecContext(ctx, `INSERT INTO weather
(dataset_id, src_pk, visit_src, at, is_daylight, temperature_c, apparent_c, humidity,
precipitation_amount, precipitation_chance, wind_speed, visibility, uv_index,
condition, symbol, uv_category, wind_direction)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,NULLIF($14,''),NULLIF($15,''),NULLIF($16,''),NULLIF($17,''))
ON CONFLICT DO NOTHING`,
dsID, pk, vsrc, w.At, w.IsDaylight, w.TempC, w.ApparentC, w.Humidity,
w.PrecipAmt, w.PrecipPct, w.WindSpeed, w.Visibility, w.UVIndex,
w.Condition, w.Symbol, w.UVCategory, w.WindDir); err != nil {
return fmt.Errorf("新增天气: %w", err)
}
st.Weather++
if err := writeRawRow(ctx, ex, dsID, "ZHOURLYWEATHER", pk, b.RawRows["ZHOURLYWEATHER"][w.SrcPK]); err != nil {
return err
}
}
return nil
}
// applyDeletions 处理「仅站内有」里被勾选删除的记录。顺序是到访 → 活动 → 地点:
// 删地点会级联删掉它名下的全部到访(外键 ON DELETE CASCADE),所以反过来会误删;
// 而且**有到访引用时直接拒绝**,让站长先处理到访,而不是悄悄带走一批。
func (s *Server) applyDeletions(ctx context.Context, ex execer, dsID int64, plan rond.MergePlan, st *mergeStats) error {
for _, sv := range plan.DelVisits {
if _, err := ex.ExecContext(ctx, `DELETE FROM movements WHERE dataset_id=$1
AND (from_visit_src=$2 OR to_visit_src=$2)`, dsID, sv.SrcPK); err != nil {
return fmt.Errorf("删除到访的位移: %w", err)
}
// 天气保留、只解除归属——与站点 editVisitDelete 的处理一致
if _, err := ex.ExecContext(ctx, `UPDATE weather SET visit_src=NULL
WHERE dataset_id=$1 AND visit_src=$2`, dsID, sv.SrcPK); err != nil {
return fmt.Errorf("解除天气归属: %w", err)
}
if _, err := ex.ExecContext(ctx, `DELETE FROM visits WHERE dataset_id=$1 AND src_pk=$2`,
dsID, sv.SrcPK); err != nil {
return fmt.Errorf("删除到访: %w", err)
}
if _, err := ex.ExecContext(ctx, `DELETE FROM entity_raw WHERE dataset_id=$1
AND entity='ZVISIT' AND src_pk=$2`, dsID, sv.SrcPK); err != nil {
return err
}
st.DelVisits++
}
for _, sa := range plan.DelActs {
// 到访的 activity_id 是 ON DELETE SET NULL,删活动不会带走到访
if _, err := ex.ExecContext(ctx, `DELETE FROM activities WHERE dataset_id=$1 AND src_pk=$2`,
dsID, sa.SrcPK); err != nil {
return fmt.Errorf("删除活动: %w", err)
}
if _, err := ex.ExecContext(ctx, `DELETE FROM entity_raw WHERE dataset_id=$1
AND entity='ZACTIVITY' AND src_pk=$2`, dsID, sa.SrcPK); err != nil {
return err
}
st.DelActs++
}
for _, sp := range plan.DelPlaces {
var n int
if err := ex.QueryRowContext(ctx, `SELECT count(*) FROM visits v
JOIN places p ON p.id = v.place_id
WHERE p.dataset_id=$1 AND p.src_pk=$2`, dsID, sp.SrcPK).Scan(&n); err != nil {
return err
}
if n > 0 {
return fmt.Errorf("地点「%s」名下还有 %d 条到访,删它会连带删掉那些到访——请先处理这些到访",
sp.Name, n)
}
if _, err := ex.ExecContext(ctx, `DELETE FROM places WHERE dataset_id=$1 AND src_pk=$2`,
dsID, sp.SrcPK); err != nil {
return fmt.Errorf("删除地点: %w", err)
}
if _, err := ex.ExecContext(ctx, `DELETE FROM entity_raw WHERE dataset_id=$1
AND entity='ZLOCATION' AND src_pk=$2`, dsID, sp.SrcPK); err != nil {
return err
}
st.DelPlaces++
}
return nil
}
// recountPlaces 重算地点聚合与数据集汇总。全量重算而不是只动受影响的行:
// 367 个地点两条 SQL 就够,省得漏掉任何一条派生列。
func (s *Server) recountPlaces(ctx context.Context, ex execer, dsID int64) error {
if _, err := ex.ExecContext(ctx, `UPDATE places SET visit_count=0, dwell_minutes=0,
first_visit_at=NULL, last_visit_at=NULL WHERE dataset_id=$1`, dsID); err != nil {
return err
}
if _, err := ex.ExecContext(ctx, `UPDATE places p SET visit_count=sub.c, dwell_minutes=sub.d,
first_visit_at=sub.f, last_visit_at=sub.l
FROM (SELECT place_id, count(*) c,
COALESCE(sum(LEAST(COALESCE(duration_min,0),10080)),0) d,
min(arrival) f, max(arrival) l
FROM visits WHERE dataset_id=$1 AND place_id IS NOT NULL GROUP BY place_id) sub
WHERE p.id = sub.place_id`, dsID); err != nil {
return err
}
_, err := ex.ExecContext(ctx, `UPDATE datasets SET
visit_count=(SELECT count(*) FROM visits WHERE dataset_id=$1),
place_count=(SELECT count(*) FROM places WHERE dataset_id=$1),
first_visit_at=(SELECT min(arrival) FROM visits WHERE dataset_id=$1),
last_visit_at=(SELECT max(arrival) FROM visits WHERE dataset_id=$1)
WHERE id=$1`, dsID)
return err
}
// writeRawRow 给新插入的行留一份原始行:导出时要靠它逐列还原,
// 再次比对时要靠它认出 UUID(否则只能退到「时间 + 坐标」兜底)。
func writeRawRow(ctx context.Context, ex execer, dsID int64, entity string, pk int, raw map[string]any) error {
if len(raw) == 0 {
return nil
}
blob, err := json.Marshal(raw)
if err != nil {
log.Printf("留档 %s#%d 序列化失败: %v", entity, pk, err)
return nil
}
_, err = ex.ExecContext(ctx, `INSERT INTO entity_raw (dataset_id, entity, src_pk, raw, skipped)
VALUES ($1,$2,$3,$4,false)
ON CONFLICT (dataset_id, entity, src_pk) DO UPDATE SET raw=EXCLUDED.raw`,
dsID, entity, pk, string(blob))
return err
}
func visitDuration(v rond.Visit) any {
if v.Arrival == nil || v.Departure == nil {
return nil
}
m := int(v.Departure.Sub(*v.Arrival).Minutes())
if m < 0 {
m = 0
}
return m
}
+273
View File
@@ -0,0 +1,273 @@
package web
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
)
// mergeBackup 走真实的合并处理函数;失败(重定向里带 err=)直接判错。
func mergeBackup(t *testing.T, f *movementFixture, token, body string) {
t.Helper()
if loc := mergeBackupAllowErr(t, f, token, body); strings.Contains(loc, "err=") {
t.Fatalf("合并失败: %s", loc)
}
}
// mergeBackupAllowErr 同上,但把落点交出来,供「预期会失败」的用例断言。
func mergeBackupAllowErr(t *testing.T, f *movementFixture, token, body string) string {
t.Helper()
rec := httptest.NewRecorder()
req := httptest.NewRequest("POST", "/admin/preview/"+token+"/merge", strings.NewReader(body))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.SetPathValue("token", token)
f.srv.adminBackupMerge(rec, req.WithContext(f.ctx))
if rec.Code != http.StatusSeeOther {
t.Fatalf("合并应跳回后台: code=%d body=%s", rec.Code, rec.Body.String())
}
return rec.Header().Get("Location")
}
// addExtraVisit 造一条「仅站内有」的到访:没有留档,于是没有 UUID,比对时必然落在仅站内有里。
func (f *movementFixture) addExtraVisit(t *testing.T, pk int, remark string) {
t.Helper()
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO visits
(dataset_id, src_pk, arrival, departure, is_user_added, remark)
VALUES ($1,$2,now(),now(),true,$3)`, f.dsID, pk, remark); err != nil {
t.Fatalf("插入补录到访失败: %v", err)
}
}
func (f *movementFixture) scalar(t *testing.T, q string, args ...any) int {
t.Helper()
var n int
if err := f.pg.QueryRowContext(f.ctx, q, args...).Scan(&n); err != nil {
t.Fatalf("查询失败: %v", err)
}
return n
}
// 合并把备份里多出来的记录写进去,而且写完要「复原」——同一份备份再比一次必须一条不差。
//
// 这条测试同时钉住三件事:
// - 新增行用 nextSrcPK 发新号,且**写进 entity_raw**(否则下次比对认不出 UUID);
// - 位移/天气的外键指向库里的新编号(写错就成孤儿);
// - 合并是幂等的:再传同一份备份,差异归零。
func TestBackupMergeRestoresMissingVisits(t *testing.T) {
f := newMovementFixture(t)
visitCount := func() int {
return f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1`, f.dsID)
}
all := visitCount()
movCount := f.scalar(t, `SELECT count(*) FROM movements WHERE dataset_id=$1`, f.dsID)
weatherCount := f.scalar(t, `SELECT count(*) FROM weather WHERE dataset_id=$1`, f.dsID)
// 原始数据里就有指向「因缺到达时间而没入库」的到访的位移,先记下基线再比
orphanQ := `SELECT count(*) FROM movements m WHERE m.dataset_id=$1
AND (m.from_visit_src IS NULL OR NOT EXISTS (
SELECT 1 FROM visits v WHERE v.dataset_id=$1 AND v.src_pk=m.from_visit_src))`
orphans0 := f.scalar(t, orphanQ, f.dsID)
// 制造差异:按**站点删到访的方式**删掉最早的两条(见 editVisitDelete)——
// 位移删掉、天气的归属置空、到访删掉。直接 DELETE visits 会留下孤儿天气,
// 那是测试手法造成的,不是产品行为。
var pk1, pk2 int
if err := f.pg.QueryRowContext(f.ctx, `SELECT
(SELECT src_pk FROM visits WHERE dataset_id=$1 ORDER BY arrival LIMIT 1),
(SELECT src_pk FROM visits WHERE dataset_id=$1 ORDER BY arrival OFFSET 1 LIMIT 1)`,
f.dsID).Scan(&pk1, &pk2); err != nil {
t.Fatalf("取待删到访失败: %v", err)
}
for _, q := range []string{
`DELETE FROM movements WHERE dataset_id=$1 AND (from_visit_src IN ($2,$3) OR to_visit_src IN ($2,$3))`,
`UPDATE weather SET visit_src=NULL WHERE dataset_id=$1 AND visit_src IN ($2,$3)`,
`DELETE FROM visits WHERE dataset_id=$1 AND src_pk IN ($2,$3)`,
} {
if _, err := f.pg.ExecContext(f.ctx, q, f.dsID, pk1, pk2); err != nil {
t.Fatalf("预置删除失败: %v", err)
}
}
if got := visitCount(); got != all-2 {
t.Fatalf("预置失败:应有 %d 条,实际 %d", all-2, got)
}
token, page := previewBackupToken(t, f, f.src)
if a, _, _ := visitCounts(t, page); a != 2 {
t.Fatalf("差异页应报 2 条新增,实际 %d", a)
}
mergeBackup(t, f, token, "add_new=1")
if got := visitCount(); got != all {
t.Fatalf("合并后到访数应回到 %d,实际 %d", all, got)
}
// 新补进来的那两条必须各有一份留档(下次比对就靠它认 UUID)。
// 被删掉的行会留下孤儿留档——那是老数据,不影响,所以只查新编号。
if got := f.scalar(t, `SELECT count(*) FROM entity_raw er
WHERE er.dataset_id=$1 AND er.entity='ZVISIT' AND EXISTS (
SELECT 1 FROM visits v WHERE v.dataset_id=$1 AND v.src_pk=er.src_pk)`, f.dsID); got != all {
t.Errorf("每条在库的到访都该有原始行留档,实际 %d / %d", got, all)
}
if got := f.scalar(t, `SELECT count(*) FROM movements WHERE dataset_id=$1`, f.dsID); got != movCount {
t.Errorf("位移应恢复到 %d,实际 %d", movCount, got)
}
if got := f.scalar(t, `SELECT count(*) FROM weather WHERE dataset_id=$1`, f.dsID); got != weatherCount {
t.Errorf("天气应恢复到 %d,实际 %d", weatherCount, got)
}
// 合并不该新增孤儿位移(原始数据里本来就有一些,见 orphanQ 的注释)
if orphans := f.scalar(t, orphanQ, f.dsID); orphans > orphans0 {
t.Errorf("孤儿位移变多了:%d → %d", orphans0, orphans)
}
// 幂等:再比一次应当完全一致
page = previewBackup(t, f, f.src)
if a, g, c := visitCounts(t, page); a != 0 || g != 0 || c != 0 {
t.Fatalf("合并后再比对应当没有差异,实际 新增 %d / 仅站内有 %d / 冲突 %d", a, g, c)
}
}
// 冲突逐条选「用实机」就把备份的值写进去,选「用站内」就原样不动。
func TestBackupMergeConflictChoice(t *testing.T) {
f := newMovementFixture(t)
var pk int
if err := f.pg.QueryRowContext(f.ctx, `UPDATE visits SET remark='站内改过的备注'
WHERE id = (SELECT id FROM visits WHERE dataset_id=$1 ORDER BY src_pk LIMIT 1)
RETURNING src_pk`, f.dsID).Scan(&pk); err != nil {
t.Fatalf("改备注失败: %v", err)
}
// 这条的 UUID 就是页面上的冲突键
var b64 string
if err := f.pg.QueryRowContext(f.ctx, `SELECT COALESCE(raw #>> '{ZIDENTIFIER_,$blob}','')
FROM entity_raw WHERE dataset_id=$1 AND entity='ZVISIT' AND src_pk=$2`, f.dsID, pk).Scan(&b64); err != nil {
t.Fatalf("取 UUID 失败: %v", err)
}
uuid := uuidFromBase64(b64)
if uuid == "" {
t.Fatal("这条到访应当有 UUID")
}
remark := func() string {
var s string
if err := f.pg.QueryRowContext(f.ctx, `SELECT COALESCE(remark,'') FROM visits
WHERE dataset_id=$1 AND src_pk=$2`, f.dsID, pk).Scan(&s); err != nil {
t.Fatalf("读备注失败: %v", err)
}
return s
}
// 用站内:备注保持站里那份
token, _ := previewBackupToken(t, f, f.src)
mergeBackup(t, f, token, "vc_"+uuid+"=db")
if got := remark(); got != "站内改过的备注" {
t.Fatalf("选「用站内」不该改动,实际 %q", got)
}
// 用实机:备注被备份里的值(空)覆盖回去
token, _ = previewBackupToken(t, f, f.src)
mergeBackup(t, f, token, "vc_"+uuid+"=backup")
if got := remark(); got != "" {
t.Fatalf("选「用实机」应覆盖成备份里的值,实际 %q", got)
}
}
// 什么都不勾、冲突全跳过时,合并应当什么都不写。
func TestBackupMergeNoopKeepsData(t *testing.T) {
f := newMovementFixture(t)
before := f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1`, f.dsID)
token, _ := previewBackupToken(t, f, f.src)
mergeBackup(t, f, token, "add_new=0")
if got := f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1`, f.dsID); got != before {
t.Fatalf("没勾任何改动却动了数据: %d → %d", before, got)
}
}
// 删除必须「勾了才删」:没勾的一个都不动,勾了的那条(连带它的位移)才消失。
func TestBackupMergeDeletesOnlyChecked(t *testing.T) {
f := newMovementFixture(t)
visitCount := func() int {
return f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1`, f.dsID)
}
before := visitCount()
f.addExtraVisit(t, 900001, "稍后删掉")
f.addExtraVisit(t, 900002, "留着不动")
if got := visitCount(); got != before+2 {
t.Fatalf("预置失败: %d", got)
}
// 只勾一条
token, page := previewBackupToken(t, f, f.src)
if _, g, _ := visitCounts(t, page); g != 2 {
t.Fatalf("差异页应报 2 条仅站内有,实际 %d", g)
}
mergeBackup(t, f, token, "dv_900001=1")
if got := visitCount(); got != before+1 {
t.Fatalf("应只删掉勾选的那条,实际剩 %d(原 %d)", got, before)
}
if n := f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1 AND src_pk=900002`, f.dsID); n != 1 {
t.Error("没勾的那条不该被动")
}
// 删掉的到访不该留下留档
if n := f.scalar(t, `SELECT count(*) FROM entity_raw WHERE dataset_id=$1
AND entity='ZVISIT' AND src_pk=900001`, f.dsID); n != 0 {
t.Error("删除到访时要连它的原始行留档一起清掉")
}
// 再合并一次、一条都不勾:剩下那条必须还在
token, _ = previewBackupToken(t, f, f.src)
mergeBackup(t, f, token, "add_new=0")
if got := visitCount(); got != before+1 {
t.Fatalf("不勾就不该删,实际 %d", got)
}
}
// 名下有到访的地点必须拒绝删除——那个外键是 ON DELETE CASCADE,
// 真删下去会把到访一起带走。拒绝后还要整体回滚,别留下半截。
func TestBackupMergeRefusesDeletingPlaceInUse(t *testing.T) {
f := newMovementFixture(t)
visitCount := func() int {
return f.scalar(t, `SELECT count(*) FROM visits WHERE dataset_id=$1`, f.dsID)
}
beforeVisits := visitCount()
// 一个没有到访的空地点:允许删
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO places (dataset_id, src_pk, name, lat, lon)
VALUES ($1, 910001, '临时空地点', 30.0, 120.0)`, f.dsID); err != nil {
t.Fatalf("插入地点失败: %v", err)
}
// 一个有到访的地点:必须拒绝
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO places (dataset_id, src_pk, name, lat, lon)
VALUES ($1, 910002, '有到访的地点', 31.0, 121.0)`, f.dsID); err != nil {
t.Fatalf("插入地点失败: %v", err)
}
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO visits (dataset_id, src_pk, place_id, arrival, departure)
VALUES ($1, 910003, (SELECT id FROM places WHERE dataset_id=$1 AND src_pk=910002), now(), now())`,
f.dsID); err != nil {
t.Fatalf("插入到访失败: %v", err)
}
// 先删空地点:应当成功
token, _ := previewBackupToken(t, f, f.src)
mergeBackup(t, f, token, "dp_910001=1")
if n := f.scalar(t, `SELECT count(*) FROM places WHERE dataset_id=$1 AND src_pk=910001`, f.dsID); n != 0 {
t.Fatal("空地点应当被删掉")
}
// 再删有到访的地点:应当被拒绝
token, _ = previewBackupToken(t, f, f.src)
loc := mergeBackupAllowErr(t, f, token, "dp_910002=1")
if !strings.Contains(loc, "err=") {
t.Fatal("名下有到访的地点应当拒绝删除")
}
if n := f.scalar(t, `SELECT count(*) FROM places WHERE dataset_id=$1 AND src_pk=910002`, f.dsID); n != 1 {
t.Error("被拒绝后地点不该消失")
}
if got := visitCount(); got != beforeVisits+1 {
t.Errorf("被拒绝后到访数不该变,实际 %d(期望 %d)", got, beforeVisits+1)
}
}
+149 -16
View File
@@ -3,11 +3,13 @@ package web
import (
"bytes"
"context"
"errors"
"fmt"
"image"
"image/png"
"io"
"log"
"math"
"net/http"
"path/filepath"
"sort"
@@ -111,41 +113,57 @@ func fogTileCacheControl(isAdmin bool) string {
return "public, max-age=3600"
}
// adminFogUpload 导入 .fwss:mode=overlay 按位或合并进现有迷雾,
// mode=cover 清空后整体替换。
func (s *Server) adminFogUpload(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
u := userFrom(ctx)
// readFogSnapshot 从 multipart 表单里读出 .fwss 的字节并做基本校验。
func readFogSnapshot(w http.ResponseWriter, r *http.Request) ([]byte, error) {
r.Body = http.MaxBytesReader(w, r.Body, maxFogUploadBytes+1<<20)
if err := r.ParseMultipartForm(32 << 20); err != nil {
s.adminFlash(w, r, "err", "读取上传内容失败:"+err.Error())
return
return nil, fmt.Errorf("读取上传内容失败:%w", err)
}
file, hdr, err := r.FormFile("fwss")
if err != nil {
s.adminFlash(w, r, "err", "请选择要上传的 .fwss 迷雾快照")
return
return nil, errors.New("请选择要上传的 .fwss 迷雾快照")
}
defer file.Close()
name := strings.ToLower(filepath.Base(hdr.Filename))
if !strings.HasSuffix(name, ".fwss") && !strings.HasSuffix(name, ".zip") {
s.adminFlash(w, r, "err", "文件类型不正确,请上传 Fog of World 导出的 .fwss 快照")
return
return nil, errors.New("文件类型不正确,请上传 Fog of World 导出的 .fwss 快照")
}
data, err := io.ReadAll(io.LimitReader(file, maxFogUploadBytes+1))
if err != nil {
return nil, fmt.Errorf("读取文件失败:%w", err)
}
if int64(len(data)) > maxFogUploadBytes {
return nil, errors.New("迷雾快照超过 64MB,疑似文件有误")
}
return data, nil
}
// adminFogUpload 导入 .fwss:mode=overlay 按位或合并进现有迷雾,
// mode=cover 清空后整体替换。
//
// 带 stash 令牌的请求来自差异页的确认按钮,快照已经在暂存目录里,
// 不用让浏览器把几十 MB 再传一遍。
func (s *Server) adminFogUpload(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
u := userFrom(ctx)
mode := r.PostFormValue("mode")
if mode != "overlay" && mode != "cover" {
mode = "overlay"
}
data, err := io.ReadAll(io.LimitReader(file, maxFogUploadBytes+1))
if err != nil {
s.adminFlash(w, r, "err", "读取文件失败:"+err.Error())
token := r.URL.Query().Get("stash")
var data []byte
var err error
if token != "" {
if data, err = s.loadStashedFog(token); err != nil {
s.adminFlash(w, r, "err", err.Error())
return
}
if int64(len(data)) > maxFogUploadBytes {
s.adminFlash(w, r, "err", "迷雾快照超过 64MB,疑似文件有误")
} else if data, err = readFogSnapshot(w, r); err != nil {
s.adminFlash(w, r, "err", err.Error())
return
}
blocks, cells, err := fog.Decode(data)
if err != nil {
s.adminFlash(w, r, "err", "解析迷雾快照失败:"+err.Error())
@@ -181,10 +199,125 @@ func (s *Server) adminFogUpload(w http.ResponseWriter, r *http.Request) {
if mode == "cover" {
verb = "整体替换"
}
// 导入成功才丢掉暂存件;失败时留着,站长可以回差异页重试
if token != "" {
s.dropStashedFog(token)
}
s.adminFlash(w, r, "ok", fmt.Sprintf(
"迷雾导入成功(%s):本次写入 %d 块 / %d 格,当前共 %d 块 / %d 格", verb, applied, cells, b, c))
}
// adminFogPreview 先把快照收下暂存,不写库,跳到差异页——
// 叠加能补多少、整体替换会丢多少,都得让站长在下手之前看见。
func (s *Server) adminFogPreview(w http.ResponseWriter, r *http.Request) {
data, err := readFogSnapshot(w, r)
if err != nil {
s.adminFlash(w, r, "err", err.Error())
return
}
// 先解一遍确认是真快照,免得把坏文件暂存下来、到差异页才发现
if _, _, err := fog.Decode(data); err != nil {
s.adminFlash(w, r, "err", "解析迷雾快照失败:"+err.Error())
return
}
token, err := s.stashFog(data)
if err != nil {
s.serverError(w, r, err)
return
}
http.Redirect(w, r, "/admin/fog/preview/"+token, http.StatusSeeOther)
}
// adminFogPreviewPage 渲染差异页。每次都重新算一遍:库里可能刚被别的操作改过。
func (s *Server) adminFogPreviewPage(w http.ResponseWriter, r *http.Request) {
p, err := s.pageBase(r, "admin", "迷雾导入预览")
if err != nil {
s.serverError(w, r, err)
return
}
token := r.PathValue("token")
data, err := s.loadStashedFog(token)
if err != nil {
s.adminFlash(w, r, "err", err.Error())
return
}
blocks, _, err := fog.Decode(data)
if err != nil {
s.adminFlash(w, r, "err", "解析迷雾快照失败:"+err.Error())
return
}
if len(blocks) == 0 {
s.adminFlash(w, r, "err", "快照里没有找到迷雾瓦片数据")
return
}
layers, err := fog.RawLayers(data)
if err != nil {
log.Printf("解析快照的其它图层失败: %v", err)
}
ix, err := s.fogIndex()
if err != nil {
s.serverError(w, r, err)
return
}
var cur []fog.Block
ix.View(func(m map[[2]int][]byte) {
cur = make([]fog.Block, 0, len(m))
for k, bm := range m {
cur = append(cur, fog.Block{GX: k[0], GY: k[1], Bitmap: bm})
}
})
tilde, hash, other := fogLayerCounts(layers)
d := FogPreviewData{
Page: p, Token: token, Diff: fog.DiffBlocks(cur, blocks),
TildeLayers: tilde, HashLayers: hash, OtherLayers: other,
}
d.MinLat, d.MinLon, d.MaxLat, d.MaxLon, d.HasBounds = diffBounds(cur, blocks, s.cfg.MapCRS)
// 布局按这个标记引入 Leaflet,漏了地图整块不出现
d.Page.HasMap = d.HasBounds
s.render(w, "admin/fog-preview", d)
}
// diffBounds 返回两边块集覆盖范围的包围盒,供差异地图定位。迷雾网格是 WGS-84,
// 高德这类底图按 GCJ-02 摆放,所以坐标要跟着底图走,否则视野会偏出几百米。
func diffBounds(cur, inc []fog.Block, crs string) (minLat, minLon, maxLat, maxLon float64, ok bool) {
visit := func(gbx, gby int) {
lat, lon := fog.BlockLonLat(gbx, gby)
if crs == "gcj02" {
lat, lon = geo.WGS84ToGCJ02(lat, lon)
}
if !ok {
minLat, maxLat, minLon, maxLon, ok = lat, lat, lon, lon, true
return
}
minLat, maxLat = math.Min(minLat, lat), math.Max(maxLat, lat)
minLon, maxLon = math.Min(minLon, lon), math.Max(maxLon, lon)
}
for _, b := range cur {
visit(b.GX, b.GY)
}
for _, b := range inc {
visit(b.GX, b.GY)
}
return
}
// fogLayerCounts 按前缀归类快照里「已探索层之外」的条目。
// Model/~ 与 Model/# 两层站点不解析、只原样留存,但整体替换会把它们一并换掉,
// 差异页要把这件事说清楚。
func fogLayerCounts(layers map[string][]byte) (tilde, hash, other int) {
for p := range layers {
switch {
case strings.HasPrefix(p, "Model/~/") || strings.HasPrefix(p, "Model~/"):
tilde++
case strings.HasPrefix(p, "Model/#/") || strings.HasPrefix(p, "Model#/"):
hash++
default:
other++
}
}
return
}
// ---------- 迷雾覆盖率 ----------
// FogCity 是单个城市(或省份)的迷雾点亮统计:地点里有多少个落在已探索格内。
+94
View File
@@ -0,0 +1,94 @@
package web
import (
"bytes"
"image/png"
"net/http"
"strconv"
"sync"
"time"
"rond-with-you/internal/fog"
)
// 差异瓦片要反复读同一份快照、同一份库内迷雾,单槽缓存住两边的块集即可——
// 站长一次只会看一份预览。缓存里的 map 加载完就只读,渲染时并发遍历是安全的。
var (
diffSetsMu sync.Mutex
diffSetsToken string
diffSetsAt time.Time
diffSetsCur map[[2]int][]byte
diffSetsInc map[[2]int][]byte
)
// diffSetsTTL 短于暂存的 2 小时:库内迷雾随时可能被别的操作改掉,
// 缓存太久会让差异图与页面上那组数字对不上。
const diffSetsTTL = 10 * time.Minute
func (s *Server) fogDiffSets(token string) (cur, inc map[[2]int][]byte, err error) {
diffSetsMu.Lock()
defer diffSetsMu.Unlock()
if token == diffSetsToken && time.Since(diffSetsAt) < diffSetsTTL {
return diffSetsCur, diffSetsInc, nil
}
data, err := s.loadStashedFog(token)
if err != nil {
return nil, nil, err
}
blocks, _, err := fog.Decode(data)
if err != nil {
return nil, nil, err
}
ix, err := s.fogIndex()
if err != nil {
return nil, nil, err
}
cur = map[[2]int][]byte{}
ix.View(func(m map[[2]int][]byte) {
for k, bm := range m {
cur[k] = bm
}
})
inc = make(map[[2]int][]byte, len(blocks))
for _, b := range blocks {
inc[[2]int{b.GX, b.GY}] = b.Bitmap
}
diffSetsToken, diffSetsCur, diffSetsInc, diffSetsAt = token, cur, inc, time.Now()
return cur, inc, nil
}
// adminFogDiffTile 渲染一张差异瓦片:绿=叠加会新增、灰=两边都有、红=整体替换会丢。
// 只给站长看,不涉及遮罩。
func (s *Server) adminFogDiffTile(w http.ResponseWriter, r *http.Request) {
z, err1 := strconv.Atoi(r.PathValue("z"))
x, err2 := strconv.Atoi(r.PathValue("x"))
y, err3 := strconv.Atoi(r.PathValue("y"))
if err1 != nil || err2 != nil || err3 != nil || z < 0 || z > 22 ||
x < 0 || y < 0 || x >= 1<<uint(z) || y >= 1<<uint(z) {
http.NotFound(w, r)
return
}
cur, inc, err := s.fogDiffSets(r.PathValue("token"))
if err != nil {
http.NotFound(w, r)
return
}
crs := r.URL.Query().Get("crs")
if crs != "gcj02" && crs != "wgs84" {
crs = s.cfg.MapCRS
}
img := fog.RenderDiffTile(cur, inc, z, x, y, crs)
if img == nil {
http.NotFound(w, r)
return
}
var buf bytes.Buffer
if err := png.Encode(&buf, img); err != nil {
s.serverError(w, r, err)
return
}
w.Header().Set("Content-Type", "image/png")
// 令牌是一次性的,内容也只给站长看,不落共享缓存
w.Header().Set("Cache-Control", "private, max-age=300")
w.Write(buf.Bytes())
}
+422
View File
@@ -0,0 +1,422 @@
package web
import (
"bytes"
"context"
"database/sql"
"fmt"
"image"
"image/png"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
"rond-with-you/internal/auth"
"rond-with-you/internal/config"
"rond-with-you/internal/db"
"rond-with-you/internal/fog"
)
// fogPreviewFixture 是一个迷雾为空、带一次性账号的测试库。
//
// ROND_TEST_CONF=conf/app-test.ini go test ./internal/web -run FogPreview -v
//
// 迷雾是站点级数据(公开读路径不区分归属),所以测试必须从空库出发;
// 用的是独立测试库(rond_fog_test),不会碰到开发库里的迷雾。
type fogPreviewFixture struct {
cfg *config.Config
pg *sql.DB
srv *Server
ctx context.Context
uid int64
ix *fog.Index
}
func newFogPreviewFixture(t *testing.T) *fogPreviewFixture {
t.Helper()
confPath := os.Getenv("ROND_TEST_CONF")
if confPath == "" {
t.Skip("设置 ROND_TEST_CONF 后运行迷雾导入校验")
}
chdirModuleRoot(t)
cfg, err := config.Load(confPath)
if err != nil {
t.Fatalf("加载配置 %s 失败: %v", confPath, err)
}
ctx := context.Background()
pg, err := db.Open(cfg.DSN)
if err != nil {
t.Fatalf("连接数据库失败: %v", err)
}
t.Cleanup(func() { pg.Close() })
if err := db.Migrate(ctx, pg); err != nil {
t.Fatalf("初始化数据库失败: %v", err)
}
user := fmt.Sprintf("e2e-fogpreview-%d", time.Now().UnixNano())
hash, err := auth.HashPassword("e2e-password")
if err != nil {
t.Fatalf("生成口令哈希失败: %v", err)
}
var uid int64
if err := pg.QueryRowContext(ctx, `INSERT INTO users (username, password_hash, display_name)
VALUES ($1,$2,'e2e') RETURNING id`, user, hash).Scan(&uid); err != nil {
t.Fatalf("创建测试账号失败: %v", err)
}
clearFog(t, ctx, pg)
t.Cleanup(func() {
clearFog(t, ctx, pg)
pg.ExecContext(ctx, `DELETE FROM users WHERE id=$1`, uid)
os.RemoveAll(filepath.Join(cfg.DataDir, "fog-staging"))
fogResetIndex()
})
srv, err := New(cfg, pg)
if err != nil {
t.Fatalf("初始化服务失败: %v", err)
}
f := &fogPreviewFixture{cfg: cfg, pg: pg, srv: srv, uid: uid,
ctx: ctxWithUser(&auth.User{ID: uid, Username: user})}
f.ix = f.reloadIndex(t)
return f
}
func clearFog(t *testing.T, ctx context.Context, pg *sql.DB) {
t.Helper()
for _, q := range []string{`DELETE FROM fog_blocks`, `DELETE FROM fog_raw_files`} {
if _, err := pg.ExecContext(ctx, q); err != nil {
t.Fatalf("清空迷雾失败: %v", err)
}
}
}
// reloadIndex 让内存索引与库对齐:fogIndex 的首次加载只跑一次(sync.Once),
// 测试里反复清库写库,必须手动同步,否则差异页算的是上一轮的残留。
func (f *fogPreviewFixture) reloadIndex(t *testing.T) *fog.Index {
t.Helper()
ix, err := f.srv.fogIndex()
if err != nil {
t.Fatalf("获取迷雾索引失败: %v", err)
}
if err := ix.LoadAll(f.pg); err != nil {
t.Fatalf("重载迷雾索引失败: %v", err)
}
return ix
}
// upload 走后台真实的上传处理函数,返回差异页的令牌。
func (f *fogPreviewFixture) upload(t *testing.T, data []byte) string {
t.Helper()
rec := httptest.NewRecorder()
f.srv.adminFogPreview(rec, fogUploadRequest(t, data).WithContext(f.ctx))
if rec.Code != http.StatusSeeOther {
t.Fatalf("上传应跳转到差异页: code=%d body=%s", rec.Code, rec.Body.String())
}
loc := rec.Header().Get("Location")
token := strings.TrimPrefix(loc, "/admin/fog/preview/")
if token == loc || len(token) != fogStashTokenLen {
t.Fatalf("差异页地址不对: %q", loc)
}
return token
}
// preview 渲染差异页。
func (f *fogPreviewFixture) preview(t *testing.T, token string) string {
t.Helper()
rec := httptest.NewRecorder()
req := httptest.NewRequest("GET", "/admin/fog/preview/"+token, nil).WithContext(f.ctx)
req.SetPathValue("token", token)
f.srv.adminFogPreviewPage(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("差异页渲染失败: code=%d body=%s", rec.Code, rec.Body.String())
}
return rec.Body.String()
}
// confirm 走差异页确认按钮那条路径:带 stash 令牌、只提交 mode。
func (f *fogPreviewFixture) confirm(t *testing.T, token, mode string) {
t.Helper()
rec := httptest.NewRecorder()
req := httptest.NewRequest("POST", "/admin/fog?stash="+token, strings.NewReader("mode="+mode))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
f.srv.adminFogUpload(rec, req.WithContext(f.ctx))
if rec.Code != http.StatusSeeOther {
t.Fatalf("确认导入应跳回后台: code=%d body=%s", rec.Code, rec.Body.String())
}
if loc := rec.Header().Get("Location"); !strings.HasPrefix(loc, "/admin?ok=") {
t.Fatalf("确认导入应报成功: %s", loc)
}
}
func (f *fogPreviewFixture) cells(t *testing.T) int {
t.Helper()
_, cells := f.reloadIndex(t).Stats()
return cells
}
func (f *fogPreviewFixture) blocks(t *testing.T) int {
t.Helper()
blocks, _ := f.reloadIndex(t).Stats()
return blocks
}
// TestFogPreviewShowsDiffBeforeWriting 端到端验证「上传 → 差异页 → 确认导入」:
// 差异页必须先把两种模式的后果摆出来,而且在上传那一刻格子里一行都不能写。
func TestFogPreviewShowsDiffBeforeWriting(t *testing.T) {
f := newFogPreviewFixture(t)
// 库里先有一块 3 格的区域
if _, err := f.ix.Import(f.pg, f.uid, []fog.Block{
newFogBlock(100, 200, [2]int{0, 0}, [2]int{1, 0}, [2]int{2, 0}),
}, false); err != nil {
t.Fatalf("预置迷雾失败: %v", err)
}
// 新快照:同一块补亮 2 格,外加一整块新的
inc := []fog.Block{
newFogBlock(100, 200, [2]int{0, 0}, [2]int{1, 0}, [2]int{2, 0}, [2]int{5, 5}, [2]int{6, 5}),
newFogBlock(101, 200, [2]int{1, 1}),
}
data, err := fog.Encode(inc, map[string][]byte{
"Model/~/aaa": {1, 2, 3},
"Model/#/bbb": {4, 5, 6},
})
if err != nil {
t.Fatalf("构造快照失败: %v", err)
}
token := f.upload(t, data)
// 差异页只是预览:格数必须还是 3
if got := f.cells(t); got != 3 {
t.Fatalf("上传阶段不该写库,当前格数 %d", got)
}
page := f.preview(t, token)
for _, want := range []string{
"迷雾导入预览",
"叠加导入",
"整体替换",
"会补上当前迷雾没有的", // 叠加卡片给出的实际收益
"替换不会丢数据", // 库里的 3 格都在快照里
"Model/~", // 图层提示
"/admin/fog?stash=" + token,
`value="overlay"`,
`value="cover"`,
} {
if !strings.Contains(page, want) {
t.Errorf("差异页缺少 %q", want)
}
}
if strings.Contains(page, "完全覆盖") {
t.Error("这份快照有新内容,不该报「已被完全覆盖」")
}
// 确认叠加:3 格 + 补亮的 2 格 + 新块 1 格 = 6 格,两块
f.confirm(t, token, "overlay")
if got := f.cells(t); got != 6 {
t.Fatalf("叠加后应有 6 格,实际 %d", got)
}
if got := f.blocks(t); got != 2 {
t.Fatalf("叠加后应有 2 块,实际 %d", got)
}
}
// TestFogPreviewWarnsBeforeCoverLosesData 整体替换必须在下手前算出「会丢多少」,
// 并且差异页要看得出这份快照已经被完全覆盖过(叠加没有意义)。
func TestFogPreviewWarnsBeforeCoverLosesData(t *testing.T) {
f := newFogPreviewFixture(t)
// 库里两块共 5 格
if _, err := f.ix.Import(f.pg, f.uid, []fog.Block{
newFogBlock(100, 200, [2]int{0, 0}, [2]int{1, 0}, [2]int{2, 0}),
newFogBlock(101, 200, [2]int{7, 7}, [2]int{8, 8}),
}, false); err != nil {
t.Fatalf("预置迷雾失败: %v", err)
}
// 快照只有第一块、且一格不多:叠加零收益,替换会丢掉第二块的两格
inc := []fog.Block{newFogBlock(100, 200, [2]int{0, 0}, [2]int{1, 0}, [2]int{2, 0})}
data, err := fog.Encode(inc, nil)
if err != nil {
t.Fatalf("构造快照失败: %v", err)
}
token := f.upload(t, data)
page := f.preview(t, token)
for _, want := range []string{
"已经被当前迷雾", // 叠加无收益
"从点亮变回未探索",
"Model/~", // 快照没有这两层,替换会清掉现有的
} {
if !strings.Contains(page, want) {
t.Errorf("差异页缺少 %q", want)
}
}
// 确认整体替换:只该剩下快照里的 1 块 3 格
f.confirm(t, token, "cover")
if got := f.cells(t); got != 3 {
t.Fatalf("替换后应有 3 格,实际 %d", got)
}
if got := f.blocks(t); got != 1 {
t.Fatalf("替换后应有 1 块,实际 %d", got)
}
// 暂存件用掉就该删掉,且过期令牌不该还能用
if _, err := os.Stat(filepath.Join(f.cfg.DataDir, "fog-staging", token+".fwss")); !os.IsNotExist(err) {
t.Error("确认导入后应删除暂存文件")
}
rec := httptest.NewRecorder()
req := httptest.NewRequest("GET", "/admin/fog/preview/"+token, nil).WithContext(f.ctx)
req.SetPathValue("token", token)
f.srv.adminFogPreviewPage(rec, req)
if rec.Code != http.StatusSeeOther {
t.Fatalf("过期令牌应跳回后台报错: code=%d", rec.Code)
}
}
// 差异瓦片要按请求坐标渲染出正确的三色,并挡住越界坐标与过期令牌。
func TestFogDiffTileColors(t *testing.T) {
f := newFogPreviewFixture(t)
if _, err := f.ix.Import(f.pg, f.uid, []fog.Block{
newFogBlock(100, 200, [2]int{0, 0}, [2]int{4, 0}),
}, false); err != nil {
t.Fatalf("预置迷雾失败: %v", err)
}
data, err := fog.Encode([]fog.Block{newFogBlock(100, 200, [2]int{0, 0}, [2]int{1, 0})}, nil)
if err != nil {
t.Fatalf("构造快照失败: %v", err)
}
token := f.upload(t, data)
// z=18 时一格 16 像素,块 (100,200) 完整落在瓦片 (400,800) 里
img := f.diffTile(t, token, 18, 400, 800)
if got := kindOf(pixAtImg(img, 8, 8)); got != "both" {
t.Errorf("两边都有的格应为灰色,实际 %s", got)
}
if got := kindOf(pixAtImg(img, 24, 8)); got != "gain" {
t.Errorf("只有快照有的格应为绿色,实际 %s", got)
}
if got := kindOf(pixAtImg(img, 72, 8)); got != "lost" {
t.Errorf("只有库里有、替换会丢的格应为红色,实际 %s", got)
}
if got := kindOf(pixAtImg(img, 200, 200)); got != "none" {
t.Errorf("没探索过的格应为透明,实际 %s", got)
}
// 越界坐标与过期令牌都只给 404,不能穿透到渲染
for _, c := range []struct{ z, x, y string }{
{"18", "262144", "0"}, // x 已达该级别的上限
{"18", "0", "-1"},
{"99", "0", "0"}, // z 越界
} {
rec := httptest.NewRecorder()
req := httptest.NewRequest("GET", "/admin/fog/preview/"+token+"/x", nil).WithContext(f.ctx)
req.SetPathValue("token", token)
req.SetPathValue("z", c.z)
req.SetPathValue("x", c.x)
req.SetPathValue("y", c.y)
f.srv.adminFogDiffTile(rec, req)
if rec.Code != http.StatusNotFound {
t.Errorf("越界坐标 %v 应给 404,实际 %d", c, rec.Code)
}
}
rec := httptest.NewRecorder()
req := httptest.NewRequest("GET", "/admin/fog/preview/deadbeef/18/400/800", nil).WithContext(f.ctx)
req.SetPathValue("token", strings.Repeat("f", 32))
req.SetPathValue("z", "18")
req.SetPathValue("x", "400")
req.SetPathValue("y", "800")
f.srv.adminFogDiffTile(rec, req)
if rec.Code != http.StatusNotFound {
t.Errorf("无效令牌应给 404,实际 %d", rec.Code)
}
}
func (f *fogPreviewFixture) diffTile(t *testing.T, token string, z, x, y int) image.Image {
t.Helper()
rec := httptest.NewRecorder()
req := httptest.NewRequest("GET", fmt.Sprintf("/admin/fog/preview/%s/%d/%d/%d?crs=wgs84", token, z, x, y), nil).
WithContext(f.ctx)
req.SetPathValue("token", token)
req.SetPathValue("z", strconv.Itoa(z))
req.SetPathValue("x", strconv.Itoa(x))
req.SetPathValue("y", strconv.Itoa(y))
f.srv.adminFogDiffTile(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("差异瓦片渲染失败: code=%d", rec.Code)
}
if ct := rec.Header().Get("Content-Type"); ct != "image/png" {
t.Fatalf("差异瓦片应为 PNG,实际 %q", ct)
}
img, err := png.Decode(bytes.NewReader(rec.Body.Bytes()))
if err != nil {
t.Fatalf("解码差异瓦片失败: %v", err)
}
return img
}
// kindOf 按「哪个通道最亮」判类别,不去比对精确色值——PNG 解码回来的是
// 预乘 alpha 的像素,硬比色值会被 alpha 换算绕进去。
func kindOf(c [4]byte) string {
switch {
case c[3] == 0:
return "none"
case c[1] > c[0] && c[1] > c[2]:
return "gain"
case c[0] > c[1] && c[0] > c[2]:
return "lost"
}
return "both"
}
func pixAtImg(img image.Image, px, py int) [4]byte {
r, g, b, a := img.At(px, py).RGBA()
return [4]byte{byte(r >> 8), byte(g >> 8), byte(b >> 8), byte(a >> 8)}
}
// 令牌来自 URL,必须挡死路径穿越。
func TestFogStashTokenRejectsTraversal(t *testing.T) {
s := &Server{cfg: &config.Config{DataDir: t.TempDir()}}
for _, bad := range []string{"", "..", "../../etc/passwd", strings.Repeat("a", 32) + "/../x", strings.ToUpper(strings.Repeat("a", 32))} {
if p, ok := s.fogStashPath(bad); ok {
t.Errorf("令牌 %q 不该通过校验,得到 %s", bad, p)
}
}
if _, ok := s.fogStashPath(strings.Repeat("0", 32)); !ok {
t.Error("32 位十六进制令牌应当通过")
}
}
func newFogBlock(gx, gy int, cells ...[2]int) fog.Block {
bm := make([]byte, 512)
for _, c := range cells {
bm[c[1]*8+c[0]/8] |= 0x80 >> (c[0] % 8)
}
return fog.Block{GX: gx, GY: gy, Bitmap: bm}
}
func fogUploadRequest(t *testing.T, data []byte) *http.Request {
t.Helper()
var buf bytes.Buffer
mw := multipart.NewWriter(&buf)
fw, err := mw.CreateFormFile("fwss", "Snapshot-test.fwss")
if err != nil {
t.Fatalf("构造上传表单失败: %v", err)
}
if _, err := fw.Write(data); err != nil {
t.Fatalf("写入上传内容失败: %v", err)
}
mw.Close()
req := httptest.NewRequest("POST", "/admin/fog/preview", &buf)
req.Header.Set("Content-Type", mw.FormDataContentType())
return req
}
+116
View File
@@ -0,0 +1,116 @@
package web
import (
"crypto/rand"
"encoding/hex"
"fmt"
"os"
"path/filepath"
"strings"
"time"
)
// 待确认的上传(迷雾快照、rond 备份)都先落到磁盘暂存,让站长看完差异再决定怎么导入;
// 看完直接走人的靠 TTL 回收,不至于把几十 MB 的文件一直留在盘上。
//
// 令牌只认固定长度的十六进制串,这样即便来自 URL 也拼不出 ../ 之类的路径。
const (
stashTokenLen = 32 // 16 字节的十六进制
stashTTL = 2 * time.Hour
)
func stashPut(dir, ext string, data []byte) (string, error) {
if err := os.MkdirAll(dir, 0o700); err != nil {
return "", err
}
var raw [stashTokenLen / 2]byte
if _, err := rand.Read(raw[:]); err != nil {
return "", err
}
token := hex.EncodeToString(raw[:])
if err := os.WriteFile(filepath.Join(dir, token+ext), data, 0o600); err != nil {
return "", err
}
stashSweep(dir, ext)
return token, nil
}
func stashPath(dir, ext, token string) (string, bool) {
if len(token) != stashTokenLen {
return "", false
}
for i := 0; i < len(token); i++ {
c := token[i]
if (c < '0' || c > '9') && (c < 'a' || c > 'f') {
return "", false
}
}
return filepath.Join(dir, token+ext), true
}
func stashRead(dir, ext, token, what string) ([]byte, error) {
p, ok := stashPath(dir, ext, token)
if !ok {
return nil, fmt.Errorf("预览链接无效,请重新上传%s", what)
}
data, err := os.ReadFile(p)
if err != nil {
return nil, fmt.Errorf("这次预览已过期,请重新上传%s", what)
}
return data, nil
}
func stashDrop(dir, ext, token string) {
if p, ok := stashPath(dir, ext, token); ok {
os.Remove(p)
}
}
func stashSweep(dir, ext string) {
entries, err := os.ReadDir(dir)
if err != nil {
return
}
cutoff := time.Now().Add(-stashTTL)
for _, e := range entries {
if e.IsDir() || !strings.HasSuffix(e.Name(), ext) {
continue
}
if info, err := e.Info(); err == nil && info.ModTime().After(cutoff) {
continue
}
os.Remove(filepath.Join(dir, e.Name()))
}
}
// ---------- 迷雾快照 ----------
// fogStashTokenLen 是迷雾预览令牌的长度,测试与调用方按它校验。
const fogStashTokenLen = stashTokenLen
const fogStashExt = ".fwss"
func (s *Server) fogStashDir() string {
return filepath.Join(s.cfg.DataDir, "fog-staging")
}
func (s *Server) stashFog(data []byte) (string, error) {
return stashPut(s.fogStashDir(), fogStashExt, data)
}
func (s *Server) fogStashPath(token string) (string, bool) {
return stashPath(s.fogStashDir(), fogStashExt, token)
}
func (s *Server) loadStashedFog(token string) ([]byte, error) {
return stashRead(s.fogStashDir(), fogStashExt, token, "快照")
}
func (s *Server) dropStashedFog(token string) {
stashDrop(s.fogStashDir(), fogStashExt, token)
}
func (s *Server) sweepFogStash() {
stashSweep(s.fogStashDir(), fogStashExt)
}
+5 -5
View File
@@ -769,12 +769,12 @@ func (s *Server) notesPage(w http.ResponseWriter, r *http.Request) {
case "note":
order = "n.updated_at DESC NULLS LAST, p.name"
case "talk":
order = "(COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.src_pk = p.src_pk),0)) DESC, p.visit_count DESC"
order = "(COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.place_key = " + placeKeyExpr + "),0)) DESC, p.visit_count DESC"
}
base := `FROM places p
LEFT JOIN place_notes n ON n.src_pk = p.src_pk AND n.dataset_id = p.dataset_id
LEFT JOIN place_votes v ON v.src_pk = p.src_pk AND v.dataset_id = p.dataset_id
LEFT JOIN place_notes n ON n.place_key = ` + placeKeyExpr + `
LEFT JOIN place_votes v ON v.place_key = ` + placeKeyExpr + `
WHERE ` + strings.Join(where, " AND ")
var total int
@@ -790,8 +790,8 @@ func (s *Server) notesPage(w http.ResponseWriter, r *http.Request) {
limitArgs := append(append([]any{}, args...), notesPerPage, (d.PageNo-1)*notesPerPage)
q2 := `SELECT p.src_pk, COALESCE(n.alias,''), COALESCE(n.note,''), COALESCE(n.rond_note,''), COALESCE(v.verdict,0),
COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.src_pk=p.src_pk AND vv.vote=1),0),
COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.src_pk=p.src_pk AND vv.vote=-1),0),
COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.place_key = ` + placeKeyExpr + ` AND vv.vote=1),0),
COALESCE((SELECT count(*) FROM place_visitor_votes vv WHERE vv.place_key = ` + placeKeyExpr + ` AND vv.vote=-1),0),
COALESCE(n.updated_at, to_timestamp(0)), p.id, COALESCE(p.name,''), COALESCE(p.city,''), COALESCE(p.visit_count,0)
` + base + ` ORDER BY ` + order + fmt.Sprintf(` LIMIT $%d OFFSET $%d`, len(args)+1, len(args)+2)
+1
View File
@@ -1742,6 +1742,7 @@ var (
activitySeq = srcPKSeq{"activities", "src_pk", "Activity", "ZACTIVITY"}
weatherSeq = srcPKSeq{"weather", "src_pk", "HourlyWeather", "ZHOURLYWEATHER"}
movementSeq = srcPKSeq{"movements", "src_pk", "Movement", "ZMOVEMENT"}
tagSeq = srcPKSeq{"tags", "src_pk", "Tag", "ZTAG"}
transportSeq = srcPKSeq{"movements", "transport_src", "Transport", "ZTRANSPORT"}
)
+40 -6
View File
@@ -3,6 +3,7 @@ package web
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"fmt"
"io"
@@ -26,6 +27,11 @@ const (
imageUploadMem = 8 << 20 // multipart 解析用的内存上限
)
// placeKeyExpr 把 places p 那一行的坐标换算成 place_key,图片、备注、结论共用这个键。
// **必须与 rond.PlaceCoordKey 以及 notes/votes 包里的同款表达式逐位一致**:
// fmt.Sprintf("%.6f|%.6f") ↔ to_char(x,'FM999990.000000')||'|'||to_char(y,'FM999990.000000')
const placeKeyExpr = `to_char(p.lat,'FM999990.000000')||'|'||to_char(p.lon,'FM999990.000000')`
// PlaceImage 是模板里用的地点图片。
type PlaceImage struct {
ID int64
@@ -57,14 +63,18 @@ func (s *Server) savePlaceImage(ctx context.Context, datasetID int64, srcPK int,
if err := s.media.Save(ctx, backend, key, data, "image/webp"); err != nil {
return 0, "", err
}
pkey, err := s.placeKeyOf(ctx, datasetID, srcPK)
if err != nil {
return 0, "", err
}
var id int64
if err := s.db.QueryRowContext(ctx, `INSERT INTO place_images
(dataset_id, src_pk, storage, object_key, width, height, bytes, position)
VALUES ($1,$2,$3,$4,$5,$6,$7,
COALESCE((SELECT max(position)+1 FROM place_images WHERE dataset_id=$1 AND src_pk=$2), 0))
(dataset_id, src_pk, place_key, storage, object_key, width, height, bytes, position)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,
COALESCE((SELECT max(position)+1 FROM place_images WHERE place_key=$3), 0))
ON CONFLICT (dataset_id, src_pk, object_key) DO UPDATE SET bytes=EXCLUDED.bytes
RETURNING id`,
datasetID, srcPK, backend, key, w, h, int64(len(data))).Scan(&id); err != nil {
datasetID, srcPK, pkey, backend, key, w, h, int64(len(data))).Scan(&id); err != nil {
return 0, "", err
}
return id, s.media.PublicURL(backend, key), nil
@@ -95,9 +105,15 @@ func (s *Server) adminPlaceImageUpload(w http.ResponseWriter, r *http.Request) {
return
}
pkey, err := s.placeKeyOf(ctx, ds.ID, src)
if err != nil {
s.serverError(w, r, err)
return
}
var count int
// 按坐标键数:换数据集后图片仍跟着同一个地点,配额也要跟着
if err := s.db.QueryRowContext(ctx, `SELECT count(*) FROM place_images
WHERE dataset_id=$1 AND src_pk=$2`, ds.ID, src).Scan(&count); err != nil {
WHERE place_key=$1`, pkey).Scan(&count); err != nil {
s.serverError(w, r, err)
return
}
@@ -196,10 +212,28 @@ func (s *Server) imageBackend(ctx context.Context) (string, error) {
return setting.MediaBackendLocal, nil
}
// placeKeyOf 把某个数据集里的地点编号换算成坐标键。
// 换数据集之后仍靠它认出「还是同一个地点」,所以图片、备注、结论都挂在这上面。
func (s *Server) placeKeyOf(ctx context.Context, datasetID int64, srcPK int) (string, error) {
var key string
err := s.db.QueryRowContext(ctx, `SELECT `+placeKeyExpr+` FROM places p
WHERE p.dataset_id=$1 AND p.src_pk=$2 AND p.lat IS NOT NULL AND p.lon IS NOT NULL`,
datasetID, srcPK).Scan(&key)
if err == sql.ErrNoRows {
return "", fmt.Errorf("地点不存在或没有坐标")
}
return key, err
}
// placeImagesOf 读某个地点的图片,按 position 排序。
// 按坐标键查而不是 (dataset_id, src_pk):换一份备份之后图片还跟着同一个地点。
func (s *Server) placeImagesOf(ctx context.Context, datasetID int64, srcPK int) ([]PlaceImage, error) {
pkey, err := s.placeKeyOf(ctx, datasetID, srcPK)
if err != nil {
return nil, nil // 地点没有坐标就当作没有图
}
rows, err := s.db.QueryContext(ctx, `SELECT id, storage, object_key, width, height, bytes
FROM place_images WHERE dataset_id=$1 AND src_pk=$2 ORDER BY position, id`, datasetID, srcPK)
FROM place_images WHERE place_key=$1 ORDER BY position, id`, pkey)
if err != nil {
return nil, err
}
+3 -3
View File
@@ -183,7 +183,7 @@ func (s *Server) attachVotes(ctx context.Context, datasetID int64, places []stat
if err != nil {
return
}
tallies, err := s.votes.Tallies(ctx)
tallies, err := s.votes.Tallies(ctx, datasetID)
if err != nil {
return
}
@@ -539,8 +539,8 @@ func (s *Server) placePage(w http.ResponseWriter, r *http.Request) {
}
}
d.Verdict, _ = s.votes.Verdict(r.Context(), p.Dataset.ID, place.SrcPK)
d.Votes, _ = s.votes.Tally(r.Context(), place.SrcPK)
d.MyVote = s.votes.Voter(r.Context(), place.SrcPK, s.voterKeyFromRequest(r))
d.Votes, _ = s.votes.Tally(r.Context(), p.Dataset.ID, place.SrcPK)
d.MyVote = s.votes.Voter(r.Context(), p.Dataset.ID, place.SrcPK, s.voterKeyFromRequest(r))
// 图片:访客也要看得到,所以不分管理员
if imgs, err := s.placeImagesOf(r.Context(), p.Dataset.ID, place.SrcPK); err == nil {
d.Images = imgs
+20 -8
View File
@@ -102,12 +102,12 @@ func (s *Server) apiVotes(w http.ResponseWriter, r *http.Request) {
s.json(w, map[string]string{"error": err.Error()})
return
}
tallies, err := s.votes.Tallies(ctx)
tallies, err := s.votes.Tallies(ctx, votesDatasetID)
if err != nil {
s.json(w, map[string]string{"error": err.Error()})
return
}
mine, err := s.votes.VoterAll(ctx, s.voterKeyFromRequest(r))
mine, err := s.votes.VoterAll(ctx, votesDatasetID, s.voterKeyFromRequest(r))
if err != nil {
s.json(w, map[string]string{"error": err.Error()})
return
@@ -145,10 +145,16 @@ func (s *Server) apiVerdict(w http.ResponseWriter, r *http.Request) {
// apiVote 记录访客对某地点的点赞 / 点踩,返回最新票数与该访客的选择。
func (s *Server) apiVote(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
if err := r.ParseForm(); err != nil {
s.json(w, map[string]string{"error": "请求解析失败"})
return
}
ds, err := s.datasetFor(ctx)
if err != nil || ds == nil {
s.json(w, map[string]string{"error": "还没有可投票的数据集"})
return
}
srcPK, err := strconv.Atoi(r.PostFormValue("src_pk"))
vote, _ := strconv.Atoi(r.PostFormValue("vote"))
if err != nil || (vote != 1 && vote != -1) {
@@ -160,7 +166,7 @@ func (s *Server) apiVote(w http.ResponseWriter, r *http.Request) {
s.json(w, map[string]string{"error": "无法识别访客"})
return
}
t, mine, err := s.votes.Cast(r.Context(), srcPK, key, vote)
t, mine, err := s.votes.Cast(ctx, ds.ID, srcPK, key, vote)
if err != nil {
s.json(w, map[string]string{"error": err.Error()})
return
@@ -170,11 +176,17 @@ func (s *Server) apiVote(w http.ResponseWriter, r *http.Request) {
// voteTotals 汇总站长结论数量与访客票数,供统计页展示。
func (s *Server) voteTotals(ctx context.Context, datasetID int64) (rec, avoid, up, down int, err error) {
// 两张票表都挂坐标键,统计要按「本数据集的地点」圈定范围,
// 不能再按它们自己的 dataset_id 过滤——那个只是最后写入时留下的线索。
err = s.db.QueryRowContext(ctx, `SELECT
(SELECT count(*) FROM place_votes WHERE dataset_id=$1 AND verdict=1),
(SELECT count(*) FROM place_votes WHERE dataset_id=$1 AND verdict=-1),
(SELECT count(*) FROM place_visitor_votes WHERE vote=1),
(SELECT count(*) FROM place_visitor_votes WHERE vote=-1)`, datasetID).
(SELECT count(*) FROM place_votes v JOIN places p ON v.place_key = `+placeKeyExpr+`
WHERE p.dataset_id=$1 AND v.verdict=1),
(SELECT count(*) FROM place_votes v JOIN places p ON v.place_key = `+placeKeyExpr+`
WHERE p.dataset_id=$1 AND v.verdict=-1),
(SELECT count(*) FROM place_visitor_votes vv JOIN places p ON vv.place_key = `+placeKeyExpr+`
WHERE p.dataset_id=$1 AND vv.vote=1),
(SELECT count(*) FROM place_visitor_votes vv JOIN places p ON vv.place_key = `+placeKeyExpr+`
WHERE p.dataset_id=$1 AND vv.vote=-1)`, datasetID).
Scan(&rec, &avoid, &up, &down)
return
}
@@ -183,7 +195,7 @@ func (s *Server) voteTotals(ctx context.Context, datasetID int64) (rec, avoid, u
func (s *Server) verdictPlaces(ctx context.Context, datasetID int64, verdict, limit int) ([]stats.Place, error) {
q := `SELECT ` + stats.PlaceCols + ` FROM places p
WHERE p.dataset_id=$1 AND EXISTS (SELECT 1 FROM place_votes pv
WHERE pv.src_pk=p.src_pk AND pv.dataset_id=p.dataset_id AND pv.verdict=$2)
WHERE pv.place_key = ` + placeKeyExpr + ` AND pv.verdict=$2)
ORDER BY p.visit_count DESC LIMIT ` + strconv.Itoa(limit)
rows, err := s.db.QueryContext(ctx, q, datasetID, verdict)
if err != nil {
+149
View File
@@ -0,0 +1,149 @@
package web
import (
"fmt"
"testing"
"time"
"rond-with-you/internal/votes"
)
// testPlace 在两个数据集里各建一个**同坐标**的地点,返回它们的编号与第二个数据集。
//
// 坐标是随手挑的专属值:备注按坐标共享,而读取按设计不按 user_id 过滤(单站长站点),
// 借用真实数据里的坐标会把别的测试残留的记录串进来。
func testPlace(t *testing.T, f *movementFixture, uid int64, lat, lon float64) (pkA, pkB int, dsB int64) {
t.Helper()
const pkAVal, pkBVal = 660001, 660002
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO places (dataset_id, src_pk, name, lat, lon)
VALUES ($1,$2,'甲侧地点',$3,$4)`, f.dsID, pkAVal, lat, lon); err != nil {
t.Fatalf("建地点失败: %v", err)
}
sha := fmt.Sprintf("%064d", time.Now().UnixNano())
if err := f.pg.QueryRowContext(f.ctx, `INSERT INTO datasets (user_id, original_name, sha256, status, is_active)
VALUES ($1,'另一个数据集',$2,'done',false) RETURNING id`, uid, sha).Scan(&dsB); err != nil {
t.Fatalf("建数据集失败: %v", err)
}
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO places (dataset_id, src_pk, name, lat, lon)
VALUES ($1,$2,'乙侧地点',$3,$4)`, dsB, pkBVal, lat, lon); err != nil {
t.Fatalf("建地点失败: %v", err)
}
key := fmt.Sprintf("%.6f|%.6f", lat, lon)
t.Cleanup(func() {
f.pg.ExecContext(f.ctx, `DELETE FROM place_notes WHERE place_key=$1`, key)
f.pg.ExecContext(f.ctx, `DELETE FROM place_votes WHERE place_key=$1`, key)
f.pg.ExecContext(f.ctx, `DELETE FROM place_visitor_votes WHERE place_key=$1`, key)
f.pg.ExecContext(f.ctx, `DELETE FROM datasets WHERE id=$1`, dsB)
})
return pkAVal, pkBVal, dsB
}
// 访客票也一样:它是站点自己的数据(rondbackup 里没有这一项),
// 但同样要认得出「是哪个地点」,所以也挂坐标键。
func TestVisitorVotesFollowPlaceAcrossDatasets(t *testing.T) {
f := newMovementFixture(t)
uid := userFrom(f.ctx).ID
pkA, pkB, dsB := testPlace(t, f, uid, 77.777777, 88.888888)
if _, mine, err := f.srv.votes.Cast(f.ctx, f.dsID, pkA, "visitor-a", 1); err != nil || mine != 1 {
t.Fatalf("投票失败: mine=%d err=%v", mine, err)
}
tally, err := f.srv.votes.Tally(f.ctx, dsB, pkB)
if err != nil {
t.Fatalf("读票数失败: %v", err)
}
if tally.Up != 1 || tally.Down != 0 {
t.Fatalf("换数据集后票数应照旧,实际 %+v", tally)
}
if v := f.srv.votes.Voter(f.ctx, dsB, pkB, "visitor-a"); v != 1 {
t.Fatalf("「我投的」也该认得出来,实际 %d", v)
}
// 再点一次是撤销:从哪一侧点都该撤掉同一张票
if _, mine, err := f.srv.votes.Cast(f.ctx, dsB, pkB, "visitor-a", 1); err != nil || mine != 0 {
t.Fatalf("再点一次应撤销: mine=%d err=%v", mine, err)
}
if tally, err := f.srv.votes.Tally(f.ctx, f.dsID, pkA); err != nil || tally.Up != 0 {
t.Fatalf("撤销后另一边也该归零: %+v err=%v", tally, err)
}
}
// 备注、结论、图片挂的是坐标键,所以**换数据集之后仍然跟着同一个地点**——
// 这正是这次迁移要解决的问题。老结构挂 (dataset_id, src_pk),
// 换一份备份同编号就是另一个地点,内容全部失联。
func TestNotesAndVerdictsFollowPlaceAcrossDatasets(t *testing.T) {
f := newMovementFixture(t)
uid := userFrom(f.ctx).ID
pkA, pkB, dsB := testPlace(t, f, uid, 11.111111, 22.222222)
if err := f.srv.notes.Save(f.ctx, uid, f.dsID, pkA, "别名甲", "备注甲"); err != nil {
t.Fatalf("写备注失败: %v", err)
}
if err := f.srv.votes.SaveVerdict(f.ctx, uid, f.dsID, pkA, votes.VerdictRecommend); err != nil {
t.Fatalf("写结论失败: %v", err)
}
// 从「另一个数据集」读:编号完全不同,但坐标一致,应当照样读得到
all, err := f.srv.notes.All(f.ctx, dsB)
if err != nil {
t.Fatalf("读备注失败: %v", err)
}
got, ok := all[pkB]
if !ok {
t.Fatal("换数据集后应当仍能读到这个地点的备注")
}
if got.Alias != "别名甲" || got.Content != "备注甲" {
t.Fatalf("备注应原样跟过来,实际 alias=%q note=%q", got.Alias, got.Content)
}
v, err := f.srv.votes.Verdict(f.ctx, dsB, pkB)
if err != nil {
t.Fatalf("读结论失败: %v", err)
}
if v != votes.VerdictRecommend {
t.Fatalf("结论应跟过来,实际 %d", v)
}
}
// 两个数据集里的编号完全不同,但备注与结论是同一份——改一处两边都变。
func TestNotesUpdateVisibleFromOtherDataset(t *testing.T) {
f := newMovementFixture(t)
uid := userFrom(f.ctx).ID
pkA, pkB, dsB := testPlace(t, f, uid, 33.333333, 44.444444)
if err := f.srv.notes.Save(f.ctx, uid, dsB, pkB, "别名乙", "备注乙"); err != nil {
t.Fatalf("在新数据集写备注失败: %v", err)
}
back, err := f.srv.notes.All(f.ctx, f.dsID)
if err != nil {
t.Fatalf("回读失败: %v", err)
}
if back[pkA].Content != "备注乙" {
t.Fatalf("备注是同一份数据,改一处两边都该变,实际 %q", back[pkA].Content)
}
}
// 坐标不同的地点绝不能互相认领——键是坐标,不是「大概位置」。
func TestNotesDoNotLeakBetweenDifferentPlaces(t *testing.T) {
f := newMovementFixture(t)
uid := userFrom(f.ctx).ID
pkA, _, dsB := testPlace(t, f, uid, 55.555555, 66.666666)
if err := f.srv.notes.Save(f.ctx, uid, f.dsID, pkA, "别名甲", "备注甲"); err != nil {
t.Fatalf("写备注失败: %v", err)
}
// 在乙数据集里放一个坐标差 0.001 度(约 100m)的地点,是完全不同的地方
const otherPK = 660003
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO places (dataset_id, src_pk, name, lat, lon)
VALUES ($1,$2,'隔壁那家', 55.556555, 66.666666)`, dsB, otherPK); err != nil {
t.Fatalf("建地点失败: %v", err)
}
all, err := f.srv.notes.All(f.ctx, dsB)
if err != nil {
t.Fatalf("读备注失败: %v", err)
}
if _, ok := all[otherPK]; ok {
t.Fatal("坐标不同的地点不该认领到别人的备注")
}
}
+6 -4
View File
@@ -317,14 +317,16 @@ func TestVisitorPlacePageCarriesImageURLs(t *testing.T) {
var placeID int64
var srcPK int
if err := f.pg.QueryRowContext(f.ctx, `SELECT id, src_pk FROM places
WHERE dataset_id=$1 ORDER BY id LIMIT 1`, f.dsID).Scan(&placeID, &srcPK); err != nil {
WHERE dataset_id=$1 AND lat IS NOT NULL ORDER BY id LIMIT 1`, f.dsID).Scan(&placeID, &srcPK); err != nil {
t.Fatalf("找不到地点: %v", err)
}
const key = "9/visitor-page-test.webp"
// 这里只验发下去的地址,不验能否取回字节,所以不用真的写文件
// 这里只验发下去的地址,不验能否取回字节,所以不用真的写文件。
// place_key 得照真实写入路径一并算好——读取是认坐标键的,不认 (dataset_id, src_pk)。
if _, err := f.pg.ExecContext(f.ctx, `INSERT INTO place_images
(dataset_id, src_pk, storage, object_key, width, height, bytes, position)
VALUES ($1,$2,'local',$3,1200,900,1024,0)
(dataset_id, src_pk, place_key, storage, object_key, width, height, bytes, position)
SELECT $1, $2, `+placeKeyExpr+`, 'local', $3, 1200, 900, 1024, 0
FROM places p WHERE p.dataset_id=$1 AND p.src_pk=$2
ON CONFLICT (dataset_id, src_pk, object_key) DO NOTHING`,
f.dsID, srcPK, key); err != nil {
t.Fatalf("插入图片记录失败: %v", err)
+8
View File
@@ -54,6 +54,8 @@ var pageFiles = map[string]struct {
"admin/twofa-login": {[]string{"templates/admin/twofa-login.html"}, "login2"},
"admin/twofa": {[]string{"templates/layout.html", "templates/admin/twofa.html"}, "layout"},
"admin/fog-mask": {[]string{"templates/layout.html", "templates/admin/fog-mask.html"}, "layout"},
"admin/fog-preview": {[]string{"templates/layout.html", "templates/admin/fog-preview.html"}, "layout"},
"admin/backup-preview": {[]string{"templates/layout.html", "templates/admin/backup-preview.html"}, "layout"},
"admin/notes": {[]string{"templates/layout.html", "templates/admin/notes.html"}, "layout"},
"admin/edit": {[]string{"templates/layout.html", "templates/admin/edit.html"}, "layout"},
"admin/topics": {[]string{"templates/layout.html", "templates/admin/topics.html"}, "layout"},
@@ -229,6 +231,9 @@ func (s *Server) routes() {
s.mux.HandleFunc("POST /admin/twofa/disable", s.requireAdmin(s.twofaDisable))
s.mux.HandleFunc("GET /admin", s.requireAdmin(s.adminHome))
s.mux.HandleFunc("POST /admin/upload", s.requireAdmin(s.adminUpload))
s.mux.HandleFunc("POST /admin/preview", s.requireAdmin(s.adminBackupPreview))
s.mux.HandleFunc("GET /admin/preview/{token}", s.requireAdmin(s.adminBackupPreviewPage))
s.mux.HandleFunc("POST /admin/preview/{token}/merge", s.requireAdmin(s.adminBackupMerge))
s.mux.HandleFunc("POST /admin/datasets/{id}/activate", s.requireAdmin(s.adminActivate))
s.mux.HandleFunc("POST /admin/datasets/{id}/delete", s.requireAdmin(s.adminDelete))
s.mux.HandleFunc("POST /admin/place/note", s.requireAdmin(s.adminPlaceNote))
@@ -261,6 +266,9 @@ func (s *Server) routes() {
s.mux.HandleFunc("POST /admin/edit/weather", s.requireAdmin(s.editWeatherCreate))
s.mux.HandleFunc("POST /admin/gpx", s.requireAdmin(s.adminGpxUpload))
s.mux.HandleFunc("POST /admin/gpx/delete", s.requireAdmin(s.adminGpxDelete))
s.mux.HandleFunc("POST /admin/fog/preview", s.requireAdmin(s.adminFogPreview))
s.mux.HandleFunc("GET /admin/fog/preview/{token}", s.requireAdmin(s.adminFogPreviewPage))
s.mux.HandleFunc("GET /admin/fog/preview/{token}/{z}/{x}/{y}", s.requireAdmin(s.adminFogDiffTile))
s.mux.HandleFunc("POST /admin/fog", s.requireAdmin(s.adminFogUpload))
s.mux.HandleFunc("POST /admin/fog/clear", s.requireAdmin(s.adminFogClear))
s.mux.HandleFunc("GET /admin/fog/export", s.requireAdmin(s.adminFogExport))
@@ -0,0 +1,310 @@
{{define "content"}}
<div class="wrap">
<div class="page-head">
<div>
<h1>备份差异预览</h1>
<div class="sub">
这份备份与当前展示的「{{.DatasetName}}」逐条比对的结果。<strong>还没有导入</strong>,站内数据一个字节都没动。
</div>
</div>
<div class="row"><a class="btn" href="/admin">← 返回后台</a></div>
</div>
{{if .Identical}}
<div class="card">
<div class="alert ok">这份备份与当前数据集<strong>完全一致</strong>:没有新增、没有被删、也没有冲突。</div>
</div>
{{end}}
{{if .Truncated}}
<div class="card"><div class="alert err">差异较多,下面每类只列出前 200 条,确切数量看概览表。</div></div>
{{end}}
<div class="card">
<h2>概览</h2>
<div class="tw">
<table class="tb tb-stack">
<thead>
<tr>
<th>实体</th><th class="num">备份</th><th class="num">站内</th>
<th class="num">备份新增</th><th class="num">仅站内有</th><th class="num">冲突</th><th class="num">一致</th>
</tr>
</thead>
<tbody>
<tr>
<td>到访</td>
<td class="num" data-label="备份">{{num .BackupVisits}}</td>
<td class="num" data-label="站内">{{num .StoredVisits}}</td>
<td class="num" data-label="备份新增">{{num .VisitCounts.Added}}</td>
<td class="num" data-label="仅站内有">{{num .VisitCounts.Removed}}</td>
<td class="num" data-label="冲突">{{num .VisitCounts.Changed}}</td>
<td class="num" data-label="一致">{{num .VisitCounts.Same}}</td>
</tr>
<tr>
<td>地点</td>
<td class="num" data-label="备份">{{num .BackupPlaces}}</td>
<td class="num" data-label="站内">{{num .StoredPlaces}}</td>
<td class="num" data-label="备份新增">{{num .PlaceCounts.Added}}</td>
<td class="num" data-label="仅站内有">{{num .PlaceCounts.Removed}}</td>
<td class="num" data-label="冲突">{{num .PlaceCounts.Changed}}</td>
<td class="num" data-label="一致">{{num .PlaceCounts.Same}}</td>
</tr>
<tr>
<td>活动类型</td>
<td class="num" data-label="备份">{{num .BackupActs}}</td>
<td class="num" data-label="站内">{{num .StoredActs}}</td>
<td class="num" data-label="备份新增">{{num .ActCounts.Added}}</td>
<td class="num" data-label="仅站内有">{{num .ActCounts.Removed}}</td>
<td class="num" data-label="冲突">{{num .ActCounts.Changed}}</td>
<td class="num" data-label="一致">{{num .ActCounts.Same}}</td>
</tr>
</tbody>
</table>
</div>
<div class="card-sub" style="margin-top:10px">
对齐按每条记录的 <strong>UUID</strong> 做:rond 每次导出都会重排记录编号(同样的 822 条到访,两次导出的编号 100% 不同,UUID 却逐条一模一样),
所以只有 UUID 认得出来是同一条;地点没有 UUID,退到「名称 + 坐标」。
站内补录的记录不会出现在真机备份里,一律归到「仅站内有」。
</div>
</div>
<div class="card">
<h2>到访</h2>
<div class="card-sub"><strong>备份里多出来的({{num .VisitCounts.Added}})</strong> —— 目前站内没有这些记录</div>
{{if .VisitAdded}}
<div class="tw">
<table class="tb tb-stack">
<thead><tr><th>到达</th><th>离开</th><th>地点</th><th>活动</th><th>备注</th></tr></thead>
<tbody>
{{range .VisitAdded}}
<tr>
<td class="mono">{{dtt .Arrival}}</td>
<td class="mono" data-label="离开">{{dtt .Departure}}</td>
<td data-label="地点">{{if .Place}}{{.Place}}{{else}}<span class="muted">—</span>{{end}}</td>
<td data-label="活动">{{if .Activity}}{{.Activity}}{{else}}<span class="muted">—</span>{{end}}</td>
<td data-label="备注">{{if .Remark}}{{.Remark}}{{else}}<span class="muted">—</span>{{end}}</td>
</tr>
{{end}}
</tbody>
</table>
</div>
{{else}}<div class="muted" style="margin-bottom:14px">没有</div>{{end}}
<div class="card-sub" style="margin-top:16px">
<strong>只在站内有({{num .VisitCounts.Removed}})</strong> —— 备份里找不到;可能是你在这里补录的,也可能是实机上删掉的。
<strong>默认一条都不删</strong>;确实想清掉的,勾上左列的「合并时删除」。
</div>
{{if .VisitGone}}
<div class="tw">
<table class="tb tb-stack">
<thead><tr><th>到达</th><th>离开</th><th>地点</th><th>活动</th><th>备注</th><th>来源</th><th>合并时删除</th></tr></thead>
<tbody>
{{range .VisitGone}}
<tr>
<td class="mono">{{dtt .Arrival}}</td>
<td class="mono" data-label="离开">{{dtt .Departure}}</td>
<td data-label="地点">{{if .Place}}{{.Place}}{{else}}<span class="muted">—</span>{{end}}</td>
<td data-label="活动">{{if .Activity}}{{.Activity}}{{else}}<span class="muted">—</span>{{end}}</td>
<td data-label="备注">{{if .Remark}}{{.Remark}}{{else}}<span class="muted">—</span>{{end}}</td>
<td data-label="来源">{{if .UserAdded}}<span>站内补录</span>{{else}}<span class="muted">备份里已无</span>{{end}}</td>
<td data-label="合并时删除">
<input type="checkbox" form="merge-form" name="dv_{{.SrcPK}}" value="1" title="勾上=合并时删掉这条到访">
</td>
</tr>
{{end}}
</tbody>
</table>
</div>
{{else}}<div class="muted" style="margin-bottom:14px">没有</div>{{end}}
<div class="card-sub" style="margin-top:16px"><strong>两边都有但不一样({{num .VisitCounts.Changed}})</strong> —— 左边是站内当前的值,右边是备份里的值</div>
{{if .VisitClash}}
<div class="tw">
<table class="tb tb-stack">
<thead><tr><th>到达</th><th>地点</th><th>活动</th><th>差异</th><th>合并时</th></tr></thead>
<tbody>
{{range .VisitClash}}
<tr>
<td class="mono">{{dtt .Arrival}}</td>
<td data-label="地点">{{if .Place}}{{.Place}}{{else}}<span class="muted">—</span>{{end}}</td>
<td data-label="活动">{{if .Activity}}{{.Activity}}{{else}}<span class="muted">—</span>{{end}}</td>
<td data-label="差异">
{{range .Fields}}
<div style="font-size:12.5px;line-height:1.7">
<span class="muted">{{.Label}}</span>
<span style="text-decoration:line-through;opacity:.65">{{if .From}}{{.From}}{{else}}(空){{end}}</span>
<span style="opacity:.5">→</span>
<strong>{{if .To}}{{.To}}{{else}}(空){{end}}</strong>
</div>
{{end}}
</td>
<td data-label="合并时" style="white-space:nowrap">
<label style="margin-right:8px"><input type="radio" form="merge-form" name="vc_{{.Key}}" value="backup" checked> 用实机</label>
<label style="margin-right:8px"><input type="radio" form="merge-form" name="vc_{{.Key}}" value="db"> 用站内</label>
<label><input type="radio" form="merge-form" name="vc_{{.Key}}" value="skip"> 跳过</label>
</td>
</tr>
{{end}}
</tbody>
</table>
</div>
{{else}}<div class="muted">没有</div>{{end}}
</div>
<div class="card">
<h2>地点</h2>
<div class="card-sub"><strong>备份里多出来的({{num .PlaceCounts.Added}})</strong></div>
{{if .PlaceAdded}}
<div class="tw">
<table class="tb tb-stack">
<thead><tr><th>名称</th><th>省 / 市</th><th>分类</th><th class="num">坐标</th></tr></thead>
<tbody>
{{range .PlaceAdded}}
<tr>
<td>{{if .Name}}{{.Name}}{{else}}<span class="muted">(未命名)</span>{{end}}</td>
<td data-label="省 / 市">{{.Province}} {{.City}}</td>
<td data-label="分类">{{if .POICategory}}{{.POICategory}}{{else}}<span class="muted">—</span>{{end}}</td>
<td class="num mono" data-label="坐标">{{printf "%.5f" .Lat}}, {{printf "%.5f" .Lon}}</td>
</tr>
{{end}}
</tbody>
</table>
</div>
{{else}}<div class="muted" style="margin-bottom:14px">没有</div>{{end}}
<div class="card-sub" style="margin-top:16px">
<strong>只在站内有({{num .PlaceCounts.Removed}})</strong>
—— 名下还有到访的地点删不掉(会连带删掉那些到访),会直接报错让你先处理。
</div>
{{if .PlaceGone}}
<div class="tw">
<table class="tb tb-stack">
<thead><tr><th>名称</th><th>省 / 市</th><th class="num">到访</th><th class="num">坐标</th><th>合并时删除</th></tr></thead>
<tbody>
{{range .PlaceGone}}
<tr>
<td>{{if .Name}}{{.Name}}{{else}}<span class="muted">(未命名)</span>{{end}}</td>
<td data-label="省 / 市">{{.Province}} {{.City}}</td>
<td class="num" data-label="到访">{{num .VisitCount}}</td>
<td class="num mono" data-label="坐标">{{printf "%.5f" .Lat}}, {{printf "%.5f" .Lon}}</td>
<td data-label="合并时删除">
{{if gt .VisitCount 0}}
<span class="muted" style="font-size:12.5px">还有到访,不能删</span>
{{else}}
<input type="checkbox" form="merge-form" name="dp_{{.SrcPK}}" value="1">
{{end}}
</td>
</tr>
{{end}}
</tbody>
</table>
</div>
{{else}}<div class="muted" style="margin-bottom:14px">没有</div>{{end}}
<div class="card-sub" style="margin-top:16px"><strong>两边都有但地址不一样({{num .PlaceCounts.Changed}})</strong></div>
{{if .PlaceClash}}
<div class="tw">
<table class="tb tb-stack">
<thead><tr><th>名称</th><th>差异</th><th>合并时</th></tr></thead>
<tbody>
{{range .PlaceClash}}
<tr>
<td>{{if .DB.Name}}{{.DB.Name}}{{else}}<span class="muted">(未命名)</span>{{end}}</td>
<td data-label="差异">
{{range .Fields}}
<div style="font-size:12.5px;line-height:1.7">
<span class="muted">{{.Label}}</span>
<span style="text-decoration:line-through;opacity:.65">{{if .From}}{{.From}}{{else}}(空){{end}}</span>
<span style="opacity:.5">→</span>
<strong>{{if .To}}{{.To}}{{else}}(空){{end}}</strong>
</div>
{{end}}
</td>
<td data-label="合并时" style="white-space:nowrap">
<label style="margin-right:8px"><input type="radio" form="merge-form" name="pc_{{.DB.Key}}" value="backup" checked> 用实机</label>
<label style="margin-right:8px"><input type="radio" form="merge-form" name="pc_{{.DB.Key}}" value="db"> 用站内</label>
<label><input type="radio" form="merge-form" name="pc_{{.DB.Key}}" value="skip"> 跳过</label>
</td>
</tr>
{{end}}
</tbody>
</table>
</div>
{{else}}<div class="muted">没有</div>{{end}}
</div>
<div class="card">
<h2>活动类型</h2>
<div class="card-sub">
新增 {{num .ActCounts.Added}} · 仅站内有 {{num .ActCounts.Removed}} · 冲突 {{num .ActCounts.Changed}} · 一致 {{num .ActCounts.Same}}
</div>
<div class="row" style="gap:24px;align-items:flex-start;flex-wrap:wrap">
<div style="flex:1 1 220px">
<div class="muted" style="font-size:12.5px;margin-bottom:4px">备份新增</div>
{{range .ActAdded}}<div>{{.Name}}</div>{{else}}<div class="muted">没有</div>{{end}}
</div>
<div style="flex:1 1 220px">
<div class="muted" style="font-size:12.5px;margin-bottom:4px">仅站内有(勾上=合并时删除)</div>
{{range .ActGone}}
<label style="display:block"><input type="checkbox" form="merge-form" name="da_{{.SrcPK}}" value="1"> {{.Name}}</label>
{{else}}<div class="muted">没有</div>{{end}}
</div>
<div style="flex:1 1 260px">
<div class="muted" style="font-size:12.5px;margin-bottom:4px">名称不同</div>
{{range .ActClash}}
<div style="margin-bottom:6px">
{{range .Fields}}<div><span style="text-decoration:line-through;opacity:.65">{{.From}}</span> → <strong>{{.To}}</strong></div>{{end}}
<div style="margin-top:4px">
<label style="margin-right:8px"><input type="radio" form="merge-form" name="ac_{{actKey .DB}}" value="backup" checked> 用实机</label>
<label style="margin-right:8px"><input type="radio" form="merge-form" name="ac_{{actKey .DB}}" value="db"> 用站内</label>
<label><input type="radio" form="merge-form" name="ac_{{actKey .DB}}" value="skip"> 跳过</label>
</div>
</div>
{{else}}<div class="muted">没有</div>{{end}}
</div>
</div>
</div>
<div class="card">
<h2>其它实体(只列数量)</h2>
<div class="tw">
<table class="tb tb-stack">
<thead><tr><th>实体</th><th class="num">备份</th><th class="num">站内</th></tr></thead>
<tbody>
<tr><td>位移</td><td class="num" data-label="备份">{{num .BackupMovements}}</td><td class="num" data-label="站内">{{num .StoredMovements}}</td></tr>
<tr><td>小时天气</td><td class="num" data-label="备份">{{num .BackupWeather}}</td><td class="num" data-label="站内">{{num .StoredWeather}}</td></tr>
<tr><td>原始定位点</td><td class="num" data-label="备份">{{num .BackupRaw}}</td><td class="num" data-label="站内"><span class="muted">不单独落库</span></td></tr>
</tbody>
</table>
</div>
<div class="card-sub" style="margin-top:10px">
位移跟着它的起止到访走、天气跟着到访走,所以这两项按到访对齐结果一并处理,不单独做逐条比对。
</div>
</div>
<div class="card">
<h2>执行合并</h2>
<div class="card-sub">
把选中的改动写进当前数据集「{{.DatasetName}}」。整个过程跑在一个事务里,任何一步失败都会<strong>整体回滚</strong>,数据不会半改。
<br>「仅站内有」的记录<strong>默认一条都不删</strong>;确实要清掉的,在上面各表里逐条勾「合并时删除」。
名下有到访的地点会被拒绝删除(否则会连带删掉那些到访),先处理到访再来。
</div>
<form id="merge-form" method="post" action="/admin/preview/{{.Token}}/merge"></form>
<label style="display:flex;align-items:flex-start;gap:8px;margin:4px 0 12px">
<input type="checkbox" form="merge-form" name="add_new" value="1" checked style="margin-top:3px">
<span>把备份里多出来的
<strong>{{num .VisitCounts.Added}}</strong> 条到访、
<strong>{{num .PlaceCounts.Added}}</strong> 个地点、
<strong>{{num .ActCounts.Added}}</strong> 个活动并进来
<span class="muted">(新到访会连它的地点、活动、标签、位移与天气一起带过来)</span>
</span>
</label>
<button class="btn primary" form="merge-form" type="submit"
onclick="return confirm('确认把选中的改动写进当前数据集?出错会自动整体回滚。')">执行合并</button>
<div class="card-sub" style="margin:10px 0 0">
合并后再传一次同一份备份,差异应当变成「完全一致」——那说明对齐键(UUID 与坐标)都写进去了。
</div>
</div>
</div>
{{end}}
@@ -0,0 +1,174 @@
{{define "content"}}
<div class="wrap">
<div class="page-head">
<div>
<h1>迷雾导入预览</h1>
<div class="sub">这份快照<strong>还没有写入</strong>。先看清两种导入方式各自的后果,再决定用哪一种。</div>
</div>
<div class="row">
<a class="btn" href="/admin">← 返回后台</a>
</div>
</div>
<div class="card">
<h2>数量对比</h2>
<div class="tw">
<table class="tb tb-stack">
<thead>
<tr><th>状态</th><th class="num">块</th><th class="num">格</th><th class="num">面积</th></tr>
</thead>
<tbody>
<tr>
<td>当前迷雾</td>
<td class="num" data-label="块">{{num .Diff.CurBlocks}}</td>
<td class="num" data-label="格">{{num .Diff.CurCells}}</td>
<td class="num" data-label="面积">{{printf "%.1f" .Diff.CurAreaKm2}} km²</td>
</tr>
<tr>
<td>这份快照</td>
<td class="num" data-label="块">{{num .Diff.IncBlocks}}</td>
<td class="num" data-label="格">{{num .Diff.IncCells}}</td>
<td class="num" data-label="面积">{{printf "%.1f" .Diff.IncAreaKm2}} km²</td>
</tr>
<tr>
<td>叠加导入后</td>
<td class="num" data-label="块">{{num .Diff.CurBlocks}}<span class="muted"> +{{num .Diff.GainBlocks}}</span></td>
<td class="num" data-label="格">{{num .Diff.CurCells}}<span class="muted"> +{{num .Diff.GainCells}}</span></td>
<td class="num" data-label="面积">{{printf "%.1f" .Diff.MergeAreaKm2}} km²</td>
</tr>
<tr>
<td>整体替换后</td>
<td class="num" data-label="块">{{num .Diff.IncBlocks}}</td>
<td class="num" data-label="格">{{num .Diff.IncCells}}</td>
<td class="num" data-label="面积">{{printf "%.1f" .Diff.CoverAreaKm2}} km²</td>
</tr>
</tbody>
</table>
</div>
<div class="card-sub" style="margin-top:10px">
两个块坐标相同就只按位合并,所以「叠加后」的块数不会等于两边相加——快照里有 {{num .Diff.OverlapBlocks}} 个块与当前迷雾落在同一块,
其中 {{num .Diff.SameBlocks}} 个一格都不会变。
</div>
</div>
{{if .HasBounds}}
<div class="card">
<h2>差异地图</h2>
<div class="card-sub">在真实底图上核对差异落在哪些区域,可以缩放平移。</div>
<div class="row" style="gap:16px;margin:2px 0 10px;font-size:12.5px;flex-wrap:wrap">
<span style="display:flex;align-items:center;gap:6px">
<i style="width:13px;height:13px;border-radius:3px;background:#24a45a;display:inline-block"></i>叠加会新增
</span>
<span style="display:flex;align-items:center;gap:6px">
<i style="width:13px;height:13px;border-radius:3px;background:#7e8692;display:inline-block"></i>两边都有
</span>
<span style="display:flex;align-items:center;gap:6px">
<i style="width:13px;height:13px;border-radius:3px;background:#d63838;display:inline-block"></i>整体替换会丢掉
</span>
</div>
<div id="fog-diff-map" style="height:400px;border-radius:10px;background:var(--line-soft)"
data-token="{{.Token}}"
data-min-lat="{{printf "%.6f" .MinLat}}" data-min-lon="{{printf "%.6f" .MinLon}}"
data-max-lat="{{printf "%.6f" .MaxLat}}" data-max-lon="{{printf "%.6f" .MaxLon}}"></div>
</div>
{{end}}
<div class="card">
<h2>叠加导入</h2>
{{if .Diff.Changed}}
<div class="alert ok">
会补上当前迷雾没有的 <strong>{{num .Diff.GainBlocks}}</strong> 块、<strong>{{num .Diff.GainCells}}</strong> 格
(约 {{printf "%.1f" .Diff.GainAreaKm2}} km²)。叠加只做按位合并,<strong>不会丢失任何已有数据</strong>。
</div>
{{else}}
<div class="alert err">
这份快照的点亮区域已经被当前迷雾<strong>完全覆盖</strong>——叠加导入不会带来任何变化,
多半是同一批数据又导出了一次。
</div>
{{end}}
<form method="post" action="/admin/fog?stash={{.Token}}"
onsubmit="this.querySelector('button').disabled=true">
<input type="hidden" name="mode" value="overlay">
<button class="btn primary" type="submit">叠加导入</button>
</form>
</div>
<div class="card">
<h2>整体替换</h2>
{{if gt .Diff.LostCells 0}}
<div class="alert err">
⚠️ 快照里<strong>没有</strong>当前迷雾的 <strong>{{num .Diff.LostBlocks}}</strong> 块、<strong>{{num .Diff.LostCells}}</strong> 格
(约 {{printf "%.1f" .Diff.LostAreaKm2}} km²)。整体替换后,这些区域会<strong>从点亮变回未探索</strong>,且无法恢复。
</div>
{{else}}
<div class="alert ok">
这份快照完整包含了当前迷雾的全部点亮区域,替换不会丢数据。
</div>
{{end}}
{{if and (eq .TildeLayers 0) (eq .HashLayers 0)}}
<div class="alert err">
⚠️ 快照里没有 <span class="mono">Model/~</span> 与 <span class="mono">Model/#</span> 两层数据,替换会把现有的这两层一并清掉。
</div>
{{else}}
<div class="muted" style="font-size:12.5px;margin-bottom:8px">
另外,站点不解析但会留存的 <span class="mono">Model/~</span>({{.TildeLayers}} 项)与
<span class="mono">Model/#</span>({{.HashLayers}} 项)也会一并换成这份快照里的{{if .OtherLayers}},另有 {{.OtherLayers}} 项其它条目{{end}}。
</div>
{{end}}
<form method="post" action="/admin/fog?stash={{.Token}}"
onsubmit="return confirm('整体替换会清空当前迷雾。{{if gt .Diff.LostCells 0}}快照里缺 {{num .Diff.LostCells}} 格,替换后这些区域会变回未探索,且不可恢复。{{end}}确认替换?')">
<input type="hidden" name="mode" value="cover">
<button class="btn danger" type="submit">整体替换</button>
</form>
</div>
</div>
<script>
(function () {
// Leaflet 由 layout 在 content 之后引入,脚本执行时 window.L 还没有,
// 必须等 load 之后再初始化,否则地图整个不出现(与迷雾遮罩页同一条约定)
function whenReady(cb) {
if (document.readyState === 'complete') { setTimeout(cb, 0); return; }
window.addEventListener('load', cb);
}
var el = document.getElementById('fog-diff-map');
if (!el) return;
// 底图走站点配置(与迷雾遮罩页同一套取法):带密钥的图层由服务端转发,
// 免密钥的是浏览器直连的 {s} 模板,不能硬写 /tiles/ 路由
var fallback = {
def: { url: 'https://webrd0{s}.is.autonavi.com/appmaptile?lang=zh_cn&size=1&scale=1&style=8&x={x}&y={y}&z={z}', maxZoom: 18, attr: '© 高德地图', crs: 'gcj02' },
subs: ['1', '2', '3', '4']
};
function init(cfg) {
var def = cfg.def;
var map = L.map(el, { scrollWheelZoom: false });
L.tileLayer(def.url, { subdomains: cfg.subs, maxZoom: def.maxZoom || 18, minZoom: 3, attribution: def.attr || '' }).addTo(map);
// 差异瓦片必须和底图同坐标系:迷雾网格是 WGS-84,服务端按 crs 逐块纠偏
L.tileLayer('/admin/fog/preview/' + el.dataset.token + '/{z}/{x}/{y}?crs=' + (def.crs || 'gcj02'), {
minZoom: 3, maxZoom: 18, opacity: 0.95
}).addTo(map);
var b = [
[parseFloat(el.dataset.minLat), parseFloat(el.dataset.minLon)],
[parseFloat(el.dataset.maxLat), parseFloat(el.dataset.maxLon)]
];
// 只有一个块时两个角重合,不设上限会一路放大到最大级别
map.fitBounds(b, { padding: [24, 24], maxZoom: 13 });
L.control.scale({ imperial: false }).addTo(map);
}
whenReady(function () {
if (!window.L) return;
fetch('/api/map-config', { headers: { 'Accept': 'application/json' } })
.then(function (r) { return r.json(); })
.then(function (c) {
var def = (c && c.base && c.base.length) ? c.base[0] : fallback.def;
init({ def: def, subs: (c && c.subs && c.subs.length) ? c.subs : fallback.subs });
})
.catch(function () { init(fallback); });
});
})();
</script>
{{end}}
+11 -10
View File
@@ -29,6 +29,7 @@
<b>上传是以备份为准重建数据</b>:站点里删掉的地点、改过的记录,只要下次传一份
<b>内容不同</b>的备份,就会被备份里的原样覆盖回来(重复传同一份文件不会,会提示「完全一致」)。
在站点上新加的地点属于当时那个数据集,传新备份后要看它们,可在下面的数据集列表里切回。
<br><b>拿不准就先用「分析差异」</b>:它会逐条比对这份备份与站内当前数据,告诉你哪些是新增、哪些只在站内有、哪些两边不一样,看完再决定要不要导入。
</div>
<form method="post" action="/admin/upload" enctype="multipart/form-data">
<label class="f" style="max-width:520px">
@@ -36,7 +37,10 @@
<input type="file" name="backup" accept=".rondbackup,.zip" required
style="padding:8px;border:1px dashed var(--line);border-radius:8px;background:var(--bg-2)">
</label>
<button class="btn primary" type="submit">上传并导入</button>
<div class="row">
<button class="btn primary" type="submit" formaction="/admin/preview">分析差异</button>
<button class="btn" type="submit" formaction="/admin/upload">直接导入</button>
</div>
</form>
</div>
@@ -53,21 +57,18 @@
<b>尚未导入迷雾</b>——把 Fog of World 的 <span class="mono">.fwss</span>(在 App 内导出,通常在 zip 里)上传即可点亮地图。
{{end}}
</div>
<form method="post" action="/admin/fog" enctype="multipart/form-data">
<form method="post" action="/admin/fog/preview" enctype="multipart/form-data">
<label class="f" style="max-width:520px">
<span>选择快照</span>
<input type="file" name="fwss" accept=".fwss,.zip" required
style="padding:8px;border:1px dashed var(--line);border-radius:8px;background:var(--bg-2)">
</label>
<label class="f" style="max-width:520px">
<span>导入方式</span>
<select name="mode">
<option value="overlay" selected>叠加 — 保留现有迷雾,把新路线合并进来</option>
<option value="cover">覆盖 — 清空现有迷雾,整体替换</option>
</select>
</label>
<button class="btn primary" type="submit">导入迷雾</button>
<button class="btn primary" type="submit">分析差异</button>
</form>
<div class="muted" style="font-size:12.5px;margin-top:8px">
上传后<strong>先不写入</strong>:会先给出这份快照能补多少新区域、以及「整体替换」会丢掉多少已有区域,
确认之后才真正导入。
</div>
{{if gt .FogBlocks 0}}
<div style="margin:10px 0 0;display:flex;gap:8px;flex-wrap:wrap">
<a class="btn" href="/admin/fog/export">导出当前迷雾(.fwss)</a>
+71
View File
@@ -14,6 +14,7 @@ import (
"rond-with-you/internal/auth"
"rond-with-you/internal/fog"
"rond-with-you/internal/rond"
"rond-with-you/internal/setting"
"rond-with-you/internal/stats"
"rond-with-you/internal/votes"
@@ -637,6 +638,74 @@ type AdminData struct {
DefaultPW bool
}
// VisitRow 是一条到访摊平后的展示行。地点与活动写名称而不是编号——
// 两边的编号空间完全不同,编号对人没有任何意义。
type VisitRow struct {
// Key 是这条冲突的对齐键(到访 UUID),合并表单拿它当字段名
Key string
// SrcPK 只有「仅站内有」的行有意义:要删哪条就勾哪条
SrcPK int
Arrival *time.Time
Departure *time.Time
Place string
Activity string
Remark string
Emoji string
UserAdded bool // 站点补录的行
Fields []rond.FieldChange
}
// BackupDiffData 是「备份差异预览」页的数据。
//
// 上限说明:列表按 diffListLimit 截断,页面上只提示「还有更多」,不做分页——
// 预演是给人下判断用的,看前 200 条足够判断要不要继续;真正要逐条取舍的是
// 后面那个合并流程(那里才有逐条操作)。
type BackupDiffData struct {
Page
Token string
DatasetName string
VisitCounts rond.Counts
PlaceCounts rond.Counts
ActCounts rond.Counts
VisitAdded, VisitGone, VisitClash []VisitRow
PlaceAdded []rond.Location
PlaceGone []rond.StoredPlace
PlaceClash []rond.PlaceConflict
ActAdded []rond.Activity
ActGone []rond.StoredActivity
ActClash []rond.ActivityConflict
// 两侧总量,用于「有没有少数据」的直觉判断
BackupVisits, StoredVisits int
BackupPlaces, StoredPlaces int
BackupActs, StoredActs int
BackupMovements, StoredMovements int
BackupWeather, StoredWeather int
BackupRaw int
Truncated bool
Identical bool
}
// FogPreviewData 是「迷雾导入预览」页的数据:先把差异摆出来,再决定叠加还是整体替换。
type FogPreviewData struct {
Page
Token string
Diff fog.Diff
// 快照里不参与渲染、但整体替换会一并换掉的图层条目数
TildeLayers int
HashLayers int
OtherLayers int
// 差异地图的初始视野:两边块集覆盖范围的包围盒(已按底图坐标系换算)。
// 没有任何块时 HasBounds 为假,模板不画地图。
HasBounds bool
MinLat, MinLon float64
MaxLat, MaxLon float64
}
type NotesData struct {
Page
Rows []NoteRow
@@ -1247,6 +1316,8 @@ func templateFuncs() template.FuncMap {
"km": func(f float64) string { return fmt.Sprintf("%.1f", f) },
"f1": func(f float64) string { return fmt.Sprintf("%.1f", f) },
"dt": func(t time.Time) string { return t.Local().Format("2006-01-02 15:04") },
// actKey 给活动冲突生成表单字段名,与处理函数共用同一套键
"actKey": rond.ActivityKey,
// dtt 接受 *time.Time,nil 返回空串,便于可选时间字段直接渲染
"dtt": func(t *time.Time) string {
if t == nil {
+89
View File
@@ -0,0 +1,89 @@
// 一次性探针:验证 ZLOCATION.ZPLACEID / ZVISIT.ZEVENTIDENTIFIER 是否跨备份稳定。
package main
import (
"fmt"
"os"
"rond-with-you/internal/rond"
)
type snap struct {
placeID map[string]int
eventID map[string]int
nulls int
total int
evTotal int
}
func load(p string) *snap {
dir, err := os.MkdirTemp("", "colprobe")
if err != nil {
panic(err)
}
defer os.RemoveAll(dir)
db, err := rond.Open(p, dir)
if err != nil {
panic(err)
}
defer db.Close()
s := &snap{placeID: map[string]int{}, eventID: map[string]int{}}
rows, err := db.Query(`SELECT Z_PK, COALESCE(ZPLACEID,'') FROM ZLOCATION`)
if err != nil {
panic(err)
}
for rows.Next() {
var pk int
var id string
rows.Scan(&pk, &id)
s.total++
if id == "" {
s.nulls++
} else {
s.placeID[id] = pk
}
}
rows.Close()
rows, err = db.Query(`SELECT COALESCE(ZEVENTIDENTIFIER,'') FROM ZVISIT`)
if err != nil {
panic(err)
}
for rows.Next() {
var id string
rows.Scan(&id)
if id != "" {
s.eventID[id] = 1
s.evTotal++
}
}
rows.Close()
fmt.Printf("%s: ZLOCATION %d 个(ZPLACEID 空 %d),ZEVENTIDENTIFIER 非空 %d\n",
p, s.total, s.nulls, s.evTotal)
return s
}
func main() {
a := load(os.Args[1])
b := load(os.Args[2])
inter, samePK := 0, 0
for k, pa := range a.placeID {
if pb, ok := b.placeID[k]; ok {
inter++
if pa == pb {
samePK++
}
}
}
fmt.Printf("\nZPLACEID 交集 %d(A 独有 %d,B 独有 %d),其中 Z_PK 相同 %d\n",
inter, len(a.placeID)-inter, len(b.placeID)-inter, samePK)
ei := 0
for k := range a.eventID {
if _, ok := b.eventID[k]; ok {
ei++
}
}
fmt.Printf("ZEVENTIDENTIFIER 交集 %d(A %d / B %d)\n", ei, len(a.eventID), len(b.eventID))
}
+438
View File
@@ -0,0 +1,438 @@
// 一次性探针:把一份 .rondbackup 与 PG 里某个数据集的记录逐条对齐,输出差异画像。
// 用完即可删,不属于站点功能。
package main
import (
"context"
"database/sql"
"fmt"
"os"
"sort"
"time"
_ "github.com/jackc/pgx/v5/stdlib"
"rond-with-you/internal/rond"
)
func main() {
if len(os.Args) < 4 {
fmt.Println("用法: diffprobe <archive.rondbackup> <dsn> <datasetID>")
os.Exit(2)
}
archive, dsn := os.Args[1], os.Args[2]
var datasetID int64
fmt.Sscanf(os.Args[3], "%d", &datasetID)
tmp, err := os.MkdirTemp("", "diffprobe")
must(err)
defer os.RemoveAll(tmp)
sqlDB, err := rond.Open(archive, tmp)
must(err)
b, err := rond.Parse(sqlDB)
sqlDB.Close()
must(err)
pg, err := sql.Open("pgx", dsn)
must(err)
defer pg.Close()
ctx := context.Background()
must(pg.PingContext(ctx))
fmt.Printf("=== 备份 %s ===\n", archive)
fmt.Printf("备份: 到访 %d / 地点 %d / 位移 %d / 天气 %d / 活动 %d / 标签 %d / 原始定位 %d\n",
len(b.Visits), len(b.Locations), len(b.Movements), len(b.Weather),
len(b.Activities), len(b.Tags), b.RawVisitNum)
var bFirst, bLast sql.NullTime
for _, v := range b.Visits {
if v.Arrival == nil {
continue
}
if !bFirst.Valid || v.Arrival.Before(bFirst.Time) {
bFirst = sql.NullTime{Time: *v.Arrival, Valid: true}
}
if !bLast.Valid || v.Arrival.After(bLast.Time) {
bLast = sql.NullTime{Time: *v.Arrival, Valid: true}
}
}
fmt.Printf("备份时间跨度: %s ~ %s\n", ts(bFirst), ts(bLast))
fmt.Printf("\n=== 库内 dataset #%d ===\n", datasetID)
var name string
var cnt, pcnt, raw int
var gFirst, gLast sql.NullTime
must(pg.QueryRowContext(ctx, `SELECT original_name, visit_count, place_count, raw_visit_count,
first_visit_at, last_visit_at FROM datasets WHERE id=$1`, datasetID).
Scan(&name, &cnt, &pcnt, &raw, &gFirst, &gLast))
fmt.Printf("%s: 到访 %d / 地点 %d / 原始定位 %d,跨度 %s ~ %s\n", name, cnt, pcnt, raw, ts(gFirst), ts(gLast))
diffVisits(ctx, pg, datasetID, b)
diffPlaces(ctx, pg, datasetID, b)
diffMovements(ctx, pg, datasetID, b)
diffWeather(ctx, pg, datasetID, b)
diffActivities(ctx, pg, datasetID, b)
diffTags(ctx, pg, datasetID, b)
}
func ts(t sql.NullTime) string {
if !t.Valid {
return "-"
}
return t.Time.Local().Format("2006-01-02 15:04")
}
type visitRow struct {
pk, placeSrc, actSrc int
arrival, departure *time.Time
remark, emoji string
bookmarked, userAdd bool
}
func diffVisits(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
fmt.Printf("\n---------- 到访 ----------\n")
rows, err := pg.QueryContext(ctx, `SELECT v.src_pk, COALESCE(p.src_pk,0), COALESCE(a.src_pk,0),
v.arrival, v.departure, COALESCE(v.remark,''), COALESCE(v.emoji,''), v.bookmarked, v.is_user_added
FROM visits v LEFT JOIN places p ON p.id=v.place_id LEFT JOIN activities a ON a.id=v.activity_id
WHERE v.dataset_id=$1`, dsID)
must(err)
db := map[int]visitRow{}
for rows.Next() {
var r visitRow
var arr time.Time
var dep sql.NullTime
must(rows.Scan(&r.pk, &r.placeSrc, &r.actSrc, &arr, &dep, &r.remark, &r.emoji, &r.bookmarked, &r.userAdd))
r.arrival = &arr
if dep.Valid {
r.departure = &dep.Time
}
db[r.pk] = r
}
rows.Close()
bk := map[int]rond.Visit{}
for _, v := range b.Visits {
bk[v.SrcPK] = v
}
var onlyBackup, onlyDB, bothDiff, same []int
for pk, v := range bk {
r, ok := db[pk]
if !ok {
onlyBackup = append(onlyBackup, pk)
continue
}
if vdiff(v, r) != "" {
bothDiff = append(bothDiff, pk)
continue
}
same = append(same, pk)
}
for pk := range db {
if _, ok := bk[pk]; !ok {
onlyDB = append(onlyDB, pk)
}
}
sort.Ints(onlyBackup)
sort.Ints(onlyDB)
sort.Ints(bothDiff)
fmt.Printf("同号且一致: %d\n", len(same))
fmt.Printf("仅备份有(实机新增,库里没有): %d\n", len(onlyBackup))
fmt.Printf("仅库里有(站点侧新增/实机已删除): %d\n", len(onlyDB))
fmt.Printf("同号但内容不同(冲突): %d\n", len(bothDiff))
// 按到达时间做多重集匹配:判断「同号冲突」到底是同批数据换了主键,还是两批无关数据
byTime := map[string][]int{}
for pk, r := range db {
if r.arrival != nil {
byTime[r.arrival.UTC().Format(time.RFC3339)] = append(byTime[r.arrival.UTC().Format(time.RFC3339)], pk)
}
}
matched, miss := 0, 0
pair := [][2]int{}
for pk, v := range bk {
if v.Arrival == nil {
continue
}
k := v.Arrival.UTC().Format(time.RFC3339)
if list := byTime[k]; len(list) > 0 {
matched++
pair = append(pair, [2]int{pk, list[0]})
byTime[k] = list[1:]
} else {
miss++
}
}
fmt.Printf("\n[时间对齐] 备份到访按「到达时刻」能在库内找到同刻记录: %d 条,找不到: %d 条\n", matched, miss)
sort.Slice(pair, func(i, j int) bool { return pair[i][0] < pair[j][0] })
fmt.Printf("[时间对齐] 同刻对的 (备份pk→库pk) 前 12 组: ")
for i, p := range pair {
if i >= 12 {
break
}
fmt.Printf("%d→%d ", p[0], p[1])
}
fmt.Println()
nDiff := 0
for _, p := range pair {
if p[0] != p[1] {
nDiff++
}
}
fmt.Printf("[时间对齐] 同刻但主键不同: %d / %d\n", nDiff, len(pair))
show := func(label string, pks []int, f func(pk int) string) {
if len(pks) == 0 {
return
}
fmt.Printf("\n[%s] 共 %d 条,前 8 条:\n", label, len(pks))
for i, pk := range pks {
if i >= 8 {
break
}
fmt.Printf(" #%d %s\n", pk, f(pk))
}
}
show("仅备份有", onlyBackup, func(pk int) string {
v := bk[pk]
return fmt.Sprintf("到访 %s 离开 %s 地点src=%s", ft(v.Arrival), ft(v.Departure), ps(v.LocationSrcPK))
})
show("仅库里有", onlyDB, func(pk int) string {
r := db[pk]
return fmt.Sprintf("到访 %s 离开 %s 地点src=%d 站点新增=%v 备注=%q",
ft(r.arrival), ftp(r.departure), r.placeSrc, r.userAdd, r.remark)
})
show("冲突", bothDiff, func(pk int) string {
return vdiff(bk[pk], db[pk])
})
}
func vdiff(v rond.Visit, r visitRow) string {
var d string
if !eq(v.Arrival, r.arrival) {
d += fmt.Sprintf("到达 %s→%s ", ft(v.Arrival), ft(r.arrival))
}
if !eq(v.Departure, r.departure) {
d += fmt.Sprintf("离开 %s→%s ", ft(v.Departure), ft(r.departure))
}
if pv(v.LocationSrcPK) != r.placeSrc {
d += fmt.Sprintf("地点 %s→%d ", ps(v.LocationSrcPK), r.placeSrc)
}
if av(v.ActivitySrcPK) != r.actSrc {
d += fmt.Sprintf("活动 %s→%d ", ps(v.ActivitySrcPK), r.actSrc)
}
if v.Remark != r.remark {
d += fmt.Sprintf("备注 %q→%q ", v.Remark, r.remark)
}
if v.Emoji != r.emoji {
d += fmt.Sprintf("表情 %q→%q ", v.Emoji, r.emoji)
}
if v.Bookmarked != r.bookmarked {
d += fmt.Sprintf("收藏 %v→%v ", v.Bookmarked, r.bookmarked)
}
return d
}
func diffPlaces(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
fmt.Printf("\n---------- 地点 ----------\n")
type prow struct {
name, poi, cc, prov, city, dist, sub, thor, tz string
lat, lon float64
}
rows, err := pg.QueryContext(ctx, `SELECT src_pk, COALESCE(name,''), COALESCE(poi_category,''),
COALESCE(country_code,''), COALESCE(province,''), COALESCE(city,''), COALESCE(district,''),
COALESCE(sublocality,''), COALESCE(thoroughfare,''), COALESCE(timezone,''),
COALESCE(lat,0), COALESCE(lon,0) FROM places WHERE dataset_id=$1`, dsID)
must(err)
db := map[int]prow{}
for rows.Next() {
var pk int
var r prow
must(rows.Scan(&pk, &r.name, &r.poi, &r.cc, &r.prov, &r.city, &r.dist, &r.sub, &r.thor, &r.tz, &r.lat, &r.lon))
db[pk] = r
}
rows.Close()
bk := map[int]rond.Location{}
for _, l := range b.Locations {
bk[l.SrcPK] = l
}
var onlyB, onlyD, diff []int
for pk, l := range bk {
r, ok := db[pk]
if !ok {
onlyB = append(onlyB, pk)
continue
}
if l.Name != r.name || l.POICategory != r.poi || l.Province != r.prov || l.City != r.city ||
l.District != r.dist || l.Sublocality != r.sub || l.Thoroughfare != r.thor || l.Timezone != r.tz ||
l.CountryCode != r.cc || !near(l.Lat, r.lat) || !near(l.Lon, r.lon) {
diff = append(diff, pk)
}
}
for pk := range db {
if _, ok := bk[pk]; !ok {
onlyD = append(onlyD, pk)
}
}
sort.Ints(onlyB)
sort.Ints(onlyD)
sort.Ints(diff)
fmt.Printf("仅备份有: %d,仅库里有: %d,同号内容不同: %d\n", len(onlyB), len(onlyD), len(diff))
for i, pk := range onlyB {
if i >= 6 {
break
}
fmt.Printf(" 仅备份 #%d %s (%s %s %s)\n", pk, bk[pk].Name, bk[pk].Province, bk[pk].City, bk[pk].POICategory)
}
for i, pk := range onlyD {
if i >= 6 {
break
}
fmt.Printf(" 仅库内 #%d %s (%s %s)\n", pk, db[pk].name, db[pk].prov, db[pk].city)
}
for i, pk := range diff {
if i >= 6 {
break
}
l, r := bk[pk], db[pk]
fmt.Printf(" 冲突 #%d 备份[name=%q city=%q %.6f,%.6f] 库[name=%q city=%q %.6f,%.6f]\n",
pk, l.Name, l.City, l.Lat, l.Lon, r.name, r.city, r.lat, r.lon)
}
}
func diffMovements(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
fmt.Printf("\n---------- 位移 ----------\n")
rows, err := pg.QueryContext(ctx, `SELECT src_pk, COALESCE(transport_src,0),
COALESCE(from_visit_src,0), COALESCE(to_visit_src,0), started_at, ended_at
FROM movements WHERE dataset_id=$1`, dsID)
must(err)
type mrow struct {
t, f, to int
s, e *time.Time
}
db := map[int]mrow{}
for rows.Next() {
var pk int
var r mrow
var s, e sql.NullTime
must(rows.Scan(&pk, &r.t, &r.f, &r.to, &s, &e))
if s.Valid {
r.s = &s.Time
}
if e.Valid {
r.e = &e.Time
}
db[pk] = r
}
rows.Close()
var onlyB, onlyD int
for _, m := range b.Movements {
if _, ok := db[m.SrcPK]; !ok {
onlyB++
}
}
for pk := range db {
found := false
for _, m := range b.Movements {
if m.SrcPK == pk {
found = true
break
}
}
if !found {
onlyD++
}
}
fmt.Printf("备份 %d 条 / 库内 %d 条;仅备份有 %d,仅库里有 %d\n", len(b.Movements), len(db), onlyB, onlyD)
}
func diffWeather(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
var n int
must(pg.QueryRowContext(ctx, `SELECT count(*) FROM weather WHERE dataset_id=$1`, dsID).Scan(&n))
fmt.Printf("\n---------- 天气 ----------\n备份 %d 条 / 库内 %d 条\n", len(b.Weather), n)
}
func diffActivities(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
fmt.Printf("\n---------- 活动类型 ----------\n")
rows, err := pg.QueryContext(ctx, `SELECT src_pk, name FROM activities WHERE dataset_id=$1`, dsID)
must(err)
db := map[int]string{}
for rows.Next() {
var pk int
var n string
must(rows.Scan(&pk, &n))
db[pk] = n
}
rows.Close()
for _, a := range b.Activities {
n, ok := db[a.SrcPK]
if !ok {
fmt.Printf(" 仅备份 #%d %q\n", a.SrcPK, a.Name)
} else if n != a.Name {
fmt.Printf(" 冲突 #%d 备份=%q 库=%q\n", a.SrcPK, a.Name, n)
}
}
for pk, n := range db {
found := false
for _, a := range b.Activities {
if a.SrcPK == pk {
found = true
}
}
if !found {
fmt.Printf(" 仅库内 #%d %q\n", pk, n)
}
}
}
func diffTags(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
var n int
must(pg.QueryRowContext(ctx, `SELECT count(*) FROM tags WHERE dataset_id=$1`, dsID).Scan(&n))
fmt.Printf("\n---------- 标签 ----------\n备份 %d 条 / 库内 %d 条\n", len(b.Tags), n)
}
func eq(a, b *time.Time) bool {
if a == nil || b == nil {
return a == nil && b == nil
}
return a.Equal(*b)
}
func pv(p *int) int {
if p == nil {
return 0
}
return *p
}
func av(p *int) int { return pv(p) }
func ps(p *int) string {
if p == nil {
return "-"
}
return fmt.Sprint(*p)
}
func ft(t *time.Time) string {
if t == nil {
return "-"
}
return t.Local().Format("01-02 15:04")
}
func ftp(t *time.Time) string { return ft(t) }
func near(a, b float64) bool {
d := a - b
return d < 1e-9 && d > -1e-9
}
func must(err error) {
if err != nil {
fmt.Fprintln(os.Stderr, "错误:", err)
os.Exit(1)
}
}
+40
View File
@@ -0,0 +1,40 @@
// 一次性探针:调用站点的导出逻辑产出一份 .rondbackup,用于核对
// 「站点导出 → 导入 rond → rond 再导出」这条链路上 UUID 与 Z_PK 是否还连贯。
package main
import (
"context"
"database/sql"
"fmt"
"os"
"strconv"
_ "github.com/jackc/pgx/v5/stdlib"
"rond-with-you/internal/rond"
)
func main() {
if len(os.Args) < 5 {
fmt.Println("用法: exportprobe <dsn> <datasetID> <baseline.sqlite> <out.rondbackup>")
os.Exit(2)
}
dsID, err := strconv.ParseInt(os.Args[2], 10, 64)
must(err)
db, err := sql.Open("pgx", os.Args[1])
must(err)
defer db.Close()
f, err := os.Create(os.Args[4])
must(err)
defer f.Close()
must(rond.ExportRondbackup(context.Background(), db, dsID, os.Args[3], f))
fmt.Printf("已导出到 %s\n", os.Args[4])
}
func must(err error) {
if err != nil {
fmt.Fprintln(os.Stderr, "错误:", err)
os.Exit(1)
}
}
+133
View File
@@ -0,0 +1,133 @@
// 一次性探针:把一份 .fwss 与 PG 里的 fog_blocks 逐块比对,量化「叠加 / 替换」两种模式的影响。
package main
import (
"database/sql"
"encoding/hex"
"fmt"
"os"
"strconv"
"strings"
_ "github.com/jackc/pgx/v5/stdlib"
"rond-with-you/internal/fog"
)
func main() {
if len(os.Args) < 3 {
fmt.Println("用法: fogprobe <snapshot.fwss> <dsn> [userID]")
os.Exit(2)
}
uid := int64(1)
if len(os.Args) > 3 {
uid, _ = strconv.ParseInt(os.Args[3], 10, 64)
}
data, err := os.ReadFile(os.Args[1])
must(err)
inc, incCells, err := fog.Decode(data)
must(err)
fmt.Printf("快照 %s: %d 块 / %d 格\n", os.Args[1], len(inc), incCells)
pg, err := sql.Open("pgx", os.Args[2])
must(err)
defer pg.Close()
rows, err := pg.Query(`SELECT gbx, gby, bitmap FROM fog_blocks WHERE user_id=$1`, uid)
must(err)
local := map[[2]int][]byte{}
localCells := 0
for rows.Next() {
var gx, gy int
var bm []byte
must(rows.Scan(&gx, &gy, &bm))
local[[2]int{gx, gy}] = bm
localCells += popcount(bm)
}
rows.Close()
fmt.Printf("库内: %d 块 / %d 格\n", len(local), localCells)
var newBlocks, newCells int
var novBlock, overlap int
var overlapGain int
for _, b := range inc {
k := [2]int{b.GX, b.GY}
old, ok := local[k]
if !ok {
newBlocks++
newCells += popcount(b.Bitmap)
continue
}
novBlock++
gain := 0
for i := range b.Bitmap {
gain += poptab[b.Bitmap[i]&^old[i]]
}
if gain > 0 {
overlap++
overlapGain += gain
}
}
var lostBlocks, lostCells, lostFull int
for k, bm := range local {
found := false
for _, b := range inc {
if b.GX == k[0] && b.GY == k[1] {
found = true
break
}
}
if !found {
lostBlocks++
lostCells += popcount(bm)
lostFull++
}
}
fmt.Printf("\n【叠加合并 overlay】\n")
fmt.Printf(" 新增整块: %d 块(+%d 格)\n", newBlocks, newCells)
fmt.Printf(" 与已有块重叠: %d 块,其中 %d 块含新格(+%d 格)\n", novBlock, overlap, overlapGain)
fmt.Printf(" 合计新增格: %d,合并后总量: %d 块 / %d 格\n",
newCells+overlapGain, len(local)+newBlocks, localCells+newCells+overlapGain)
fmt.Printf("\n【整体替换 cover】\n")
fmt.Printf(" 库里现有 %d 块会被丢弃(快照里完全没有这些块),损失 %d 格\n", lostFull, lostCells)
fmt.Printf(" 替换后总量: %d 块 / %d 格\n", len(inc), incCells)
fmt.Printf(" 净变化: %+d 块 / %+d 格\n", len(inc)-len(local), incCells-localCells)
fmt.Printf("\n【其它图层】\n")
layers, err := fog.RawLayers(data)
if err == nil {
for k, v := range layers {
fmt.Printf(" %s (%d 字节, %s)\n", k, len(v), hex.EncodeToString(v[:min(8, len(v))]))
}
}
}
func min(a, b int) int {
if a < b {
return a
}
return b
}
var poptab = func() [256]int {
var t [256]int
for i := range t {
t[i] = strings.Count(strconv.FormatUint(uint64(i), 2), "1")
}
return t
}()
func popcount(b []byte) int {
n := 0
for _, v := range b {
n += poptab[v]
}
return n
}
func must(err error) {
if err != nil {
fmt.Fprintln(os.Stderr, "错误:", err)
os.Exit(1)
}
}
+171
View File
@@ -0,0 +1,171 @@
// 一次性探针:检查 rond 备份里是否存在「跨备份稳定」的对齐键。
// 用完即可删。
package main
import (
"database/sql"
"fmt"
"os"
"strings"
"rond-with-you/internal/rond"
)
func main() {
if len(os.Args) < 3 {
fmt.Println("用法: idprobe <a.rondbackup> <b.rondbackup>")
os.Exit(2)
}
a := load(os.Args[1])
b := load(os.Args[2])
fmt.Printf("\n=== A=%s ===\n=== B=%s ===\n", os.Args[1], os.Args[2])
fmt.Printf("\n---------- ZVISIT.ZIDENTIFIER_ 稳定性 ----------\n")
fmt.Printf("A 有 UUID %d / %d,B 有 UUID %d / %d\n", len(a.visitUUID), len(a.visitPK), len(b.visitUUID), len(b.visitPK))
inter := 0
for u := range a.visitUUID {
if _, ok := b.visitUUID[u]; ok {
inter++
}
}
fmt.Printf("两次备份 UUID 交集: %d(A 独有 %d,B 独有 %d)\n",
inter, len(a.visitUUID)-inter, len(b.visitUUID)-inter)
// UUID 相同则对比 pk 是否相同
samePK, diffPK := 0, 0
for u, pa := range a.visitUUID {
if pb, ok := b.visitUUID[u]; ok {
if pa == pb {
samePK++
} else {
diffPK++
}
}
}
fmt.Printf("UUID 相同者中:Z_PK 也相同 %d,Z_PK 已变 %d\n", samePK, diffPK)
fmt.Printf("\n---------- ZACTIVITY.ZUID_ ----------\n")
fmt.Printf("A UUID %d / %d,B UUID %d / %d\n", len(a.actUUID), len(a.actPK), len(b.actUUID), len(b.actPK))
same, dif := 0, 0
for u, pa := range a.actUUID {
if pb, ok := b.actUUID[u]; ok {
if pa == pb {
same++
} else {
dif++
}
}
}
fmt.Printf("交集配对:Z_PK 相同 %d,Z_PK 已变 %d\n", same, dif)
fmt.Printf("\n---------- ZLOCATION 可用稳定列 ----------\n")
fmt.Printf("A 列: %s\n", strings.Join(a.locCols, ", "))
fmt.Printf("A 地点 %d,B 地点 %d\n", len(a.locPK), len(b.locPK))
// 名称+粗坐标是否可作键
type lk struct {
name string
lat float64
lon float64
}
keyA := map[string][]int{}
for pk, l := range a.locPK {
k := fmt.Sprintf("%s|%.5f|%.5f", l.name, l.lat, l.lon)
keyA[k] = append(keyA[k], pk)
}
hit, miss := 0, 0
for pk, l := range b.locPK {
k := fmt.Sprintf("%s|%.5f|%.5f", l.name, l.lat, l.lon)
if _, ok := keyA[k]; ok {
hit++
} else {
miss++
if miss <= 5 {
fmt.Printf(" B 独有地点 #%d %q %.6f,%.6f\n", pk, l.name, l.lat, l.lon)
}
}
}
fmt.Printf("按「名称+坐标(1e-5)」键:B 命中 A %d,未命中 %d\n", hit, miss)
}
type loc struct {
name string
lat, lon float64
}
type snap struct {
visitUUID map[string]int
visitPK map[int]bool
actUUID map[string]int
actPK map[int]bool
locPK map[int]loc
locCols []string
}
func load(path string) *snap {
dir, err := os.MkdirTemp("", "idprobe")
must(err)
defer os.RemoveAll(dir)
db, err := rond.Open(path, dir)
must(err)
defer db.Close()
s := &snap{visitUUID: map[string]int{}, visitPK: map[int]bool{},
actUUID: map[string]int{}, actPK: map[int]bool{}, locPK: map[int]loc{}}
rows, err := db.Query(`SELECT Z_PK, hex(ZIDENTIFIER_) FROM ZVISIT`)
must(err)
for rows.Next() {
var pk int
var u sql.NullString
must(rows.Scan(&pk, &u))
s.visitPK[pk] = true
if u.Valid && u.String != "" && strings.Trim(u.String, "0") != "" {
s.visitUUID[u.String] = pk
}
}
rows.Close()
rows, err = db.Query(`SELECT Z_PK, hex(ZUID_), COALESCE(ZNAME_,'') FROM ZACTIVITY`)
must(err)
for rows.Next() {
var pk int
var u sql.NullString
var n string
must(rows.Scan(&pk, &u, &n))
s.actPK[pk] = true
if u.Valid && u.String != "" && strings.Trim(u.String, "0") != "" {
s.actUUID[u.String] = pk
}
}
rows.Close()
rows, err = db.Query(`PRAGMA table_info(ZLOCATION)`)
must(err)
for rows.Next() {
var cid int
var name, typ string
var notnull int
var dflt sql.NullString
var pk int
must(rows.Scan(&cid, &name, &typ, &notnull, &dflt, &pk))
s.locCols = append(s.locCols, name)
}
rows.Close()
rows, err = db.Query(`SELECT Z_PK, COALESCE(ZNAME_,''), COALESCE(ZLATITUDE,0), COALESCE(ZLONGITUDE,0) FROM ZLOCATION`)
must(err)
for rows.Next() {
var pk int
var l loc
must(rows.Scan(&pk, &l.name, &l.lat, &l.lon))
s.locPK[pk] = l
}
rows.Close()
return s
}
func must(err error) {
if err != nil {
fmt.Fprintln(os.Stderr, "错误:", err)
os.Exit(1)
}
}
+57
View File
@@ -0,0 +1,57 @@
// 一次性探针:在 rond 备份的 SQLite 上跑一条查询并打印结果(BLOB 打十六进制)。
package main
import (
"fmt"
"os"
"strings"
"rond-with-you/internal/rond"
)
func main() {
if len(os.Args) < 3 {
fmt.Println(`用法: sqlprobe <backup> "SELECT ..."`)
os.Exit(2)
}
dir, err := os.MkdirTemp("", "sqlprobe")
if err != nil {
panic(err)
}
defer os.RemoveAll(dir)
db, err := rond.Open(os.Args[1], dir)
if err != nil {
panic(err)
}
defer db.Close()
rows, err := db.Query(os.Args[2])
if err != nil {
panic(err)
}
defer rows.Close()
cols, _ := rows.Columns()
fmt.Println(strings.Join(cols, " | "))
for rows.Next() {
vals := make([]any, len(cols))
ptrs := make([]any, len(cols))
for i := range vals {
ptrs[i] = &vals[i]
}
if err := rows.Scan(ptrs...); err != nil {
panic(err)
}
parts := make([]string, len(cols))
for i, v := range vals {
switch t := v.(type) {
case nil:
parts[i] = "-"
case []byte:
parts[i] = fmt.Sprintf("%x", t)
default:
parts[i] = fmt.Sprint(t)
}
}
fmt.Println(strings.Join(parts, " | "))
}
}