备份与迷雾的差异分析、合并执行,附属数据改挂坐标键

备份(.rondbackup)与迷雾(.fwss)各加一层「先看差异再动手」:

- 迷雾:上传先出差异页,说清叠加能补多少、整体替换会丢多少,并在真实底图上
  用绿/灰/红标出会新增 / 不变 / 会丢的区域
- 备份:与当前数据集按 UUID 逐条比对(rond 每次导出都会重排 Z_PK,编号认不出人),
  新增合入、冲突逐条选、仅站内有逐条勾选删除,全在一个事务里执行;
  合入的行写回 entity_raw,保证再传同一份备份时差异归零
- 地点按坐标对齐:站点的「补全地址」会把地点改名,拿名称当键会让一整个地点的
  到访全变成假冲突
- 附属数据(备注/结论/访客票/图片)从 dataset_id+src_pk 换成坐标键,换数据集不再失联;
  访客票没有 dataset_id,回填按活跃数据集认领
- 修正 contentHash 拿 Z_PK 当指纹的隐患,以及两处「src_pk 跨快照稳定」的错误注释

导出格式(.rondbackup / .fwss)保持原样,往返测试通过。
This commit is contained in:
tamakyi
2026-09-19 16:04:00 +08:00
parent 9e01923244
commit 30c502ce7b
43 changed files with 5968 additions and 170 deletions
+90
View File
@@ -0,0 +1,90 @@
package fog
// Diff 是一份待导入快照与当前迷雾之间的差异画像。
//
// 导入有两种语义完全不同的模式:叠加只做按位或、不可能丢数据;整体替换会先清空,
// 快照里没有的块一律消失。以前两种模式在界面上只是一个下拉框,站长在下手之前
// 看不到「这次能补多少」和「这次会丢多少」,实测里整体替换静默丢掉过 14% 的已探索区域。
type Diff struct {
CurBlocks, CurCells int // 库里现有
IncBlocks, IncCells int // 快照本身
// GainBlocks/GainCells 是叠加模式真正新增的部分:快照里当前完全没有的整块,
// 加上重叠块里补亮的格。
GainBlocks int
GainCells int
// OverlapBlocks 是与现有块坐标相同的块数,其中 SameBlocks 是一格都不会变的。
OverlapBlocks int
SameBlocks int
// LostBlocks/LostCells 是快照里完全没有、只有整体替换才会消失的部分。
LostBlocks int
LostCells int
curAreaM2, incAreaM2, gainAreaM2, lostAreaM2 float64
}
// DiffBlocks 比对当前块集与一份待导入快照。入参块集不要求有序。
func DiffBlocks(cur, inc []Block) Diff {
var d Diff
d.CurBlocks, d.IncBlocks = len(cur), len(inc)
curMap := make(map[[2]int][]byte, len(cur))
for _, b := range cur {
curMap[[2]int{b.GX, b.GY}] = b.Bitmap
n := popcountSum(b.Bitmap)
d.CurCells += n
d.curAreaM2 += float64(n) * cellAreaM2(b.GX, b.GY)
}
incSet := make(map[[2]int]struct{}, len(inc))
for _, b := range inc {
incSet[[2]int{b.GX, b.GY}] = struct{}{}
n := popcountSum(b.Bitmap)
d.IncCells += n
d.incAreaM2 += float64(n) * cellAreaM2(b.GX, b.GY)
old, ok := curMap[[2]int{b.GX, b.GY}]
if !ok {
d.GainBlocks++
d.GainCells += n
d.gainAreaM2 += float64(n) * cellAreaM2(b.GX, b.GY)
continue
}
d.OverlapBlocks++
gain := 0
for i := range b.Bitmap {
gain += int(popcount[b.Bitmap[i]&^old[i]])
}
if gain == 0 {
d.SameBlocks++
continue
}
d.GainCells += gain
d.gainAreaM2 += float64(gain) * cellAreaM2(b.GX, b.GY)
}
for _, b := range cur {
if _, ok := incSet[[2]int{b.GX, b.GY}]; ok {
continue
}
n := popcountSum(b.Bitmap)
d.LostBlocks++
d.LostCells += n
d.lostAreaM2 += float64(n) * cellAreaM2(b.GX, b.GY)
}
return d
}
func (d Diff) CurAreaKm2() float64 { return d.curAreaM2 / 1e6 }
func (d Diff) IncAreaKm2() float64 { return d.incAreaM2 / 1e6 }
func (d Diff) GainAreaKm2() float64 { return d.gainAreaM2 / 1e6 }
func (d Diff) LostAreaKm2() float64 { return d.lostAreaM2 / 1e6 }
// MergeAreaKm2 是叠加后的总量,CoverAreaKm2 是整体替换后的总量。
func (d Diff) MergeAreaKm2() float64 { return (d.curAreaM2 + d.gainAreaM2) / 1e6 }
func (d Diff) CoverAreaKm2() float64 { return d.incAreaM2 / 1e6 }
// Changed 报告叠加是否会让地图有任何变化——为假说明这份快照已经被完全覆盖过。
func (d Diff) Changed() bool { return d.GainCells > 0 }
+88
View File
@@ -0,0 +1,88 @@
package fog
import "testing"
// blk 造一个块,cells 是块内的 (列, 行) 坐标。
func blk(gx, gy int, cells ...[2]int) Block {
bm := make([]byte, bitmapSize)
for _, c := range cells {
bm[c[1]*8+c[0]/8] |= 0x80 >> (c[0] % 8)
}
return Block{GX: gx, GY: gy, Bitmap: bm}
}
func TestDiffEmptyBase(t *testing.T) {
d := DiffBlocks(nil, []Block{blk(10, 20, [2]int{0, 0}, [2]int{1, 0})})
if d.CurBlocks != 0 || d.CurCells != 0 {
t.Fatalf("空库不该有现有数据: %+v", d)
}
if d.GainBlocks != 1 || d.GainCells != 2 || d.IncCells != 2 {
t.Fatalf("空库叠加应为全部新增: %+v", d)
}
if d.LostBlocks != 0 || d.LostCells != 0 || !d.Changed() {
t.Fatalf("空库不该有丢失: %+v", d)
}
}
func TestDiffIdentical(t *testing.T) {
cur := []Block{blk(10, 20, [2]int{0, 0}, [2]int{1, 0}), blk(11, 20, [2]int{5, 5})}
d := DiffBlocks(cur, cur)
if d.GainCells != 0 || d.Changed() {
t.Fatalf("同一份快照再叠一次不该有任何新增: %+v", d)
}
if d.LostBlocks != 0 || d.LostCells != 0 {
t.Fatalf("同一份快照不该有可丢的部分: %+v", d)
}
if d.OverlapBlocks != 2 || d.SameBlocks != 2 {
t.Fatalf("两个块都应算作「重叠且无变化」: %+v", d)
}
}
func TestDiffOverlapGain(t *testing.T) {
d := DiffBlocks(
[]Block{blk(10, 20, [2]int{0, 0})},
[]Block{blk(10, 20, [2]int{0, 0}, [2]int{3, 4})})
if d.GainBlocks != 0 || d.GainCells != 1 {
t.Fatalf("重叠块应只补亮多出的那一格: %+v", d)
}
if d.OverlapBlocks != 1 || d.SameBlocks != 0 {
t.Fatalf("重叠但补亮了,不该记为无变化: %+v", d)
}
if d.LostBlocks != 0 {
t.Fatalf("同一个块不该被判为会丢失: %+v", d)
}
}
func TestDiffDisjoint(t *testing.T) {
d := DiffBlocks(
[]Block{blk(10, 20, [2]int{0, 0})},
[]Block{blk(30, 40, [2]int{1, 1})})
if d.GainBlocks != 1 || d.GainCells != 1 {
t.Fatalf("另一块应是新增: %+v", d)
}
if d.LostBlocks != 1 || d.LostCells != 1 {
t.Fatalf("库里那块在快照里没有,整体替换会丢掉: %+v", d)
}
}
// 面积按块中心纬度做 cos 修正,越靠近两极越小;总量关系也要自洽。
func TestDiffAreas(t *testing.T) {
equator := DiffBlocks(nil, []Block{blk(100, WorldCells/BlockCells/2, [2]int{1, 1})})
high := DiffBlocks(nil, []Block{blk(100, 20000, [2]int{1, 1})})
if high.GainAreaKm2() <= 0 {
t.Fatalf("高纬单格面积应为正: %v", high.GainAreaKm2())
}
if !(equator.GainAreaKm2() > high.GainAreaKm2()) {
t.Fatalf("赤道单格应大于高纬: %v vs %v", equator.GainAreaKm2(), high.GainAreaKm2())
}
m := DiffBlocks(
[]Block{blk(10, 20, [2]int{0, 0})},
[]Block{blk(10, 20, [2]int{0, 0}, [2]int{9, 9})})
if diff := m.MergeAreaKm2() - (m.CurAreaKm2() + m.GainAreaKm2()); diff > 1e-12 || diff < -1e-12 {
t.Fatalf("叠加后总量应等于现有加新增: %v", diff)
}
if m.CoverAreaKm2() != m.IncAreaKm2() {
t.Fatalf("整体替换后总量应等于快照本身: %v vs %v", m.CoverAreaKm2(), m.IncAreaKm2())
}
}
+6
View File
@@ -180,6 +180,12 @@ func mercY(lat float64) float64 {
return (math.Pi - math.Asinh(math.Tan(rad))) / (2 * math.Pi) * WorldCells
}
// BlockLonLat 返回某块中心的 WGS-84 经纬度,供差异地图定位视野。
func BlockLonLat(gbx, gby int) (lat, lon float64) {
lon, lat = cellLonLat(gbx*BlockCells+BlockCells/2, gby*BlockCells+BlockCells/2)
return
}
func cellLonLat(cx, cy int) (float64, float64) {
lon := (float64(cx)+0.5)/WorldCells*360.0 - 180.0
n := 2 * math.Pi * (float64(cy) + 0.5) / WorldCells
+200
View File
@@ -0,0 +1,200 @@
package fog
import (
"image"
"math"
)
// 差异图的三种归属。改这里的色值要同步 templates/admin/fog-preview.html 的图例。
var (
DiffBothColor = [4]byte{126, 134, 146, 148} // 两边都有:导入不影响
DiffGainColor = [4]byte{36, 164, 90, 205} // 只有快照有:叠加会新增
DiffLostColor = [4]byte{214, 56, 56, 210} // 只有库里有:整体替换会丢
)
const (
diffNone uint8 = iota
diffBoth
diffGain
diffLost
)
// RenderDiffTile 渲染一张 256x256 的差异瓦片,供站长在真实底图上核对
// 「这份快照能补上哪里」与「整体替换会丢掉哪里」。
//
// 与 RenderTile 一样按底图坐标系做 WGS84→GCJ02 纠偏;不做描边——三种颜色
// 已经够区分,再描边在粗缩放级下会糊成一片。
//
// 逐格判定只在放大到「一格不小于一像素」(z>=14)时做;更粗的级别下一个块还不到
// 一个像素,按块内占多数的类别整块涂色。绘制顺序 both→gain→lost,让「会新增」
// 和「会丢」压住底色,重叠处不至于被灰吃掉。
func RenderDiffTile(cur, inc map[[2]int][]byte, z, x, y int, crs string) *image.NRGBA {
if z < 0 || z > 22 {
return nil
}
step := WorldCells >> z
scale := 256.0 / float64(step)
ox, oy := float64(x*step), float64(y*step)
layer := make([]uint8, 256*256)
mark := func(px, py, w, h int, kind uint8) {
for r := 0; r < h; r++ {
yy := py + r
if yy < 0 || yy > 255 {
continue
}
row := yy * 256
for c := 0; c < w; c++ {
xx := px + c
if xx < 0 || xx > 255 {
continue
}
layer[row+xx] = kind
}
}
}
// 直接遍历块表而不是全球块范围:低缩放级下范围循环会膨胀到数十亿次空查询。
// GCJ 偏移最多约 700m(74 格),粗裁剪时两侧各放宽两块保证不漏。
const margin = 2 * BlockCells
lo, hi := float64(x*step-margin), float64((x+1)*step+margin)
rlo, rhi := float64(y*step-margin), float64((y+1)*step+margin)
shift := crs == "gcj02"
draw := func(pass uint8, gbx, gby int, cb, ib []byte) {
bx, by := float64(gbx*BlockCells), float64(gby*BlockCells)
if bx+BlockCells <= lo || bx >= hi || by+BlockCells <= rlo || by >= rhi {
return
}
ddx, ddy := 0.0, 0.0
if shift {
ddx, ddy = blockShift(gbx, gby)
}
u0 := (bx + ddx - ox) * scale
v0 := (by + ddy - oy) * scale
if scale >= 1 {
w := int(math.Ceil(scale))
for row := 0; row < BlockCells; row++ {
base := row * 8
for colB := 0; colB < 8; colB++ {
var cbv, ibv byte
if cb != nil {
cbv = cb[base+colB]
}
if ib != nil {
ibv = ib[base+colB]
}
if cbv|ibv == 0 {
continue
}
py := int(math.Floor(v0 + float64(row)*scale))
for bit := 0; bit < 8; bit++ {
m := byte(0x80 >> bit)
kind := cellKind(cbv&m != 0, ibv&m != 0)
if kind != pass {
continue
}
px := int(math.Floor(u0 + float64(colB*8+bit)*scale))
mark(px, py, w, w, kind)
}
}
}
return
}
// 一格不足一像素:整块按多数类涂,三种混在一起的块以数量最多的为准
if dominantKind(blockDiffCounts(cb, ib)) != pass {
return
}
u1 := (bx + BlockCells + ddx - ox) * scale
v1 := (by + BlockCells + ddy - oy) * scale
px0, py0 := int(math.Floor(u0)), int(math.Floor(v0))
px1, py1 := int(math.Ceil(u1)), int(math.Ceil(v1))
if px1-px0 < 1 {
px1 = px0 + 1
}
if py1-py0 < 1 {
py1 = py0 + 1
}
mark(px0, py0, px1-px0, py1-py0, pass)
}
for _, pass := range []uint8{diffBoth, diffGain, diffLost} {
for key, cb := range cur {
draw(pass, key[0], key[1], cb, inc[key])
}
// 只在快照里的块单独走一遍,两边都有的块不再重复画
for key, ib := range inc {
if _, dup := cur[key]; dup {
continue
}
draw(pass, key[0], key[1], nil, ib)
}
}
img := image.NewNRGBA(image.Rect(0, 0, 256, 256))
for i, kind := range layer {
if kind == diffNone {
continue
}
var c [4]byte
switch kind {
case diffBoth:
c = DiffBothColor
case diffGain:
c = DiffGainColor
default:
c = DiffLostColor
}
o := i * 4
img.Pix[o], img.Pix[o+1], img.Pix[o+2], img.Pix[o+3] = c[0], c[1], c[2], c[3]
}
return img
}
func cellKind(inCur, inInc bool) uint8 {
switch {
case inCur && inInc:
return diffBoth
case inInc:
return diffGain
case inCur:
return diffLost
}
return diffNone
}
// blockDiffCounts 数一个块里三类各占多少格。两个位图都可能缺失(只有一边有这个块)。
func blockDiffCounts(cb, ib []byte) (both, gain, lost int) {
n := len(cb)
if len(ib) > n {
n = len(ib)
}
for i := 0; i < n; i++ {
var c, in byte
if i < len(cb) {
c = cb[i]
}
if i < len(ib) {
in = ib[i]
}
both += int(popcount[c&in])
gain += int(popcount[in&^c])
lost += int(popcount[c&^in])
}
return
}
// dominantKind 取三类里占得最多的一类,用来在粗缩放级下决定整块的着色。
func dominantKind(both, gain, lost int) uint8 {
switch {
case gain > 0 && gain >= both && gain >= lost:
return diffGain
case lost > 0 && lost >= both:
return diffLost
case both > 0:
return diffBoth
}
return diffNone
}
+114
View File
@@ -0,0 +1,114 @@
package fog
import (
"image"
"math"
"testing"
)
func bmOf(cells ...[2]int) []byte {
bm := make([]byte, bitmapSize)
for _, c := range cells {
bm[c[1]*8+c[0]/8] |= 0x80 >> (c[0] % 8)
}
return bm
}
func pixAt(img *image.NRGBA, px, py int) [4]byte {
i := (py*256 + px) * 4
return [4]byte{img.Pix[i], img.Pix[i+1], img.Pix[i+2], img.Pix[i+3]}
}
// z=18 时一格正好 16 像素,块 (100,200) 完整落在瓦片 (400,800) 里,
// 格 (col,row) 的中心像素是 (col*16+8, row*16+8)。
func TestRenderDiffTilePerCell(t *testing.T) {
key := [2]int{100, 200}
cur := map[[2]int][]byte{key: bmOf([2]int{0, 0}, [2]int{4, 0})}
inc := map[[2]int][]byte{key: bmOf([2]int{0, 0}, [2]int{1, 0})}
img := RenderDiffTile(cur, inc, 18, 400, 800, "wgs84")
if img == nil {
t.Fatal("瓦片不该为空")
}
if got := pixAt(img, 8, 8); got != DiffBothColor {
t.Errorf("两边都有的格应为灰色,实际 %v", got)
}
if got := pixAt(img, 24, 8); got != DiffGainColor {
t.Errorf("只有快照有的格应为绿色,实际 %v", got)
}
if got := pixAt(img, 72, 8); got != DiffLostColor {
t.Errorf("只有库里有、整体替换会丢的格应为红色,实际 %v", got)
}
if got := pixAt(img, 200, 200); got[3] != 0 {
t.Errorf("没被探索的格应保持透明,实际 %v", got)
}
}
func TestRenderDiffTileSingleSided(t *testing.T) {
key := [2]int{100, 200}
onlyCur := RenderDiffTile(map[[2]int][]byte{key: bmOf([2]int{3, 3})}, nil, 18, 400, 800, "wgs84")
if got := pixAt(onlyCur, 56, 56); got != DiffLostColor {
t.Errorf("快照里完全没有的块应整块为红,实际 %v", got)
}
onlyInc := RenderDiffTile(nil, map[[2]int][]byte{key: bmOf([2]int{3, 3})}, 18, 400, 800, "wgs84")
if got := pixAt(onlyInc, 56, 56); got != DiffGainColor {
t.Errorf("库里完全没有的块应整块为绿,实际 %v", got)
}
}
// z=10 时一格不足一像素,块只占 4x4 像素,按块内多数类涂色。
func TestRenderDiffTileCoarseZoomUsesDominantKind(t *testing.T) {
key := [2]int{100, 200}
cur := map[[2]int][]byte{key: bmOf([2]int{0, 0})}
inc := map[[2]int][]byte{key: bmOf([2]int{1, 1}, [2]int{2, 2}, [2]int{3, 3})}
img := RenderDiffTile(cur, inc, 10, 1, 3, "wgs84")
const step, scale = WorldCells >> 10, 256.0 / (WorldCells >> 10)
bx, by := float64(key[0]*BlockCells), float64(key[1]*BlockCells)
px := int(math.Floor((bx - 1*step) * scale))
py := int(math.Floor((by - 3*step) * scale))
if got := pixAt(img, px+1, py+1); got != DiffGainColor {
t.Errorf("三格新增、一格不变的块应以绿色为主,实际 %v", got)
}
if w := int(math.Ceil(float64(BlockCells) * scale)); w != 4 {
t.Fatalf("z=10 下块应占 4 像素,实际 %d", w)
}
}
func TestRenderDiffTileEdgeCases(t *testing.T) {
if RenderDiffTile(nil, nil, 18, 400, 800, "wgs84") == nil {
t.Error("两边都空也应返回一张空瓦片")
}
if RenderDiffTile(nil, nil, 23, 0, 0, "wgs84") != nil {
t.Error("超出缩放范围应返回 nil")
}
}
func TestDominantKind(t *testing.T) {
cases := []struct {
both, gain, lost int
want uint8
}{
{0, 0, 0, diffNone},
{5, 1, 0, diffBoth},
{1, 5, 0, diffGain},
{1, 0, 5, diffLost},
{2, 2, 0, diffGain}, // 平局让「会新增」胜出
{3, 1, 1, diffBoth},
}
for _, c := range cases {
if got := dominantKind(c.both, c.gain, c.lost); got != c.want {
t.Errorf("dominantKind(%d,%d,%d)=%d,期望 %d", c.both, c.gain, c.lost, got, c.want)
}
}
}
func TestBlockDiffCounts(t *testing.T) {
both, gain, lost := blockDiffCounts(bmOf([2]int{0, 0}, [2]int{1, 0}), bmOf([2]int{0, 0}, [2]int{2, 0}))
if both != 1 || gain != 1 || lost != 1 {
t.Errorf("一块里应各有一格:both=%d gain=%d lost=%d", both, gain, lost)
}
both, gain, lost = blockDiffCounts(nil, bmOf([2]int{5, 5}))
if both != 0 || gain != 1 || lost != 0 {
t.Errorf("只有快照有该块:both=%d gain=%d lost=%d", both, gain, lost)
}
}