tamakyi
30c502ce7b
备份（.rondbackup）与迷雾（.fwss）各加一层「先看差异再动手」： - 迷雾：上传先出差异页，说清叠加能补多少、整体替换会丢多少，并在真实底图上 用绿/灰/红标出会新增 / 不变 / 会丢的区域 - 备份：与当前数据集按 UUID 逐条比对（rond 每次导出都会重排 Z_PK，编号认不出人）， 新增合入、冲突逐条选、仅站内有逐条勾选删除，全在一个事务里执行； 合入的行写回 entity_raw，保证再传同一份备份时差异归零 - 地点按坐标对齐：站点的「补全地址」会把地点改名，拿名称当键会让一整个地点的 到访全变成假冲突 - 附属数据（备注/结论/访客票/图片）从 dataset_id+src_pk 换成坐标键，换数据集不再失联； 访客票没有 dataset_id，回填按活跃数据集认领 - 修正 contentHash 拿 Z_PK 当指纹的隐患，以及两处「src_pk 跨快照稳定」的错误注释 导出格式（.rondbackup / .fwss）保持原样，往返测试通过。
439 lines
12 KiB
Go
439 lines
12 KiB
Go
// 一次性探针：把一份 .rondbackup 与 PG 里某个数据集的记录逐条对齐，输出差异画像。
|
|
// 用完即可删，不属于站点功能。
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"os"
|
|
"sort"
|
|
"time"
|
|
|
|
_ "github.com/jackc/pgx/v5/stdlib"
|
|
|
|
"rond-with-you/internal/rond"
|
|
)
|
|
|
|
func main() {
|
|
if len(os.Args) < 4 {
|
|
fmt.Println("用法: diffprobe <archive.rondbackup> <dsn> <datasetID>")
|
|
os.Exit(2)
|
|
}
|
|
archive, dsn := os.Args[1], os.Args[2]
|
|
var datasetID int64
|
|
fmt.Sscanf(os.Args[3], "%d", &datasetID)
|
|
|
|
tmp, err := os.MkdirTemp("", "diffprobe")
|
|
must(err)
|
|
defer os.RemoveAll(tmp)
|
|
|
|
sqlDB, err := rond.Open(archive, tmp)
|
|
must(err)
|
|
b, err := rond.Parse(sqlDB)
|
|
sqlDB.Close()
|
|
must(err)
|
|
|
|
pg, err := sql.Open("pgx", dsn)
|
|
must(err)
|
|
defer pg.Close()
|
|
ctx := context.Background()
|
|
must(pg.PingContext(ctx))
|
|
|
|
fmt.Printf("=== 备份 %s ===\n", archive)
|
|
fmt.Printf("备份: 到访 %d / 地点 %d / 位移 %d / 天气 %d / 活动 %d / 标签 %d / 原始定位 %d\n",
|
|
len(b.Visits), len(b.Locations), len(b.Movements), len(b.Weather),
|
|
len(b.Activities), len(b.Tags), b.RawVisitNum)
|
|
var bFirst, bLast sql.NullTime
|
|
for _, v := range b.Visits {
|
|
if v.Arrival == nil {
|
|
continue
|
|
}
|
|
if !bFirst.Valid || v.Arrival.Before(bFirst.Time) {
|
|
bFirst = sql.NullTime{Time: *v.Arrival, Valid: true}
|
|
}
|
|
if !bLast.Valid || v.Arrival.After(bLast.Time) {
|
|
bLast = sql.NullTime{Time: *v.Arrival, Valid: true}
|
|
}
|
|
}
|
|
fmt.Printf("备份时间跨度: %s ~ %s\n", ts(bFirst), ts(bLast))
|
|
|
|
fmt.Printf("\n=== 库内 dataset #%d ===\n", datasetID)
|
|
var name string
|
|
var cnt, pcnt, raw int
|
|
var gFirst, gLast sql.NullTime
|
|
must(pg.QueryRowContext(ctx, `SELECT original_name, visit_count, place_count, raw_visit_count,
|
|
first_visit_at, last_visit_at FROM datasets WHERE id=$1`, datasetID).
|
|
Scan(&name, &cnt, &pcnt, &raw, &gFirst, &gLast))
|
|
fmt.Printf("%s: 到访 %d / 地点 %d / 原始定位 %d，跨度 %s ~ %s\n", name, cnt, pcnt, raw, ts(gFirst), ts(gLast))
|
|
|
|
diffVisits(ctx, pg, datasetID, b)
|
|
diffPlaces(ctx, pg, datasetID, b)
|
|
diffMovements(ctx, pg, datasetID, b)
|
|
diffWeather(ctx, pg, datasetID, b)
|
|
diffActivities(ctx, pg, datasetID, b)
|
|
diffTags(ctx, pg, datasetID, b)
|
|
}
|
|
|
|
func ts(t sql.NullTime) string {
|
|
if !t.Valid {
|
|
return "-"
|
|
}
|
|
return t.Time.Local().Format("2006-01-02 15:04")
|
|
}
|
|
|
|
type visitRow struct {
|
|
pk, placeSrc, actSrc int
|
|
arrival, departure *time.Time
|
|
remark, emoji string
|
|
bookmarked, userAdd bool
|
|
}
|
|
|
|
func diffVisits(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
|
|
fmt.Printf("\n---------- 到访 ----------\n")
|
|
rows, err := pg.QueryContext(ctx, `SELECT v.src_pk, COALESCE(p.src_pk,0), COALESCE(a.src_pk,0),
|
|
v.arrival, v.departure, COALESCE(v.remark,''), COALESCE(v.emoji,''), v.bookmarked, v.is_user_added
|
|
FROM visits v LEFT JOIN places p ON p.id=v.place_id LEFT JOIN activities a ON a.id=v.activity_id
|
|
WHERE v.dataset_id=$1`, dsID)
|
|
must(err)
|
|
db := map[int]visitRow{}
|
|
for rows.Next() {
|
|
var r visitRow
|
|
var arr time.Time
|
|
var dep sql.NullTime
|
|
must(rows.Scan(&r.pk, &r.placeSrc, &r.actSrc, &arr, &dep, &r.remark, &r.emoji, &r.bookmarked, &r.userAdd))
|
|
r.arrival = &arr
|
|
if dep.Valid {
|
|
r.departure = &dep.Time
|
|
}
|
|
db[r.pk] = r
|
|
}
|
|
rows.Close()
|
|
|
|
bk := map[int]rond.Visit{}
|
|
for _, v := range b.Visits {
|
|
bk[v.SrcPK] = v
|
|
}
|
|
|
|
var onlyBackup, onlyDB, bothDiff, same []int
|
|
for pk, v := range bk {
|
|
r, ok := db[pk]
|
|
if !ok {
|
|
onlyBackup = append(onlyBackup, pk)
|
|
continue
|
|
}
|
|
if vdiff(v, r) != "" {
|
|
bothDiff = append(bothDiff, pk)
|
|
continue
|
|
}
|
|
same = append(same, pk)
|
|
}
|
|
for pk := range db {
|
|
if _, ok := bk[pk]; !ok {
|
|
onlyDB = append(onlyDB, pk)
|
|
}
|
|
}
|
|
sort.Ints(onlyBackup)
|
|
sort.Ints(onlyDB)
|
|
sort.Ints(bothDiff)
|
|
|
|
fmt.Printf("同号且一致: %d\n", len(same))
|
|
fmt.Printf("仅备份有（实机新增，库里没有）: %d\n", len(onlyBackup))
|
|
fmt.Printf("仅库里有（站点侧新增/实机已删除）: %d\n", len(onlyDB))
|
|
fmt.Printf("同号但内容不同（冲突）: %d\n", len(bothDiff))
|
|
|
|
// 按到达时间做多重集匹配：判断「同号冲突」到底是同批数据换了主键，还是两批无关数据
|
|
byTime := map[string][]int{}
|
|
for pk, r := range db {
|
|
if r.arrival != nil {
|
|
byTime[r.arrival.UTC().Format(time.RFC3339)] = append(byTime[r.arrival.UTC().Format(time.RFC3339)], pk)
|
|
}
|
|
}
|
|
matched, miss := 0, 0
|
|
pair := [][2]int{}
|
|
for pk, v := range bk {
|
|
if v.Arrival == nil {
|
|
continue
|
|
}
|
|
k := v.Arrival.UTC().Format(time.RFC3339)
|
|
if list := byTime[k]; len(list) > 0 {
|
|
matched++
|
|
pair = append(pair, [2]int{pk, list[0]})
|
|
byTime[k] = list[1:]
|
|
} else {
|
|
miss++
|
|
}
|
|
}
|
|
fmt.Printf("\n[时间对齐] 备份到访按「到达时刻」能在库内找到同刻记录: %d 条，找不到: %d 条\n", matched, miss)
|
|
sort.Slice(pair, func(i, j int) bool { return pair[i][0] < pair[j][0] })
|
|
fmt.Printf("[时间对齐] 同刻对的 (备份pk→库pk) 前 12 组: ")
|
|
for i, p := range pair {
|
|
if i >= 12 {
|
|
break
|
|
}
|
|
fmt.Printf("%d→%d ", p[0], p[1])
|
|
}
|
|
fmt.Println()
|
|
nDiff := 0
|
|
for _, p := range pair {
|
|
if p[0] != p[1] {
|
|
nDiff++
|
|
}
|
|
}
|
|
fmt.Printf("[时间对齐] 同刻但主键不同: %d / %d\n", nDiff, len(pair))
|
|
|
|
show := func(label string, pks []int, f func(pk int) string) {
|
|
if len(pks) == 0 {
|
|
return
|
|
}
|
|
fmt.Printf("\n[%s] 共 %d 条，前 8 条：\n", label, len(pks))
|
|
for i, pk := range pks {
|
|
if i >= 8 {
|
|
break
|
|
}
|
|
fmt.Printf(" #%d %s\n", pk, f(pk))
|
|
}
|
|
}
|
|
show("仅备份有", onlyBackup, func(pk int) string {
|
|
v := bk[pk]
|
|
return fmt.Sprintf("到访 %s 离开 %s 地点src=%s", ft(v.Arrival), ft(v.Departure), ps(v.LocationSrcPK))
|
|
})
|
|
show("仅库里有", onlyDB, func(pk int) string {
|
|
r := db[pk]
|
|
return fmt.Sprintf("到访 %s 离开 %s 地点src=%d 站点新增=%v 备注=%q",
|
|
ft(r.arrival), ftp(r.departure), r.placeSrc, r.userAdd, r.remark)
|
|
})
|
|
show("冲突", bothDiff, func(pk int) string {
|
|
return vdiff(bk[pk], db[pk])
|
|
})
|
|
}
|
|
|
|
func vdiff(v rond.Visit, r visitRow) string {
|
|
var d string
|
|
if !eq(v.Arrival, r.arrival) {
|
|
d += fmt.Sprintf("到达 %s→%s ", ft(v.Arrival), ft(r.arrival))
|
|
}
|
|
if !eq(v.Departure, r.departure) {
|
|
d += fmt.Sprintf("离开 %s→%s ", ft(v.Departure), ft(r.departure))
|
|
}
|
|
if pv(v.LocationSrcPK) != r.placeSrc {
|
|
d += fmt.Sprintf("地点 %s→%d ", ps(v.LocationSrcPK), r.placeSrc)
|
|
}
|
|
if av(v.ActivitySrcPK) != r.actSrc {
|
|
d += fmt.Sprintf("活动 %s→%d ", ps(v.ActivitySrcPK), r.actSrc)
|
|
}
|
|
if v.Remark != r.remark {
|
|
d += fmt.Sprintf("备注 %q→%q ", v.Remark, r.remark)
|
|
}
|
|
if v.Emoji != r.emoji {
|
|
d += fmt.Sprintf("表情 %q→%q ", v.Emoji, r.emoji)
|
|
}
|
|
if v.Bookmarked != r.bookmarked {
|
|
d += fmt.Sprintf("收藏 %v→%v ", v.Bookmarked, r.bookmarked)
|
|
}
|
|
return d
|
|
}
|
|
|
|
func diffPlaces(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
|
|
fmt.Printf("\n---------- 地点 ----------\n")
|
|
type prow struct {
|
|
name, poi, cc, prov, city, dist, sub, thor, tz string
|
|
lat, lon float64
|
|
}
|
|
rows, err := pg.QueryContext(ctx, `SELECT src_pk, COALESCE(name,''), COALESCE(poi_category,''),
|
|
COALESCE(country_code,''), COALESCE(province,''), COALESCE(city,''), COALESCE(district,''),
|
|
COALESCE(sublocality,''), COALESCE(thoroughfare,''), COALESCE(timezone,''),
|
|
COALESCE(lat,0), COALESCE(lon,0) FROM places WHERE dataset_id=$1`, dsID)
|
|
must(err)
|
|
db := map[int]prow{}
|
|
for rows.Next() {
|
|
var pk int
|
|
var r prow
|
|
must(rows.Scan(&pk, &r.name, &r.poi, &r.cc, &r.prov, &r.city, &r.dist, &r.sub, &r.thor, &r.tz, &r.lat, &r.lon))
|
|
db[pk] = r
|
|
}
|
|
rows.Close()
|
|
|
|
bk := map[int]rond.Location{}
|
|
for _, l := range b.Locations {
|
|
bk[l.SrcPK] = l
|
|
}
|
|
var onlyB, onlyD, diff []int
|
|
for pk, l := range bk {
|
|
r, ok := db[pk]
|
|
if !ok {
|
|
onlyB = append(onlyB, pk)
|
|
continue
|
|
}
|
|
if l.Name != r.name || l.POICategory != r.poi || l.Province != r.prov || l.City != r.city ||
|
|
l.District != r.dist || l.Sublocality != r.sub || l.Thoroughfare != r.thor || l.Timezone != r.tz ||
|
|
l.CountryCode != r.cc || !near(l.Lat, r.lat) || !near(l.Lon, r.lon) {
|
|
diff = append(diff, pk)
|
|
}
|
|
}
|
|
for pk := range db {
|
|
if _, ok := bk[pk]; !ok {
|
|
onlyD = append(onlyD, pk)
|
|
}
|
|
}
|
|
sort.Ints(onlyB)
|
|
sort.Ints(onlyD)
|
|
sort.Ints(diff)
|
|
fmt.Printf("仅备份有: %d，仅库里有: %d，同号内容不同: %d\n", len(onlyB), len(onlyD), len(diff))
|
|
for i, pk := range onlyB {
|
|
if i >= 6 {
|
|
break
|
|
}
|
|
fmt.Printf(" 仅备份 #%d %s (%s %s %s)\n", pk, bk[pk].Name, bk[pk].Province, bk[pk].City, bk[pk].POICategory)
|
|
}
|
|
for i, pk := range onlyD {
|
|
if i >= 6 {
|
|
break
|
|
}
|
|
fmt.Printf(" 仅库内 #%d %s (%s %s)\n", pk, db[pk].name, db[pk].prov, db[pk].city)
|
|
}
|
|
for i, pk := range diff {
|
|
if i >= 6 {
|
|
break
|
|
}
|
|
l, r := bk[pk], db[pk]
|
|
fmt.Printf(" 冲突 #%d 备份[name=%q city=%q %.6f,%.6f] 库[name=%q city=%q %.6f,%.6f]\n",
|
|
pk, l.Name, l.City, l.Lat, l.Lon, r.name, r.city, r.lat, r.lon)
|
|
}
|
|
}
|
|
|
|
func diffMovements(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
|
|
fmt.Printf("\n---------- 位移 ----------\n")
|
|
rows, err := pg.QueryContext(ctx, `SELECT src_pk, COALESCE(transport_src,0),
|
|
COALESCE(from_visit_src,0), COALESCE(to_visit_src,0), started_at, ended_at
|
|
FROM movements WHERE dataset_id=$1`, dsID)
|
|
must(err)
|
|
type mrow struct {
|
|
t, f, to int
|
|
s, e *time.Time
|
|
}
|
|
db := map[int]mrow{}
|
|
for rows.Next() {
|
|
var pk int
|
|
var r mrow
|
|
var s, e sql.NullTime
|
|
must(rows.Scan(&pk, &r.t, &r.f, &r.to, &s, &e))
|
|
if s.Valid {
|
|
r.s = &s.Time
|
|
}
|
|
if e.Valid {
|
|
r.e = &e.Time
|
|
}
|
|
db[pk] = r
|
|
}
|
|
rows.Close()
|
|
var onlyB, onlyD int
|
|
for _, m := range b.Movements {
|
|
if _, ok := db[m.SrcPK]; !ok {
|
|
onlyB++
|
|
}
|
|
}
|
|
for pk := range db {
|
|
found := false
|
|
for _, m := range b.Movements {
|
|
if m.SrcPK == pk {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
onlyD++
|
|
}
|
|
}
|
|
fmt.Printf("备份 %d 条 / 库内 %d 条；仅备份有 %d，仅库里有 %d\n", len(b.Movements), len(db), onlyB, onlyD)
|
|
}
|
|
|
|
func diffWeather(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
|
|
var n int
|
|
must(pg.QueryRowContext(ctx, `SELECT count(*) FROM weather WHERE dataset_id=$1`, dsID).Scan(&n))
|
|
fmt.Printf("\n---------- 天气 ----------\n备份 %d 条 / 库内 %d 条\n", len(b.Weather), n)
|
|
}
|
|
|
|
func diffActivities(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
|
|
fmt.Printf("\n---------- 活动类型 ----------\n")
|
|
rows, err := pg.QueryContext(ctx, `SELECT src_pk, name FROM activities WHERE dataset_id=$1`, dsID)
|
|
must(err)
|
|
db := map[int]string{}
|
|
for rows.Next() {
|
|
var pk int
|
|
var n string
|
|
must(rows.Scan(&pk, &n))
|
|
db[pk] = n
|
|
}
|
|
rows.Close()
|
|
for _, a := range b.Activities {
|
|
n, ok := db[a.SrcPK]
|
|
if !ok {
|
|
fmt.Printf(" 仅备份 #%d %q\n", a.SrcPK, a.Name)
|
|
} else if n != a.Name {
|
|
fmt.Printf(" 冲突 #%d 备份=%q 库=%q\n", a.SrcPK, a.Name, n)
|
|
}
|
|
}
|
|
for pk, n := range db {
|
|
found := false
|
|
for _, a := range b.Activities {
|
|
if a.SrcPK == pk {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
fmt.Printf(" 仅库内 #%d %q\n", pk, n)
|
|
}
|
|
}
|
|
}
|
|
|
|
func diffTags(ctx context.Context, pg *sql.DB, dsID int64, b *rond.Backup) {
|
|
var n int
|
|
must(pg.QueryRowContext(ctx, `SELECT count(*) FROM tags WHERE dataset_id=$1`, dsID).Scan(&n))
|
|
fmt.Printf("\n---------- 标签 ----------\n备份 %d 条 / 库内 %d 条\n", len(b.Tags), n)
|
|
}
|
|
|
|
func eq(a, b *time.Time) bool {
|
|
if a == nil || b == nil {
|
|
return a == nil && b == nil
|
|
}
|
|
return a.Equal(*b)
|
|
}
|
|
|
|
func pv(p *int) int {
|
|
if p == nil {
|
|
return 0
|
|
}
|
|
return *p
|
|
}
|
|
|
|
func av(p *int) int { return pv(p) }
|
|
|
|
func ps(p *int) string {
|
|
if p == nil {
|
|
return "-"
|
|
}
|
|
return fmt.Sprint(*p)
|
|
}
|
|
|
|
func ft(t *time.Time) string {
|
|
if t == nil {
|
|
return "-"
|
|
}
|
|
return t.Local().Format("01-02 15:04")
|
|
}
|
|
|
|
func ftp(t *time.Time) string { return ft(t) }
|
|
|
|
func near(a, b float64) bool {
|
|
d := a - b
|
|
return d < 1e-9 && d > -1e-9
|
|
}
|
|
|
|
func must(err error) {
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "错误:", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|