From f26537b1e94de7431ef975a367273a72e74e29f4 Mon Sep 17 00:00:00 2001 From: tamakyi Date: Tue, 29 Sep 2026 11:50:06 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AF=BC=E5=87=BA=EF=BC=9A=E6=96=B0=E5=BB=BA?= =?UTF-8?q?=E6=B4=BB=E5=8A=A8=E6=B2=A1=E9=80=89=E9=A2=9C=E8=89=B2=E6=97=B6?= =?UTF-8?q?=E8=A1=A5=E5=8D=A0=E4=BD=8D=E8=89=B2=EF=BC=8C=E5=88=AB=E5=86=8D?= =?UTF-8?q?=E5=86=99=20ZCOLOR=5F=3DNULL=EF=BC=88=E7=9C=9F=E6=9C=BA?= =?UTF-8?q?=E6=8B=92=E6=94=B6=E6=95=B4=E5=8C=85=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 症状：生产导出的包导不回 rond，提示「错误，备份恢复失败，你的当前数据未被更改」。 根因链：站内「地点补全」新建活动时写的是 color=''、icon=''（fillup.go 的 INSERT），而导出侧 ns() 把空串也当空（返回 nil）——图标那侧早就有 📍 兜底（注释还写着「站点自己渲染时缺省用 📍」）， 颜色这侧漏了，于是写出 ZACTIVITY.ZCOLOR_ = NULL。真机这一列在任何包里都是 0 NULL/0 空串 （扫了 data/ 与 storage/ 下全部 15 个包，只有出事的那份有 1 个空），Core Data 拿到非可选属性上的 NULL 会判成损坏数据 → 整包拒收。 修复：加 activityPlaceholderColor = "gray"（与 transportPlaceholderColor 同一套写法）， 活动颜色为空时兜底。 用例：TestExportNeverNullsColumnsTheDeviceAlwaysFills 补上「没选颜色的活动」场景——它守的通用 不变量是「基线里 0 个 NULL 的列，导出新行也不许 NULL」，但此前只造过交通方式，没覆盖活动。 变异验证：去掉兜底 → 两条断言都红（提示语正好复刻生产现象）。 另新增 tools/audit_export_nulls.py：拿参照包做逐列空值画像 + 全部关系列悬空 + Z_MAX 检查， 是排查「导出被拒收」最快的一步（这次就是它一击命中的）。注意判据必须是「参照里 0 个 NULL」—— ZLOCATION.ZISOCOUNTRYCODE 真机自己就漏过 1 个 NULL，说明它容忍 NULL，不要当违规去修。 全量验收 293 PASS / 0 FAIL / 0 SKIP。 --- internal/rond/export.go | 14 +- internal/web/export_placeholder_test.go | 29 +++- tools/audit_export_nulls.py | 170 ++++++++++++++++++++++++ 3 files changed, 209 insertions(+), 4 deletions(-) create mode 100644 tools/audit_export_nulls.py diff --git a/internal/rond/export.go b/internal/rond/export.go index fba923c..3c10726 100644 --- a/internal/rond/export.go +++ b/internal/rond/export.go @@ -86,11 +86,12 @@ var keepNilCols = map[string][]string{ "ZTRANSPORT": {"ZNAME_", "ZCOLOR_", "ZICON_"}, } -// 新建交通方式没选颜色/图标时的占位值。真机这两列从来不空，写 NULL 是它没见过的形状； -// 颜色取 rond 系统色名，图标取一个必定存在的 SF Symbol。 +// 新建行没选颜色/图标时的占位值。真机这几列从来不空（NULL 和空串都没有），写 NULL 是 +// 它没见过的形状；颜色取 rond 系统色名，图标取一个必定存在的 SF Symbol。 const ( transportPlaceholderColor = "gray" transportPlaceholderIcon = "circle.fill" + activityPlaceholderColor = "gray" ) // movementTypeDefault 给站点新建的位移补 ZTYPE_：真机里这条规律覆盖 757/791 行 @@ -308,12 +309,19 @@ func patchSQLite(ctx context.Context, db *sql.DB, datasetID int64, sqlitePath st if icon == nil { icon = "📍" } + // 颜色同样要兜底：ns() 把空串也当空，而「地点补全」新建的活动 color 就是空串 + // （见 fillup.go），不兜底这里会写出 ZCOLOR_=NULL。真机这列从来不空——2026-09-29 + // 生产导出就是因为这一格被 rond 判成损坏数据，提示「备份恢复失败，你的当前数据未被更改」。 + color := ns(a.Color) + if color == nil { + color = activityPlaceholderColor + } // 活动列表里的「最近活跃时间」：同 ZLOCATION，取该活动最晚一次到访 var last any if t, ok := actLast[int64(a.ID)]; ok { last = appleSeconds(t) } - if err := act.write(a.SrcPK, []any{ns(a.Name), ns(a.Color), icon, + if err := act.write(a.SrcPK, []any{ns(a.Name), color, icon, bint(a.Home), bint(a.Work), bint(a.Excluded), bint(a.Archived), last, coreDataUUID(!known["ZACTIVITY"][a.SrcPK], "ZACTIVITY", a.SrcPK)}, raw); err != nil { diff --git a/internal/web/export_placeholder_test.go b/internal/web/export_placeholder_test.go index 270aa6d..78bc52a 100644 --- a/internal/web/export_placeholder_test.go +++ b/internal/web/export_placeholder_test.go @@ -43,6 +43,17 @@ func TestExportNeverNullsColumnsTheDeviceAlwaysFills(t *testing.T) { t.Fatal("新建交通方式没有分配编号") } + // 活动同理：站点「地点补全」新建活动时 color/icon 写的是空串（见 fillup.go 的 INSERT）， + // 而导出侧的 ns() 把空串也当空——图标那侧早就有 📍 兜底，颜色这侧曾经漏了。 + var newAct int + if err := pg.QueryRowContext(ctx, `INSERT INTO activities (dataset_id, src_pk, name, color, icon) + VALUES ($1, 950001, '测试无色活动', '', '') RETURNING src_pk`, dsID).Scan(&newAct); err != nil { + t.Fatalf("插入无色活动失败: %v", err) + } + t.Cleanup(func() { + pg.ExecContext(ctx, `DELETE FROM activities WHERE dataset_id=$1 AND src_pk=950001`, dsID) + }) + out := filepath.Join(t.TempDir(), "export.rondbackup") if err := os.WriteFile(out, f.export(t), 0o644); err != nil { t.Fatal(err) @@ -69,7 +80,23 @@ func TestExportNeverNullsColumnsTheDeviceAlwaysFills(t *testing.T) { } } - // 2) 通用判据：基线里一个空值都没有的列，导出里的新行也不许出现空值 + // 2) 新建的那条「没选颜色」的活动也必须自带可用的颜色与图标 + var actName, actColor, actIcon sql.NullString + if err := edb.QueryRow(`SELECT ZNAME_, ZCOLOR_, ZICON_ FROM ZACTIVITY WHERE Z_PK=?`, newAct). + Scan(&actName, &actColor, &actIcon); err != nil { + t.Fatalf("导出包里找不到新建的活动 #%d: %v", newAct, err) + } + for _, c := range []struct { + col string + v sql.NullString + }{{"ZNAME_", actName}, {"ZCOLOR_", actColor}, {"ZICON_", actIcon}} { + if !c.v.Valid || c.v.String == "" { + t.Errorf("新建活动的 %s 是空的（%q）：真机这列从不留空，"+ + "写 NULL 会被当成损坏的数据而拒收整包", c.col, c.v.String) + } + } + + // 3) 通用判据：基线里一个空值都没有的列，导出里的新行也不许出现空值 for _, tbl := range []string{"ZVISIT", "ZLOCATION", "ZACTIVITY", "ZMOVEMENT", "ZTRANSPORT", "ZTAG"} { var baseMax sql.NullInt64 if err := srcDB.QueryRow(`SELECT MAX(Z_PK) FROM ` + tbl).Scan(&baseMax); err != nil || !baseMax.Valid { diff --git a/tools/audit_export_nulls.py b/tools/audit_export_nulls.py new file mode 100644 index 0000000..df85f19 --- /dev/null +++ b/tools/audit_export_nulls.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""对导出的 .rondbackup 做「能不能被 Core Data 读回」的深度取证。 + +与 tools/verify_export_structure.py 互补：那个比结构与部分不变量，这个专查两件事—— + 1. 逐列空值画像：**参照包里某列一个 NULL 都没有，导出包里却出现 NULL/空值** ⇒ + Core Data 会以「备份恢复失败」拒收（非可选属性上它自己从不产生 NULL）。 + 2. 全部关系列的悬空引用（技能里列的每一列都查）。 +用法：python tools/audit_export_nulls.py <参照.rondbackup> <导出.rondbackup> +""" +import os +import sqlite3 +import sys +import tempfile +import zipfile + +MANAGED = ["ZVISIT", "ZLOCATION", "ZACTIVITY", "ZMOVEMENT", "ZHOURLYWEATHER", "ZTAG", "ZTRANSPORT"] + +# 关系列 -> 目标表（按技能 1.45 节） +RELS = [ + ("ZVISIT", "ZLOCATION", "ZLOCATION"), + ("ZVISIT", "ZACTIVITY_", "ZACTIVITY"), + ("ZVISIT", "ZMERGEDTO", "ZVISIT"), + ("ZVISIT", "ZPARENT", "ZVISIT"), + ("ZVISIT", "ZTRIPSEGMENT_", "ZTRIPSEGMENT"), + ("ZRAWVISIT", "ZVISIT", "ZVISIT"), + ("ZRAWVISIT", "ZLOCATION", "ZLOCATION"), + ("ZRAWVISIT", "ZACTIVITY", "ZACTIVITY"), + ("ZSTAYSEGMENT", "ZLOCATION", "ZLOCATION"), + ("ZSTAYSEGMENT", "ZPARENTVISIT", "ZVISIT"), + ("ZUSERPHOTO", "ZVISIT_", "ZVISIT"), + ("ZVISITPHOTOFAVORITE", "ZVISIT_", "ZVISIT"), + ("ZKEYWORD", "ZACTIVITY_", "ZACTIVITY"), + ("ZPOICATEGORY", "ZACTIVITY_", "ZACTIVITY"), + ("ZMOVEMENT", "ZVISITFROM_", "ZVISIT"), + ("ZMOVEMENT", "ZVISITTO_", "ZVISIT"), + ("ZMOVEMENT", "ZTRANSPORT_", "ZTRANSPORT"), + ("ZHOURLYWEATHER", "ZVISIT", "ZVISIT"), + ("ZTAG", "ZGROUP", "ZTAGGROUP"), + ("Z_11VISITS_", "Z_18VISITS_", "ZVISIT"), + ("Z_11VISITS_", "Z_11TAGS_5", "ZTAG"), + ("Z_6TAGS_", "Z_6LOCATIONS_", "ZLOCATION"), + ("Z_6TAGS_", "Z_11TAGS_2", "ZTAG"), + ("Z_1TAGS_", "Z_1ACTIVITIES_", "ZACTIVITY"), + ("Z_1TAGS_", "Z_11TAGS_", "ZTAG"), +] + + +def extract(path): + d = tempfile.mkdtemp(prefix="audit") + with zipfile.ZipFile(path) as z: + z.extractall(d) + main = [os.path.join(r, f) for r, _, fs in os.walk(d) for f in fs if f.endswith(".sqlite")][0] + con = sqlite3.connect(main) + con.execute("PRAGMA wal_checkpoint(TRUNCATE)") + con.commit() + return con, d + + +def cols(con, table): + return [(r[1], r[2]) for r in con.execute('PRAGMA table_info("%s")' % table)] + + +def blob_as_zeros(con, table, col): + """BLOB 列里全 0x00 也算「空」——Core Data 的 UUID 不会是全 0。""" + n = 0 + for (v,) in con.execute('SELECT "%s" FROM "%s" WHERE "%s" IS NOT NULL' % (col, table, col)): + if isinstance(v, bytes) and v and set(v) == {0}: + n += 1 + return n + + +def profile(con, ref): + """列 -> (NULL 数, 空串数, 全零 BLOB 数)；ref=True 时额外统计行数基线""" + out = {} + for t in [r[0] for r in con.execute("SELECT name FROM sqlite_master WHERE type='table'")]: + for c, ctype in cols(con, t): + q = 'SELECT count(*), sum(CASE WHEN "%s" IS NULL THEN 1 ELSE 0 END), sum(CASE WHEN "%s" = \'\' THEN 1 ELSE 0 END) FROM "%s"' % (c, c, t) + try: + total, nulls, empties = con.execute(q).fetchone() + except sqlite3.Error: + continue + nulls = nulls or 0 + empties = empties or 0 + z = blob_as_zeros(con, t, c) if "BLOB" in (ctype or "").upper() else 0 + out[(t, c)] = (total, nulls, empties, z) + return out + + +def main(): + ref_path, out_path = sys.argv[1], sys.argv[2] + ref, d1 = extract(ref_path) + out, d2 = extract(out_path) + + print("=" * 78) + print("A. 逐列空值画像（参照 = %s）" % os.path.basename(ref_path)) + print("=" * 78) + pr = profile(ref, True) + po = profile(out, False) + bad = 0 + for (t, c), (total_r, nulls_r, emp_r, zero_r) in sorted(pr.items()): + if (t, c) not in po: + print(" !! 导出缺列 %s.%s" % (t, c)) + bad += 1 + continue + total_o, nulls_o, emp_o, zero_o = po[(t, c)] + if nulls_r == 0 and nulls_o > 0: + print(" !! %s.%s：参照里 0 个 NULL，导出里 %d 个 NULL（共 %d 行）" % (t, c, nulls_o, total_o)) + bad += 1 + if emp_r == 0 and emp_o > 0 and nulls_r == 0: + print(" ~ %s.%s：参照里 0 个空串，导出里 %d 个空串" % (t, c, emp_o)) + if zero_r == 0 and zero_o > 0: + print(" !! %s.%s：参照里没有全零 BLOB，导出里 %d 个" % (t, c, zero_o)) + bad += 1 + if bad == 0: + print(" 未发现「真机不留空、导出留空」的列 ✓") + + print() + print("=" * 78) + print("B. 关系完整性（悬空引用）") + print("=" * 78) + dangling = 0 + for tbl, col, target in RELS: + if tbl not in [r[0] for r in out.execute("SELECT name FROM sqlite_master WHERE type='table'")]: + continue + if col not in [c for c, _ in cols(out, tbl)]: + continue + q = ('SELECT count(*) FROM "%s" a WHERE a."%s" IS NOT NULL AND a."%s" <> 0 ' + 'AND NOT EXISTS (SELECT 1 FROM "%s" b WHERE b.Z_PK = a."%s")') % (tbl, col, col, target, col) + try: + n = out.execute(q).fetchone()[0] + except sqlite3.Error as e: + print(" 跳过 %s.%s（%s）" % (tbl, col, e)) + continue + if n: + print(" !! %s.%s -> %s：%d 条悬空" % (tbl, col, target, n)) + for row in out.execute( + 'SELECT a.Z_PK, a."%s" FROM "%s" a WHERE a."%s" IS NOT NULL AND a."%s" <> 0 ' + 'AND NOT EXISTS (SELECT 1 FROM "%s" b WHERE b.Z_PK = a."%s") LIMIT 5' + % (col, tbl, col, col, target, col)): + print(" %s #%s -> %s" % (tbl, row[0], row[1])) + dangling += n + if dangling == 0: + print(" 所有关系列悬空引用 0 ✓") + + print() + print("=" * 78) + print("C. Z_PRIMARYKEY.Z_MAX 是否够大（不够就会让 rond 下次插入撞键）") + print("=" * 78) + ents = {r[1]: (r[0], r[2]) for r in out.execute("SELECT Z_ENT, Z_NAME, Z_MAX FROM Z_PRIMARYKEY")} + badmax = 0 + for name, (ent, mx) in sorted(ents.items()): + if not name or not out.execute("SELECT count(*) FROM sqlite_master WHERE type='table' AND name=?", ("Z" + name,)).fetchone()[0]: + continue + tbl = "Z" + name + try: + real = out.execute('SELECT COALESCE(MAX(Z_PK),0) FROM "%s"' % tbl).fetchone()[0] + except sqlite3.Error: + continue + if real > (mx or 0): + print(" !! %s：Z_MAX=%s 但最大 Z_PK=%s" % (name, mx, real)) + badmax += 1 + if badmax == 0: + print(" Z_MAX 全部 ≥ 实际最大 Z_PK ✓") + + print() + print("结论：空值画像问题 %d 项，悬空引用 %d 条，Z_MAX 问题 %d 项" % (bad, dangling, badmax)) + + +if __name__ == "__main__": + main()