导出：新建活动没选颜色时补占位色，别再写 ZCOLOR_=NULL（真机拒收整包）
症状：生产导出的包导不回 rond，提示「错误，备份恢复失败，你的当前数据未被更改」。 根因链：站内「地点补全」新建活动时写的是 color=''、icon=''（fillup.go 的 INSERT），而导出侧 ns() 把空串也当空（返回 nil）——图标那侧早就有 📍 兜底（注释还写着「站点自己渲染时缺省用 📍」）， 颜色这侧漏了，于是写出 ZACTIVITY.ZCOLOR_ = NULL。真机这一列在任何包里都是 0 NULL/0 空串 （扫了 data/ 与 storage/ 下全部 15 个包，只有出事的那份有 1 个空），Core Data 拿到非可选属性上的 NULL 会判成损坏数据 → 整包拒收。 修复：加 activityPlaceholderColor = "gray"（与 transportPlaceholderColor 同一套写法）， 活动颜色为空时兜底。 用例：TestExportNeverNullsColumnsTheDeviceAlwaysFills 补上「没选颜色的活动」场景——它守的通用 不变量是「基线里 0 个 NULL 的列，导出新行也不许 NULL」，但此前只造过交通方式，没覆盖活动。 变异验证：去掉兜底 → 两条断言都红（提示语正好复刻生产现象）。 另新增 tools/audit_export_nulls.py：拿参照包做逐列空值画像 + 全部关系列悬空 + Z_MAX 检查， 是排查「导出被拒收」最快的一步（这次就是它一击命中的）。注意判据必须是「参照里 0 个 NULL」—— ZLOCATION.ZISOCOUNTRYCODE 真机自己就漏过 1 个 NULL，说明它容忍 NULL，不要当违规去修。 全量验收 293 PASS / 0 FAIL / 0 SKIP。
This commit is contained in:
tamakyi
+11
-3
@@ -86,11 +86,12 @@ var keepNilCols = map[string][]string{
|
|||||||
"ZTRANSPORT": {"ZNAME_", "ZCOLOR_", "ZICON_"},
|
"ZTRANSPORT": {"ZNAME_", "ZCOLOR_", "ZICON_"},
|
||||||
}
|
}
|
||||||
|
|
||||||
// 新建交通方式没选颜色/图标时的占位值。真机这两列从来不空，写 NULL 是它没见过的形状；
|
// 新建行没选颜色/图标时的占位值。真机这几列从来不空（NULL 和空串都没有），写 NULL 是
|
||||||
// 颜色取 rond 系统色名，图标取一个必定存在的 SF Symbol。
|
// 它没见过的形状；颜色取 rond 系统色名，图标取一个必定存在的 SF Symbol。
|
||||||
const (
|
const (
|
||||||
transportPlaceholderColor = "gray"
|
transportPlaceholderColor = "gray"
|
||||||
transportPlaceholderIcon = "circle.fill"
|
transportPlaceholderIcon = "circle.fill"
|
||||||
|
activityPlaceholderColor = "gray"
|
||||||
)
|
)
|
||||||
|
|
||||||
// movementTypeDefault 给站点新建的位移补 ZTYPE_：真机里这条规律覆盖 757/791 行
|
// movementTypeDefault 给站点新建的位移补 ZTYPE_：真机里这条规律覆盖 757/791 行
|
||||||
@@ -308,12 +309,19 @@ func patchSQLite(ctx context.Context, db *sql.DB, datasetID int64, sqlitePath st
|
|||||||
if icon == nil {
|
if icon == nil {
|
||||||
icon = "📍"
|
icon = "📍"
|
||||||
}
|
}
|
||||||
|
// 颜色同样要兜底：ns() 把空串也当空，而「地点补全」新建的活动 color 就是空串
|
||||||
|
// （见 fillup.go），不兜底这里会写出 ZCOLOR_=NULL。真机这列从来不空——2026-09-29
|
||||||
|
// 生产导出就是因为这一格被 rond 判成损坏数据，提示「备份恢复失败，你的当前数据未被更改」。
|
||||||
|
color := ns(a.Color)
|
||||||
|
if color == nil {
|
||||||
|
color = activityPlaceholderColor
|
||||||
|
}
|
||||||
// 活动列表里的「最近活跃时间」：同 ZLOCATION，取该活动最晚一次到访
|
// 活动列表里的「最近活跃时间」：同 ZLOCATION，取该活动最晚一次到访
|
||||||
var last any
|
var last any
|
||||||
if t, ok := actLast[int64(a.ID)]; ok {
|
if t, ok := actLast[int64(a.ID)]; ok {
|
||||||
last = appleSeconds(t)
|
last = appleSeconds(t)
|
||||||
}
|
}
|
||||||
if err := act.write(a.SrcPK, []any{ns(a.Name), ns(a.Color), icon,
|
if err := act.write(a.SrcPK, []any{ns(a.Name), color, icon,
|
||||||
bint(a.Home), bint(a.Work), bint(a.Excluded), bint(a.Archived), last,
|
bint(a.Home), bint(a.Work), bint(a.Excluded), bint(a.Archived), last,
|
||||||
coreDataUUID(!known["ZACTIVITY"][a.SrcPK], "ZACTIVITY", a.SrcPK)},
|
coreDataUUID(!known["ZACTIVITY"][a.SrcPK], "ZACTIVITY", a.SrcPK)},
|
||||||
raw); err != nil {
|
raw); err != nil {
|
||||||
|
|||||||
@@ -43,6 +43,17 @@ func TestExportNeverNullsColumnsTheDeviceAlwaysFills(t *testing.T) {
|
|||||||
t.Fatal("新建交通方式没有分配编号")
|
t.Fatal("新建交通方式没有分配编号")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 活动同理：站点「地点补全」新建活动时 color/icon 写的是空串（见 fillup.go 的 INSERT），
|
||||||
|
// 而导出侧的 ns() 把空串也当空——图标那侧早就有 📍 兜底，颜色这侧曾经漏了。
|
||||||
|
var newAct int
|
||||||
|
if err := pg.QueryRowContext(ctx, `INSERT INTO activities (dataset_id, src_pk, name, color, icon)
|
||||||
|
VALUES ($1, 950001, '测试无色活动', '', '') RETURNING src_pk`, dsID).Scan(&newAct); err != nil {
|
||||||
|
t.Fatalf("插入无色活动失败: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
pg.ExecContext(ctx, `DELETE FROM activities WHERE dataset_id=$1 AND src_pk=950001`, dsID)
|
||||||
|
})
|
||||||
|
|
||||||
out := filepath.Join(t.TempDir(), "export.rondbackup")
|
out := filepath.Join(t.TempDir(), "export.rondbackup")
|
||||||
if err := os.WriteFile(out, f.export(t), 0o644); err != nil {
|
if err := os.WriteFile(out, f.export(t), 0o644); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -69,7 +80,23 @@ func TestExportNeverNullsColumnsTheDeviceAlwaysFills(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2) 通用判据：基线里一个空值都没有的列，导出里的新行也不许出现空值
|
// 2) 新建的那条「没选颜色」的活动也必须自带可用的颜色与图标
|
||||||
|
var actName, actColor, actIcon sql.NullString
|
||||||
|
if err := edb.QueryRow(`SELECT ZNAME_, ZCOLOR_, ZICON_ FROM ZACTIVITY WHERE Z_PK=?`, newAct).
|
||||||
|
Scan(&actName, &actColor, &actIcon); err != nil {
|
||||||
|
t.Fatalf("导出包里找不到新建的活动 #%d: %v", newAct, err)
|
||||||
|
}
|
||||||
|
for _, c := range []struct {
|
||||||
|
col string
|
||||||
|
v sql.NullString
|
||||||
|
}{{"ZNAME_", actName}, {"ZCOLOR_", actColor}, {"ZICON_", actIcon}} {
|
||||||
|
if !c.v.Valid || c.v.String == "" {
|
||||||
|
t.Errorf("新建活动的 %s 是空的（%q）：真机这列从不留空，"+
|
||||||
|
"写 NULL 会被当成损坏的数据而拒收整包", c.col, c.v.String)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3) 通用判据：基线里一个空值都没有的列，导出里的新行也不许出现空值
|
||||||
for _, tbl := range []string{"ZVISIT", "ZLOCATION", "ZACTIVITY", "ZMOVEMENT", "ZTRANSPORT", "ZTAG"} {
|
for _, tbl := range []string{"ZVISIT", "ZLOCATION", "ZACTIVITY", "ZMOVEMENT", "ZTRANSPORT", "ZTAG"} {
|
||||||
var baseMax sql.NullInt64
|
var baseMax sql.NullInt64
|
||||||
if err := srcDB.QueryRow(`SELECT MAX(Z_PK) FROM ` + tbl).Scan(&baseMax); err != nil || !baseMax.Valid {
|
if err := srcDB.QueryRow(`SELECT MAX(Z_PK) FROM ` + tbl).Scan(&baseMax); err != nil || !baseMax.Valid {
|
||||||
|
|||||||
@@ -0,0 +1,170 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""对导出的 .rondbackup 做「能不能被 Core Data 读回」的深度取证。
|
||||||
|
|
||||||
|
与 tools/verify_export_structure.py 互补：那个比结构与部分不变量，这个专查两件事——
|
||||||
|
1. 逐列空值画像：**参照包里某列一个 NULL 都没有，导出包里却出现 NULL/空值** ⇒
|
||||||
|
Core Data 会以「备份恢复失败」拒收（非可选属性上它自己从不产生 NULL）。
|
||||||
|
2. 全部关系列的悬空引用（技能里列的每一列都查）。
|
||||||
|
用法：python tools/audit_export_nulls.py <参照.rondbackup> <导出.rondbackup>
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
MANAGED = ["ZVISIT", "ZLOCATION", "ZACTIVITY", "ZMOVEMENT", "ZHOURLYWEATHER", "ZTAG", "ZTRANSPORT"]
|
||||||
|
|
||||||
|
# 关系列 -> 目标表（按技能 1.45 节）
|
||||||
|
RELS = [
|
||||||
|
("ZVISIT", "ZLOCATION", "ZLOCATION"),
|
||||||
|
("ZVISIT", "ZACTIVITY_", "ZACTIVITY"),
|
||||||
|
("ZVISIT", "ZMERGEDTO", "ZVISIT"),
|
||||||
|
("ZVISIT", "ZPARENT", "ZVISIT"),
|
||||||
|
("ZVISIT", "ZTRIPSEGMENT_", "ZTRIPSEGMENT"),
|
||||||
|
("ZRAWVISIT", "ZVISIT", "ZVISIT"),
|
||||||
|
("ZRAWVISIT", "ZLOCATION", "ZLOCATION"),
|
||||||
|
("ZRAWVISIT", "ZACTIVITY", "ZACTIVITY"),
|
||||||
|
("ZSTAYSEGMENT", "ZLOCATION", "ZLOCATION"),
|
||||||
|
("ZSTAYSEGMENT", "ZPARENTVISIT", "ZVISIT"),
|
||||||
|
("ZUSERPHOTO", "ZVISIT_", "ZVISIT"),
|
||||||
|
("ZVISITPHOTOFAVORITE", "ZVISIT_", "ZVISIT"),
|
||||||
|
("ZKEYWORD", "ZACTIVITY_", "ZACTIVITY"),
|
||||||
|
("ZPOICATEGORY", "ZACTIVITY_", "ZACTIVITY"),
|
||||||
|
("ZMOVEMENT", "ZVISITFROM_", "ZVISIT"),
|
||||||
|
("ZMOVEMENT", "ZVISITTO_", "ZVISIT"),
|
||||||
|
("ZMOVEMENT", "ZTRANSPORT_", "ZTRANSPORT"),
|
||||||
|
("ZHOURLYWEATHER", "ZVISIT", "ZVISIT"),
|
||||||
|
("ZTAG", "ZGROUP", "ZTAGGROUP"),
|
||||||
|
("Z_11VISITS_", "Z_18VISITS_", "ZVISIT"),
|
||||||
|
("Z_11VISITS_", "Z_11TAGS_5", "ZTAG"),
|
||||||
|
("Z_6TAGS_", "Z_6LOCATIONS_", "ZLOCATION"),
|
||||||
|
("Z_6TAGS_", "Z_11TAGS_2", "ZTAG"),
|
||||||
|
("Z_1TAGS_", "Z_1ACTIVITIES_", "ZACTIVITY"),
|
||||||
|
("Z_1TAGS_", "Z_11TAGS_", "ZTAG"),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def extract(path):
|
||||||
|
d = tempfile.mkdtemp(prefix="audit")
|
||||||
|
with zipfile.ZipFile(path) as z:
|
||||||
|
z.extractall(d)
|
||||||
|
main = [os.path.join(r, f) for r, _, fs in os.walk(d) for f in fs if f.endswith(".sqlite")][0]
|
||||||
|
con = sqlite3.connect(main)
|
||||||
|
con.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
||||||
|
con.commit()
|
||||||
|
return con, d
|
||||||
|
|
||||||
|
|
||||||
|
def cols(con, table):
|
||||||
|
return [(r[1], r[2]) for r in con.execute('PRAGMA table_info("%s")' % table)]
|
||||||
|
|
||||||
|
|
||||||
|
def blob_as_zeros(con, table, col):
|
||||||
|
"""BLOB 列里全 0x00 也算「空」——Core Data 的 UUID 不会是全 0。"""
|
||||||
|
n = 0
|
||||||
|
for (v,) in con.execute('SELECT "%s" FROM "%s" WHERE "%s" IS NOT NULL' % (col, table, col)):
|
||||||
|
if isinstance(v, bytes) and v and set(v) == {0}:
|
||||||
|
n += 1
|
||||||
|
return n
|
||||||
|
|
||||||
|
|
||||||
|
def profile(con, ref):
|
||||||
|
"""列 -> (NULL 数, 空串数, 全零 BLOB 数)；ref=True 时额外统计行数基线"""
|
||||||
|
out = {}
|
||||||
|
for t in [r[0] for r in con.execute("SELECT name FROM sqlite_master WHERE type='table'")]:
|
||||||
|
for c, ctype in cols(con, t):
|
||||||
|
q = 'SELECT count(*), sum(CASE WHEN "%s" IS NULL THEN 1 ELSE 0 END), sum(CASE WHEN "%s" = \'\' THEN 1 ELSE 0 END) FROM "%s"' % (c, c, t)
|
||||||
|
try:
|
||||||
|
total, nulls, empties = con.execute(q).fetchone()
|
||||||
|
except sqlite3.Error:
|
||||||
|
continue
|
||||||
|
nulls = nulls or 0
|
||||||
|
empties = empties or 0
|
||||||
|
z = blob_as_zeros(con, t, c) if "BLOB" in (ctype or "").upper() else 0
|
||||||
|
out[(t, c)] = (total, nulls, empties, z)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ref_path, out_path = sys.argv[1], sys.argv[2]
|
||||||
|
ref, d1 = extract(ref_path)
|
||||||
|
out, d2 = extract(out_path)
|
||||||
|
|
||||||
|
print("=" * 78)
|
||||||
|
print("A. 逐列空值画像（参照 = %s）" % os.path.basename(ref_path))
|
||||||
|
print("=" * 78)
|
||||||
|
pr = profile(ref, True)
|
||||||
|
po = profile(out, False)
|
||||||
|
bad = 0
|
||||||
|
for (t, c), (total_r, nulls_r, emp_r, zero_r) in sorted(pr.items()):
|
||||||
|
if (t, c) not in po:
|
||||||
|
print(" !! 导出缺列 %s.%s" % (t, c))
|
||||||
|
bad += 1
|
||||||
|
continue
|
||||||
|
total_o, nulls_o, emp_o, zero_o = po[(t, c)]
|
||||||
|
if nulls_r == 0 and nulls_o > 0:
|
||||||
|
print(" !! %s.%s：参照里 0 个 NULL，导出里 %d 个 NULL（共 %d 行）" % (t, c, nulls_o, total_o))
|
||||||
|
bad += 1
|
||||||
|
if emp_r == 0 and emp_o > 0 and nulls_r == 0:
|
||||||
|
print(" ~ %s.%s：参照里 0 个空串，导出里 %d 个空串" % (t, c, emp_o))
|
||||||
|
if zero_r == 0 and zero_o > 0:
|
||||||
|
print(" !! %s.%s：参照里没有全零 BLOB，导出里 %d 个" % (t, c, zero_o))
|
||||||
|
bad += 1
|
||||||
|
if bad == 0:
|
||||||
|
print(" 未发现「真机不留空、导出留空」的列 ✓")
|
||||||
|
|
||||||
|
print()
|
||||||
|
print("=" * 78)
|
||||||
|
print("B. 关系完整性（悬空引用）")
|
||||||
|
print("=" * 78)
|
||||||
|
dangling = 0
|
||||||
|
for tbl, col, target in RELS:
|
||||||
|
if tbl not in [r[0] for r in out.execute("SELECT name FROM sqlite_master WHERE type='table'")]:
|
||||||
|
continue
|
||||||
|
if col not in [c for c, _ in cols(out, tbl)]:
|
||||||
|
continue
|
||||||
|
q = ('SELECT count(*) FROM "%s" a WHERE a."%s" IS NOT NULL AND a."%s" <> 0 '
|
||||||
|
'AND NOT EXISTS (SELECT 1 FROM "%s" b WHERE b.Z_PK = a."%s")') % (tbl, col, col, target, col)
|
||||||
|
try:
|
||||||
|
n = out.execute(q).fetchone()[0]
|
||||||
|
except sqlite3.Error as e:
|
||||||
|
print(" 跳过 %s.%s（%s）" % (tbl, col, e))
|
||||||
|
continue
|
||||||
|
if n:
|
||||||
|
print(" !! %s.%s -> %s：%d 条悬空" % (tbl, col, target, n))
|
||||||
|
for row in out.execute(
|
||||||
|
'SELECT a.Z_PK, a."%s" FROM "%s" a WHERE a."%s" IS NOT NULL AND a."%s" <> 0 '
|
||||||
|
'AND NOT EXISTS (SELECT 1 FROM "%s" b WHERE b.Z_PK = a."%s") LIMIT 5'
|
||||||
|
% (col, tbl, col, col, target, col)):
|
||||||
|
print(" %s #%s -> %s" % (tbl, row[0], row[1]))
|
||||||
|
dangling += n
|
||||||
|
if dangling == 0:
|
||||||
|
print(" 所有关系列悬空引用 0 ✓")
|
||||||
|
|
||||||
|
print()
|
||||||
|
print("=" * 78)
|
||||||
|
print("C. Z_PRIMARYKEY.Z_MAX 是否够大（不够就会让 rond 下次插入撞键）")
|
||||||
|
print("=" * 78)
|
||||||
|
ents = {r[1]: (r[0], r[2]) for r in out.execute("SELECT Z_ENT, Z_NAME, Z_MAX FROM Z_PRIMARYKEY")}
|
||||||
|
badmax = 0
|
||||||
|
for name, (ent, mx) in sorted(ents.items()):
|
||||||
|
if not name or not out.execute("SELECT count(*) FROM sqlite_master WHERE type='table' AND name=?", ("Z" + name,)).fetchone()[0]:
|
||||||
|
continue
|
||||||
|
tbl = "Z" + name
|
||||||
|
try:
|
||||||
|
real = out.execute('SELECT COALESCE(MAX(Z_PK),0) FROM "%s"' % tbl).fetchone()[0]
|
||||||
|
except sqlite3.Error:
|
||||||
|
continue
|
||||||
|
if real > (mx or 0):
|
||||||
|
print(" !! %s：Z_MAX=%s 但最大 Z_PK=%s" % (name, mx, real))
|
||||||
|
badmax += 1
|
||||||
|
if badmax == 0:
|
||||||
|
print(" Z_MAX 全部 ≥ 实际最大 Z_PK ✓")
|
||||||
|
|
||||||
|
print()
|
||||||
|
print("结论：空值画像问题 %d 项，悬空引用 %d 条，Z_MAX 问题 %d 项" % (bad, dangling, badmax))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user