工具：tools/bisect_export.py —— 把被真机拒收的导出包做行级二分
生产导出被拒（「备份恢复失败，你的当前数据未被修改」），但同一台设备导入「纯净往返包」成功 ⇒ 导出链路与 App 都没问题，问题在数据集的数据里。静态把技能里记的全部判据都查了一遍 （打包层位标志、Core Data 骨架逐字节、逐列空值画像、关系完整性、类型分布、Z_OPT/UUID/BLOB、 行级时间不变量、取值域与数值范围、SQLite 文件头）——一处违规都没有，只能靠真机二分。 脚本以被拒的包为底生成三个对照包（都清了悬空引用、按 zip.Store 带长度与 CRC 重新打包）： V1 去掉站内新建的到访；V2 = V1 + 去掉站内新建的地点；V3 手机里也有的行按手机原值重置。 README 与记忆里记了怎么读这三个结果（哪一版开始能进，问题就在上一版删/改掉的那一类里）。
This commit is contained in:
tamakyi
@@ -0,0 +1,188 @@
|
||||
#!/usr/bin/env python3
|
||||
"""从「被真机拒收的导出包」里做行级二分，生成对照包，好把范围一刀刀切小。
|
||||
|
||||
背景（2026-09-30）：生产导出被真机拒收（「备份恢复失败，你的当前数据未被修改」），但
|
||||
同一台设备导入「纯净往返包」（手机原包导入后立刻导出）成功 ⇒ 导出链路没问题，问题在数据集的数据里。
|
||||
|
||||
做法：拿被拒的包当底，按「这一行是不是手机包里就有的」逐类删掉站内新建的行，
|
||||
每个变体都清理悬空引用、重新打包（zip.Store，本地头带长度与 CRC）。
|
||||
用户逐个导入，哪一版开始能进，问题就出在上一版删掉的那一类里。
|
||||
|
||||
用法：
|
||||
python tools/bisect_export.py --export data/20260930.rondbackup --phone data/20260929v2.rondbackup \
|
||||
--out-dir data/bisect
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import zipfile
|
||||
|
||||
|
||||
def open_pkg(path, tag):
|
||||
d = tempfile.mkdtemp(prefix=tag)
|
||||
with zipfile.ZipFile(path) as z:
|
||||
z.extractall(d)
|
||||
main = [os.path.join(r, f) for r, _, fs in os.walk(d) for f in fs if f.endswith('.sqlite')][0]
|
||||
con = sqlite3.connect(main)
|
||||
con.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
||||
con.commit()
|
||||
return con, main
|
||||
|
||||
|
||||
def coord_key(lat, lon):
|
||||
return "%.6f|%.6f" % (round(lat or 0, 6), round(lon or 0, 6))
|
||||
|
||||
|
||||
def phone_identities(con):
|
||||
uuids = {bytes(u).hex() for (u,) in
|
||||
con.execute("SELECT ZIDENTIFIER_ FROM ZVISIT WHERE ZIDENTIFIER_ IS NOT NULL")}
|
||||
places = {coord_key(lat, lon) for (lat, lon)
|
||||
in con.execute("SELECT ZLATITUDE, ZLONGITUDE FROM ZLOCATION")}
|
||||
acts = {n for (n,) in con.execute("SELECT ZNAME_ FROM ZACTIVITY WHERE ZNAME_ IS NOT NULL")}
|
||||
trans = {n for (n,) in con.execute("SELECT ZNAME_ FROM ZTRANSPORT WHERE ZNAME_ IS NOT NULL")}
|
||||
return uuids, places, acts, trans
|
||||
|
||||
|
||||
def delete_rows(con, sql, args=()):
|
||||
cur = con.execute(sql, args)
|
||||
n = cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
|
||||
con.commit()
|
||||
return n
|
||||
|
||||
|
||||
def cleanup_dangling(con):
|
||||
"""按关系删掉指向已不存在目标的行 / 把自引用置空（Core Data 的表没有外键约束，只能手查）。"""
|
||||
fixed = {}
|
||||
fixed['ZRAWVISIT(ZVISIT)'] = delete_rows(con, """DELETE FROM ZRAWVISIT WHERE ZVISIT IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM ZVISIT v WHERE v.Z_PK=ZRAWVISIT.ZVISIT)""")
|
||||
fixed['ZRAWVISIT(ZLOCATION)'] = delete_rows(con, """DELETE FROM ZRAWVISIT WHERE ZLOCATION IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM ZLOCATION l WHERE l.Z_PK=ZRAWVISIT.ZLOCATION)""")
|
||||
fixed['ZRAWVISIT(ZACTIVITY)'] = delete_rows(con, """DELETE FROM ZRAWVISIT WHERE ZACTIVITY IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM ZACTIVITY a WHERE a.Z_PK=ZRAWVISIT.ZACTIVITY)""")
|
||||
fixed['ZMOVEMENT(端点到访)'] = delete_rows(con, """DELETE FROM ZMOVEMENT WHERE
|
||||
(ZVISITFROM_ IS NOT NULL AND NOT EXISTS (SELECT 1 FROM ZVISIT v WHERE v.Z_PK=ZMOVEMENT.ZVISITFROM_))
|
||||
OR (ZVISITTO_ IS NOT NULL AND NOT EXISTS (SELECT 1 FROM ZVISIT v WHERE v.Z_PK=ZMOVEMENT.ZVISITTO_))""")
|
||||
fixed['ZHOURLYWEATHER(到访)'] = delete_rows(con, """DELETE FROM ZHOURLYWEATHER WHERE ZVISIT IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM ZVISIT v WHERE v.Z_PK=ZHOURLYWEATHER.ZVISIT)""")
|
||||
# 到访自引用：目标没了就置空（别留悬空）
|
||||
for col in ('ZMERGEDTO', 'ZPARENT'):
|
||||
fixed['ZVISIT.%s' % col] = delete_rows(con, f"""UPDATE ZVISIT SET {col}=NULL WHERE {col} IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM ZVISIT v WHERE v.Z_PK=ZVISIT.{col})""")
|
||||
return {k: v for k, v in fixed.items() if v}
|
||||
|
||||
|
||||
def _revert(con, phone_con, table, key_col, keyfn=lambda *a: a[0]):
|
||||
"""把 table 里「手机包里也有」的行按手机的值重置（保留 Z_PK）。返回重置的行数。
|
||||
|
||||
key_col 为 None 时按坐标键对齐（地点的身份就是坐标，与合并/差异同一套规则）。
|
||||
"""
|
||||
cols = [r[1] for r in phone_con.execute(f'PRAGMA table_info("{table}")') if r[1] != 'Z_PK']
|
||||
sel = [key_col] + cols if key_col else ['ZLATITUDE', 'ZLONGITUDE'] + cols
|
||||
mapping = {}
|
||||
for row in phone_con.execute('SELECT ' + ','.join(f'"{c}"' for c in sel) + f' FROM "{table}"'):
|
||||
k = keyfn(row[0]) if key_col else keyfn(row[0], row[1])
|
||||
if k is None:
|
||||
continue
|
||||
mapping[k] = dict(zip(cols, row[len(sel) - len(cols):]))
|
||||
n = 0
|
||||
for pk, *vals in con.execute('SELECT Z_PK,' + ','.join(f'"{c}"' for c in cols) + f' FROM "{table}"'):
|
||||
v = dict(zip(cols, vals))
|
||||
k = keyfn(v[key_col]) if key_col else keyfn(v['ZLATITUDE'], v['ZLONGITUDE'])
|
||||
if k in mapping:
|
||||
con.execute(f'UPDATE "{table}" SET ' + ', '.join(f'"{c}"=?' for c in cols) + ' WHERE Z_PK=?',
|
||||
[mapping[k][c] for c in cols] + [pk])
|
||||
n += 1
|
||||
con.commit()
|
||||
return n
|
||||
|
||||
|
||||
def pack(con, src_sqlite, dst):
|
||||
con.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
||||
con.commit()
|
||||
con.close()
|
||||
d = os.path.dirname(src_sqlite)
|
||||
empty = os.path.join(d, '_empty')
|
||||
open(empty, 'wb').close()
|
||||
with zipfile.ZipFile(dst, 'w', zipfile.ZIP_STORED) as z:
|
||||
z.write(src_sqlite, 'LifeEasy.sqlite')
|
||||
z.write(empty, 'LifeEasy.sqlite-wal')
|
||||
z.write(empty, 'LifeEasy.sqlite-shm')
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--export', required=True)
|
||||
ap.add_argument('--phone', required=True)
|
||||
ap.add_argument('--out-dir', default='data/bisect')
|
||||
args = ap.parse_args()
|
||||
os.makedirs(args.out_dir, exist_ok=True)
|
||||
|
||||
pcon, _ = open_pkg(args.phone, 'ph-')
|
||||
uuids, place_keys, acts, trans = phone_identities(pcon)
|
||||
pcon.close()
|
||||
print(f"手机包身份：到访 UUID {len(uuids)}、地点坐标键 {len(place_keys)}、活动 {len(acts)}、交通方式 {len(trans)}")
|
||||
|
||||
def build(tag, drop_site_visits, drop_site_places, revert_values=False):
|
||||
con, main_db = open_pkg(args.export, 'ex-')
|
||||
out = [f"== 变体 {tag}（{'去掉站内新建的到访' if drop_site_visits else '保留全部到访'}"
|
||||
f"{' + 站内新建的地点' if drop_site_places else ''}"
|
||||
f"{'；手机里也有的行按手机原值重置' if revert_values else ''}）"]
|
||||
if revert_values:
|
||||
# 把「手机包里也有」的到访 / 地点 / 活动按手机的值原样重置，站内新建的行留着。
|
||||
# 重置后若某行的 ZLOCATION / ZACTIVITY_ 指向的目标在本包里不存在（手机有、我们删过），
|
||||
# 就还原成我们原来的值——否则会造出新的悬空引用，变体本身就不合格了。
|
||||
keep = {pk: (loc, act) for pk, loc, act in
|
||||
con.execute("SELECT Z_PK, ZLOCATION, ZACTIVITY_ FROM ZVISIT")}
|
||||
locs = {pk for (pk,) in con.execute("SELECT Z_PK FROM ZLOCATION")}
|
||||
acts = {pk for (pk,) in con.execute("SELECT Z_PK FROM ZACTIVITY")}
|
||||
pcon2, _ = open_pkg(args.phone, 'ph2-')
|
||||
out.append(f" 重置到访 {_revert(con, pcon2, 'ZVISIT', 'ZIDENTIFIER_',
|
||||
lambda v: bytes(v).hex() if v is not None else None)} 行")
|
||||
out.append(f" 重置地点 {_revert(con, pcon2, 'ZLOCATION', None,
|
||||
lambda lat, lon: coord_key(lat, lon))} 行")
|
||||
out.append(f" 重置活动 {_revert(con, pcon2, 'ZACTIVITY', 'ZNAME_')} 行")
|
||||
pcon2.close()
|
||||
back = 0
|
||||
for pk, loc, act in con.execute("SELECT Z_PK,ZLOCATION,ZACTIVITY_ FROM ZVISIT"):
|
||||
if pk not in keep:
|
||||
continue
|
||||
if (loc is not None and loc not in locs) or (act is not None and act not in acts):
|
||||
con.execute("UPDATE ZVISIT SET ZLOCATION=?, ZACTIVITY_=? WHERE Z_PK=?",
|
||||
(keep[pk][0], keep[pk][1], pk))
|
||||
back += 1
|
||||
con.commit()
|
||||
out.append(f" 引用解不开、还原成我们的值：{back} 行")
|
||||
if drop_site_visits:
|
||||
site = [pk for pk, u in con.execute(
|
||||
"SELECT Z_PK, ZIDENTIFIER_ FROM ZVISIT")
|
||||
if u is not None and bytes(u).hex() not in uuids]
|
||||
if site:
|
||||
q = ','.join(str(p) for p in site)
|
||||
out.append(f" 删站内新建到访 {delete_rows(con, f'DELETE FROM ZVISIT WHERE Z_PK IN ({q})')} 条")
|
||||
if drop_site_places:
|
||||
# 先删这些地点名下的到访，再删地点本身
|
||||
site_places = [pk for pk, lat, lon in con.execute(
|
||||
"SELECT Z_PK, ZLATITUDE, ZLONGITUDE FROM ZLOCATION")
|
||||
if coord_key(lat, lon) not in place_keys]
|
||||
if site_places:
|
||||
q = ','.join(str(p) for p in site_places)
|
||||
out.append(f" 删站内新建地点名下的到访 {delete_rows(con, f'DELETE FROM ZVISIT WHERE ZLOCATION IN ({q})')} 条")
|
||||
out.append(f" 删站内新建地点 {delete_rows(con, f'DELETE FROM ZLOCATION WHERE Z_PK IN ({q})')} 个")
|
||||
fixed = cleanup_dangling(con)
|
||||
if fixed:
|
||||
out.append(f" 清理悬空引用：{fixed}")
|
||||
dst = os.path.join(args.out_dir, f"{os.path.basename(args.export).replace('.rondbackup','')}-{tag}.rondbackup")
|
||||
pack(con, main_db, dst)
|
||||
n = len(open(dst, 'rb').read())
|
||||
out.append(f" 已写出 {dst}（{n//1024} KB）")
|
||||
print('\n'.join(out))
|
||||
|
||||
build('V1', drop_site_visits=True, drop_site_places=False)
|
||||
build('V2', drop_site_visits=True, drop_site_places=True)
|
||||
build('V3', drop_site_visits=False, drop_site_places=False, revert_values=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user