Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9e112d55f9 | ||
|
|
0e670aa1cf |
@@ -35,16 +35,20 @@
|
||||
|
||||
### 数据编辑页上的「重复到访」
|
||||
|
||||
站点里可能残留**历史副本**:同样一次到访存了两份,差别只在 UUID(站点侧生成的那份,
|
||||
在手机包里找不到)。合并与差异页都是按 UUID 认人的,于是它们会被当成「备份新增」,
|
||||
导出到 rond 后就是成对的两条。
|
||||
站点里可能多出一条一模一样的到访,成因有两种,卡片上会分别标出来:
|
||||
|
||||
判定以**这份数据集的基线(导入时留下的手机包)**为参照:到访的 UUID 在基线里找不到、
|
||||
而基线里有一条「到达时间 + 地点」完全相同的,判为副本。卡片只在检出时出现在数据编辑页,
|
||||
默认全选 + 二次确认;清理会连它的位移 / 天气 / 留档一起删,并把指向它的合并标记改指到保留项
|
||||
(不动就会在导出时留下悬空引用)。基线不可用或与本站一条都对不上时,卡片只给说明、不给按钮。
|
||||
- **站点侧副本**:同一次到访在站点存了两份,差别只在 UUID(站点侧生成的那份在手机包里找不到)。
|
||||
合并与差异页都是按 UUID 认人的,于是它被当成「备份新增」。
|
||||
- **补录行被合并回来**:站点**补录**的到访没有留档、也就没有 UUID;导出时会被补上一个确定性的
|
||||
派生 UUID(`v5("ZVISIT/<编号>")`)写进包里,手机于是把它当成一条真记录。这份包再合并回站点时,
|
||||
站点这侧没有 UUID 可对齐,又插了一条。
|
||||
|
||||
命令行也有一条等价的工具:`tools/dedup_site_visits.py`(默认只预演,加 `--apply` 才动手)。
|
||||
判定以**这份数据集的基线(导入时留下的手机包)**为参照(第二类不用基线)。卡片只在检出时出现在
|
||||
数据编辑页,默认全选 + 二次确认;清理会连它的位移 / 天气 / 留档一起清——「补录行被合并回来」那类
|
||||
会把位移 / 天气**改挂**到保留项(补录行自己名下没有位移,跟着删就白丢里程),并把指向它的合并
|
||||
标记改指到保留项。基线不可用或与本站一条都对不上时,只判第二类,另给一行说明。
|
||||
|
||||
命令行有一条等价的工具 **只覆盖第一类**:`tools/dedup_site_visits.py`(默认只预演,加 `--apply` 才动手)。
|
||||
|
||||
### 为什么有些到访在页面上看不到
|
||||
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
#!/usr/bin/env python3
|
||||
"""从「被真机拒收的导出包」里做行级二分,生成对照包,好把范围一刀刀切小。
|
||||
|
||||
背景(2026-09-30):生产导出被真机拒收(「备份恢复失败,你的当前数据未被修改」),但
|
||||
同一台设备导入「纯净往返包」(手机原包导入后立刻导出)成功 ⇒ 导出链路没问题,问题在数据集的数据里。
|
||||
|
||||
做法:拿被拒的包当底,按「这一行是不是手机包里就有的」逐类删掉站内新建的行,
|
||||
每个变体都清理悬空引用、重新打包(zip.Store,本地头带长度与 CRC)。
|
||||
用户逐个导入,哪一版开始能进,问题就出在上一版删掉的那一类里。
|
||||
|
||||
用法:
|
||||
python tools/bisect_export.py --export data/20260930.rondbackup --phone data/20260929v2.rondbackup \
|
||||
--out-dir data/bisect
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import zipfile
|
||||
|
||||
|
||||
def open_pkg(path, tag):
|
||||
d = tempfile.mkdtemp(prefix=tag)
|
||||
with zipfile.ZipFile(path) as z:
|
||||
z.extractall(d)
|
||||
main = [os.path.join(r, f) for r, _, fs in os.walk(d) for f in fs if f.endswith('.sqlite')][0]
|
||||
con = sqlite3.connect(main)
|
||||
con.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
||||
con.commit()
|
||||
return con, main
|
||||
|
||||
|
||||
def coord_key(lat, lon):
|
||||
return "%.6f|%.6f" % (round(lat or 0, 6), round(lon or 0, 6))
|
||||
|
||||
|
||||
def phone_identities(con):
|
||||
uuids = {bytes(u).hex() for (u,) in
|
||||
con.execute("SELECT ZIDENTIFIER_ FROM ZVISIT WHERE ZIDENTIFIER_ IS NOT NULL")}
|
||||
places = {coord_key(lat, lon) for (lat, lon)
|
||||
in con.execute("SELECT ZLATITUDE, ZLONGITUDE FROM ZLOCATION")}
|
||||
acts = {n for (n,) in con.execute("SELECT ZNAME_ FROM ZACTIVITY WHERE ZNAME_ IS NOT NULL")}
|
||||
trans = {n for (n,) in con.execute("SELECT ZNAME_ FROM ZTRANSPORT WHERE ZNAME_ IS NOT NULL")}
|
||||
return uuids, places, acts, trans
|
||||
|
||||
|
||||
def delete_rows(con, sql, args=()):
|
||||
cur = con.execute(sql, args)
|
||||
n = cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
|
||||
con.commit()
|
||||
return n
|
||||
|
||||
|
||||
def cleanup_dangling(con):
|
||||
"""按关系删掉指向已不存在目标的行 / 把自引用置空(Core Data 的表没有外键约束,只能手查)。"""
|
||||
fixed = {}
|
||||
fixed['ZRAWVISIT(ZVISIT)'] = delete_rows(con, """DELETE FROM ZRAWVISIT WHERE ZVISIT IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM ZVISIT v WHERE v.Z_PK=ZRAWVISIT.ZVISIT)""")
|
||||
fixed['ZRAWVISIT(ZLOCATION)'] = delete_rows(con, """DELETE FROM ZRAWVISIT WHERE ZLOCATION IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM ZLOCATION l WHERE l.Z_PK=ZRAWVISIT.ZLOCATION)""")
|
||||
fixed['ZRAWVISIT(ZACTIVITY)'] = delete_rows(con, """DELETE FROM ZRAWVISIT WHERE ZACTIVITY IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM ZACTIVITY a WHERE a.Z_PK=ZRAWVISIT.ZACTIVITY)""")
|
||||
fixed['ZMOVEMENT(端点到访)'] = delete_rows(con, """DELETE FROM ZMOVEMENT WHERE
|
||||
(ZVISITFROM_ IS NOT NULL AND NOT EXISTS (SELECT 1 FROM ZVISIT v WHERE v.Z_PK=ZMOVEMENT.ZVISITFROM_))
|
||||
OR (ZVISITTO_ IS NOT NULL AND NOT EXISTS (SELECT 1 FROM ZVISIT v WHERE v.Z_PK=ZMOVEMENT.ZVISITTO_))""")
|
||||
fixed['ZHOURLYWEATHER(到访)'] = delete_rows(con, """DELETE FROM ZHOURLYWEATHER WHERE ZVISIT IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM ZVISIT v WHERE v.Z_PK=ZHOURLYWEATHER.ZVISIT)""")
|
||||
# 到访自引用:目标没了就置空(别留悬空)
|
||||
for col in ('ZMERGEDTO', 'ZPARENT'):
|
||||
fixed['ZVISIT.%s' % col] = delete_rows(con, f"""UPDATE ZVISIT SET {col}=NULL WHERE {col} IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM ZVISIT v WHERE v.Z_PK=ZVISIT.{col})""")
|
||||
return {k: v for k, v in fixed.items() if v}
|
||||
|
||||
|
||||
def _revert(con, phone_con, table, key_col, keyfn=lambda *a: a[0]):
|
||||
"""把 table 里「手机包里也有」的行按手机的值重置(保留 Z_PK)。返回重置的行数。
|
||||
|
||||
key_col 为 None 时按坐标键对齐(地点的身份就是坐标,与合并/差异同一套规则)。
|
||||
"""
|
||||
cols = [r[1] for r in phone_con.execute(f'PRAGMA table_info("{table}")') if r[1] != 'Z_PK']
|
||||
sel = [key_col] + cols if key_col else ['ZLATITUDE', 'ZLONGITUDE'] + cols
|
||||
mapping = {}
|
||||
for row in phone_con.execute('SELECT ' + ','.join(f'"{c}"' for c in sel) + f' FROM "{table}"'):
|
||||
k = keyfn(row[0]) if key_col else keyfn(row[0], row[1])
|
||||
if k is None:
|
||||
continue
|
||||
mapping[k] = dict(zip(cols, row[len(sel) - len(cols):]))
|
||||
n = 0
|
||||
for pk, *vals in con.execute('SELECT Z_PK,' + ','.join(f'"{c}"' for c in cols) + f' FROM "{table}"'):
|
||||
v = dict(zip(cols, vals))
|
||||
k = keyfn(v[key_col]) if key_col else keyfn(v['ZLATITUDE'], v['ZLONGITUDE'])
|
||||
if k in mapping:
|
||||
con.execute(f'UPDATE "{table}" SET ' + ', '.join(f'"{c}"=?' for c in cols) + ' WHERE Z_PK=?',
|
||||
[mapping[k][c] for c in cols] + [pk])
|
||||
n += 1
|
||||
con.commit()
|
||||
return n
|
||||
|
||||
|
||||
def pack(con, src_sqlite, dst):
|
||||
con.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
||||
con.commit()
|
||||
con.close()
|
||||
d = os.path.dirname(src_sqlite)
|
||||
empty = os.path.join(d, '_empty')
|
||||
open(empty, 'wb').close()
|
||||
with zipfile.ZipFile(dst, 'w', zipfile.ZIP_STORED) as z:
|
||||
z.write(src_sqlite, 'LifeEasy.sqlite')
|
||||
z.write(empty, 'LifeEasy.sqlite-wal')
|
||||
z.write(empty, 'LifeEasy.sqlite-shm')
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--export', required=True)
|
||||
ap.add_argument('--phone', required=True)
|
||||
ap.add_argument('--out-dir', default='data/bisect')
|
||||
args = ap.parse_args()
|
||||
os.makedirs(args.out_dir, exist_ok=True)
|
||||
|
||||
pcon, _ = open_pkg(args.phone, 'ph-')
|
||||
uuids, place_keys, acts, trans = phone_identities(pcon)
|
||||
pcon.close()
|
||||
print(f"手机包身份:到访 UUID {len(uuids)}、地点坐标键 {len(place_keys)}、活动 {len(acts)}、交通方式 {len(trans)}")
|
||||
|
||||
def build(tag, drop_site_visits, drop_site_places, revert_values=False):
|
||||
con, main_db = open_pkg(args.export, 'ex-')
|
||||
out = [f"== 变体 {tag}({'去掉站内新建的到访' if drop_site_visits else '保留全部到访'}"
|
||||
f"{' + 站内新建的地点' if drop_site_places else ''}"
|
||||
f"{';手机里也有的行按手机原值重置' if revert_values else ''})"]
|
||||
if revert_values:
|
||||
# 把「手机包里也有」的到访 / 地点 / 活动按手机的值原样重置,站内新建的行留着。
|
||||
# 重置后若某行的 ZLOCATION / ZACTIVITY_ 指向的目标在本包里不存在(手机有、我们删过),
|
||||
# 就还原成我们原来的值——否则会造出新的悬空引用,变体本身就不合格了。
|
||||
keep = {pk: (loc, act) for pk, loc, act in
|
||||
con.execute("SELECT Z_PK, ZLOCATION, ZACTIVITY_ FROM ZVISIT")}
|
||||
locs = {pk for (pk,) in con.execute("SELECT Z_PK FROM ZLOCATION")}
|
||||
acts = {pk for (pk,) in con.execute("SELECT Z_PK FROM ZACTIVITY")}
|
||||
pcon2, _ = open_pkg(args.phone, 'ph2-')
|
||||
out.append(f" 重置到访 {_revert(con, pcon2, 'ZVISIT', 'ZIDENTIFIER_',
|
||||
lambda v: bytes(v).hex() if v is not None else None)} 行")
|
||||
out.append(f" 重置地点 {_revert(con, pcon2, 'ZLOCATION', None,
|
||||
lambda lat, lon: coord_key(lat, lon))} 行")
|
||||
out.append(f" 重置活动 {_revert(con, pcon2, 'ZACTIVITY', 'ZNAME_')} 行")
|
||||
pcon2.close()
|
||||
back = 0
|
||||
for pk, loc, act in con.execute("SELECT Z_PK,ZLOCATION,ZACTIVITY_ FROM ZVISIT"):
|
||||
if pk not in keep:
|
||||
continue
|
||||
if (loc is not None and loc not in locs) or (act is not None and act not in acts):
|
||||
con.execute("UPDATE ZVISIT SET ZLOCATION=?, ZACTIVITY_=? WHERE Z_PK=?",
|
||||
(keep[pk][0], keep[pk][1], pk))
|
||||
back += 1
|
||||
con.commit()
|
||||
out.append(f" 引用解不开、还原成我们的值:{back} 行")
|
||||
if drop_site_visits:
|
||||
site = [pk for pk, u in con.execute(
|
||||
"SELECT Z_PK, ZIDENTIFIER_ FROM ZVISIT")
|
||||
if u is not None and bytes(u).hex() not in uuids]
|
||||
if site:
|
||||
q = ','.join(str(p) for p in site)
|
||||
out.append(f" 删站内新建到访 {delete_rows(con, f'DELETE FROM ZVISIT WHERE Z_PK IN ({q})')} 条")
|
||||
if drop_site_places:
|
||||
# 先删这些地点名下的到访,再删地点本身
|
||||
site_places = [pk for pk, lat, lon in con.execute(
|
||||
"SELECT Z_PK, ZLATITUDE, ZLONGITUDE FROM ZLOCATION")
|
||||
if coord_key(lat, lon) not in place_keys]
|
||||
if site_places:
|
||||
q = ','.join(str(p) for p in site_places)
|
||||
out.append(f" 删站内新建地点名下的到访 {delete_rows(con, f'DELETE FROM ZVISIT WHERE ZLOCATION IN ({q})')} 条")
|
||||
out.append(f" 删站内新建地点 {delete_rows(con, f'DELETE FROM ZLOCATION WHERE Z_PK IN ({q})')} 个")
|
||||
fixed = cleanup_dangling(con)
|
||||
if fixed:
|
||||
out.append(f" 清理悬空引用:{fixed}")
|
||||
dst = os.path.join(args.out_dir, f"{os.path.basename(args.export).replace('.rondbackup','')}-{tag}.rondbackup")
|
||||
pack(con, main_db, dst)
|
||||
n = len(open(dst, 'rb').read())
|
||||
out.append(f" 已写出 {dst}({n//1024} KB)")
|
||||
print('\n'.join(out))
|
||||
|
||||
build('V1', drop_site_visits=True, drop_site_places=False)
|
||||
build('V2', drop_site_visits=True, drop_site_places=True)
|
||||
build('V3', drop_site_visits=False, drop_site_places=False, revert_values=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user