diff --git a/internal/rond/export.go b/internal/rond/export.go index 73dcaa1..97811ec 100644 --- a/internal/rond/export.go +++ b/internal/rond/export.go @@ -39,7 +39,8 @@ var managedCols = map[string][]string{ "ZTAG": {"ZNAME_", "ZCOLOR_", "ZGROUP", "ZLASTARRIVALDATE_"}, "ZTRANSPORT": {"ZNAME_", "ZCOLOR_", "ZICON_"}, "ZVISIT": {"ZLOCATION", "ZACTIVITY_", "ZARRIVALDATE_", "ZDEPARTUREDATE_", - "ZBOOKMARKED", "ZUSERADDED", "ZREMARK_", "ZEMOJINAME_", "ZWEATHERSYMBOL_", "ZIDENTIFIER_"}, + "ZBOOKMARKED", "ZUSERADDED", "ZREMARK_", "ZEMOJINAME_", "ZWEATHERSYMBOL_", "ZIDENTIFIER_", + "ZMERGEDTO", "ZPARENT"}, "ZMOVEMENT": {"ZTRANSPORT_", "ZVISITFROM_", "ZVISITTO_", "ZSTART_", "ZEND_", "ZTYPE_"}, "ZHOURLYWEATHER": {"ZVISIT", "ZDATE_", "ZISDAYLIGHT", "ZTEMPERATURE_", "ZAPPARENTTEMPERATURE_", "ZHUMIDITY_", "ZPRECIPITATIONAMOUNT_", "ZPRECIPITATIONCHANCE_", "ZWINDSPEED_", "ZVISIBILITY_", "ZUVINDEXVALUE_", @@ -241,7 +242,13 @@ func patchSQLite(ctx context.Context, db *sql.DB, datasetID int64, sqlitePath st return err } loc := w("ZLOCATION", "Location") + // 编号对齐(见 export_align.go):把站点编号映射到基线的编号空间 + placeAlign, err := alignPlaces(ctx, tx, places) + if err != nil { + return fmt.Errorf("对齐地点编号: %w", err) + } for _, p := range places { + outPK := placeAlign[p.SrcPK] placeSrcByID[int64(p.ID)] = p.SrcPK raw := raws["ZLOCATION"][p.SrcPK] // 地点详情的「最近活跃时间」= 该地点最晚一次到访的到达时间,没有到访就不写 @@ -253,13 +260,13 @@ func patchSQLite(ctx context.Context, db *sql.DB, datasetID int64, sqlitePath st // 有留档就用真机那份(它才是真正的原始定位,哪怕用户后来挪过点); // 站点新建的地点没有留档,用显示坐标反算一个,否则这列会空着。 var rawLat, rawLon any - if p.Lat.Valid && p.Lon.Valid && !known["ZLOCATION"][p.SrcPK] { + if p.Lat.Valid && p.Lon.Valid && !known["ZLOCATION"][outPK] { rawLat, rawLon = geo.GCJ02ToWGS84(p.Lat.Float64, p.Lon.Float64) } // 国家码写成 NULL,**不要**补空串:2026-09-29 实测——只把这一列从 NULL 改成 '' // 就让真机从「能导入」变成「备份恢复失败」(同一份包、同一台设备,唯一差异就是这列)。 // 站点新建的地点这一项本来就是 NULL,保持原样最安全。 - if err := loc.write(p.SrcPK, []any{ns(p.Name), ns(p.Category), nf(p.Lat), nf(p.Lon), + if err := loc.write(outPK, []any{ns(p.Name), ns(p.Category), nf(p.Lat), nf(p.Lon), ns(p.Country), ns(p.Province), ns(p.City), ns(p.Sublocality), ns(p.Thoroughfare), ns(p.Timezone), lastArrival, rawLat, rawLon}, raw); err != nil { @@ -277,7 +284,7 @@ func patchSQLite(ctx context.Context, db *sql.DB, datasetID int64, sqlitePath st var pks []any for _, p := range places { if p.CoordSource.Valid && p.CoordSource.String == "photo" { - pks = append(pks, p.SrcPK) + pks = append(pks, placeAlign[p.SrcPK]) } } for len(pks) > 0 { @@ -306,7 +313,12 @@ func patchSQLite(ctx context.Context, db *sql.DB, datasetID int64, sqlitePath st return err } act := w("ZACTIVITY", "Activity") + actAlign, err := alignUUIDs(ctx, db, tx, datasetID, "ZACTIVITY", "ZUID_", actsPKs(activities)) + if err != nil { + return fmt.Errorf("对齐活动编号: %w", err) + } for _, a := range activities { + outPK := actAlign[a.SrcPK] actSrcByID[int64(a.ID)] = a.SrcPK raw := raws["ZACTIVITY"][a.SrcPK] // 站点没有图标入口,新建的活动在 rond 里会是空图标;站点自己渲染时缺省用 📍 @@ -326,9 +338,9 @@ func patchSQLite(ctx context.Context, db *sql.DB, datasetID int64, sqlitePath st if t, ok := actLast[int64(a.ID)]; ok { last = appleSeconds(t) } - if err := act.write(a.SrcPK, []any{ns(a.Name), color, icon, + if err := act.write(outPK, []any{ns(a.Name), color, icon, bint(a.Home), bint(a.Work), bint(a.Excluded), bint(a.Archived), last, - coreDataUUID(!known["ZACTIVITY"][a.SrcPK], "ZACTIVITY", a.SrcPK)}, + coreDataUUID(!known["ZACTIVITY"][outPK], "ZACTIVITY", a.SrcPK)}, raw); err != nil { return fmt.Errorf("写入活动类型失败: %w", err) } @@ -385,6 +397,10 @@ func patchSQLite(ctx context.Context, db *sql.DB, datasetID int64, sqlitePath st return err } tag := w("ZTAG", "Tag") + tagAlign, err := alignTags(ctx, tx, tags) + if err != nil { + return fmt.Errorf("对齐标签编号: %w", err) + } tagLast, err := queryTagMaxArrival(ctx, db, datasetID) if err != nil { return err @@ -402,7 +418,7 @@ func patchSQLite(ctx context.Context, db *sql.DB, datasetID int64, sqlitePath st if tm, ok := tagLast[int64(t.SrcPK)]; ok { last = appleSeconds(tm) } - if err := tag.write(t.SrcPK, []any{ns(t.Name), ns(t.Color), grp, last}, + if err := tag.write(tagAlign[t.SrcPK], []any{ns(t.Name), ns(t.Color), grp, last}, raws["ZTAG"][t.SrcPK]); err != nil { return fmt.Errorf("写入标签失败: %w", err) } @@ -412,13 +428,17 @@ func patchSQLite(ctx context.Context, db *sql.DB, datasetID int64, sqlitePath st } // 6) 交通方式 ZTRANSPORT(从位移里聚合而来) - transports, transByName, err := queryTransports(ctx, db, datasetID, maxPKOf(known["ZTRANSPORT"], nil)+1) + transports, transByNameSite, err := queryTransports(ctx, db, datasetID, maxPKOf(known["ZTRANSPORT"], nil)+1) if err != nil { return err } tr := w("ZTRANSPORT", "Transport") // 没有任何行程引用的方式保持基线原样,不当成「已删除」 tr.keepMissing = true + transAlign, err := alignTransports(ctx, tx, transports) + if err != nil { + return fmt.Errorf("对齐交通方式编号: %w", err) + } for _, t := range transports { name, color, icon := ns(t.Name), ns(t.Color), ns(t.Icon) // 真机这三列从来没出现过空值(NULL 和空串都没有),而站点不单独维护交通方式表、 @@ -426,7 +446,7 @@ func patchSQLite(ctx context.Context, db *sql.DB, datasetID int64, sqlitePath st // 否则会把 NULL 写进包里——rond 拿到的是它自己从来不会产生的形状 // (2026-09-20:新建的「步行」就是这样导出的,与新建活动补 blue/📍 同一套做法)。 // 基线里已有的行不碰:那种情况交给 keepNilCols 保留真机原值。 - if !known["ZTRANSPORT"][t.SrcPK] { + if !known["ZTRANSPORT"][transAlign[t.SrcPK]] { if name == nil { name = "未命名" } @@ -437,7 +457,7 @@ func patchSQLite(ctx context.Context, db *sql.DB, datasetID int64, sqlitePath st icon = transportPlaceholderIcon } } - if err := tr.write(t.SrcPK, []any{name, color, icon}, raws["ZTRANSPORT"][t.SrcPK]); err != nil { + if err := tr.write(transAlign[t.SrcPK], []any{name, color, icon}, raws["ZTRANSPORT"][t.SrcPK]); err != nil { return fmt.Errorf("写入交通方式失败: %w", err) } } @@ -451,21 +471,30 @@ func patchSQLite(ctx context.Context, db *sql.DB, datasetID int64, sqlitePath st return err } visit := w("ZVISIT", "Visit") + visitAlign, err := alignUUIDs(ctx, db, tx, datasetID, "ZVISIT", "ZIDENTIFIER_", visitsPKs(visits)) + if err != nil { + return fmt.Errorf("对齐到访编号: %w", err) + } for _, v := range visits { + outPK := visitAlign[v.SrcPK] var locPK, actPK any if v.PlaceID.Valid { if sp, ok := placeSrcByID[v.PlaceID.Int64]; ok { - locPK = sp + locPK = placeAlign[sp] } } if v.ActivityID.Valid { if sp, ok := actSrcByID[v.ActivityID.Int64]; ok { - actPK = sp + actPK = actAlign[sp] } } - if err := visit.write(v.SrcPK, []any{locPK, actPK, appleSeconds(v.Arrival), nt(v.Departure), + // ZMERGEDTO / ZPARENT 是自引用:留档里存的是站点编号,对齐后必须跟着改挂, + // 否则会指到别的到访、甚至不存在的行。 + mergedTo := alignedVisitRef(raws["ZVISIT"][v.SrcPK], "ZMERGEDTO", visitAlign) + parent := alignedVisitRef(raws["ZVISIT"][v.SrcPK], "ZPARENT", visitAlign) + if err := visit.write(outPK, []any{locPK, actPK, appleSeconds(v.Arrival), nt(v.Departure), bint(v.Bookmarked), bint(v.UserAdded), ns(v.Remark), ns(v.Emoji), ns(v.WeatherSymbol), - coreDataUUID(!known["ZVISIT"][v.SrcPK], "ZVISIT", v.SrcPK)}, + coreDataUUID(!known["ZVISIT"][outPK], "ZVISIT", v.SrcPK), mergedTo, parent}, raws["ZVISIT"][v.SrcPK]); err != nil { return fmt.Errorf("写入访问记录失败: %w", err) } @@ -517,29 +546,40 @@ func patchSQLite(ctx context.Context, db *sql.DB, datasetID int64, sqlitePath st return err } mov := w("ZMOVEMENT", "Movement") + movVisit := map[int]int{} for _, m := range movements { + if m.FromVisit.Valid { + movVisit[m.SrcPK] = visitAlign[int(m.FromVisit.Int64)] + } + } + movAlign, err := alignByVisitRef(ctx, tx, "ZMOVEMENT", "ZVISITFROM_", movsPKs(movements), movVisit) + if err != nil { + return fmt.Errorf("对齐位移编号: %w", err) + } + for _, m := range movements { + outPK := movAlign[m.SrcPK] var trv, fromv, tov any if m.Transport.Valid { // 方式按**这条位移自己的名称**归并(见 queryTransports):同一个 src 在站点里可能 // 带着两种含义,只有当条的名称才能认出它到底是哪一种,否则会挂到错的方式上。 - if x, ok := transByName[m.TransportName]; ok { - trv = x + if x, ok := transByNameSite[m.TransportName]; ok { + trv = transAlign[x] } else { - trv = m.Transport.Int64 + trv = transAlign[int(m.Transport.Int64)] } } if m.FromVisit.Valid { - fromv = m.FromVisit.Int64 + fromv = visitAlign[int(m.FromVisit.Int64)] } if m.ToVisit.Valid { - tov = m.ToVisit.Int64 + tov = visitAlign[int(m.ToVisit.Int64)] } // ZTYPE_ 只有站点新建的行才补(已在基线里的行保持真机原值,见 keepNilCols) var mvType any - if !known["ZMOVEMENT"][m.SrcPK] { + if !known["ZMOVEMENT"][outPK] { mvType = movementTypeDefault(m.Transport.Valid) } - if err := mov.write(m.SrcPK, []any{trv, fromv, tov, nt(m.Start), nt(m.End), mvType}, + if err := mov.write(outPK, []any{trv, fromv, tov, nt(m.Start), nt(m.End), mvType}, raws["ZMOVEMENT"][m.SrcPK]); err != nil { return fmt.Errorf("写入位移失败: %w", err) } @@ -554,12 +594,23 @@ func patchSQLite(ctx context.Context, db *sql.DB, datasetID int64, sqlitePath st return err } wx := w("ZHOURLYWEATHER", "HourlyWeather") + wxVisit := map[int]int{} for _, row := range weather { + if row.VisitSrc.Valid { + wxVisit[row.SrcPK] = visitAlign[int(row.VisitSrc.Int64)] + } + } + wxAlign, err := alignByVisitRef(ctx, tx, "ZHOURLYWEATHER", "ZVISIT", wxPKs(weather), wxVisit) + if err != nil { + return fmt.Errorf("对齐天气编号: %w", err) + } + for _, row := range weather { + outPK := wxAlign[row.SrcPK] var vsrc any if row.VisitSrc.Valid { - vsrc = row.VisitSrc.Int64 + vsrc = visitAlign[int(row.VisitSrc.Int64)] } - if err := wx.write(row.SrcPK, []any{vsrc, nt(row.At), bint(row.Daylight), + if err := wx.write(outPK, []any{vsrc, nt(row.At), bint(row.Daylight), nf(row.Temp), nf(row.App), nf(row.Hum), nf(row.Pa), nf(row.Pc), nf(row.Ws), nf(row.Vis), ni(row.UV), ns(row.Cond), ns(row.Symbol), ns(row.UVCat), ns(row.WindDir)}, raws["ZHOURLYWEATHER"][row.SrcPK]); err != nil { @@ -619,13 +670,13 @@ func patchSQLite(ctx context.Context, db *sql.DB, datasetID int64, sqlitePath st // 11) 回写 Z_PRIMARYKEY.Z_MAX:只上调,避免 rond 下次插入撞键 bumps := map[string]int{ - "Location": maxPKOf(known["ZLOCATION"], placesPKs(places)), - "Activity": maxPKOf(known["ZACTIVITY"], actsPKs(activities)), - "Tag": maxPKOf(known["ZTAG"], tagsPKs(tags)), - "Transport": maxPKOf(known["ZTRANSPORT"], transPKs(transports)), - "Visit": maxPKOf(known["ZVISIT"], visitsPKs(visits)), - "Movement": maxPKOf(known["ZMOVEMENT"], movsPKs(movements)), - "HourlyWeather": maxPKOf(known["ZHOURLYWEATHER"], wxPKs(weather)), + "Location": maxPKOf(known["ZLOCATION"], mapValues(placeAlign)), + "Activity": maxPKOf(known["ZACTIVITY"], mapValues(actAlign)), + "Tag": maxPKOf(known["ZTAG"], mapValues(tagAlign)), + "Transport": maxPKOf(known["ZTRANSPORT"], mapValues(transAlign)), + "Visit": maxPKOf(known["ZVISIT"], mapValues(visitAlign)), + "Movement": maxPKOf(known["ZMOVEMENT"], mapValues(movAlign)), + "HourlyWeather": maxPKOf(known["ZHOURLYWEATHER"], mapValues(wxAlign)), "TagGroup": maxGroupPK, } for name, mx := range bumps { diff --git a/internal/rond/export_align.go b/internal/rond/export_align.go new file mode 100644 index 0000000..bae27be --- /dev/null +++ b/internal/rond/export_align.go @@ -0,0 +1,459 @@ +package rond + +import ( + "context" + "database/sql" + "encoding/base64" + "fmt" + "os" + "sort" + "strings" + + "rond-with-you/internal/geo" +) + +// 导出时的**编号对齐**。 +// +// 背景:Core Data 的 store 里**地点与活动没有 UUID**,app 只能靠 Z_PK 解引用 +// (`ZVISIT.ZLOCATION` / `ZVISIT.ZACTIVITY_`)。而站点的 `src_pk` 与手机 store 的 `Z_PK` +// 并不是同一套编号——站点的基线本身可能是「导出包」,编号会在导入-导出的循环里漂移。 +// 导出若把漂移过的编号原样写回,手机那边每条到访都会指向错的地点/活动(或指向不存在的行) +// ⇒ 导入后「落不进去 / 看不见数据」。 +// +// 2026-09-29 实测(真机 09-28 包 vs 站点导出,按到访 UUID 对齐):地点引用 920/920 不同、 +// 活动引用 903/920 不同,而引用到的地点按「名称 + 坐标」完全一致——语义相同、编号不同。 +// 真机自己导出的包永远没问题(编号自洽)。 +// +// 做法:把托管表的行**按身份**重新编号到「基线的编号空间」(基线就是这份数据集从手机那儿 +// 来的那个 store,所以对齐基线 = 对齐手机): +// +// 地点:坐标键优先(多值索引),退到「同名且相距 ≤1km」 +// 活动 / 到访:UUID +// 标签 / 交通方式:名称 +// 位移 / 天气:所属到访(已对齐) +// +// 基线里找不到身份对应的(站点新建的)行,取「基线最大编号之上」的新号;分配顺序固定, +// 保证同一份数据每次导出编号一致。 +// +// 设 ROND_ALIGN_DEBUG=1 会在 stderr 打出每张表的映射规模,便于排查。 + +// pkAlloc 按「基线最大编号 + 1」起逐个发号,避免与基线的行相撞。 +type pkAlloc struct { + next int + used map[int]bool + cache map[int]int +} + +func newPKAlloc(baseMax int) *pkAlloc { + return &pkAlloc{next: baseMax + 1, used: map[int]bool{}, cache: map[int]int{}} +} + +func (al *pkAlloc) take(srcPK int, reserved map[int]bool) int { + if v, ok := al.cache[srcPK]; ok { + return v + } + for al.used[al.next] || reserved[al.next] { + al.next++ + } + al.used[al.next] = true + al.cache[srcPK] = al.next + al.next++ + return al.cache[srcPK] +} + +func alignDebug(kind string, base, site, rows, matched int) { + if os.Getenv("ROND_ALIGN_DEBUG") == "" { + return + } + fmt.Fprintf(os.Stderr, "ALIGN %-12s 基线 %4d / 站点留档 %4d / 待写 %4d / 身份匹配 %4d\n", + kind, base, site, rows, matched) +} + +// alignPlaces 地点:坐标键优先(多值索引),退到同名且 ≤ nameMatchMaxKm。 +func alignPlaces(ctx context.Context, tx *sql.Tx, places []pgPlace) (map[int]int, error) { + byCoord, byName := map[string][]int{}, map[string][]int{} + base := map[int]StoredPlace{} + maxPK := 0 + // 多值索引:真机一个坐标下有多条同坐标同名的地点(实测「柳州站」3 条), + // 单槽位会让其余行永远配不上,引用就会错位。ORDER BY 保证结果确定。 + rows, err := tx.QueryContext(ctx, `SELECT Z_PK, COALESCE(ZNAME_,''), COALESCE(ZLATITUDE,0), COALESCE(ZLONGITUDE,0) + FROM ZLOCATION ORDER BY Z_PK`) + if err != nil { + return nil, err + } + for rows.Next() { + var sp StoredPlace + if err := rows.Scan(&sp.SrcPK, &sp.Name, &sp.Lat, &sp.Lon); err != nil { + rows.Close() + return nil, err + } + if sp.Lat != 0 || sp.Lon != 0 { + sp.Key = PlaceCoordKey(sp.Lat, sp.Lon) + byCoord[sp.Key] = append(byCoord[sp.Key], sp.SrcPK) + } + if sp.Name != "" { + byName[sp.Name] = append(byName[sp.Name], sp.SrcPK) + } + base[sp.SrcPK] = sp + if sp.SrcPK > maxPK { + maxPK = sp.SrcPK + } + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + + out := make(map[int]int, len(places)) + used := map[int]bool{} + alloc := newPKAlloc(maxPK) + var leftover []pgPlace + matched := 0 + for _, p := range places { + key := "" + if p.Lat.Valid && p.Lon.Valid && (p.Lat.Float64 != 0 || p.Lon.Float64 != 0) { + key = PlaceCoordKey(p.Lat.Float64, p.Lon.Float64) + } + picked := 0 + for _, id := range byCoord[key] { + if !used[id] { + picked = id + break + } + } + if picked != 0 { + used[picked] = true + out[p.SrcPK] = picked + matched++ + continue + } + leftover = append(leftover, p) + } + for _, p := range leftover { + if id, ok := matchPlaceByName(byName, base, used, p); ok { + used[id] = true + out[p.SrcPK] = id + matched++ + continue + } + if !used[p.SrcPK] { + used[p.SrcPK] = true + out[p.SrcPK] = p.SrcPK + continue + } + out[p.SrcPK] = alloc.take(p.SrcPK, used) + } + alignDebug("ZLOCATION", maxPK, len(places), len(places), matched) + return out, nil +} + +func matchPlaceByName(byName map[string][]int, base map[int]StoredPlace, used map[int]bool, p pgPlace) (int, bool) { + if !p.Name.Valid || p.Name.String == "" { + return 0, false + } + for _, id := range byName[p.Name.String] { + if used[id] { + continue + } + sp, ok := base[id] + if !ok { + continue + } + if p.Lat.Valid && p.Lon.Valid && (sp.Lat != 0 || sp.Lon != 0) { + if geo.Haversine(sp.Lat, sp.Lon, p.Lat.Float64, p.Lon.Float64) > nameMatchMaxKm { + continue + } + } + return id, true + } + return 0, false +} + +// alignUUIDs 活动 / 到访:按留档里的 UUID 对齐到基线同一 UUID 的那一行。 +func alignUUIDs(ctx context.Context, db *sql.DB, tx *sql.Tx, datasetID int64, + entity, col string, srcPKs []int) (map[int]int, error) { + base, maxPK, err := baselineUUIDMap(ctx, tx, entity, col) + if err != nil { + return nil, err + } + site, err := siteUUIDMap(ctx, db, datasetID, entity, col) + if err != nil { + return nil, err + } + out := make(map[int]int, len(srcPKs)) + used := map[int]bool{} + alloc := newPKAlloc(maxPK) + matched := 0 + for _, pk := range srcPKs { + if u, ok := site[pk]; ok { + if id, ok := base[u]; ok && !used[id] { + used[id] = true + out[pk] = id + matched++ + continue + } + } + if !used[pk] { + used[pk] = true + out[pk] = pk + continue + } + out[pk] = alloc.take(pk, used) + } + alignDebug(entity, len(base), len(site), len(srcPKs), matched) + return out, nil +} + +func baselineUUIDMap(ctx context.Context, tx *sql.Tx, table, col string) (map[string]int, int, error) { + out := map[string]int{} + maxPK := 0 + q := fmt.Sprintf(`SELECT Z_PK, hex(%s) FROM %s WHERE %s IS NOT NULL ORDER BY Z_PK`, col, table, col) + rows, err := tx.QueryContext(ctx, q) + if err != nil { + return nil, 0, err + } + defer rows.Close() + for rows.Next() { + var pk int + var hex string + if err := rows.Scan(&pk, &hex); err != nil { + return nil, 0, err + } + if hex != "" { + out[strings.ToLower(hex)] = pk + } + if pk > maxPK { + maxPK = pk + } + } + return out, maxPK, rows.Err() +} + +// siteUUIDMap 读站点留档里的「src_pk -> UUID(hex)」。站点新建的行没有留档 ⇒ 不在结果里, +// 那些行本来就该拿新编号。留档里的 UUID 是 base64 的 16 字节 blob(见 loadStoredVisits)。 +func siteUUIDMap(ctx context.Context, db *sql.DB, datasetID int64, entity, col string) (map[int]string, error) { + out := map[int]string{} + q := fmt.Sprintf(`SELECT src_pk, raw #>> '{%s,$blob}' FROM entity_raw + WHERE dataset_id=$1 AND entity=$2 AND (raw #>> '{%s,$blob}') IS NOT NULL`, col, col) + rows, err := db.QueryContext(ctx, q, datasetID, entity) + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + var pk int + var b64 string + if err := rows.Scan(&pk, &b64); err != nil { + return nil, err + } + if h := base64ToHex(b64); h != "" { + out[pk] = h + } + } + return out, rows.Err() +} + +// base64ToHex 把留档里的 base64 blob 转成小写 hex(与基线 `hex(ZIDENTIFIER_)` 同一形式)。 +func base64ToHex(s string) string { + b, err := base64.StdEncoding.DecodeString(s) + if err != nil || len(b) == 0 { + return "" + } + return strings.ToLower(fmt.Sprintf("%x", b)) +} + +// alignTags 标签:按名称(同名取基线上编号最小的那条)。 +func alignTags(ctx context.Context, tx *sql.Tx, tags []pgTag) (map[int]int, error) { + base := map[string]int{} + maxPK := 0 + rows, err := tx.QueryContext(ctx, `SELECT Z_PK, COALESCE(ZNAME_,'') FROM ZTAG ORDER BY Z_PK`) + if err != nil { + return nil, err + } + for rows.Next() { + var pk int + var name string + if err := rows.Scan(&pk, &name); err != nil { + rows.Close() + return nil, err + } + if name != "" { + if id, ok := base[name]; !ok || pk < id { + base[name] = pk + } + } + if pk > maxPK { + maxPK = pk + } + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + out := make(map[int]int, len(tags)) + used := map[int]bool{} + alloc := newPKAlloc(maxPK) + matched := 0 + for _, t := range tags { + if t.Name.Valid && t.Name.String != "" { + if id, ok := base[t.Name.String]; ok && !used[id] { + used[id] = true + out[t.SrcPK] = id + matched++ + continue + } + } + if !used[t.SrcPK] { + used[t.SrcPK] = true + out[t.SrcPK] = t.SrcPK + continue + } + out[t.SrcPK] = alloc.take(t.SrcPK, used) + } + alignDebug("ZTAG", maxPK, len(base), len(tags), matched) + return out, nil +} + +// alignTransports 交通方式:按名称对齐到基线(基线同名方式的编号)。 +func alignTransports(ctx context.Context, tx *sql.Tx, transports []pgTransport) (map[int]int, error) { + base := map[string]int{} + maxPK := 0 + rows, err := tx.QueryContext(ctx, `SELECT Z_PK, COALESCE(ZNAME_,'') FROM ZTRANSPORT ORDER BY Z_PK`) + if err != nil { + return nil, err + } + for rows.Next() { + var pk int + var name string + if err := rows.Scan(&pk, &name); err != nil { + rows.Close() + return nil, err + } + if name != "" { + if id, ok := base[name]; !ok || pk < id { + base[name] = pk + } + } + if pk > maxPK { + maxPK = pk + } + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + out := make(map[int]int, len(transports)) + used := map[int]bool{} + alloc := newPKAlloc(maxPK) + matched := 0 + for _, t := range transports { + if t.Name.Valid && t.Name.String != "" { + if id, ok := base[t.Name.String]; ok && !used[id] { + used[id] = true + out[t.SrcPK] = id + matched++ + continue + } + } + if !used[t.SrcPK] { + used[t.SrcPK] = true + out[t.SrcPK] = t.SrcPK + continue + } + out[t.SrcPK] = alloc.take(t.SrcPK, used) + } + alignDebug("ZTRANSPORT", maxPK, len(base), len(transports), matched) + return out, nil +} + +// alignByVisitRef 位移 / 天气:按「所属到访(已对齐后的编号)」在基线里找没被占用的行。 +// 同一到访下有多条时按编号顺序依次认领,保证确定性。 +func alignByVisitRef(ctx context.Context, tx *sql.Tx, table, visitCol string, + srcPKs []int, visitOf map[int]int) (map[int]int, error) { + base := map[int][]int{} + maxPK := 0 + q := fmt.Sprintf(`SELECT Z_PK, COALESCE(%s,0) FROM %s ORDER BY Z_PK`, visitCol, table) + rows, err := tx.QueryContext(ctx, q) + if err != nil { + return nil, err + } + for rows.Next() { + var pk, v int + if err := rows.Scan(&pk, &v); err != nil { + rows.Close() + return nil, err + } + base[v] = append(base[v], pk) + if pk > maxPK { + maxPK = pk + } + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + out := make(map[int]int, len(srcPKs)) + used := map[int]bool{} + alloc := newPKAlloc(maxPK) + matched := 0 + for _, pk := range srcPKs { + for _, id := range base[visitOf[pk]] { + if !used[id] { + used[id] = true + out[pk] = id + matched++ + break + } + } + if _, ok := out[pk]; !ok { + if !used[pk] { + used[pk] = true + out[pk] = pk + } else { + out[pk] = alloc.take(pk, used) + } + } + } + alignDebug(table, maxPK, len(base), len(srcPKs), matched) + return out, nil +} + +// alignedVisitRef 把留档里某个「指向到访的列」改挂到对齐后的编号:留档存的是站点编号, +// 对齐之后直接写回会指到别的到访(甚至不存在的行)。空值仍写 NULL。 +func alignedVisitRef(raw map[string]any, col string, visitAlign map[int]int) any { + if raw == nil { + return nil + } + v, ok := raw[col] + if !ok || v == nil { + return nil + } + var pk int + switch t := v.(type) { + case float64: + pk = int(t) + case int64: + pk = int(t) + case int: + pk = t + default: + return nil + } + if pk <= 0 { + return nil + } + if out, ok := visitAlign[pk]; ok { + return out + } + return nil // 找不到对应(那条到访已不在站点数据里)——宁可置空也不要指错 +} + +// mapValues 取映射的值集合,给 Z_PRIMARYKEY 的 Z_MAX 用(要覆盖实际写入的编号)。 +func mapValues(m map[int]int) []int { + out := make([]int, 0, len(m)) + for _, v := range m { + out = append(out, v) + } + sort.Ints(out) + return out +} diff --git a/internal/web/export_align_test.go b/internal/web/export_align_test.go new file mode 100644 index 0000000..19f7488 --- /dev/null +++ b/internal/web/export_align_test.go @@ -0,0 +1,145 @@ +package web + +import ( + "database/sql" + "fmt" + "os" + "path/filepath" + "testing" + + "rond-with-you/internal/rond" +) + +// 导出的**编号必须与基线对齐**。 +// +// 为什么:Core Data 的 store 里地点与活动**没有 UUID**,app 只能靠 Z_PK 解引用 +// (`ZVISIT.ZLOCATION` / `ZVISIT.ZACTIVITY_`)。站点这边的编号是从「导入→导出」循环里滚出来的, +// 与手机 store 的编号不是一套——2026-09-29 实测:同一批到访里 920/920 的地点引用、903/920 的 +// 活动引用与真机包不同(引用到的地点按名称+坐标完全一致),结果是「导入成功但看不到数据」。 +// +// 这里把库里的编号整体平移(模拟「数据来自编号完全不同的备份」),再导出,要求导出结果回到基线的编号。 +func TestExportAlignsPKsToBaseline(t *testing.T) { + f := newMovementFixture(t) + f.shiftSrcPKs(t, 500000) // 库内编号整体平移 ⇒ 与基线完全不同 + + out := filepath.Join(t.TempDir(), "export.rondbackup") + if err := os.WriteFile(out, f.export(t), 0o644); err != nil { + t.Fatal(err) + } + edb, err := rond.Open(out, t.TempDir()) + if err != nil { + t.Fatalf("打开导出包失败: %v", err) + } + t.Cleanup(func() { edb.Close() }) + base := f.openSource(t) // 基线 = 导入用的那份包 + + type ref struct { + placePK, actPK sql.NullInt64 + } + load := func(db *sql.DB) map[string]ref { + rows, err := db.Query(`SELECT hex(ZIDENTIFIER_), ZLOCATION, ZACTIVITY_ FROM ZVISIT + WHERE ZIDENTIFIER_ IS NOT NULL`) + if err != nil { + t.Fatalf("读到访失败: %v", err) + } + defer rows.Close() + m := map[string]ref{} + for rows.Next() { + var u string + var r ref + if err := rows.Scan(&u, &r.placePK, &r.actPK); err != nil { + t.Fatal(err) + } + m[u] = r + } + return m + } + describe := func(db *sql.DB, placePK, actPK sql.NullInt64) (string, string) { + p := "<空>" + if placePK.Valid { + var n string + var la, lo float64 + if err := db.QueryRow(`SELECT COALESCE(ZNAME_,''), COALESCE(ZLATITUDE,0), COALESCE(ZLONGITUDE,0) + FROM ZLOCATION WHERE Z_PK=?`, placePK.Int64).Scan(&n, &la, &lo); err != nil { + return "<地点不存在>", "" + } + p = fmt.Sprintf("%s %.5f,%.5f", n, la, lo) + } + a := "<空>" + if actPK.Valid { + var n string + if err := db.QueryRow(`SELECT COALESCE(ZNAME_,'') FROM ZACTIVITY WHERE Z_PK=?`, actPK.Int64).Scan(&n); err != nil { + return p, "<活动不存在>" + } + a = n + } + return p, a + } + + baseVisits, expVisits := load(base), load(edb) + if len(baseVisits) == 0 || len(expVisits) == 0 { + t.Fatalf("包里的到访为空:基线 %d 条 / 导出 %d 条", len(baseVisits), len(expVisits)) + } + common, placeOK, actOK := 0, 0, 0 + for u, br := range baseVisits { + er, ok := expVisits[u] + if !ok { + continue + } + common++ + bp, ba := describe(base, br.placePK, br.actPK) + ep, ea := describe(edb, er.placePK, er.actPK) + // 关键:**编号本身**要和基线一致(app 就是拿这个编号解引用的)。只看「名称+坐标」 + // 查不出问题——编号不一致时引用依然自洽,但手机那边解不出来。 + samePK := br.placePK.Valid == er.placePK.Valid && + (!br.placePK.Valid || br.placePK.Int64 == er.placePK.Int64) + if samePK { + placeOK++ + } else if placeOK+3 > common { // 只记前几条,避免刷屏 + // 少数不一致是正常的:真机自己就有重名的重复行(如同名车站两条), + // 地点没有 UUID,1:1 归属只能任选其一。所以例子只记日志,断言看比例。 + t.Logf("到访 %s 的地点编号不同:基线 #%v(%q) / 导出 #%v(%q)", + u[:8], br.placePK.Int64, bp, er.placePK.Int64, ep) + } + sameAct := br.actPK.Valid == er.actPK.Valid && + (!br.actPK.Valid || br.actPK.Int64 == er.actPK.Int64) + if sameAct { + actOK++ + } else if actOK+3 > common { + t.Logf("到访 %s 的活动编号不同:基线 #%v(%q) / 导出 #%v(%q)", + u[:8], br.actPK.Int64, ba, er.actPK.Int64, ea) + } + } + if common == 0 { + t.Fatal("基线与导出没有可对齐的到访") + } + if placeOK*100 < common*95 { + t.Errorf("地点引用与基线一致率过低:%d/%d(应 ≥95%%)", placeOK, common) + } + if actOK*100 < common*95 { + t.Errorf("活动引用与基线一致率过低:%d/%d(应 ≥95%%)", actOK, common) + } + t.Logf("按 UUID 对齐共同到访 %d 条:地点引用一致 %d、活动引用一致 %d", common, placeOK, actOK) + + // 对齐之后不许出现悬空引用(含自引用 ZMERGEDTO / ZPARENT) + for _, q := range []struct{ name, sql string }{ + {"visit→location", `SELECT count(*) FROM ZVISIT a WHERE a.ZLOCATION IS NOT NULL AND a.ZLOCATION<>0 + AND NOT EXISTS (SELECT 1 FROM ZLOCATION b WHERE b.Z_PK=a.ZLOCATION)`}, + {"visit→activity", `SELECT count(*) FROM ZVISIT a WHERE a.ZACTIVITY_ IS NOT NULL AND a.ZACTIVITY_<>0 + AND NOT EXISTS (SELECT 1 FROM ZACTIVITY b WHERE b.Z_PK=a.ZACTIVITY_)`}, + {"visit→mergedTo", `SELECT count(*) FROM ZVISIT a WHERE a.ZMERGEDTO IS NOT NULL AND a.ZMERGEDTO<>0 + AND NOT EXISTS (SELECT 1 FROM ZVISIT b WHERE b.Z_PK=a.ZMERGEDTO)`}, + {"visit→parent", `SELECT count(*) FROM ZVISIT a WHERE a.ZPARENT IS NOT NULL AND a.ZPARENT<>0 + AND NOT EXISTS (SELECT 1 FROM ZVISIT b WHERE b.Z_PK=a.ZPARENT)`}, + {"movement→visit", `SELECT count(*) FROM ZMOVEMENT a WHERE a.ZVISITFROM_ IS NOT NULL AND a.ZVISITFROM_<>0 + AND NOT EXISTS (SELECT 1 FROM ZVISIT b WHERE b.Z_PK=a.ZVISITFROM_)`}, + {"movement→transport", `SELECT count(*) FROM ZMOVEMENT a WHERE a.ZTRANSPORT_ IS NOT NULL AND a.ZTRANSPORT_<>0 + AND NOT EXISTS (SELECT 1 FROM ZTRANSPORT b WHERE b.Z_PK=a.ZTRANSPORT_)`}, + {"weather→visit", `SELECT count(*) FROM ZHOURLYWEATHER a WHERE a.ZVISIT IS NOT NULL AND a.ZVISIT<>0 + AND NOT EXISTS (SELECT 1 FROM ZVISIT b WHERE b.Z_PK=a.ZVISIT)`}, + } { + if n := countNull(t, edb, q.sql); n != 0 { + t.Errorf("编号对齐后出现 %d 条悬空引用:%s", n, q.name) + } + } +}