diff --git a/internal/db/schema.sql b/internal/db/schema.sql index a05b58c..d86e9b7 100644 --- a/internal/db/schema.sql +++ b/internal/db/schema.sql @@ -558,3 +558,37 @@ END $$; CREATE INDEX IF NOT EXISTS idx_place_images_key ON place_images(place_key, position); CREATE INDEX IF NOT EXISTS idx_pvv_key ON place_visitor_votes(place_key); + +-- 已并入 / 子段到访不收进地点的到访数与停留（2026-09-29） +-- +-- 真机里两行到访可能「不独立成立」：ZVISIT.ZMERGEDTO 指向它被并入的那条， +-- ZPARENT 说明它只是某个父段的一部分。app 对这两种行既不单独显示也不计数，本站以前 +-- 当成普通到访 ⇒ 时间线上出现「同一时刻同一地点两条」、日停留重复计入（实测数据集 9 +-- 有 54 + 10 条，26 条的到达时间与原件完全相同）。 +-- +-- 判据与 internal/stats/stats.go 的 SoloVisitSQL 同口径，只在**读**的一侧收口： +-- 导出的包照旧原样带着这些行（rond 靠它们认这次合并发生过），所以这里只动 +-- visit_count / dwell_minutes 两列，first_visit_at / last_visit_at 保持全量 +-- （后者会被导出成 ZLOCATION.ZLASTARRIVALDATE_）。 +-- +-- 只动「名下真有这种行」的地点，且幂等：重跑到收敛，没有这种行的库一次也不改行。 +UPDATE places p SET + visit_count = sub.c, + dwell_minutes = sub.d +FROM ( + SELECT v.place_id, + count(*) FILTER (WHERE NOT EXISTS (SELECT 1 FROM entity_raw er + WHERE er.dataset_id = v.dataset_id AND er.entity = 'ZVISIT' AND er.src_pk = v.src_pk + AND (er.raw->>'ZMERGEDTO' IS NOT NULL OR er.raw->>'ZPARENT' IS NOT NULL))) AS c, + COALESCE(sum(LEAST(COALESCE(v.duration_min,0), 10080)) FILTER (WHERE NOT EXISTS (SELECT 1 FROM entity_raw er + WHERE er.dataset_id = v.dataset_id AND er.entity = 'ZVISIT' AND er.src_pk = v.src_pk + AND (er.raw->>'ZMERGEDTO' IS NOT NULL OR er.raw->>'ZPARENT' IS NOT NULL))), 0) AS d + FROM visits v + WHERE v.place_id IS NOT NULL + GROUP BY v.place_id +) sub +WHERE p.id = sub.place_id + AND EXISTS (SELECT 1 FROM visits v JOIN entity_raw er + ON er.dataset_id = v.dataset_id AND er.entity = 'ZVISIT' AND er.src_pk = v.src_pk + WHERE v.place_id = p.id + AND (er.raw->>'ZMERGEDTO' IS NOT NULL OR er.raw->>'ZPARENT' IS NOT NULL)); diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 7c2dfd6..e16563d 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -17,6 +17,7 @@ import ( "rond-with-you/internal/geo" "rond-with-you/internal/rond" + "rond-with-you/internal/stats" ) type Stats struct { @@ -205,13 +206,6 @@ func Import(ctx context.Context, db *sql.DB, userID int64, originalName, sha str // 地点 placeID := map[int]int64{} - type locAgg struct { - count int - dwell int64 - first *time.Time - last *time.Time - } - agg := map[int]*locAgg{} locCoord := map[int][2]float64{} for _, l := range b.Locations { var id int64 @@ -225,7 +219,6 @@ func Import(ctx context.Context, db *sql.DB, userID int64, originalName, sha str return 0, nil, fmt.Errorf("写入地点: %w", err) } placeID[l.SrcPK] = id - agg[l.SrcPK] = &locAgg{} locCoord[l.SrcPK] = [2]float64{l.Lat, l.Lon} } @@ -282,18 +275,6 @@ func Import(ctx context.Context, db *sql.DB, userID int64, originalName, sha str vcount++ if v.LocationSrcPK != nil { - if a, ok := agg[*v.LocationSrcPK]; ok { - a.count++ - if dur != nil { - a.dwell += int64(dur.(int)) - } - if a.first == nil || v.Arrival.Before(*a.first) { - a.first = v.Arrival - } - if a.last == nil || v.Arrival.After(*a.last) { - a.last = v.Arrival - } - } visitPlace[v.SrcPK] = *v.LocationSrcPK } if first == nil || v.Arrival.Before(*first) { @@ -374,14 +355,6 @@ func Import(ctx context.Context, db *sql.DB, userID int64, originalName, sha str } } - // 回填地点聚合 - for src, a := range agg { - if _, err := tx.ExecContext(ctx, `UPDATE places SET visit_count=$1, dwell_minutes=$2, - first_visit_at=$3, last_visit_at=$4 WHERE id=$5`, a.count, a.dwell, a.first, a.last, placeID[src]); err != nil { - return 0, nil, fmt.Errorf("回填地点统计: %w", err) - } - } - // 汇总 st := &Stats{Visits: vcount, Skipped: skipped, Places: len(b.Locations), RawVisits: b.RawVisitNum, FirstVisit: first, LastVisit: last} if _, err := tx.ExecContext(ctx, `UPDATE datasets SET status='done', visit_count=$1, place_count=$2, @@ -403,6 +376,23 @@ func Import(ctx context.Context, db *sql.DB, userID int64, originalName, sha str return 0, nil, err } + // 回填地点聚合：走与刷新路径（refreshPlaceAgg / recountPlaces）同一条 SQL， + // 口径才不会在两处漂移——到访数与停留按访客口径（stats.SoloVisitSQL，不含已并入 / 子段）， + // 首末到访保持全量。原来这里在 Go 里另算一遍，还漏了停留上限（LEAST(...,10080)）。 + // + // ⚠️ 必须排在 insertEntityRaw **之后**：判据读的是 entity_raw，留档还没写盘时 + // 每一行都会被当成独立到访（实测会整整多算一个「已并入」的量，26 条）。 + if _, err := tx.ExecContext(ctx, `UPDATE places p SET visit_count=sub.c, dwell_minutes=sub.d, + first_visit_at=sub.f, last_visit_at=sub.l + FROM (SELECT vis.place_id, + count(*) FILTER (WHERE `+stats.SoloVisitSQL("vis")+`) c, + COALESCE(sum(LEAST(COALESCE(vis.duration_min,0),10080)) FILTER (WHERE `+stats.SoloVisitSQL("vis")+`),0) d, + min(vis.arrival) f, max(vis.arrival) l + FROM visits vis WHERE vis.dataset_id=$1 AND vis.place_id IS NOT NULL GROUP BY vis.place_id) sub + WHERE p.id = sub.place_id`, datasetID); err != nil { + return 0, nil, fmt.Errorf("回填地点统计: %w", err) + } + // 坐标来源留档要等 entity_raw 写完才能做（原始 GPS 只在留档行里）。 if _, err := BackfillCoordSource(ctx, tx, datasetID); err != nil { return 0, nil, err diff --git a/internal/stats/stats.go b/internal/stats/stats.go index 77aecd7..3e4960f 100644 --- a/internal/stats/stats.go +++ b/internal/stats/stats.go @@ -108,11 +108,40 @@ func (w *where) andSQL() string { return " AND " + strings.Join(w.conds, " AND ") } -// visitsWhere 构造作用于 visits 表别名 v 的过滤条件。 +// SoloVisitSQL 生成「这次到访是独立成立的」SQL 条件（alias 是 visits 表的别名）。 +// +// 真机把它**并入了别的到访**（ZVISIT.ZMERGEDTO），或它**只是某个父段的一部分**（ZVISIT.ZPARENT）时， +// app 既不单独显示它、也不把它算进次数与停留——本站以前把这两种行当成普通到访， +// 于是时间线上出现「同一时刻同一地点两条」（实测数据集 9 有 54 + 10 条，26 条的到达时间与原件完全相同）， +// 日停留也跟着重复计入。 +// +// ⚠️ 只在**读**的一侧收口：导出包必须原样带着这些行（rond 靠它们认这次合并发生过）， +// 所以 internal/rond 与 backup_diff / backup_merge 一律不加这个条件。 +// +// 判据只此一份，别在别处手抄——抄一份迟早跟这里漂移（区域限制那套就犯过一次）。 +func SoloVisitSQL(alias string) string { + return `NOT EXISTS (SELECT 1 FROM entity_raw er WHERE er.dataset_id=` + alias + `.dataset_id` + + ` AND er.entity='ZVISIT' AND er.src_pk=` + alias + `.src_pk` + + ` AND (er.raw->>'ZMERGEDTO' IS NOT NULL OR er.raw->>'ZPARENT' IS NOT NULL))` +} + +// visitsWhere 构造作用于 visits 表别名 v 的过滤条件（**访客口径**：已并入 / 子段的到访不算数）。 // off 用于嵌套：子查询里的占位符从 off+1 开始编号，避免与外层冲突。 -func (f Filter) visitsWhere(off int) *where { +func (f Filter) visitsWhere(off int) *where { return f.visitConds(off, true) } + +// visitConds 是 visitsWhere 的实现。 +// +// solo=false 时不筛「已并入 / 子段」——天气与位移用的是这一支：那几小时、那段路是真的 +// 发生过的，不该因为这次到访在 app 里不单独显示就从统计里消失（实测数据集 9 有 12 条 +// 天气行、5 条位移挂在被并入的行上）。 +func (f Filter) visitConds(off int, solo bool) *where { w := &where{off: off} w.cond("v.dataset_id = ?", f.DatasetID) + // 已并入 / 子段不算一条独立到访（判据见 SoloVisitSQL）。放在这里是为了**全域一致**： + // 时间线、首页、统计、城市 / 省份、活动计数、星期分布、地点列的到访数都从这份条件派生。 + if solo { + w.cond(SoloVisitSQL("v")) + } if f.From != nil { w.cond("v.arrival >= ?", *f.From) } @@ -204,7 +233,7 @@ func (f Filter) weatherWhere() *where { if f.hasVisitConds() { rest := f rest.From, rest.To = nil, nil // 时间已经按 w.at 判过了 - vw := rest.visitsWhere(len(w.args)) + vw := rest.visitConds(len(w.args), false) w.conds = append(w.conds, "EXISTS (SELECT 1 FROM visits v WHERE v.src_pk = w.visit_src AND "+vw.expr()+")") w.args = append(w.args, vw.args...) @@ -221,13 +250,14 @@ func (f Filter) weatherWhere() *where { // 清掉三类条件，剩下的正是可以按到访判的： // - 时间：位移按 movements.started_at 判，再按到访的 arrival 判一次就是另一套口径； // - 区域：位移要按**两端 places** 判（任一端命中即算），见 movementsWhere； -// - 家 / 工作：那边是「两端都不是」的 NOT EXISTS，塞进这里的 EXISTS 会松成「任一端不是」。 +// - 家 / 工作：那边是「两端都不是」的 NOT EXISTS，塞进这里的 EXISTS 会松成「任一端不是」； +// - 已并入 / 子段：位移照算（里程与轨迹是真实发生过的），见 visitConds 的 solo 参数。 func (f Filter) movementVisitConds(off int) *where { rest := f rest.From, rest.To = nil, nil rest.Regions = nil rest.ExcludeHome, rest.ExcludeWork = false, false - return rest.visitsWhere(off) + return rest.visitConds(off, false) } // hasMovementVisitConds 表示「带上了只有到访才判得出的筛选条件」。 @@ -735,23 +765,28 @@ type Place struct { // 每个相关子查询都带上 dataset_id = p.dataset_id：它是 idx_visits_place(dataset_id, place_id) // 的前导列，缺了它每次循环都要为库里每个 dataset 各下一次索引（实测 Buffer 命中 23737 → 5669）。 // 谓词本身冗余（到访与所属地点的 dataset 必然相同），纯粹是给计划器的。 -const placeCols = `p.id, COALESCE(p.name,''), COALESCE(p.poi_category,''), COALESCE(p.lat,0), COALESCE(p.lon,0), +var placeCols = `p.id, COALESCE(p.name,''), COALESCE(p.poi_category,''), COALESCE(p.lat,0), COALESCE(p.lon,0), COALESCE(p.city,''), COALESCE(p.province,''), COALESCE(p.district,''), COALESCE(p.timezone,''), - p.visit_count, p.dwell_minutes, p.first_visit_at, p.last_visit_at, + p.visit_count, p.dwell_minutes, + -- 首末到访这两列**存的是全量**（last_visit_at 会写成导出包里的 ZLOCATION.ZLASTARRIVALDATE_， + -- 不能在库里改口径）。于是「到访数按访客口径、日期按全量」会凑出「0 次却有最近到访」这种 + -- 自相矛盾的行（实测数据集 9 有 4 个地点名下只剩已并入 / 子段到访），这里按可见到访数兜一下。 + CASE WHEN p.visit_count > 0 THEN p.first_visit_at END, + CASE WHEN p.visit_count > 0 THEN p.last_visit_at END, COALESCE((SELECT a.name FROM visits v2 JOIN activities a ON a.id=v2.activity_id - WHERE v2.place_id=p.id AND v2.dataset_id=p.dataset_id GROUP BY a.name ORDER BY count(*) DESC LIMIT 1),''), + WHERE v2.place_id=p.id AND v2.dataset_id=p.dataset_id AND ` + SoloVisitSQL("v2") + ` GROUP BY a.name ORDER BY count(*) DESC LIMIT 1),''), COALESCE((SELECT a.color FROM visits v2 JOIN activities a ON a.id=v2.activity_id - WHERE v2.place_id=p.id AND v2.dataset_id=p.dataset_id GROUP BY a.color ORDER BY count(*) DESC LIMIT 1),''), + WHERE v2.place_id=p.id AND v2.dataset_id=p.dataset_id AND ` + SoloVisitSQL("v2") + ` GROUP BY a.color ORDER BY count(*) DESC LIMIT 1),''), COALESCE((SELECT a.icon FROM visits v2 JOIN activities a ON a.id=v2.activity_id - WHERE v2.place_id=p.id AND v2.dataset_id=p.dataset_id GROUP BY a.icon ORDER BY count(*) DESC LIMIT 1),''), - EXISTS (SELECT 1 FROM visits v3 WHERE v3.place_id=p.id AND v3.dataset_id=p.dataset_id AND v3.is_home), - EXISTS (SELECT 1 FROM visits v4 WHERE v4.place_id=p.id AND v4.dataset_id=p.dataset_id AND v4.is_work), + WHERE v2.place_id=p.id AND v2.dataset_id=p.dataset_id AND ` + SoloVisitSQL("v2") + ` GROUP BY a.icon ORDER BY count(*) DESC LIMIT 1),''), + EXISTS (SELECT 1 FROM visits v3 WHERE v3.place_id=p.id AND v3.dataset_id=p.dataset_id AND v3.is_home AND ` + SoloVisitSQL("v3") + `), + EXISTS (SELECT 1 FROM visits v4 WHERE v4.place_id=p.id AND v4.dataset_id=p.dataset_id AND v4.is_work AND ` + SoloVisitSQL("v4") + `), p.src_pk, EXISTS (SELECT 1 FROM place_images pi WHERE pi.place_key = to_char(p.lat,'FM999990.000000')||'|'||to_char(p.lon,'FM999990.000000'))` // PlaceCols 与 ScanPlace 配对，供包外构造只取单个地点的查询。 -const PlaceCols = placeCols +var PlaceCols = placeCols func scanPlace(s interface{ Scan(...any) error }) (Place, error) { var p Place @@ -946,7 +981,8 @@ func (d DB) VisitsByPlace(ctx context.Context, datasetID, placeID int64, limit i q := `SELECT ` + visitCols + ` FROM visits v LEFT JOIN places p ON p.id = v.place_id LEFT JOIN activities a ON a.id = v.activity_id - WHERE v.dataset_id=$1 AND v.place_id=$2 ORDER BY v.arrival DESC LIMIT ` + fmt.Sprint(limit) + WHERE v.dataset_id=$1 AND v.place_id=$2 AND ` + SoloVisitSQL("v") + + ` ORDER BY v.arrival DESC LIMIT ` + fmt.Sprint(limit) rows, err := d.QueryContext(ctx, q, datasetID, placeID) if err != nil { return nil, err @@ -967,7 +1003,8 @@ func (d DB) Bookmarked(ctx context.Context, datasetID int64, limit int) ([]Visit q := `SELECT ` + visitCols + ` FROM visits v LEFT JOIN places p ON p.id = v.place_id LEFT JOIN activities a ON a.id = v.activity_id - WHERE v.dataset_id=$1 AND v.bookmarked ORDER BY v.arrival DESC LIMIT ` + fmt.Sprint(limit) + WHERE v.dataset_id=$1 AND v.bookmarked AND ` + SoloVisitSQL("v") + + ` ORDER BY v.arrival DESC LIMIT ` + fmt.Sprint(limit) rows, err := d.QueryContext(ctx, q, datasetID) if err != nil { return nil, err diff --git a/internal/stats/stats_test.go b/internal/stats/stats_test.go index b56234b..a78ced9 100644 --- a/internal/stats/stats_test.go +++ b/internal/stats/stats_test.go @@ -43,9 +43,16 @@ func TestOverviewMatchesManualSQL(t *testing.T) { } var places, cities, days, visits, movements int var dist float64 + // 判据按**访客口径**在手写 SQL 里独立写一遍（不复用 stats.SoloVisitSQL）：已并入 / 子段的 + // 到访在 app 里既不算次数也不占天数——独立写一份，正好在口径写错时把它抓出来。 + // 位移/里程那两行**不**加这个条件：那段路是真的发生过的，收口只管到访本身。 if err := db.QueryRow(`SELECT count(DISTINCT v.place_id), count(DISTINCT p.city), count(DISTINCT (v.arrival AT TIME ZONE 'Asia/Shanghai')::date), count(*) - FROM visits v LEFT JOIN places p ON p.id = v.place_id WHERE v.dataset_id=$1`, dsID). + FROM visits v LEFT JOIN places p ON p.id = v.place_id + WHERE v.dataset_id=$1 + AND NOT EXISTS (SELECT 1 FROM entity_raw er + WHERE er.dataset_id=v.dataset_id AND er.entity='ZVISIT' AND er.src_pk=v.src_pk + AND (er.raw->>'ZMERGEDTO' IS NOT NULL OR er.raw->>'ZPARENT' IS NOT NULL))`, dsID). Scan(&places, &cities, &days, &visits); err != nil { t.Fatal(err) } diff --git a/internal/web/backup_merge.go b/internal/web/backup_merge.go index 0a16ec5..282d4b7 100644 --- a/internal/web/backup_merge.go +++ b/internal/web/backup_merge.go @@ -14,6 +14,7 @@ import ( "rond-with-you/internal/geo" "rond-with-you/internal/ingest" "rond-with-you/internal/rond" + "rond-with-you/internal/stats" ) // 差异页确认后的合并执行。 @@ -755,6 +756,9 @@ func (s *Server) applyDeletions(ctx context.Context, ex execer, dsID int64, plan // recountPlaces 重算地点聚合与数据集汇总。全量重算而不是只动受影响的行： // 367 个地点两条 SQL 就够，省得漏掉任何一条派生列。 +// +// 到访数与停留按**访客口径**（不含已并入 / 子段，见 stats.SoloVisitSQL），与 stats.Filter 一致； +// 首末到访与数据集计数保持全量（last_visit_at 会写进导出包，datasets.visit_count 是后台口径）。 func (s *Server) recountPlaces(ctx context.Context, ex execer, dsID int64) error { if _, err := ex.ExecContext(ctx, `UPDATE places SET visit_count=0, dwell_minutes=0, first_visit_at=NULL, last_visit_at=NULL WHERE dataset_id=$1`, dsID); err != nil { @@ -762,10 +766,11 @@ func (s *Server) recountPlaces(ctx context.Context, ex execer, dsID int64) error } if _, err := ex.ExecContext(ctx, `UPDATE places p SET visit_count=sub.c, dwell_minutes=sub.d, first_visit_at=sub.f, last_visit_at=sub.l - FROM (SELECT place_id, count(*) c, - COALESCE(sum(LEAST(COALESCE(duration_min,0),10080)),0) d, - min(arrival) f, max(arrival) l - FROM visits WHERE dataset_id=$1 AND place_id IS NOT NULL GROUP BY place_id) sub + FROM (SELECT vis.place_id, + count(*) FILTER (WHERE `+stats.SoloVisitSQL("vis")+`) c, + COALESCE(sum(LEAST(COALESCE(vis.duration_min,0),10080)) FILTER (WHERE `+stats.SoloVisitSQL("vis")+`),0) d, + min(vis.arrival) f, max(vis.arrival) l + FROM visits vis WHERE vis.dataset_id=$1 AND vis.place_id IS NOT NULL GROUP BY vis.place_id) sub WHERE p.id = sub.place_id`, dsID); err != nil { return err } diff --git a/internal/web/edit_duplicates.go b/internal/web/edit_duplicates.go index 664e1b8..0c8469f 100644 --- a/internal/web/edit_duplicates.go +++ b/internal/web/edit_duplicates.go @@ -80,7 +80,7 @@ type dupPlace struct { // 模式（VACUUM INTO 的产物），只做 SELECT 不会落下任何旁写文件，再加 query_only 兜住 // 「误写基线库」这条路——基线是导出的底稿，写坏了整个数据集就导不出去了。 func readDupBaseline(ctx context.Context, path string) (*dupBaseline, error) { - sdb, err := sql.Open("sqlite", "file:"+filepath.ToSlash(path)+"?_pragma=query_only(1)") + sdb, err := sql.Open("sqlite", "file:"+filepath.ToSlash(path)+"?_pragma=query_only(1)&_pragma=busy_timeout(5000)") if err != nil { return nil, err } diff --git a/internal/web/handlers_edit.go b/internal/web/handlers_edit.go index d7a8f33..71605bf 100644 --- a/internal/web/handlers_edit.go +++ b/internal/web/handlers_edit.go @@ -15,6 +15,7 @@ import ( "rond-with-you/internal/geo" "rond-with-you/internal/rond" + "rond-with-you/internal/stats" ) // ---------- 数据编辑：手动补录没带手机时的记录点，改完可导出回 rondbackup ---------- @@ -282,8 +283,13 @@ func (s *Server) editPage(w http.ResponseWriter, r *http.Request) { vargs = append(vargs, editPerPage, (d.VisitPageNo-1)*editPerPage) vrows, err := s.db.QueryContext(ctx, `SELECT v.src_pk, COALESCE(p.name,''), COALESCE(p.city,''), COALESCE(a.name,''), COALESCE(a.id,0), v.arrival, v.departure, COALESCE(v.remark,''), COALESCE(v.emoji,''), - v.bookmarked, v.is_user_added + v.bookmarked, v.is_user_added, + -- 真机里「不独立成立」的那两种行（别处已并入 / 只是某个父段的一部分）：访客侧不显示它们， + -- 这一页要看得到全部原始行，所以只打标、不过滤（口径见 stats.SoloVisitSQL） + CASE WHEN er.raw->>'ZMERGEDTO' IS NOT NULL THEN '已并入 #'||(er.raw->>'ZMERGEDTO') + WHEN er.raw->>'ZPARENT' IS NOT NULL THEN '子段 #'||(er.raw->>'ZPARENT') ELSE '' END FROM visits v LEFT JOIN places p ON p.id=v.place_id LEFT JOIN activities a ON a.id=v.activity_id + LEFT JOIN entity_raw er ON er.dataset_id=v.dataset_id AND er.entity='ZVISIT' AND er.src_pk=v.src_pk WHERE `+vwhere+fmt.Sprintf(` ORDER BY v.arrival DESC LIMIT $%d OFFSET $%d`, len(vargs)-1, len(vargs)), vargs...) if err != nil { @@ -294,7 +300,7 @@ func (s *Server) editPage(w http.ResponseWriter, r *http.Request) { var e EditVisit var dep sql.NullTime if err := vrows.Scan(&e.SrcPK, &e.PlaceName, &e.City, &e.ActivityName, &e.ActivityID, - &e.Arrival, &dep, &e.Remark, &e.Emoji, &e.Bookmarked, &e.UserAdded); err != nil { + &e.Arrival, &dep, &e.Remark, &e.Emoji, &e.Bookmarked, &e.UserAdded, &e.Hidden); err != nil { vrows.Close() s.serverError(w, r, err) return @@ -2261,13 +2267,21 @@ func (s *Server) saveVisitTags(ctx context.Context, ex execer, datasetID int64, } // refreshPlaceAgg 重算某地点的到访聚合（次数 / 停留 / 首末时间）。 +// +// 次数与停留按**访客口径**算（不含已并入 / 子段，判据见 stats.SoloVisitSQL）——编辑页删掉一条 +// 普通到访后，「地点库」上那个「到访 N 次」要跟时间线对得上。 +// first_visit_at / last_visit_at 保持全量：last_visit_at 会被导出成 ZLOCATION.ZLASTARRIVALDATE_， +// 这里改口径就等于动导出包（实测数据集 9 有 7 个地点的「最晚到达」是由被并入的行决定的）。 func (s *Server) refreshPlaceAgg(ctx context.Context, ex execer, placeID int64) error { if _, err := ex.ExecContext(ctx, `UPDATE places p SET visit_count = COALESCE(v.c,0), dwell_minutes = COALESCE(v.d,0), first_visit_at = v.f, last_visit_at = v.l FROM (SELECT $1::bigint AS pid) x - LEFT JOIN (SELECT place_id, count(*) c, COALESCE(sum(LEAST(COALESCE(duration_min,0), 10080)),0) d, min(arrival) f, max(arrival) l - FROM visits WHERE place_id=$1 GROUP BY place_id) v ON v.place_id = x.pid + LEFT JOIN (SELECT vis.place_id, + count(*) FILTER (WHERE `+stats.SoloVisitSQL("vis")+`) c, + COALESCE(sum(LEAST(COALESCE(vis.duration_min,0), 10080)) FILTER (WHERE `+stats.SoloVisitSQL("vis")+`),0) d, + min(vis.arrival) f, max(vis.arrival) l + FROM visits vis WHERE vis.place_id=$1 GROUP BY vis.place_id) v ON v.place_id = x.pid WHERE p.id = x.pid`, placeID); err != nil { return fmt.Errorf("重算地点（id=%d）聚合失败: %w", placeID, err) } diff --git a/internal/web/handlers_public.go b/internal/web/handlers_public.go index 66aa251..f44bf3a 100644 --- a/internal/web/handlers_public.go +++ b/internal/web/handlers_public.go @@ -867,7 +867,7 @@ func (s *Server) firstArrivalCities(ctx context.Context, datasetID int64, from, regSQL := regionSQL("p", regions, exclude, &args) q := `SELECT COALESCE(p.city,''), COALESCE(p.province,''), min(v.arrival)::date FROM visits v JOIN places p ON p.id = v.place_id - WHERE v.dataset_id = $1 AND p.city IS NOT NULL AND p.city <> ''` + regSQL + ` + WHERE v.dataset_id = $1 AND p.city IS NOT NULL AND p.city <> '' AND ` + stats.SoloVisitSQL("v") + regSQL + ` GROUP BY p.city, p.province HAVING min(v.arrival) >= $2 AND min(v.arrival) < $3 ORDER BY min(v.arrival) ASC` @@ -899,7 +899,8 @@ func (s *Server) extremePlace(ctx context.Context, datasetID int64, from, to tim args := []any{datasetID, from, to} q := `SELECT ` + stats.PlaceCols + ` FROM places p WHERE p.dataset_id=$1 AND p.lat IS NOT NULL - AND EXISTS (SELECT 1 FROM visits v WHERE v.place_id=p.id AND v.arrival >= $2 AND v.arrival < $3)` + + AND EXISTS (SELECT 1 FROM visits v WHERE v.place_id=p.id AND v.arrival >= $2 AND v.arrival < $3 + AND ` + stats.SoloVisitSQL("v") + `)` + regionSQL("p", regions, exclude, &args) + ` ORDER BY p.lat ` + dir + ` LIMIT 1` if err := stats.ScanPlace(s.db.QueryRowContext(ctx, q, args...), &p); err != nil { diff --git a/internal/web/handlers_topics.go b/internal/web/handlers_topics.go index c878d17..d67ffb7 100644 --- a/internal/web/handlers_topics.go +++ b/internal/web/handlers_topics.go @@ -12,6 +12,8 @@ import ( "strconv" "strings" "time" + + "rond-with-you/internal/stats" ) // 专题 = 某段时间 + 某个区域内的足迹。地点来自「时间范围内的到访」， @@ -118,14 +120,14 @@ func (s *Server) topicContent(ctx context.Context, t *Topic, datasetID int64, q := `SELECT p.id, p.src_pk, COALESCE(p.name,''), COALESCE(p.lat,0), COALESCE(p.lon,0), COALESCE(p.city,''), COALESCE((SELECT a.name FROM visits v2 LEFT JOIN activities a ON a.id=v2.activity_id - WHERE v2.place_id=p.id AND v2.activity_id IS NOT NULL + WHERE v2.place_id=p.id AND v2.activity_id IS NOT NULL AND ` + stats.SoloVisitSQL("v2") + ` GROUP BY a.name ORDER BY count(*) DESC LIMIT 1),''), COALESCE((SELECT a.color FROM visits v2 LEFT JOIN activities a ON a.id=v2.activity_id - WHERE v2.place_id=p.id AND v2.activity_id IS NOT NULL + WHERE v2.place_id=p.id AND v2.activity_id IS NOT NULL AND ` + stats.SoloVisitSQL("v2") + ` GROUP BY a.color ORDER BY count(*) DESC LIMIT 1),''), count(*), COALESCE(sum(LEAST(COALESCE(v.duration_min,0), 10080)),0), max(v.arrival) FROM places p JOIN visits v ON v.place_id = p.id - WHERE p.dataset_id=$1 AND v.arrival >= $2 AND v.arrival <= $3` + area + + WHERE p.dataset_id=$1 AND v.arrival >= $2 AND v.arrival <= $3 AND ` + stats.SoloVisitSQL("v") + area + regionSQL("p", regions, exclude, &args) + ` GROUP BY p.id ORDER BY count(*) DESC, p.id` rows, err := s.db.QueryContext(ctx, q, args...) @@ -168,7 +170,7 @@ func (s *Server) topicContent(ctx context.Context, t *Topic, datasetID int64, daysRegion := regionSQL("p", regions, exclude, &dargs) if err := s.db.QueryRowContext(ctx, `SELECT count(DISTINCT (v.arrival AT TIME ZONE 'Asia/Shanghai')::date) FROM visits v JOIN places p ON p.id = v.place_id - WHERE p.dataset_id=$1 AND v.arrival >= $2 AND v.arrival <= $3`+area+daysRegion, dargs...).Scan(&days); err != nil { + WHERE p.dataset_id=$1 AND v.arrival >= $2 AND v.arrival <= $3 AND `+stats.SoloVisitSQL("v")+area+daysRegion, dargs...).Scan(&days); err != nil { return nil, 0, 0, 0, err } return points, totalVisits, days, len(cities), nil @@ -201,7 +203,7 @@ func (s *Server) fillTopicStatsBatch(ctx context.Context, list []Topic, datasetI count(DISTINCT NULLIF(p.city,'')), COALESCE(sum(LEAST(COALESCE(v.duration_min,0),10080)),0) FROM places p JOIN visits v ON v.place_id = p.id - WHERE p.dataset_id=$%d AND v.arrival >= $%d AND v.arrival <= $%d%s%s`, + WHERE p.dataset_id=$%d AND v.arrival >= $%d AND v.arrival <= $%d AND `+stats.SoloVisitSQL("v")+`%s%s`, i, base+1, base+2, base+3, area, region) } rows, err := s.db.QueryContext(ctx, b.String(), args...) diff --git a/internal/web/hidden_visit_test.go b/internal/web/hidden_visit_test.go new file mode 100644 index 0000000..6705b6a --- /dev/null +++ b/internal/web/hidden_visit_test.go @@ -0,0 +1,250 @@ +package web + +import ( + "database/sql" + "fmt" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "rond-with-you/internal/db" + "rond-with-you/internal/stats" +) + +// 真机里「不独立成立」的到访（已并入别处 ZMERGEDTO / 某个父段的子段 ZPARENT）只在**读**的一侧收口： +// 时间线、统计、地点计数不显示它们，编辑页照样看得到（打标），而导出的包一个字节都不能少 +// ——rond 靠这些行认「这次合并发生过」，删掉或丢掉就是毁真机的记录。 + +// exportVisitRow 在导出包里按 UUID 找一行到访，返回它在包里的 Z_PK 与 ZMERGEDTO。 +// 必须按 UUID 找：（编号也要对，但按编号找会因为对齐而误判）。 +func exportVisitRow(t *testing.T, edb *sql.DB, uuidHex string) (int, sql.NullInt64, bool) { + t.Helper() + var pk int + var merged sql.NullInt64 + err := edb.QueryRow(`SELECT Z_PK, ZMERGEDTO FROM ZVISIT WHERE hex(ZIDENTIFIER_)=?`, + strings.ToUpper(uuidHex)).Scan(&pk, &merged) + if err == sql.ErrNoRows { + return 0, merged, false + } + if err != nil { + t.Fatalf("查导出包失败: %v", err) + } + return pk, merged, true +} + +func TestHiddenVisitStaysInExportButLeavesReads(t *testing.T) { + f := newMovementFixture(t) + ctx, pg, dsID := f.ctx, f.pg, f.dsID + srv := f.srv + + // 夹具导入的是真机包，本来就带这类行；没有的话下面的增量断言就不成立 + var fixtureHidden int + if err := pg.QueryRowContext(ctx, `SELECT count(*) FROM entity_raw + WHERE dataset_id=$1 AND entity='ZVISIT' + AND (raw->>'ZMERGEDTO' IS NOT NULL OR raw->>'ZPARENT' IS NOT NULL)`, dsID).Scan(&fixtureHidden); err != nil { + t.Fatal(err) + } + t.Logf("夹具自带「不独立成立」的到访 %d 条", fixtureHidden) + + // 挑一条正常到访，把它标成「已并入 #target」 + var victim, target int + var placeID int64 + var durMin int + if err := pg.QueryRowContext(ctx, `SELECT v.src_pk, COALESCE(v.place_id,0), COALESCE(v.duration_min,0) + FROM visits v JOIN entity_raw er ON er.dataset_id=v.dataset_id AND er.entity='ZVISIT' AND er.src_pk=v.src_pk + WHERE v.dataset_id=$1 AND v.place_id IS NOT NULL + AND er.raw->>'ZMERGEDTO' IS NULL AND er.raw->>'ZPARENT' IS NULL + ORDER BY v.src_pk DESC LIMIT 1`, dsID).Scan(&victim, &placeID, &durMin); err != nil { + t.Fatalf("挑不出可标记的到访: %v", err) + } + if err := pg.QueryRowContext(ctx, `SELECT src_pk FROM visits + WHERE dataset_id=$1 AND src_pk<>$2 ORDER BY src_pk LIMIT 1`, dsID, victim).Scan(&target); err != nil { + t.Fatal(err) + } + victimUUID := visitUUIDHex(t, srv, ctx, dsID, victim) + if victimUUID == "" { + t.Fatal("挑中的到访没有留档 UUID，用例前提不成立") + } + + // 标记前：访客侧看得到它，导出包里它没有合并标记 + flt := stats.Filter{DatasetID: dsID} + _, totalBefore, err := srv.q.Visits(ctx, flt, 5, 0, "desc") + if err != nil { + t.Fatal(err) + } + var pcBefore, dwellBefore int64 + if err := pg.QueryRowContext(ctx, `SELECT visit_count, dwell_minutes FROM places WHERE id=$1`, placeID). + Scan(&pcBefore, &dwellBefore); err != nil { + t.Fatal(err) + } + var mvCount int + var mvKm float64 + pg.QueryRowContext(ctx, `SELECT count(*), COALESCE(sum(distance_km),0) FROM movements WHERE dataset_id=$1`, dsID). + Scan(&mvCount, &mvKm) + var dsCountBefore int + pg.QueryRowContext(ctx, `SELECT visit_count FROM datasets WHERE id=$1`, dsID).Scan(&dsCountBefore) + + edbBefore := openExport(t, f.export(t)) + if _, merged, ok := exportVisitRow(t, edbBefore, victimUUID); !ok { + t.Fatalf("标记前导出包里就找不到 #%d", victim) + } else if merged.Valid { + t.Fatalf("标记前 ZMERGEDTO 就不该有值，实际 %v", merged.Int64) + } + + // 标记成「已并入 #target」 + if _, err := pg.ExecContext(ctx, `UPDATE entity_raw + SET raw = jsonb_set(raw, '{ZMERGEDTO}', to_jsonb($3::int), true) + WHERE dataset_id=$1 AND entity='ZVISIT' AND src_pk=$2`, dsID, victim, target); err != nil { + t.Fatalf("标记失败: %v", err) + } + + // 1) 访客侧：时间线少了一条，且被标记的那条不再出现 + rows, totalAfter, err := srv.q.Visits(ctx, flt, 5, 0, "desc") + if err != nil { + t.Fatal(err) + } + if totalAfter != totalBefore-1 { + t.Errorf("时间线总数应为 %d，实际 %d", totalBefore-1, totalAfter) + } + for _, r := range rows { + if r.SrcPK == victim { + t.Errorf("被标记为「已并入」的 #%d 仍出现在时间线里", victim) + } + } + // 地点详情页的到访列表、收藏列表走的是另外两条自建 where，也要一起收口 + if vs, err := srv.q.VisitsByPlace(ctx, dsID, placeID, 500); err != nil { + t.Fatal(err) + } else { + for _, v := range vs { + if v.SrcPK == victim { + t.Errorf("地点页的到访列表里仍有 #%d", victim) + } + } + } + + // 2) 地点的到访数 / 停留跟着少（编辑页删改之后也会重算，口径必须一致） + if err := srv.refreshPlaceAgg(ctx, pg, placeID); err != nil { + t.Fatalf("重算地点聚合失败: %v", err) + } + var pcAfter, dwellAfter int64 + if err := pg.QueryRowContext(ctx, `SELECT visit_count, dwell_minutes FROM places WHERE id=$1`, placeID). + Scan(&pcAfter, &dwellAfter); err != nil { + t.Fatal(err) + } + capped := durMin + if capped > 10080 { + capped = 10080 + } + if pcAfter != pcBefore-1 || dwellAfter != dwellBefore-int64(capped) { + t.Errorf("地点聚合应为 %d 次 / %d 分，实际 %d 次 / %d 分", + pcBefore-1, dwellBefore-int64(capped), pcAfter, dwellAfter) + } + + // 3) 位移（里程 / 轨迹）不动：那条到访不单独显示，但它参与过的行程真实发生过 + var mvCountAfter int + var mvKmAfter float64 + pg.QueryRowContext(ctx, `SELECT count(*), COALESCE(sum(distance_km),0) FROM movements WHERE dataset_id=$1`, dsID). + Scan(&mvCountAfter, &mvKmAfter) + if mvCountAfter != mvCount || mvKmAfter != mvKm { + t.Errorf("位移不该受影响：%d 段 / %.3f km → %d 段 / %.3f km", mvCount, mvKm, mvCountAfter, mvKmAfter) + } + // 数据集计数是后台口径（原始行数），不跟着访客口径变 + var dsCountAfter int + pg.QueryRowContext(ctx, `SELECT visit_count FROM datasets WHERE id=$1`, dsID).Scan(&dsCountAfter) + if dsCountAfter != dsCountBefore { + t.Errorf("datasets.visit_count 是后台口径，不该变成 %d（原 %d）", dsCountAfter, dsCountBefore) + } + + // 4) 标了之后仍然导得出去，而且这一行连合并标记一起原样在包里 + edbAfter := openExport(t, f.export(t)) + pk, merged, ok := exportVisitRow(t, edbAfter, victimUUID) + if !ok { + t.Fatalf("导出包里丢了被标记的到访 #%d —— 读侧收口绝不能影响导出", victim) + } + if !merged.Valid || int(merged.Int64) != target { + t.Errorf("导出包里 ZMERGEDTO 应为 #%d，实际 %v", target, merged) + } + if pk != victim { + t.Errorf("包内编号应为 %d，实际 %d", victim, pk) + } + + // 5) 管理端（数据编辑页）照样看得到它，并带上说明——不然站长会以为数据丢了 + rec := httptest.NewRecorder() + srv.editPage(rec, httptest.NewRequest("GET", fmt.Sprintf("/admin/edit?vq=%d", victim), nil).WithContext(ctx)) + if rec.Code != 200 { + t.Fatalf("渲染数据编辑页失败: code=%d", rec.Code) + } + if body := rec.Body.String(); !strings.Contains(body, fmt.Sprintf("已并入 #%d", target)) { + t.Errorf("编辑页没有给 #%d 打上「已并入 #%d」的标记", victim, target) + } +} + +// TestHiddenVisitBackfillFixesStaleAggregates 存量库的那段回填（schema.sql）必须把旧口径的数字拉正。 +// +// 只在读的一侧收口的话，已经算好的 places.visit_count / dwell_minutes 会一直偏大 +// （要等到有人编辑那个地点才重算），所以迁移里带一段幂等的重算。这里真的跑一次 Migrate， +// 而不是另抄一份 SQL——抄一份就等于没测那段。 +func TestHiddenVisitBackfillFixesStaleAggregates(t *testing.T) { + f := newMovementFixture(t) + ctx, pg, dsID := f.ctx, f.pg, f.dsID + + // 造出「旧口径」的偏大值：假设那条已并入的到访也被算进去了 + var placeID int64 + var want int64 + if err := pg.QueryRowContext(ctx, `SELECT p.id, p.visit_count FROM places p + WHERE p.dataset_id=$1 AND EXISTS (SELECT 1 FROM visits v JOIN entity_raw er + ON er.dataset_id=v.dataset_id AND er.entity='ZVISIT' AND er.src_pk=v.src_pk + WHERE v.place_id=p.id AND er.raw->>'ZMERGEDTO' IS NOT NULL) LIMIT 1`, dsID).Scan(&placeID, &want); err != nil { + t.Skipf("夹具里没有「已并入」的到访（%v）", err) + } + if _, err := pg.ExecContext(ctx, `UPDATE places SET visit_count=$2, dwell_minutes=dwell_minutes+1 WHERE id=$1`, + placeID, want+1); err != nil { + t.Fatal(err) + } + + if err := db.Migrate(ctx, pg); err != nil { + t.Fatalf("跑迁移失败: %v", err) + } + + var got int64 + if err := pg.QueryRowContext(ctx, `SELECT visit_count FROM places WHERE id=$1`, placeID).Scan(&got); err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("回填后应回到 %d 次，实际 %d 次——存量库的旧口径没被拉正", want, got) + } +} + +// TestSoloVisitCondSharedAndOffTheExport 静态守卫：这条判据只能有一份，而且不许出现在导出 / 差异侧。 +// +// 分散手抄迟早漂移（区域限制那套就漏过一次，表现是「到访藏了里程还在」）； +// 反过来，导出与合并一旦用了它，包里就会少掉真机自己的行。 +func TestSoloVisitCondSharedAndOffTheExport(t *testing.T) { + root := moduleRoot(t) + read := func(rel string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(rel))) + if err != nil { + t.Fatalf("读 %s 失败: %v", rel, err) + } + return string(b) + } + // 导出与差异必须看到全部行 + for _, rel := range []string{"internal/rond/export.go", "internal/web/backup_diff.go"} { + if strings.Contains(read(rel), "SoloVisitSQL") { + t.Errorf("%s 不该用 SoloVisitSQL：导出与差异要看到真机的全部行", rel) + } + } + // 读的一侧共用同一份判据 + for _, rel := range []string{ + "internal/stats/stats.go", "internal/ingest/ingest.go", + "internal/web/handlers_public.go", "internal/web/handlers_topics.go", + "internal/web/handlers_edit.go", "internal/web/backup_merge.go", + } { + if !strings.Contains(read(rel), "SoloVisitSQL") { + t.Errorf("%s 没有用 SoloVisitSQL：到访数与时间线会对不上", rel) + } + } +} diff --git a/internal/web/templates/admin/edit.html b/internal/web/templates/admin/edit.html index 8fb6920..4911b39 100644 --- a/internal/web/templates/admin/edit.html +++ b/internal/web/templates/admin/edit.html @@ -416,7 +416,7 @@ {{range .Visits}} {{.ArrivalStr}}{{if .DepartureStr}}
→ {{.DepartureStr}}{{else}}
（未离开）{{end}} - #{{.SrcPK}} {{if .PlaceName}}{{.PlaceName}}{{else}}—{{end}}{{if .City}} · {{.City}}{{end}} + #{{.SrcPK}} {{if .PlaceName}}{{.PlaceName}}{{else}}—{{end}}{{if .City}} · {{.City}}{{end}}{{if .Hidden}} {{.Hidden}}{{end}} {{.ActivityName}}{{if .UserAdded}} 补录{{end}} {{.Emoji}}{{if .Remark}}{{.Remark}}{{end}}
diff --git a/internal/web/view.go b/internal/web/view.go index 7f31aab..40f6318 100644 --- a/internal/web/view.go +++ b/internal/web/view.go @@ -1056,6 +1056,9 @@ type EditVisit struct { Emoji string Bookmarked bool UserAdded bool + // Hidden 非空表示这一行在真机里不独立成立（「已并入 #562」/「子段 #374」）： + // 访客侧（时间线、统计、地点计数）不显示它，编辑页要看得到全部原始行，所以只打标不过滤 + Hidden string } type SettingsData struct {