收尾：验收脚本自动清死元组、城市圈首末到访按可见到访算、补使用说明
verify.sh：测试库 ≥ VACUUM_MIN_MB（默认 200MB）时先 VACUUM (ANALYZE)——一轮全量会留几百 MB 死元组，级联删除随之变慢（同一夹具用例实测 13.6s → 10.2s）。只用普通 VACUUM：FULL 要 ACCESS EXCLUSIVE 锁，有别的服务连着测试库时会把验收整段卡住。 /api/cities 的首末到访改成按可见到访的到达时间算（内层 min/max(v.arrival)），不再取 places.first/last_visit_at——那两列是全量的，会被已并入 / 子段的到访推早推晚。 README 补两节：「数据编辑页上的重复到访」（判定口径与清理范围）、「为什么有些到访在页面上看不到」 （已并入 / 子段不计入显示与统计，但导出必须原样带着），并在开发提示里记下验收环境的死元组与 is_active 两个坑。 回归：city_agg_test.go 补了一段能区分的断言——把某地点最晚那次到访标成已并入后，城市最近到访 必须从 09-13 退到 09-08（原先只比对同口径时，两种算法在夹具数据上结果一样，是条假绿的断言）。
This commit is contained in:
tamakyi
@@ -26,6 +26,8 @@ func TestCityAggCountsOncePerPlace(t *testing.T) {
|
||||
Visits int `json:"visits"`
|
||||
Places int `json:"places"`
|
||||
Dwell int64 `json:"dwell"`
|
||||
First string `json:"first,omitempty"`
|
||||
Last string `json:"last,omitempty"`
|
||||
} `json:"cities"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil {
|
||||
@@ -38,7 +40,8 @@ func TestCityAggCountsOncePerPlace(t *testing.T) {
|
||||
// 手写一份同一口径的对照（判据独立写，不复用 stats.SoloVisitSQL）：
|
||||
// 不含已并入 / 子段，也不含「家 / 工作」地点——与 apiCityAgg 保持一致。
|
||||
rows, err := pg.QueryContext(ctx, `SELECT p.city, count(*), count(DISTINCT p.id),
|
||||
COALESCE(sum(LEAST(COALESCE(v.duration_min,0),10080)),0)
|
||||
COALESCE(sum(LEAST(COALESCE(v.duration_min,0),10080)),0),
|
||||
to_char(min(v.arrival), 'YYYY-MM-DD'), to_char(max(v.arrival), 'YYYY-MM-DD')
|
||||
FROM places p JOIN visits v ON v.place_id = p.id AND v.dataset_id = p.dataset_id
|
||||
WHERE p.dataset_id = $1 AND p.city IS NOT NULL AND p.city <> '' AND p.lat IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM entity_raw er
|
||||
@@ -54,12 +57,13 @@ func TestCityAggCountsOncePerPlace(t *testing.T) {
|
||||
type want struct {
|
||||
visits, places int
|
||||
dwell int64
|
||||
first, last string
|
||||
}
|
||||
truth := map[string]want{}
|
||||
for rows.Next() {
|
||||
var city string
|
||||
var w want
|
||||
if err := rows.Scan(&city, &w.visits, &w.places, &w.dwell); err != nil {
|
||||
if err := rows.Scan(&city, &w.visits, &w.places, &w.dwell, &w.first, &w.last); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
truth[city] = w
|
||||
@@ -78,6 +82,11 @@ func TestCityAggCountsOncePerPlace(t *testing.T) {
|
||||
t.Errorf("城市「%s」聚合对不上：到访 %d/%d、地点 %d/%d、停留 %d/%d",
|
||||
c.City, c.Visits, w.visits, c.Places, w.places, c.Dwell, w.dwell)
|
||||
}
|
||||
// 首末到访必须按「可见到访的到达时间」算，不能拿 places 的全量列值
|
||||
if c.First != w.first || c.Last != w.last {
|
||||
t.Errorf("城市「%s」首末到访对不上：%s~%s，应为 %s~%s（按可见到访算）",
|
||||
c.City, c.First, c.Last, w.first, w.last)
|
||||
}
|
||||
}
|
||||
// 保底的不变量：地点数不可能多于到访数（旧写法里这两个数就换位了）
|
||||
for _, c := range payload.Cities {
|
||||
@@ -85,4 +94,86 @@ func TestCityAggCountsOncePerPlace(t *testing.T) {
|
||||
t.Errorf("城市「%s」的地点数 %d 多于到访数 %d，说明又按到访条数重复累加了", c.City, c.Places, c.Visits)
|
||||
}
|
||||
}
|
||||
|
||||
// 把某个地点**最晚的那次到访**标成「已并入」：该城市的最近到访必须跟着退到更早那条。
|
||||
// 这一条专治「偷懒直接取 places.last_visit_at」——那是全量列值，不会跟着退。
|
||||
// （光比对相同口径还不够：被并入的行大多落在父段区间内，两种算法在夹具数据上结果一样，
|
||||
// 不做这一步就是一条假绿的断言。）
|
||||
var placeID int64
|
||||
var latestVisit int
|
||||
if err := pg.QueryRowContext(ctx, `SELECT p.id, v.src_pk
|
||||
FROM visits v JOIN places p ON p.id = v.place_id
|
||||
WHERE v.dataset_id=$1 AND p.city IS NOT NULL AND p.city <> '' AND p.lat IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM visits vh WHERE vh.place_id = p.id AND (vh.is_home OR vh.is_work))
|
||||
AND (SELECT count(*) FROM visits v2 WHERE v2.place_id = p.id) >= 2
|
||||
ORDER BY v.arrival DESC LIMIT 1`, dsID).Scan(&placeID, &latestVisit); err != nil {
|
||||
t.Skipf("挑不出「同一地点有多条到访」的城市：%v", err)
|
||||
}
|
||||
var cityName string
|
||||
if err := pg.QueryRowContext(ctx, `SELECT city FROM places WHERE id=$1`, placeID).Scan(&cityName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lastBefore := ""
|
||||
lastAfterWant := ""
|
||||
for _, c := range payload.Cities {
|
||||
if c.City == cityName {
|
||||
lastBefore = c.Last
|
||||
}
|
||||
}
|
||||
// 真值（按城市取，标记前后各来一次）：与上面那条对账同一套口径，只是限定到某个城市
|
||||
truthLast := func() string {
|
||||
var s string
|
||||
if err := pg.QueryRowContext(ctx, `SELECT to_char(max(v.arrival),'YYYY-MM-DD')
|
||||
FROM places p JOIN visits v ON v.place_id = p.id AND v.dataset_id = p.dataset_id
|
||||
WHERE p.dataset_id = $1 AND p.city = $2 AND p.city <> '' AND p.lat IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM entity_raw er
|
||||
WHERE er.dataset_id=v.dataset_id AND er.entity='ZVISIT' AND er.src_pk=v.src_pk
|
||||
AND (er.raw->>'ZMERGEDTO' IS NOT NULL OR er.raw->>'ZPARENT' IS NOT NULL))
|
||||
AND NOT EXISTS (SELECT 1 FROM visits vh WHERE vh.place_id = p.id
|
||||
AND (vh.is_home OR vh.is_work))`, dsID, cityName).Scan(&s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
lastBeforeWant := truthLast()
|
||||
if lastBeforeWant != lastBefore {
|
||||
t.Fatalf("标记前该城市的最近到访就对不上（%s vs %s），用例前提不成立", lastBefore, lastBeforeWant)
|
||||
}
|
||||
if _, err := pg.ExecContext(ctx, `UPDATE entity_raw
|
||||
SET raw = jsonb_set(raw, '{ZMERGEDTO}', to_jsonb($3::int), true)
|
||||
WHERE dataset_id=$1 AND entity='ZVISIT' AND src_pk=$2`, dsID, latestVisit, latestVisit); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lastAfterWant = truthLast()
|
||||
if lastAfterWant == lastBeforeWant {
|
||||
t.Skipf("城市「%s」里最晚那次不是唯一的最晚记录，换一个再测", cityName)
|
||||
}
|
||||
|
||||
rec2 := httptest.NewRecorder()
|
||||
f.srv.apiCityAgg(rec2, httptest.NewRequest("GET", "/api/cities", nil).WithContext(ctx))
|
||||
if rec2.Code != 200 {
|
||||
t.Fatalf("取城市聚合失败: code=%d", rec2.Code)
|
||||
}
|
||||
var after struct {
|
||||
Cities []struct {
|
||||
City string `json:"city"`
|
||||
Last string `json:"last,omitempty"`
|
||||
} `json:"cities"`
|
||||
}
|
||||
if err := json.Unmarshal(rec2.Body.Bytes(), &after); err != nil {
|
||||
t.Fatalf("解析城市聚合失败: %v", err)
|
||||
}
|
||||
gotLast := ""
|
||||
for _, c := range after.Cities {
|
||||
if c.City == cityName {
|
||||
gotLast = c.Last
|
||||
}
|
||||
}
|
||||
if gotLast == lastBeforeWant {
|
||||
t.Errorf("把 #%d 标成已并入后，城市「%s」的最近到访仍是 %s（应退到 %s）—— 说明取的是 places 的全量列值而不是可见到访",
|
||||
latestVisit, cityName, gotLast, lastAfterWant)
|
||||
}
|
||||
if gotLast != lastAfterWant {
|
||||
t.Errorf("城市「%s」的最近到访应为 %s，实际 %s", cityName, lastAfterWant, gotLast)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -956,7 +956,10 @@ func (s *Server) apiCityAgg(w http.ResponseWriter, r *http.Request) {
|
||||
COALESCE(sum(pa.visits),0), count(*), COALESCE(sum(pa.dwell),0),
|
||||
min(pa.first_at), max(pa.last_at)
|
||||
FROM (
|
||||
SELECT p.id, p.city, p.province, p.lat, p.lon, p.first_visit_at AS first_at, p.last_visit_at AS last_at,
|
||||
SELECT p.id, p.city, p.province, p.lat, p.lon,
|
||||
-- 首末到访按**可见到访**的到达时间算，不取 places 列值（那两列是全量，
|
||||
-- 会被已并入 / 子段的行推早推晚），与 visit_count 之类的口径保持一致
|
||||
min(v.arrival) AS first_at, max(v.arrival) AS last_at,
|
||||
count(*) AS visits,
|
||||
COALESCE(sum(LEAST(COALESCE(v.duration_min,0),10080)),0) AS dwell
|
||||
FROM places p
|
||||
|
||||
Reference in New Issue
Block a user