修地图城市圈的聚合口径:先按地点汇总,再按城市汇总

/api/cities 原来把聚合打在 places ⋈ visits 上:一个地点有 N 条到访,它的 visit_count 与
dwell_minutes 就被累加 N 次,count(*) 数出来的到访条数又被当成「地点数」显示——实测最大的
那个城市气泡写着「到访 72661 次 · 568 个地点」,真值是 568 次 / 86 个地点(差 128 倍)。
改成内层按地点 GROUP BY、外层按城市 sum/count,并补上「已并入 / 子段不算数」的判据,
圈的位置、排序与返回列序都不变。

回归 city_agg_test.go:手写同口径 SQL 对账 + 「地点数不可能多于到访数」的不变量;
变异验证(sum(pa.visits) *2)立刻红。
This commit is contained in:
tamakyi
2026-09-29 22:07:23 +08:00
parent 4597047989
commit 3d03aae018
2 changed files with 108 additions and 10 deletions
+88
View File
@@ -0,0 +1,88 @@
package web
import (
"encoding/json"
"net/http/httptest"
"testing"
)
// 地图上的城市圈气泡(/api/cities)必须「先按地点聚合、再按城市汇总」。
//
// 旧写法把聚合打在 places ⋈ visits 上,于是每个地点的 visit_count / dwell_minutes
// 会被按它的到访条数重复累加,count(*) 又被当成「地点数」——实测最大那个城市显示
// 「到访 72661 次 · 568 个地点」,真值是 568 次 / 86 个地点。这里按同一套口径手写一遍对账。
func TestCityAggCountsOncePerPlace(t *testing.T) {
f := newMovementFixture(t)
ctx, pg, dsID := f.ctx, f.pg, f.dsID
rec := httptest.NewRecorder()
f.srv.apiCityAgg(rec, httptest.NewRequest("GET", "/api/cities", nil).WithContext(ctx))
if rec.Code != 200 {
t.Fatalf("取城市聚合失败: code=%d body=%s", rec.Code, rec.Body.String())
}
var payload struct {
Cities []struct {
City string `json:"city"`
Visits int `json:"visits"`
Places int `json:"places"`
Dwell int64 `json:"dwell"`
} `json:"cities"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil {
t.Fatalf("解析城市聚合失败: %v", err)
}
if len(payload.Cities) == 0 {
t.Skip("夹具没有带城市的到访")
}
// 手写一份同一口径的对照(判据独立写,不复用 stats.SoloVisitSQL):
// 不含已并入 / 子段,也不含「家 / 工作」地点——与 apiCityAgg 保持一致。
rows, err := pg.QueryContext(ctx, `SELECT p.city, count(*), count(DISTINCT p.id),
COALESCE(sum(LEAST(COALESCE(v.duration_min,0),10080)),0)
FROM places p JOIN visits v ON v.place_id = p.id AND v.dataset_id = p.dataset_id
WHERE p.dataset_id = $1 AND p.city IS NOT NULL AND p.city <> '' AND p.lat IS NOT NULL
AND NOT EXISTS (SELECT 1 FROM entity_raw er
WHERE er.dataset_id=v.dataset_id AND er.entity='ZVISIT' AND er.src_pk=v.src_pk
AND (er.raw->>'ZMERGEDTO' IS NOT NULL OR er.raw->>'ZPARENT' IS NOT NULL))
AND NOT EXISTS (SELECT 1 FROM visits vh WHERE vh.place_id = p.id
AND (vh.is_home OR vh.is_work))
GROUP BY p.city`, dsID)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
type want struct {
visits, places int
dwell int64
}
truth := map[string]want{}
for rows.Next() {
var city string
var w want
if err := rows.Scan(&city, &w.visits, &w.places, &w.dwell); err != nil {
t.Fatal(err)
}
truth[city] = w
}
if err := rows.Err(); err != nil {
t.Fatal(err)
}
for _, c := range payload.Cities {
w, ok := truth[c.City]
if !ok {
t.Errorf("城市「%s」不该出现(手写 SQL 里没有)", c.City)
continue
}
if c.Visits != w.visits || c.Places != w.places || c.Dwell != w.dwell {
t.Errorf("城市「%s」聚合对不上:到访 %d/%d、地点 %d/%d、停留 %d/%d",
c.City, c.Visits, w.visits, c.Places, w.places, c.Dwell, w.dwell)
}
}
// 保底的不变量:地点数不可能多于到访数(旧写法里这两个数就换位了)
for _, c := range payload.Cities {
if c.Places > c.Visits {
t.Errorf("城市「%s」的地点数 %d 多于到访数 %d,说明又按到访条数重复累加了", c.City, c.Places, c.Visits)
}
}
}
+15 -5
View File
@@ -948,16 +948,26 @@ func (s *Server) apiCityAgg(w http.ResponseWriter, r *http.Request) {
if !p.IsAdmin() { if !p.IsAdmin() {
regSQL = regionSQL("p", p.Settings.Regions, p.Settings.RegionBlacklist(), &args) regSQL = regionSQL("p", p.Settings.Regions, p.Settings.RegionBlacklist(), &args)
} }
rows, err := s.db.QueryContext(r.Context(), `SELECT COALESCE(p.city,''), COALESCE(p.province,''), // 先按**地点**聚合、再按城市汇总。直接在 places ⋈ visits 上 sum(places.visit_count) 是错的:
AVG(p.lat)::float8, AVG(p.lon)::float8, // 一个地点有 N 条到访,它的 visit_count 就会被累加 N 次(实测「到访 72661 次」,真值 568),
COALESCE(sum(p.visit_count),0), count(*), COALESCE(sum(p.dwell_minutes),0), // dwell 同理;count(*) 数出来的是到访条数,却被当成「地点数」显示。
min(p.first_visit_at), max(p.last_visit_at) rows, err := s.db.QueryContext(r.Context(), `SELECT COALESCE(pa.city,''), COALESCE(pa.province,''),
AVG(pa.lat)::float8, AVG(pa.lon)::float8,
COALESCE(sum(pa.visits),0), count(*), COALESCE(sum(pa.dwell),0),
min(pa.first_at), max(pa.last_at)
FROM (
SELECT p.id, p.city, p.province, p.lat, p.lon, p.first_visit_at AS first_at, p.last_visit_at AS last_at,
count(*) AS visits,
COALESCE(sum(LEAST(COALESCE(v.duration_min,0),10080)),0) AS dwell
FROM places p FROM places p
JOIN visits v ON v.place_id = p.id AND v.dataset_id = p.dataset_id JOIN visits v ON v.place_id = p.id AND v.dataset_id = p.dataset_id
WHERE p.dataset_id = $1 AND p.city IS NOT NULL AND p.city <> '' AND p.lat IS NOT NULL`+regSQL+` WHERE p.dataset_id = $1 AND p.city IS NOT NULL AND p.city <> '' AND p.lat IS NOT NULL`+regSQL+`
AND `+stats.SoloVisitSQL("v")+`
AND NOT EXISTS (SELECT 1 FROM visits vh WHERE vh.place_id = p.id AND NOT EXISTS (SELECT 1 FROM visits vh WHERE vh.place_id = p.id
AND (vh.is_home OR vh.is_work)) AND (vh.is_home OR vh.is_work))
GROUP BY p.city, p.province GROUP BY p.id
) pa
GROUP BY pa.city, pa.province
ORDER BY 5 DESC`, args...) ORDER BY 5 DESC`, args...)
if err != nil { if err != nil {
s.json(w, map[string]string{"error": err.Error()}) s.json(w, map[string]string{"error": err.Error()})