修地图城市圈的聚合口径:先按地点汇总,再按城市汇总
/api/cities 原来把聚合打在 places ⋈ visits 上:一个地点有 N 条到访,它的 visit_count 与 dwell_minutes 就被累加 N 次,count(*) 数出来的到访条数又被当成「地点数」显示——实测最大的 那个城市气泡写着「到访 72661 次 · 568 个地点」,真值是 568 次 / 86 个地点(差 128 倍)。 改成内层按地点 GROUP BY、外层按城市 sum/count,并补上「已并入 / 子段不算数」的判据, 圈的位置、排序与返回列序都不变。 回归 city_agg_test.go:手写同口径 SQL 对账 + 「地点数不可能多于到访数」的不变量; 变异验证(sum(pa.visits) *2)立刻红。
This commit is contained in:
@@ -0,0 +1,88 @@
|
|||||||
|
package web
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 地图上的城市圈气泡(/api/cities)必须「先按地点聚合、再按城市汇总」。
|
||||||
|
//
|
||||||
|
// 旧写法把聚合打在 places ⋈ visits 上,于是每个地点的 visit_count / dwell_minutes
|
||||||
|
// 会被按它的到访条数重复累加,count(*) 又被当成「地点数」——实测最大那个城市显示
|
||||||
|
// 「到访 72661 次 · 568 个地点」,真值是 568 次 / 86 个地点。这里按同一套口径手写一遍对账。
|
||||||
|
func TestCityAggCountsOncePerPlace(t *testing.T) {
|
||||||
|
f := newMovementFixture(t)
|
||||||
|
ctx, pg, dsID := f.ctx, f.pg, f.dsID
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
f.srv.apiCityAgg(rec, httptest.NewRequest("GET", "/api/cities", nil).WithContext(ctx))
|
||||||
|
if rec.Code != 200 {
|
||||||
|
t.Fatalf("取城市聚合失败: code=%d body=%s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
var payload struct {
|
||||||
|
Cities []struct {
|
||||||
|
City string `json:"city"`
|
||||||
|
Visits int `json:"visits"`
|
||||||
|
Places int `json:"places"`
|
||||||
|
Dwell int64 `json:"dwell"`
|
||||||
|
} `json:"cities"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil {
|
||||||
|
t.Fatalf("解析城市聚合失败: %v", err)
|
||||||
|
}
|
||||||
|
if len(payload.Cities) == 0 {
|
||||||
|
t.Skip("夹具没有带城市的到访")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 手写一份同一口径的对照(判据独立写,不复用 stats.SoloVisitSQL):
|
||||||
|
// 不含已并入 / 子段,也不含「家 / 工作」地点——与 apiCityAgg 保持一致。
|
||||||
|
rows, err := pg.QueryContext(ctx, `SELECT p.city, count(*), count(DISTINCT p.id),
|
||||||
|
COALESCE(sum(LEAST(COALESCE(v.duration_min,0),10080)),0)
|
||||||
|
FROM places p JOIN visits v ON v.place_id = p.id AND v.dataset_id = p.dataset_id
|
||||||
|
WHERE p.dataset_id = $1 AND p.city IS NOT NULL AND p.city <> '' AND p.lat IS NOT NULL
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM entity_raw er
|
||||||
|
WHERE er.dataset_id=v.dataset_id AND er.entity='ZVISIT' AND er.src_pk=v.src_pk
|
||||||
|
AND (er.raw->>'ZMERGEDTO' IS NOT NULL OR er.raw->>'ZPARENT' IS NOT NULL))
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM visits vh WHERE vh.place_id = p.id
|
||||||
|
AND (vh.is_home OR vh.is_work))
|
||||||
|
GROUP BY p.city`, dsID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
type want struct {
|
||||||
|
visits, places int
|
||||||
|
dwell int64
|
||||||
|
}
|
||||||
|
truth := map[string]want{}
|
||||||
|
for rows.Next() {
|
||||||
|
var city string
|
||||||
|
var w want
|
||||||
|
if err := rows.Scan(&city, &w.visits, &w.places, &w.dwell); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
truth[city] = w
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, c := range payload.Cities {
|
||||||
|
w, ok := truth[c.City]
|
||||||
|
if !ok {
|
||||||
|
t.Errorf("城市「%s」不该出现(手写 SQL 里没有)", c.City)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if c.Visits != w.visits || c.Places != w.places || c.Dwell != w.dwell {
|
||||||
|
t.Errorf("城市「%s」聚合对不上:到访 %d/%d、地点 %d/%d、停留 %d/%d",
|
||||||
|
c.City, c.Visits, w.visits, c.Places, w.places, c.Dwell, w.dwell)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 保底的不变量:地点数不可能多于到访数(旧写法里这两个数就换位了)
|
||||||
|
for _, c := range payload.Cities {
|
||||||
|
if c.Places > c.Visits {
|
||||||
|
t.Errorf("城市「%s」的地点数 %d 多于到访数 %d,说明又按到访条数重复累加了", c.City, c.Places, c.Visits)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -948,16 +948,26 @@ func (s *Server) apiCityAgg(w http.ResponseWriter, r *http.Request) {
|
|||||||
if !p.IsAdmin() {
|
if !p.IsAdmin() {
|
||||||
regSQL = regionSQL("p", p.Settings.Regions, p.Settings.RegionBlacklist(), &args)
|
regSQL = regionSQL("p", p.Settings.Regions, p.Settings.RegionBlacklist(), &args)
|
||||||
}
|
}
|
||||||
rows, err := s.db.QueryContext(r.Context(), `SELECT COALESCE(p.city,''), COALESCE(p.province,''),
|
// 先按**地点**聚合、再按城市汇总。直接在 places ⋈ visits 上 sum(places.visit_count) 是错的:
|
||||||
AVG(p.lat)::float8, AVG(p.lon)::float8,
|
// 一个地点有 N 条到访,它的 visit_count 就会被累加 N 次(实测「到访 72661 次」,真值 568),
|
||||||
COALESCE(sum(p.visit_count),0), count(*), COALESCE(sum(p.dwell_minutes),0),
|
// dwell 同理;count(*) 数出来的是到访条数,却被当成「地点数」显示。
|
||||||
min(p.first_visit_at), max(p.last_visit_at)
|
rows, err := s.db.QueryContext(r.Context(), `SELECT COALESCE(pa.city,''), COALESCE(pa.province,''),
|
||||||
|
AVG(pa.lat)::float8, AVG(pa.lon)::float8,
|
||||||
|
COALESCE(sum(pa.visits),0), count(*), COALESCE(sum(pa.dwell),0),
|
||||||
|
min(pa.first_at), max(pa.last_at)
|
||||||
|
FROM (
|
||||||
|
SELECT p.id, p.city, p.province, p.lat, p.lon, p.first_visit_at AS first_at, p.last_visit_at AS last_at,
|
||||||
|
count(*) AS visits,
|
||||||
|
COALESCE(sum(LEAST(COALESCE(v.duration_min,0),10080)),0) AS dwell
|
||||||
FROM places p
|
FROM places p
|
||||||
JOIN visits v ON v.place_id = p.id AND v.dataset_id = p.dataset_id
|
JOIN visits v ON v.place_id = p.id AND v.dataset_id = p.dataset_id
|
||||||
WHERE p.dataset_id = $1 AND p.city IS NOT NULL AND p.city <> '' AND p.lat IS NOT NULL`+regSQL+`
|
WHERE p.dataset_id = $1 AND p.city IS NOT NULL AND p.city <> '' AND p.lat IS NOT NULL`+regSQL+`
|
||||||
|
AND `+stats.SoloVisitSQL("v")+`
|
||||||
AND NOT EXISTS (SELECT 1 FROM visits vh WHERE vh.place_id = p.id
|
AND NOT EXISTS (SELECT 1 FROM visits vh WHERE vh.place_id = p.id
|
||||||
AND (vh.is_home OR vh.is_work))
|
AND (vh.is_home OR vh.is_work))
|
||||||
GROUP BY p.city, p.province
|
GROUP BY p.id
|
||||||
|
) pa
|
||||||
|
GROUP BY pa.city, pa.province
|
||||||
ORDER BY 5 DESC`, args...)
|
ORDER BY 5 DESC`, args...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.json(w, map[string]string{"error": err.Error()})
|
s.json(w, map[string]string{"error": err.Error()})
|
||||||
|
|||||||
Reference in New Issue
Block a user