Files
2026-09-10 18:04:19 +08:00

460 lines
18 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Copyright 2022 E99p1ant. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package censor
import (
"encoding/json"
"errors"
"strings"
"testing"
"unicode/utf8"
green20220302 "github.com/alibabacloud-go/green-20220302/v3/client"
"github.com/alibabacloud-go/tea/dara"
"github.com/stretchr/testify/assert"
)
func TestNewAliyunTextModerationRequestShape(t *testing.T) {
// 2.0(TextModeration)的接口契约:Service 指明审核场景,
// ServiceParameters 是 JSON 字符串 {"content": ...}。
req, err := newAliyunTextModerationRequest("测试文本")
assert.NoError(t, err)
assert.Equal(t, aliyunTextService, dara.StringValue(req.Service))
var serviceParameters map[string]string
assert.NoError(t, json.Unmarshal([]byte(dara.StringValue(req.ServiceParameters)), &serviceParameters))
assert.Equal(t, "测试文本", serviceParameters["content"])
}
func TestSplitAliyunText(t *testing.T) {
// 未超限时原样返回,不做无意义的切分。
assert.Equal(t, []string{"短文本"}, splitAliyunText("短文本"))
// 恰好等于上限:仍是单片。
exact := strings.Repeat("a", aliyunContentRuneLimit)
assert.Equal(t, []string{exact}, splitAliyunText(exact))
// 各档站点内容上限都要能覆盖:提问 1000 / 回答·私聊 2000 / 投票描述 5000。
for _, length := range []int{aliyunContentRuneLimit + 1, 1000, 2000, 5000} {
text := strings.Repeat("敏", length)
chunks := splitAliyunText(text)
assert.Greater(t, len(chunks), 1, "length=%d 应当被切分", length)
for i, chunk := range chunks {
// 单次不得超过 2.0 的硬上限,否则请求会被服务端拒绝。
assert.LessOrEqual(t, utf8.RuneCountInString(chunk), aliyunContentRuneLimit,
"length=%d chunk=%d 超过单次上限", length, i)
// 按 rune 切分,不能切坏多字节字符。
assert.True(t, utf8.ValidString(chunk), "length=%d chunk=%d 切坏字符", length, i)
}
// 首片必须从原文开头开始。
assert.Equal(t, string([]rune(text)[:aliyunContentRuneLimit]), chunks[0], "length=%d", length)
// 末片必须收束到原文结尾,否则尾部内容整段漏审。
assert.True(t, strings.HasSuffix(text, chunks[len(chunks)-1]),
"length=%d 最后一个分片未覆盖到原文结尾", length)
// 相邻分片必须重叠:消除「风险词正好跨切分点」的漏判盲区。
for i := 1; i < len(chunks); i++ {
previous := []rune(chunks[i-1])
current := []rune(chunks[i])
overlap := aliyunChunkOverlapRunes
if len(previous) < overlap {
overlap = len(previous)
}
assert.Equal(t, string(previous[len(previous)-overlap:]), string(current[:overlap]),
"length=%d 分片 %d 与上一片的重叠不符合预期", length, i)
}
}
}
func TestAliyunRiskLevelScore(t *testing.T) {
// 2.0 不返回逐标签置信度,只返回 riskLevel,这里折算成 0~1 供 policy.go 复用。
assert.Equal(t, 1.0, aliyunRiskLevelScore("high"))
assert.Equal(t, 1.0, aliyunRiskLevelScore("HIGH"))
assert.Equal(t, 0.6, aliyunRiskLevelScore("medium"))
assert.Equal(t, 0.3, aliyunRiskLevelScore("low"))
assert.Equal(t, 0.0, aliyunRiskLevelScore("none"))
// 未知/缺失级别取保守值 1:宁可够到辱骂门槛被判拦截,也不要静默降权后放行。
assert.Equal(t, 1.0, aliyunRiskLevelScore(""))
assert.Equal(t, 1.0, aliyunRiskLevelScore("brand-new-level"))
}
func TestAliyunRiskLevelSuggestion(t *testing.T) {
assert.Equal(t, SuggestionBlock, aliyunRiskLevelSuggestion("high"))
assert.Equal(t, SuggestionReview, aliyunRiskLevelSuggestion("medium"))
assert.Equal(t, SuggestionPass, aliyunRiskLevelSuggestion("low"))
assert.Equal(t, "", aliyunRiskLevelSuggestion("none"))
assert.Equal(t, "", aliyunRiskLevelSuggestion(""))
}
func TestAliyunHighestRiskLevel(t *testing.T) {
assert.Equal(t, "", aliyunHighestRiskLevel(nil))
assert.Equal(t, "", aliyunHighestRiskLevel([]string{"", " "}))
assert.Equal(t, "low", aliyunHighestRiskLevel([]string{"low"}))
assert.Equal(t, "medium", aliyunHighestRiskLevel([]string{"low", "medium"}))
assert.Equal(t, "high", aliyunHighestRiskLevel([]string{"low", "high", "medium"}))
// 未知级别按最高处理(方向保守),否则新级别会把整条内容静默降级。
assert.Equal(t, "high", aliyunHighestRiskLevel([]string{"low", "brand-new-level"}))
}
func TestAliyunParserRejectsLegacyFormat(t *testing.T) {
// 迁移前内容安全 1.0 的响应(/green/text/scan)必须被拒绝。
// 这是关键回归护栏:若把无法识别的载荷当成「无标签 → 通过」,
// 历史缓存就会变成绕过审核的后门。
raw := []byte(`{
"code": 200,
"data": [{"code": 200, "results": [
{"label": "porn", "rate": 99.9, "scene": "antispam", "suggestion": "block", "details": []}
]}],
"msg": "OK",
"requestId": "req-legacy"
}`)
_, err := AliyunTextCensorParser(raw)
assert.Error(t, err)
assert.Contains(t, err.Error(), "unsupported aliyun response format")
}
func TestAliyunParserCleanResponsePasses(t *testing.T) {
raw := []byte(`{"api":"TextModeration","statusCode":200,"code":200,"message":"OK","requestIds":["req-1"]}`)
parsed, err := AliyunTextCensorParser(raw)
assert.NoError(t, err)
assert.True(t, parsed.Pass)
assert.Empty(t, parsed.Details)
assert.Equal(t, ForbiddenType(""), parsed.ForbiddenType)
}
func TestAliyunEmptyCleanRawParses(t *testing.T) {
// 空内容分支(原文只有一条可信链接、被 RemoveTrustedURL 整段移除)也必须走同一个解析器,
// 保证「在线调用」与「缓存回放」两条路径产出的结构完全一致。
parsed, err := AliyunTextCensorParser(aliyunEmptyCleanRaw)
assert.NoError(t, err)
assert.True(t, parsed.Pass)
assert.Empty(t, parsed.Details)
}
func TestAliyunParserFeedsPolicy(t *testing.T) {
// 端到端校验:riskLevel=medium 折算 0.6,低于标准档的辱骂门槛(0.9),
// 因此「标准档只拦高风险辱骂」,而严格档不限置信度、照拦。
raw := []byte(`{
"api": "TextModeration",
"statusCode": 200,
"code": 200,
"message": "OK",
"requestIds": ["req-1"],
"labels": ["profanity", "negative_content"],
"riskLevel": "medium",
"hint": "示例命中词"
}`)
parsed, err := AliyunTextCensorParser(raw)
assert.NoError(t, err)
assert.Len(t, parsed.Details, 2)
assert.Equal(t, ForbiddenTypeAbuse, parsed.Details[0].Label)
assert.InDelta(t, 0.6, parsed.Details[0].Score, 1e-9)
assert.Equal(t, ForbiddenTypeHarmful, parsed.Details[1].Label)
// 首个命中的标签决定 ForbiddenType / Confidence。
assert.Equal(t, ForbiddenTypeAbuse, parsed.ForbiddenType)
assert.InDelta(t, 0.6, parsed.Confidence, 1e-9)
assert.Equal(t, "示例命中词", parsed.Hint)
assert.Equal(t, SuggestionReview, parsed.ProviderSuggestion)
assert.False(t, parsed.Pass)
decision, _ := Decide("lax", parsed)
assert.False(t, decision.Blocked())
// 中等风险不足 0.9 门槛 → 标准档放行(收敛误伤)。
decision, _ = Decide("standard", parsed)
assert.False(t, decision.Blocked())
// 严格档不限置信度 → 拦截。
decision, detail := Decide("strict", parsed)
assert.Equal(t, DecisionBlock, decision)
assert.NotNil(t, detail)
assert.Equal(t, ForbiddenTypeAbuse, detail.Label)
}
func TestAliyunParserMapsBaselineLabels(t *testing.T) {
// 底线四件套在 2.0 下必须仍然映射得到(标签名与 1.0 完全不同):
// political_content 涉政、violence 暴恐、sexual_content 色情、C_customized 自定义词库。
for _, tc := range []struct {
label string
want ForbiddenType
}{
{"political_content", ForbiddenTypePolitics},
{"regional", ForbiddenTypePolitics},
{"violence", ForbiddenTypeTerrorism},
{"sexual_content", ForbiddenTypePorn},
{"sexuality", ForbiddenTypePorn},
{"C_customized", ForbiddenTypeCustomized},
{"contraband", ForbiddenTypeContraband},
{"ad", ForbiddenTypeAd},
{"PullinTraffic", ForbiddenTypeAd},
{"ad_compliance", ForbiddenTypeAd},
{"profanity", ForbiddenTypeAbuse},
{"negative_content", ForbiddenTypeHarmful},
{"cyberbullying", ForbiddenTypeHarmful},
{"religion", ForbiddenTypeHarmful},
{"nonsense", ForbiddenTypeMeaningless},
{"spam", ForbiddenTypeSpam},
} {
assert.Equal(t, tc.want, formatAliyunForbiddenType(tc.label), "label=%s", tc.label)
// 逐一验证:只要映射到 ForbiddenType,就必须能被策略层正常消费。
mapped := formatAliyunForbiddenType(tc.label)
raw, err := json.Marshal(aliyunTextModerationRaw{
API: aliyunTextModerationAPI,
Code: 200,
Labels: []string{tc.label},
RiskLevel: "low",
})
assert.NoError(t, err)
parsed, err := AliyunTextCensorParser(raw)
assert.NoError(t, err)
assert.Len(t, parsed.Details, 1)
assert.Equal(t, mapped, parsed.Details[0].Label)
}
}
func TestAliyunBaselineLabelsBlockAtEveryLevel(t *testing.T) {
// 底线维度不受档位影响:即使审核服务给 low、置信度折算为 0.3,也必须拦。
for _, label := range []string{"political_content", "violence", "sexual_content", "C_customized"} {
raw, err := json.Marshal(aliyunTextModerationRaw{
API: aliyunTextModerationAPI,
Code: 200,
Labels: []string{label},
RiskLevel: "low",
})
assert.NoError(t, err)
parsed, err := AliyunTextCensorParser(raw)
assert.NoError(t, err)
for _, level := range []string{"lax", "standard", "strict"} {
decision, detail := Decide(level, parsed)
assert.Equal(t, DecisionBlock, decision, "label=%s level=%s", label, level)
assert.NotNil(t, detail)
assert.Equal(t, formatAliyunForbiddenType(label), detail.Label)
}
}
}
func TestAliyunUnmappedLabelSkipped(t *testing.T) {
// 未映射的标签不进入明细(无法参与按维度裁决),原始标签仍保留在 RawResponse 中供审计。
raw, err := json.Marshal(aliyunTextModerationRaw{
API: aliyunTextModerationAPI,
Code: 200,
Labels: []string{"brand_new_label"},
RiskLevel: "medium",
})
assert.NoError(t, err)
parsed, err := AliyunTextCensorParser(raw)
assert.NoError(t, err)
assert.Empty(t, parsed.Details)
assert.Equal(t, SuggestionReview, parsed.ProviderSuggestion)
}
func TestAliyunLabeledResponseWithoutRiskLevelFallsBackToBlock(t *testing.T) {
// 与七牛保持一致:审核服务判为「非通过」却没给出建议时补成 block,
// 否则策略层会因「没有明细维度 + 没有建议」把命中内容当成未命中放行。
raw, err := json.Marshal(aliyunTextModerationRaw{
API: aliyunTextModerationAPI,
Code: 200,
Labels: []string{"brand_new_label"},
})
assert.NoError(t, err)
parsed, err := AliyunTextCensorParser(raw)
assert.NoError(t, err)
assert.False(t, parsed.Pass)
assert.Equal(t, SuggestionBlock, parsed.ProviderSuggestion)
// 严格档采信整体 block 建议 → 拦截;宽松/标准档不采信 → 放行。
decision, _ := Decide("strict", parsed)
assert.Equal(t, DecisionBlock, decision)
decision, _ = Decide("standard", parsed)
assert.False(t, decision.Blocked())
decision, _ = Decide("lax", parsed)
assert.False(t, decision.Blocked())
}
func TestAliyunInvalidJSONErrorCarriesRawResponse(t *testing.T) {
// 响应不是 JSON(如网关返回 HTML 错误页)时,错误里要能看到响应原文。
raw := []byte(`<html>502 Bad Gateway</html>`)
_, err := AliyunTextCensorParser(raw)
assert.Error(t, err)
assert.Contains(t, err.Error(), "502 Bad Gateway")
}
func TestAliyunRawDigestTruncates(t *testing.T) {
assert.Equal(t, "<empty>", aliyunRawDigest([]byte(" ")))
long := strings.Repeat("敏", aliyunMaxRawLogBytes+50)
got := aliyunRawDigest([]byte(long))
suffix := "...(truncated)"
assert.True(t, strings.HasSuffix(got, suffix))
// 按字符(而非字节)截断,多字节字符不能被切成半个。
assert.True(t, utf8.ValidString(got))
assert.Equal(t, aliyunMaxRawLogBytes, len([]rune(strings.TrimSuffix(got, suffix))))
}
// aliyunSDKResponse 构造一个 SDK 形态的 TextModeration 响应,省去每个用例重复拼结构体。
func aliyunSDKResponse(statusCode, code int32, labels, reason string) *green20220302.TextModerationResponse {
response := &green20220302.TextModerationResponse{
StatusCode: dara.Int32(statusCode),
Body: &green20220302.TextModerationResponseBody{
Code: dara.Int32(code),
Message: dara.String("OK"),
RequestId: dara.String("req-1"),
},
}
if labels != "" || reason != "" {
response.Body.Data = &green20220302.TextModerationResponseBodyData{
Labels: dara.String(labels),
Reason: dara.String(reason),
}
}
return response
}
func TestExtractAliyunChunkResult(t *testing.T) {
t.Run("成功:解析标签与 Reason", func(t *testing.T) {
result, err := extractAliyunChunkResult(aliyunSDKResponse(200, 200, "profanity, negative_content",
`{"riskLevel":"medium","riskTips":"辱骂_人身攻击","riskWords":"你是个傻子"}`))
assert.NoError(t, err)
assert.Equal(t, int32(200), result.Code)
assert.Equal(t, int32(200), result.StatusCode)
assert.Equal(t, "req-1", result.RequestID)
assert.Equal(t, []string{"profanity", "negative_content"}, result.Labels)
assert.Equal(t, "medium", result.RiskLevel)
// Hint 优先取命中词。
assert.Equal(t, "你是个傻子", result.Hint)
})
t.Run("nonLabel/normal 是占位标签,不进入明细", func(t *testing.T) {
result, err := extractAliyunChunkResult(aliyunSDKResponse(200, 200, "nonLabel,normal,", ""))
assert.NoError(t, err)
assert.Empty(t, result.Labels)
})
t.Run("RiskWords 缺失时 Hint 退回 RiskTips", func(t *testing.T) {
result, err := extractAliyunChunkResult(aliyunSDKResponse(200, 200, "ad",
`{"riskLevel":"high","riskTips":"广告_引流"}`))
assert.NoError(t, err)
assert.Equal(t, "广告_引流", result.Hint)
})
t.Run("Reason 非法 JSON 不应导致整条审核失败", func(t *testing.T) {
result, err := extractAliyunChunkResult(aliyunSDKResponse(200, 200, "ad", `not-json`))
assert.NoError(t, err)
assert.Equal(t, []string{"ad"}, result.Labels)
assert.Equal(t, "", result.RiskLevel)
})
t.Run("code 非 200 必须报错,不能当成未命中", func(t *testing.T) {
// 服务未开通时阿里云会返回自己的错误码。若被当成「无标签 → 通过」,
// fail-closed 会悄悄变成 fail-open。
_, err := extractAliyunChunkResult(aliyunSDKResponse(200, 596, "", ""))
assert.Error(t, err)
assert.Contains(t, err.Error(), "596")
})
t.Run("HTTP 非 2xx 必须报错", func(t *testing.T) {
_, err := extractAliyunChunkResult(aliyunSDKResponse(502, 200, "", ""))
assert.Error(t, err)
assert.Contains(t, err.Error(), "unexpected http status")
})
t.Run("响应体缺失必须报错", func(t *testing.T) {
_, err := extractAliyunChunkResult(nil)
assert.Error(t, err)
_, err = extractAliyunChunkResult(&green20220302.TextModerationResponse{})
assert.Error(t, err)
})
}
func TestMergeAliyunChunkResults(t *testing.T) {
merged := mergeAliyunChunkResults([]aliyunChunkResult{
{StatusCode: 200, Code: 200, RequestID: "r1", Labels: []string{"profanity"}, RiskLevel: "low"},
{StatusCode: 200, Code: 200, RequestID: "r2", Labels: []string{"profanity", "political_content"},
RiskLevel: "high", Hint: "命中词"},
})
assert.Equal(t, aliyunTextModerationAPI, merged.API)
assert.Equal(t, []string{"r1", "r2"}, merged.RequestIDs)
// 标签取并集并去重,保持出现顺序。
assert.Equal(t, []string{"profanity", "political_content"}, merged.Labels)
// 风险级别取最高:任一分片命中高风险,整条内容按高风险处置。
assert.Equal(t, "high", merged.RiskLevel)
// Hint 取首个非空。
assert.Equal(t, "命中词", merged.Hint)
// 合并结果可直接被解析器消费,且高风险 + 涉政 → 任何档位都拦。
raw, err := json.Marshal(merged)
assert.NoError(t, err)
parsed, err := AliyunTextCensorParser(raw)
assert.NoError(t, err)
assert.Len(t, parsed.Details, 2)
for _, detail := range parsed.Details {
assert.InDelta(t, 1.0, detail.Score, 1e-9)
}
decision, detail := Decide("lax", parsed)
assert.Equal(t, DecisionBlock, decision)
assert.NotNil(t, detail)
assert.Equal(t, ForbiddenTypePolitics, detail.Label)
}
func TestMergeAliyunChunkResultsEmpty(t *testing.T) {
// 没有任何分片(理论上不会发生,因为空内容有单独的短路分支)时,
// 合并结果必须仍然是「无标签」,即不产生任何命中。
merged := mergeAliyunChunkResults(nil)
assert.Empty(t, merged.Labels)
assert.Equal(t, "", merged.RiskLevel)
assert.Equal(t, aliyunTextModerationAPI, merged.API)
}
func TestAliyunIsAccountIssue(t *testing.T) {
// 账号侧未就绪的各类表现都要能识别出来 —— 它们不会自愈,必须给出可操作告警。
for _, message := range []string{
"code=596 You have not opened Yundun Content Moderation Service",
"You have not opened Yundun Content Moderation Service, please go to the page",
"Forbidden.RAM: no permission to perform this action",
"InvalidAccessKeyId.NotFound: the access key id is invalid",
"SignatureDoesNotMatch: signature mismatch",
} {
assert.True(t, aliyunIsAccountIssue(errors.New(message)), "message=%q", message)
}
// 偶发/网络类问题不应被当成账号问题,否则一次性告警会被无关噪声消耗掉。
for _, message := range []string{
"dial tcp 1.2.3.4:443: i/o timeout",
"EOF",
"context deadline exceeded",
} {
assert.False(t, aliyunIsAccountIssue(errors.New(message)), "message=%q", message)
}
assert.False(t, aliyunIsAccountIssue(nil))
}
func TestAliyunIsAccountIssueFromStructuredCode(t *testing.T) {
// SDK 的结构化 code 比消息文本更稳定:即使 message 里没有特征词也要能识别。
sdkErr := dara.NewSDKError(map[string]interface{}{
"code": "Forbidden",
"message": "the request is denied",
})
assert.True(t, aliyunIsAccountIssue(sdkErr))
}