// Copyright 2022 E99p1ant. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. package censor import ( "encoding/json" "errors" "strings" "testing" "unicode/utf8" green20220302 "github.com/alibabacloud-go/green-20220302/v3/client" "github.com/alibabacloud-go/tea/dara" "github.com/stretchr/testify/assert" ) func TestNewAliyunTextModerationRequestShape(t *testing.T) { // 2.0(TextModeration)的接口契约:Service 指明审核场景, // ServiceParameters 是 JSON 字符串 {"content": ...}。 req, err := newAliyunTextModerationRequest("测试文本") assert.NoError(t, err) assert.Equal(t, aliyunTextService, dara.StringValue(req.Service)) var serviceParameters map[string]string assert.NoError(t, json.Unmarshal([]byte(dara.StringValue(req.ServiceParameters)), &serviceParameters)) assert.Equal(t, "测试文本", serviceParameters["content"]) } func TestSplitAliyunText(t *testing.T) { // 未超限时原样返回,不做无意义的切分。 assert.Equal(t, []string{"短文本"}, splitAliyunText("短文本")) // 恰好等于上限:仍是单片。 exact := strings.Repeat("a", aliyunContentRuneLimit) assert.Equal(t, []string{exact}, splitAliyunText(exact)) // 各档站点内容上限都要能覆盖:提问 1000 / 回答·私聊 2000 / 投票描述 5000。 for _, length := range []int{aliyunContentRuneLimit + 1, 1000, 2000, 5000} { text := strings.Repeat("敏", length) chunks := splitAliyunText(text) assert.Greater(t, len(chunks), 1, "length=%d 应当被切分", length) for i, chunk := range chunks { // 单次不得超过 2.0 的硬上限,否则请求会被服务端拒绝。 assert.LessOrEqual(t, utf8.RuneCountInString(chunk), aliyunContentRuneLimit, "length=%d chunk=%d 超过单次上限", length, i) // 按 rune 切分,不能切坏多字节字符。 assert.True(t, utf8.ValidString(chunk), "length=%d chunk=%d 切坏字符", length, i) } // 首片必须从原文开头开始。 assert.Equal(t, string([]rune(text)[:aliyunContentRuneLimit]), chunks[0], "length=%d", length) // 末片必须收束到原文结尾,否则尾部内容整段漏审。 assert.True(t, strings.HasSuffix(text, chunks[len(chunks)-1]), "length=%d 最后一个分片未覆盖到原文结尾", length) // 相邻分片必须重叠:消除「风险词正好跨切分点」的漏判盲区。 for i := 1; i < len(chunks); i++ { previous := []rune(chunks[i-1]) current := []rune(chunks[i]) overlap := aliyunChunkOverlapRunes if len(previous) < overlap { overlap = len(previous) } assert.Equal(t, string(previous[len(previous)-overlap:]), string(current[:overlap]), "length=%d 分片 %d 与上一片的重叠不符合预期", length, i) } } } func TestAliyunRiskLevelScore(t *testing.T) { // 2.0 不返回逐标签置信度,只返回 riskLevel,这里折算成 0~1 供 policy.go 复用。 assert.Equal(t, 1.0, aliyunRiskLevelScore("high")) assert.Equal(t, 1.0, aliyunRiskLevelScore("HIGH")) assert.Equal(t, 0.6, aliyunRiskLevelScore("medium")) assert.Equal(t, 0.3, aliyunRiskLevelScore("low")) assert.Equal(t, 0.0, aliyunRiskLevelScore("none")) // 未知/缺失级别取保守值 1:宁可够到辱骂门槛被判拦截,也不要静默降权后放行。 assert.Equal(t, 1.0, aliyunRiskLevelScore("")) assert.Equal(t, 1.0, aliyunRiskLevelScore("brand-new-level")) } func TestAliyunRiskLevelSuggestion(t *testing.T) { assert.Equal(t, SuggestionBlock, aliyunRiskLevelSuggestion("high")) assert.Equal(t, SuggestionReview, aliyunRiskLevelSuggestion("medium")) assert.Equal(t, SuggestionPass, aliyunRiskLevelSuggestion("low")) assert.Equal(t, "", aliyunRiskLevelSuggestion("none")) assert.Equal(t, "", aliyunRiskLevelSuggestion("")) } func TestAliyunHighestRiskLevel(t *testing.T) { assert.Equal(t, "", aliyunHighestRiskLevel(nil)) assert.Equal(t, "", aliyunHighestRiskLevel([]string{"", " "})) assert.Equal(t, "low", aliyunHighestRiskLevel([]string{"low"})) assert.Equal(t, "medium", aliyunHighestRiskLevel([]string{"low", "medium"})) assert.Equal(t, "high", aliyunHighestRiskLevel([]string{"low", "high", "medium"})) // 未知级别按最高处理(方向保守),否则新级别会把整条内容静默降级。 assert.Equal(t, "high", aliyunHighestRiskLevel([]string{"low", "brand-new-level"})) } func TestAliyunParserRejectsLegacyFormat(t *testing.T) { // 迁移前内容安全 1.0 的响应(/green/text/scan)必须被拒绝。 // 这是关键回归护栏:若把无法识别的载荷当成「无标签 → 通过」, // 历史缓存就会变成绕过审核的后门。 raw := []byte(`{ "code": 200, "data": [{"code": 200, "results": [ {"label": "porn", "rate": 99.9, "scene": "antispam", "suggestion": "block", "details": []} ]}], "msg": "OK", "requestId": "req-legacy" }`) _, err := AliyunTextCensorParser(raw) assert.Error(t, err) assert.Contains(t, err.Error(), "unsupported aliyun response format") } func TestAliyunParserCleanResponsePasses(t *testing.T) { raw := []byte(`{"api":"TextModeration","statusCode":200,"code":200,"message":"OK","requestIds":["req-1"]}`) parsed, err := AliyunTextCensorParser(raw) assert.NoError(t, err) assert.True(t, parsed.Pass) assert.Empty(t, parsed.Details) assert.Equal(t, ForbiddenType(""), parsed.ForbiddenType) } func TestAliyunEmptyCleanRawParses(t *testing.T) { // 空内容分支(原文只有一条可信链接、被 RemoveTrustedURL 整段移除)也必须走同一个解析器, // 保证「在线调用」与「缓存回放」两条路径产出的结构完全一致。 parsed, err := AliyunTextCensorParser(aliyunEmptyCleanRaw) assert.NoError(t, err) assert.True(t, parsed.Pass) assert.Empty(t, parsed.Details) } func TestAliyunParserFeedsPolicy(t *testing.T) { // 端到端校验:riskLevel=medium 折算 0.6,低于标准档的辱骂门槛(0.9), // 因此「标准档只拦高风险辱骂」,而严格档不限置信度、照拦。 raw := []byte(`{ "api": "TextModeration", "statusCode": 200, "code": 200, "message": "OK", "requestIds": ["req-1"], "labels": ["profanity", "negative_content"], "riskLevel": "medium", "hint": "示例命中词" }`) parsed, err := AliyunTextCensorParser(raw) assert.NoError(t, err) assert.Len(t, parsed.Details, 2) assert.Equal(t, ForbiddenTypeAbuse, parsed.Details[0].Label) assert.InDelta(t, 0.6, parsed.Details[0].Score, 1e-9) assert.Equal(t, ForbiddenTypeHarmful, parsed.Details[1].Label) // 首个命中的标签决定 ForbiddenType / Confidence。 assert.Equal(t, ForbiddenTypeAbuse, parsed.ForbiddenType) assert.InDelta(t, 0.6, parsed.Confidence, 1e-9) assert.Equal(t, "示例命中词", parsed.Hint) assert.Equal(t, SuggestionReview, parsed.ProviderSuggestion) assert.False(t, parsed.Pass) decision, _ := Decide("lax", parsed) assert.False(t, decision.Blocked()) // 中等风险不足 0.9 门槛 → 标准档放行(收敛误伤)。 decision, _ = Decide("standard", parsed) assert.False(t, decision.Blocked()) // 严格档不限置信度 → 拦截。 decision, detail := Decide("strict", parsed) assert.Equal(t, DecisionBlock, decision) assert.NotNil(t, detail) assert.Equal(t, ForbiddenTypeAbuse, detail.Label) } func TestAliyunParserMapsBaselineLabels(t *testing.T) { // 底线四件套在 2.0 下必须仍然映射得到(标签名与 1.0 完全不同): // political_content 涉政、violence 暴恐、sexual_content 色情、C_customized 自定义词库。 for _, tc := range []struct { label string want ForbiddenType }{ {"political_content", ForbiddenTypePolitics}, {"regional", ForbiddenTypePolitics}, {"violence", ForbiddenTypeTerrorism}, {"sexual_content", ForbiddenTypePorn}, {"sexuality", ForbiddenTypePorn}, {"C_customized", ForbiddenTypeCustomized}, {"contraband", ForbiddenTypeContraband}, {"ad", ForbiddenTypeAd}, {"PullinTraffic", ForbiddenTypeAd}, {"ad_compliance", ForbiddenTypeAd}, {"profanity", ForbiddenTypeAbuse}, {"negative_content", ForbiddenTypeHarmful}, {"cyberbullying", ForbiddenTypeHarmful}, {"religion", ForbiddenTypeHarmful}, {"nonsense", ForbiddenTypeMeaningless}, {"spam", ForbiddenTypeSpam}, } { assert.Equal(t, tc.want, formatAliyunForbiddenType(tc.label), "label=%s", tc.label) // 逐一验证:只要映射到 ForbiddenType,就必须能被策略层正常消费。 mapped := formatAliyunForbiddenType(tc.label) raw, err := json.Marshal(aliyunTextModerationRaw{ API: aliyunTextModerationAPI, Code: 200, Labels: []string{tc.label}, RiskLevel: "low", }) assert.NoError(t, err) parsed, err := AliyunTextCensorParser(raw) assert.NoError(t, err) assert.Len(t, parsed.Details, 1) assert.Equal(t, mapped, parsed.Details[0].Label) } } func TestAliyunBaselineLabelsBlockAtEveryLevel(t *testing.T) { // 底线维度不受档位影响:即使审核服务给 low、置信度折算为 0.3,也必须拦。 for _, label := range []string{"political_content", "violence", "sexual_content", "C_customized"} { raw, err := json.Marshal(aliyunTextModerationRaw{ API: aliyunTextModerationAPI, Code: 200, Labels: []string{label}, RiskLevel: "low", }) assert.NoError(t, err) parsed, err := AliyunTextCensorParser(raw) assert.NoError(t, err) for _, level := range []string{"lax", "standard", "strict"} { decision, detail := Decide(level, parsed) assert.Equal(t, DecisionBlock, decision, "label=%s level=%s", label, level) assert.NotNil(t, detail) assert.Equal(t, formatAliyunForbiddenType(label), detail.Label) } } } func TestAliyunUnmappedLabelSkipped(t *testing.T) { // 未映射的标签不进入明细(无法参与按维度裁决),原始标签仍保留在 RawResponse 中供审计。 raw, err := json.Marshal(aliyunTextModerationRaw{ API: aliyunTextModerationAPI, Code: 200, Labels: []string{"brand_new_label"}, RiskLevel: "medium", }) assert.NoError(t, err) parsed, err := AliyunTextCensorParser(raw) assert.NoError(t, err) assert.Empty(t, parsed.Details) assert.Equal(t, SuggestionReview, parsed.ProviderSuggestion) } func TestAliyunLabeledResponseWithoutRiskLevelFallsBackToBlock(t *testing.T) { // 与七牛保持一致:审核服务判为「非通过」却没给出建议时补成 block, // 否则策略层会因「没有明细维度 + 没有建议」把命中内容当成未命中放行。 raw, err := json.Marshal(aliyunTextModerationRaw{ API: aliyunTextModerationAPI, Code: 200, Labels: []string{"brand_new_label"}, }) assert.NoError(t, err) parsed, err := AliyunTextCensorParser(raw) assert.NoError(t, err) assert.False(t, parsed.Pass) assert.Equal(t, SuggestionBlock, parsed.ProviderSuggestion) // 严格档采信整体 block 建议 → 拦截;宽松/标准档不采信 → 放行。 decision, _ := Decide("strict", parsed) assert.Equal(t, DecisionBlock, decision) decision, _ = Decide("standard", parsed) assert.False(t, decision.Blocked()) decision, _ = Decide("lax", parsed) assert.False(t, decision.Blocked()) } func TestAliyunInvalidJSONErrorCarriesRawResponse(t *testing.T) { // 响应不是 JSON(如网关返回 HTML 错误页)时,错误里要能看到响应原文。 raw := []byte(`502 Bad Gateway`) _, err := AliyunTextCensorParser(raw) assert.Error(t, err) assert.Contains(t, err.Error(), "502 Bad Gateway") } func TestAliyunRawDigestTruncates(t *testing.T) { assert.Equal(t, "", aliyunRawDigest([]byte(" "))) long := strings.Repeat("敏", aliyunMaxRawLogBytes+50) got := aliyunRawDigest([]byte(long)) suffix := "...(truncated)" assert.True(t, strings.HasSuffix(got, suffix)) // 按字符(而非字节)截断,多字节字符不能被切成半个。 assert.True(t, utf8.ValidString(got)) assert.Equal(t, aliyunMaxRawLogBytes, len([]rune(strings.TrimSuffix(got, suffix)))) } // aliyunSDKResponse 构造一个 SDK 形态的 TextModeration 响应,省去每个用例重复拼结构体。 func aliyunSDKResponse(statusCode, code int32, labels, reason string) *green20220302.TextModerationResponse { response := &green20220302.TextModerationResponse{ StatusCode: dara.Int32(statusCode), Body: &green20220302.TextModerationResponseBody{ Code: dara.Int32(code), Message: dara.String("OK"), RequestId: dara.String("req-1"), }, } if labels != "" || reason != "" { response.Body.Data = &green20220302.TextModerationResponseBodyData{ Labels: dara.String(labels), Reason: dara.String(reason), } } return response } func TestExtractAliyunChunkResult(t *testing.T) { t.Run("成功:解析标签与 Reason", func(t *testing.T) { result, err := extractAliyunChunkResult(aliyunSDKResponse(200, 200, "profanity, negative_content", `{"riskLevel":"medium","riskTips":"辱骂_人身攻击","riskWords":"你是个傻子"}`)) assert.NoError(t, err) assert.Equal(t, int32(200), result.Code) assert.Equal(t, int32(200), result.StatusCode) assert.Equal(t, "req-1", result.RequestID) assert.Equal(t, []string{"profanity", "negative_content"}, result.Labels) assert.Equal(t, "medium", result.RiskLevel) // Hint 优先取命中词。 assert.Equal(t, "你是个傻子", result.Hint) }) t.Run("nonLabel/normal 是占位标签,不进入明细", func(t *testing.T) { result, err := extractAliyunChunkResult(aliyunSDKResponse(200, 200, "nonLabel,normal,", "")) assert.NoError(t, err) assert.Empty(t, result.Labels) }) t.Run("RiskWords 缺失时 Hint 退回 RiskTips", func(t *testing.T) { result, err := extractAliyunChunkResult(aliyunSDKResponse(200, 200, "ad", `{"riskLevel":"high","riskTips":"广告_引流"}`)) assert.NoError(t, err) assert.Equal(t, "广告_引流", result.Hint) }) t.Run("Reason 非法 JSON 不应导致整条审核失败", func(t *testing.T) { result, err := extractAliyunChunkResult(aliyunSDKResponse(200, 200, "ad", `not-json`)) assert.NoError(t, err) assert.Equal(t, []string{"ad"}, result.Labels) assert.Equal(t, "", result.RiskLevel) }) t.Run("code 非 200 必须报错,不能当成未命中", func(t *testing.T) { // 服务未开通时阿里云会返回自己的错误码。若被当成「无标签 → 通过」, // fail-closed 会悄悄变成 fail-open。 _, err := extractAliyunChunkResult(aliyunSDKResponse(200, 596, "", "")) assert.Error(t, err) assert.Contains(t, err.Error(), "596") }) t.Run("HTTP 非 2xx 必须报错", func(t *testing.T) { _, err := extractAliyunChunkResult(aliyunSDKResponse(502, 200, "", "")) assert.Error(t, err) assert.Contains(t, err.Error(), "unexpected http status") }) t.Run("响应体缺失必须报错", func(t *testing.T) { _, err := extractAliyunChunkResult(nil) assert.Error(t, err) _, err = extractAliyunChunkResult(&green20220302.TextModerationResponse{}) assert.Error(t, err) }) } func TestMergeAliyunChunkResults(t *testing.T) { merged := mergeAliyunChunkResults([]aliyunChunkResult{ {StatusCode: 200, Code: 200, RequestID: "r1", Labels: []string{"profanity"}, RiskLevel: "low"}, {StatusCode: 200, Code: 200, RequestID: "r2", Labels: []string{"profanity", "political_content"}, RiskLevel: "high", Hint: "命中词"}, }) assert.Equal(t, aliyunTextModerationAPI, merged.API) assert.Equal(t, []string{"r1", "r2"}, merged.RequestIDs) // 标签取并集并去重,保持出现顺序。 assert.Equal(t, []string{"profanity", "political_content"}, merged.Labels) // 风险级别取最高:任一分片命中高风险,整条内容按高风险处置。 assert.Equal(t, "high", merged.RiskLevel) // Hint 取首个非空。 assert.Equal(t, "命中词", merged.Hint) // 合并结果可直接被解析器消费,且高风险 + 涉政 → 任何档位都拦。 raw, err := json.Marshal(merged) assert.NoError(t, err) parsed, err := AliyunTextCensorParser(raw) assert.NoError(t, err) assert.Len(t, parsed.Details, 2) for _, detail := range parsed.Details { assert.InDelta(t, 1.0, detail.Score, 1e-9) } decision, detail := Decide("lax", parsed) assert.Equal(t, DecisionBlock, decision) assert.NotNil(t, detail) assert.Equal(t, ForbiddenTypePolitics, detail.Label) } func TestMergeAliyunChunkResultsEmpty(t *testing.T) { // 没有任何分片(理论上不会发生,因为空内容有单独的短路分支)时, // 合并结果必须仍然是「无标签」,即不产生任何命中。 merged := mergeAliyunChunkResults(nil) assert.Empty(t, merged.Labels) assert.Equal(t, "", merged.RiskLevel) assert.Equal(t, aliyunTextModerationAPI, merged.API) } func TestAliyunIsAccountIssue(t *testing.T) { // 账号侧未就绪的各类表现都要能识别出来 —— 它们不会自愈,必须给出可操作告警。 for _, message := range []string{ "code=596 You have not opened Yundun Content Moderation Service", "You have not opened Yundun Content Moderation Service, please go to the page", "Forbidden.RAM: no permission to perform this action", "InvalidAccessKeyId.NotFound: the access key id is invalid", "SignatureDoesNotMatch: signature mismatch", } { assert.True(t, aliyunIsAccountIssue(errors.New(message)), "message=%q", message) } // 偶发/网络类问题不应被当成账号问题,否则一次性告警会被无关噪声消耗掉。 for _, message := range []string{ "dial tcp 1.2.3.4:443: i/o timeout", "EOF", "context deadline exceeded", } { assert.False(t, aliyunIsAccountIssue(errors.New(message)), "message=%q", message) } assert.False(t, aliyunIsAccountIssue(nil)) } func TestAliyunIsAccountIssueFromStructuredCode(t *testing.T) { // SDK 的结构化 code 比消息文本更稳定:即使 message 里没有特征词也要能识别。 sdkErr := dara.NewSDKError(map[string]interface{}{ "code": "Forbidden", "message": "the request is denied", }) assert.True(t, aliyunIsAccountIssue(sdkErr)) }