aliyun sensor server

2026-09-10 18:04:19 +08:00
parent e3e59e7c69
commit 8e2325eda5
38 changed files with 3147 additions and 344 deletions
@@ -0,0 +1,459 @@
+// Copyright 2022 E99p1ant. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package censor
+
+import (
+	"encoding/json"
+	"errors"
+	"strings"
+	"testing"
+	"unicode/utf8"
+
+	green20220302 "github.com/alibabacloud-go/green-20220302/v3/client"
+	"github.com/alibabacloud-go/tea/dara"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewAliyunTextModerationRequestShape(t *testing.T) {
+	// 2.0（TextModeration）的接口契约：Service 指明审核场景，
+	// ServiceParameters 是 JSON 字符串 {"content": ...}。
+	req, err := newAliyunTextModerationRequest("测试文本")
+	assert.NoError(t, err)
+	assert.Equal(t, aliyunTextService, dara.StringValue(req.Service))
+
+	var serviceParameters map[string]string
+	assert.NoError(t, json.Unmarshal([]byte(dara.StringValue(req.ServiceParameters)), &serviceParameters))
+	assert.Equal(t, "测试文本", serviceParameters["content"])
+}
+
+func TestSplitAliyunText(t *testing.T) {
+	// 未超限时原样返回，不做无意义的切分。
+	assert.Equal(t, []string{"短文本"}, splitAliyunText("短文本"))
+
+	// 恰好等于上限：仍是单片。
+	exact := strings.Repeat("a", aliyunContentRuneLimit)
+	assert.Equal(t, []string{exact}, splitAliyunText(exact))
+
+	// 各档站点内容上限都要能覆盖：提问 1000 / 回答·私聊 2000 / 投票描述 5000。
+	for _, length := range []int{aliyunContentRuneLimit + 1, 1000, 2000, 5000} {
+		text := strings.Repeat("敏", length)
+		chunks := splitAliyunText(text)
+
+		assert.Greater(t, len(chunks), 1, "length=%d 应当被切分", length)
+
+		for i, chunk := range chunks {
+			// 单次不得超过 2.0 的硬上限，否则请求会被服务端拒绝。
+			assert.LessOrEqual(t, utf8.RuneCountInString(chunk), aliyunContentRuneLimit,
+				"length=%d chunk=%d 超过单次上限", length, i)
+			// 按 rune 切分，不能切坏多字节字符。
+			assert.True(t, utf8.ValidString(chunk), "length=%d chunk=%d 切坏字符", length, i)
+		}
+
+		// 首片必须从原文开头开始。
+		assert.Equal(t, string([]rune(text)[:aliyunContentRuneLimit]), chunks[0], "length=%d", length)
+		// 末片必须收束到原文结尾，否则尾部内容整段漏审。
+		assert.True(t, strings.HasSuffix(text, chunks[len(chunks)-1]),
+			"length=%d 最后一个分片未覆盖到原文结尾", length)
+
+		// 相邻分片必须重叠：消除「风险词正好跨切分点」的漏判盲区。
+		for i := 1; i < len(chunks); i++ {
+			previous := []rune(chunks[i-1])
+			current := []rune(chunks[i])
+			overlap := aliyunChunkOverlapRunes
+			if len(previous) < overlap {
+				overlap = len(previous)
+			}
+			assert.Equal(t, string(previous[len(previous)-overlap:]), string(current[:overlap]),
+				"length=%d 分片 %d 与上一片的重叠不符合预期", length, i)
+		}
+	}
+}
+
+func TestAliyunRiskLevelScore(t *testing.T) {
+	// 2.0 不返回逐标签置信度，只返回 riskLevel，这里折算成 0~1 供 policy.go 复用。
+	assert.Equal(t, 1.0, aliyunRiskLevelScore("high"))
+	assert.Equal(t, 1.0, aliyunRiskLevelScore("HIGH"))
+	assert.Equal(t, 0.6, aliyunRiskLevelScore("medium"))
+	assert.Equal(t, 0.3, aliyunRiskLevelScore("low"))
+	assert.Equal(t, 0.0, aliyunRiskLevelScore("none"))
+	// 未知/缺失级别取保守值 1：宁可够到辱骂门槛被判拦截，也不要静默降权后放行。
+	assert.Equal(t, 1.0, aliyunRiskLevelScore(""))
+	assert.Equal(t, 1.0, aliyunRiskLevelScore("brand-new-level"))
+}
+
+func TestAliyunRiskLevelSuggestion(t *testing.T) {
+	assert.Equal(t, SuggestionBlock, aliyunRiskLevelSuggestion("high"))
+	assert.Equal(t, SuggestionReview, aliyunRiskLevelSuggestion("medium"))
+	assert.Equal(t, SuggestionPass, aliyunRiskLevelSuggestion("low"))
+	assert.Equal(t, "", aliyunRiskLevelSuggestion("none"))
+	assert.Equal(t, "", aliyunRiskLevelSuggestion(""))
+}
+
+func TestAliyunHighestRiskLevel(t *testing.T) {
+	assert.Equal(t, "", aliyunHighestRiskLevel(nil))
+	assert.Equal(t, "", aliyunHighestRiskLevel([]string{"", "  "}))
+	assert.Equal(t, "low", aliyunHighestRiskLevel([]string{"low"}))
+	assert.Equal(t, "medium", aliyunHighestRiskLevel([]string{"low", "medium"}))
+	assert.Equal(t, "high", aliyunHighestRiskLevel([]string{"low", "high", "medium"}))
+	// 未知级别按最高处理（方向保守），否则新级别会把整条内容静默降级。
+	assert.Equal(t, "high", aliyunHighestRiskLevel([]string{"low", "brand-new-level"}))
+}
+
+func TestAliyunParserRejectsLegacyFormat(t *testing.T) {
+	// 迁移前内容安全 1.0 的响应（/green/text/scan）必须被拒绝。
+	// 这是关键回归护栏：若把无法识别的载荷当成「无标签 → 通过」，
+	// 历史缓存就会变成绕过审核的后门。
+	raw := []byte(`{
+		"code": 200,
+		"data": [{"code": 200, "results": [
+			{"label": "porn", "rate": 99.9, "scene": "antispam", "suggestion": "block", "details": []}
+		]}],
+		"msg": "OK",
+		"requestId": "req-legacy"
+	}`)
+
+	_, err := AliyunTextCensorParser(raw)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "unsupported aliyun response format")
+}
+
+func TestAliyunParserCleanResponsePasses(t *testing.T) {
+	raw := []byte(`{"api":"TextModeration","statusCode":200,"code":200,"message":"OK","requestIds":["req-1"]}`)
+
+	parsed, err := AliyunTextCensorParser(raw)
+	assert.NoError(t, err)
+	assert.True(t, parsed.Pass)
+	assert.Empty(t, parsed.Details)
+	assert.Equal(t, ForbiddenType(""), parsed.ForbiddenType)
+}
+
+func TestAliyunEmptyCleanRawParses(t *testing.T) {
+	// 空内容分支（原文只有一条可信链接、被 RemoveTrustedURL 整段移除）也必须走同一个解析器，
+	// 保证「在线调用」与「缓存回放」两条路径产出的结构完全一致。
+	parsed, err := AliyunTextCensorParser(aliyunEmptyCleanRaw)
+	assert.NoError(t, err)
+	assert.True(t, parsed.Pass)
+	assert.Empty(t, parsed.Details)
+}
+
+func TestAliyunParserFeedsPolicy(t *testing.T) {
+	// 端到端校验：riskLevel=medium 折算 0.6，低于标准档的辱骂门槛（0.9），
+	// 因此「标准档只拦高风险辱骂」，而严格档不限置信度、照拦。
+	raw := []byte(`{
+		"api": "TextModeration",
+		"statusCode": 200,
+		"code": 200,
+		"message": "OK",
+		"requestIds": ["req-1"],
+		"labels": ["profanity", "negative_content"],
+		"riskLevel": "medium",
+		"hint": "示例命中词"
+	}`)
+
+	parsed, err := AliyunTextCensorParser(raw)
+	assert.NoError(t, err)
+
+	assert.Len(t, parsed.Details, 2)
+	assert.Equal(t, ForbiddenTypeAbuse, parsed.Details[0].Label)
+	assert.InDelta(t, 0.6, parsed.Details[0].Score, 1e-9)
+	assert.Equal(t, ForbiddenTypeHarmful, parsed.Details[1].Label)
+	// 首个命中的标签决定 ForbiddenType / Confidence。
+	assert.Equal(t, ForbiddenTypeAbuse, parsed.ForbiddenType)
+	assert.InDelta(t, 0.6, parsed.Confidence, 1e-9)
+	assert.Equal(t, "示例命中词", parsed.Hint)
+	assert.Equal(t, SuggestionReview, parsed.ProviderSuggestion)
+	assert.False(t, parsed.Pass)
+
+	decision, _ := Decide("lax", parsed)
+	assert.False(t, decision.Blocked())
+
+	// 中等风险不足 0.9 门槛 → 标准档放行（收敛误伤）。
+	decision, _ = Decide("standard", parsed)
+	assert.False(t, decision.Blocked())
+
+	// 严格档不限置信度 → 拦截。
+	decision, detail := Decide("strict", parsed)
+	assert.Equal(t, DecisionBlock, decision)
+	assert.NotNil(t, detail)
+	assert.Equal(t, ForbiddenTypeAbuse, detail.Label)
+}
+
+func TestAliyunParserMapsBaselineLabels(t *testing.T) {
+	// 底线四件套在 2.0 下必须仍然映射得到（标签名与 1.0 完全不同）：
+	// political_content 涉政、violence 暴恐、sexual_content 色情、C_customized 自定义词库。
+	for _, tc := range []struct {
+		label string
+		want  ForbiddenType
+	}{
+		{"political_content", ForbiddenTypePolitics},
+		{"regional", ForbiddenTypePolitics},
+		{"violence", ForbiddenTypeTerrorism},
+		{"sexual_content", ForbiddenTypePorn},
+		{"sexuality", ForbiddenTypePorn},
+		{"C_customized", ForbiddenTypeCustomized},
+		{"contraband", ForbiddenTypeContraband},
+		{"ad", ForbiddenTypeAd},
+		{"PullinTraffic", ForbiddenTypeAd},
+		{"ad_compliance", ForbiddenTypeAd},
+		{"profanity", ForbiddenTypeAbuse},
+		{"negative_content", ForbiddenTypeHarmful},
+		{"cyberbullying", ForbiddenTypeHarmful},
+		{"religion", ForbiddenTypeHarmful},
+		{"nonsense", ForbiddenTypeMeaningless},
+		{"spam", ForbiddenTypeSpam},
+	} {
+		assert.Equal(t, tc.want, formatAliyunForbiddenType(tc.label), "label=%s", tc.label)
+
+		// 逐一验证：只要映射到 ForbiddenType，就必须能被策略层正常消费。
+		mapped := formatAliyunForbiddenType(tc.label)
+		raw, err := json.Marshal(aliyunTextModerationRaw{
+			API:       aliyunTextModerationAPI,
+			Code:      200,
+			Labels:    []string{tc.label},
+			RiskLevel: "low",
+		})
+		assert.NoError(t, err)
+
+		parsed, err := AliyunTextCensorParser(raw)
+		assert.NoError(t, err)
+		assert.Len(t, parsed.Details, 1)
+		assert.Equal(t, mapped, parsed.Details[0].Label)
+	}
+}
+
+func TestAliyunBaselineLabelsBlockAtEveryLevel(t *testing.T) {
+	// 底线维度不受档位影响：即使审核服务给 low、置信度折算为 0.3，也必须拦。
+	for _, label := range []string{"political_content", "violence", "sexual_content", "C_customized"} {
+		raw, err := json.Marshal(aliyunTextModerationRaw{
+			API:       aliyunTextModerationAPI,
+			Code:      200,
+			Labels:    []string{label},
+			RiskLevel: "low",
+		})
+		assert.NoError(t, err)
+
+		parsed, err := AliyunTextCensorParser(raw)
+		assert.NoError(t, err)
+
+		for _, level := range []string{"lax", "standard", "strict"} {
+			decision, detail := Decide(level, parsed)
+			assert.Equal(t, DecisionBlock, decision, "label=%s level=%s", label, level)
+			assert.NotNil(t, detail)
+			assert.Equal(t, formatAliyunForbiddenType(label), detail.Label)
+		}
+	}
+}
+
+func TestAliyunUnmappedLabelSkipped(t *testing.T) {
+	// 未映射的标签不进入明细（无法参与按维度裁决），原始标签仍保留在 RawResponse 中供审计。
+	raw, err := json.Marshal(aliyunTextModerationRaw{
+		API:       aliyunTextModerationAPI,
+		Code:      200,
+		Labels:    []string{"brand_new_label"},
+		RiskLevel: "medium",
+	})
+	assert.NoError(t, err)
+
+	parsed, err := AliyunTextCensorParser(raw)
+	assert.NoError(t, err)
+	assert.Empty(t, parsed.Details)
+	assert.Equal(t, SuggestionReview, parsed.ProviderSuggestion)
+}
+
+func TestAliyunLabeledResponseWithoutRiskLevelFallsBackToBlock(t *testing.T) {
+	// 与七牛保持一致：审核服务判为「非通过」却没给出建议时补成 block，
+	// 否则策略层会因「没有明细维度 + 没有建议」把命中内容当成未命中放行。
+	raw, err := json.Marshal(aliyunTextModerationRaw{
+		API:    aliyunTextModerationAPI,
+		Code:   200,
+		Labels: []string{"brand_new_label"},
+	})
+	assert.NoError(t, err)
+
+	parsed, err := AliyunTextCensorParser(raw)
+	assert.NoError(t, err)
+	assert.False(t, parsed.Pass)
+	assert.Equal(t, SuggestionBlock, parsed.ProviderSuggestion)
+
+	// 严格档采信整体 block 建议 → 拦截；宽松/标准档不采信 → 放行。
+	decision, _ := Decide("strict", parsed)
+	assert.Equal(t, DecisionBlock, decision)
+	decision, _ = Decide("standard", parsed)
+	assert.False(t, decision.Blocked())
+	decision, _ = Decide("lax", parsed)
+	assert.False(t, decision.Blocked())
+}
+
+func TestAliyunInvalidJSONErrorCarriesRawResponse(t *testing.T) {
+	// 响应不是 JSON（如网关返回 HTML 错误页）时，错误里要能看到响应原文。
+	raw := []byte(`<html>502 Bad Gateway</html>`)
+
+	_, err := AliyunTextCensorParser(raw)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "502 Bad Gateway")
+}
+
+func TestAliyunRawDigestTruncates(t *testing.T) {
+	assert.Equal(t, "<empty>", aliyunRawDigest([]byte("  ")))
+
+	long := strings.Repeat("敏", aliyunMaxRawLogBytes+50)
+	got := aliyunRawDigest([]byte(long))
+
+	suffix := "...(truncated)"
+	assert.True(t, strings.HasSuffix(got, suffix))
+	// 按字符（而非字节）截断，多字节字符不能被切成半个。
+	assert.True(t, utf8.ValidString(got))
+	assert.Equal(t, aliyunMaxRawLogBytes, len([]rune(strings.TrimSuffix(got, suffix))))
+}
+
+// aliyunSDKResponse 构造一个 SDK 形态的 TextModeration 响应，省去每个用例重复拼结构体。
+func aliyunSDKResponse(statusCode, code int32, labels, reason string) *green20220302.TextModerationResponse {
+	response := &green20220302.TextModerationResponse{
+		StatusCode: dara.Int32(statusCode),
+		Body: &green20220302.TextModerationResponseBody{
+			Code:      dara.Int32(code),
+			Message:   dara.String("OK"),
+			RequestId: dara.String("req-1"),
+		},
+	}
+	if labels != "" || reason != "" {
+		response.Body.Data = &green20220302.TextModerationResponseBodyData{
+			Labels: dara.String(labels),
+			Reason: dara.String(reason),
+		}
+	}
+	return response
+}
+
+func TestExtractAliyunChunkResult(t *testing.T) {
+	t.Run("成功：解析标签与 Reason", func(t *testing.T) {
+		result, err := extractAliyunChunkResult(aliyunSDKResponse(200, 200, "profanity, negative_content",
+			`{"riskLevel":"medium","riskTips":"辱骂_人身攻击","riskWords":"你是个傻子"}`))
+		assert.NoError(t, err)
+
+		assert.Equal(t, int32(200), result.Code)
+		assert.Equal(t, int32(200), result.StatusCode)
+		assert.Equal(t, "req-1", result.RequestID)
+		assert.Equal(t, []string{"profanity", "negative_content"}, result.Labels)
+		assert.Equal(t, "medium", result.RiskLevel)
+		// Hint 优先取命中词。
+		assert.Equal(t, "你是个傻子", result.Hint)
+	})
+
+	t.Run("nonLabel/normal 是占位标签，不进入明细", func(t *testing.T) {
+		result, err := extractAliyunChunkResult(aliyunSDKResponse(200, 200, "nonLabel,normal,", ""))
+		assert.NoError(t, err)
+		assert.Empty(t, result.Labels)
+	})
+
+	t.Run("RiskWords 缺失时 Hint 退回 RiskTips", func(t *testing.T) {
+		result, err := extractAliyunChunkResult(aliyunSDKResponse(200, 200, "ad",
+			`{"riskLevel":"high","riskTips":"广告_引流"}`))
+		assert.NoError(t, err)
+		assert.Equal(t, "广告_引流", result.Hint)
+	})
+
+	t.Run("Reason 非法 JSON 不应导致整条审核失败", func(t *testing.T) {
+		result, err := extractAliyunChunkResult(aliyunSDKResponse(200, 200, "ad", `not-json`))
+		assert.NoError(t, err)
+		assert.Equal(t, []string{"ad"}, result.Labels)
+		assert.Equal(t, "", result.RiskLevel)
+	})
+
+	t.Run("code 非 200 必须报错，不能当成未命中", func(t *testing.T) {
+		// 服务未开通时阿里云会返回自己的错误码。若被当成「无标签 → 通过」，
+		// fail-closed 会悄悄变成 fail-open。
+		_, err := extractAliyunChunkResult(aliyunSDKResponse(200, 596, "", ""))
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "596")
+	})
+
+	t.Run("HTTP 非 2xx 必须报错", func(t *testing.T) {
+		_, err := extractAliyunChunkResult(aliyunSDKResponse(502, 200, "", ""))
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "unexpected http status")
+	})
+
+	t.Run("响应体缺失必须报错", func(t *testing.T) {
+		_, err := extractAliyunChunkResult(nil)
+		assert.Error(t, err)
+		_, err = extractAliyunChunkResult(&green20220302.TextModerationResponse{})
+		assert.Error(t, err)
+	})
+}
+
+func TestMergeAliyunChunkResults(t *testing.T) {
+	merged := mergeAliyunChunkResults([]aliyunChunkResult{
+		{StatusCode: 200, Code: 200, RequestID: "r1", Labels: []string{"profanity"}, RiskLevel: "low"},
+		{StatusCode: 200, Code: 200, RequestID: "r2", Labels: []string{"profanity", "political_content"},
+			RiskLevel: "high", Hint: "命中词"},
+	})
+
+	assert.Equal(t, aliyunTextModerationAPI, merged.API)
+	assert.Equal(t, []string{"r1", "r2"}, merged.RequestIDs)
+	// 标签取并集并去重，保持出现顺序。
+	assert.Equal(t, []string{"profanity", "political_content"}, merged.Labels)
+	// 风险级别取最高：任一分片命中高风险，整条内容按高风险处置。
+	assert.Equal(t, "high", merged.RiskLevel)
+	// Hint 取首个非空。
+	assert.Equal(t, "命中词", merged.Hint)
+
+	// 合并结果可直接被解析器消费，且高风险 + 涉政 → 任何档位都拦。
+	raw, err := json.Marshal(merged)
+	assert.NoError(t, err)
+
+	parsed, err := AliyunTextCensorParser(raw)
+	assert.NoError(t, err)
+	assert.Len(t, parsed.Details, 2)
+	for _, detail := range parsed.Details {
+		assert.InDelta(t, 1.0, detail.Score, 1e-9)
+	}
+	decision, detail := Decide("lax", parsed)
+	assert.Equal(t, DecisionBlock, decision)
+	assert.NotNil(t, detail)
+	assert.Equal(t, ForbiddenTypePolitics, detail.Label)
+}
+
+func TestMergeAliyunChunkResultsEmpty(t *testing.T) {
+	// 没有任何分片（理论上不会发生，因为空内容有单独的短路分支）时，
+	// 合并结果必须仍然是「无标签」，即不产生任何命中。
+	merged := mergeAliyunChunkResults(nil)
+	assert.Empty(t, merged.Labels)
+	assert.Equal(t, "", merged.RiskLevel)
+	assert.Equal(t, aliyunTextModerationAPI, merged.API)
+}
+
+func TestAliyunIsAccountIssue(t *testing.T) {
+	// 账号侧未就绪的各类表现都要能识别出来 —— 它们不会自愈，必须给出可操作告警。
+	for _, message := range []string{
+		"code=596 You have not opened Yundun Content Moderation Service",
+		"You have not opened Yundun Content Moderation Service, please go to the page",
+		"Forbidden.RAM: no permission to perform this action",
+		"InvalidAccessKeyId.NotFound: the access key id is invalid",
+		"SignatureDoesNotMatch: signature mismatch",
+	} {
+		assert.True(t, aliyunIsAccountIssue(errors.New(message)), "message=%q", message)
+	}
+
+	// 偶发/网络类问题不应被当成账号问题，否则一次性告警会被无关噪声消耗掉。
+	for _, message := range []string{
+		"dial tcp 1.2.3.4:443: i/o timeout",
+		"EOF",
+		"context deadline exceeded",
+	} {
+		assert.False(t, aliyunIsAccountIssue(errors.New(message)), "message=%q", message)
+	}
+
+	assert.False(t, aliyunIsAccountIssue(nil))
+}
+
+func TestAliyunIsAccountIssueFromStructuredCode(t *testing.T) {
+	// SDK 的结构化 code 比消息文本更稳定：即使 message 里没有特征词也要能识别。
+	sdkErr := dara.NewSDKError(map[string]interface{}{
+		"code":    "Forbidden",
+		"message": "the request is denied",
+	})
+	assert.True(t, aliyunIsAccountIssue(sdkErr))
+}