aliyun sensor server
This commit is contained in:
tamakyi
+533
-112
@@ -4,14 +4,71 @@
|
||||
|
||||
package censor
|
||||
|
||||
// 阿里云文本审核通道 —— 内容安全 2.0「文本审核增强版」。
|
||||
//
|
||||
// 迁移记录（2026-09）：本通道原走「内容安全 1.0」（/green/text/scan + alibaba-cloud-sdk-go），
|
||||
// 但 1.0 的开通前提是**企业实名认证**，个人认证账号无法开通，接口会永久返回
|
||||
// `code=596 You have not opened Yundun Content Moderation Service`。
|
||||
// 现改为 2.0（TextModeration + alibabacloud-go/green-20220302/v3）：2.0 只要求实名认证
|
||||
// （个人可开通），且标签体系对底线维度（涉政/暴恐/色情/自定义词库）覆盖完整。
|
||||
//
|
||||
// 官方文档：
|
||||
// - 文本审核增强版通用服务 API：https://help.aliyun.com/practice_detail/434034
|
||||
// - 标签处理建议参考： https://help.aliyun.com/practice_detail/465315
|
||||
//
|
||||
// ⚠️ 不要把 SDK 里 `TextModerationResponseBodyData.Labels` 的内联注释当作权威标签表：
|
||||
// 那份列表是旧的且不完整（没有 political 相关标签），以官方《标签处理建议参考》为准。
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
stderrors "errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/aliyun/alibaba-cloud-sdk-go/services/green"
|
||||
openapiutil "github.com/alibabacloud-go/darabonba-openapi/v2/utils"
|
||||
green20220302 "github.com/alibabacloud-go/green-20220302/v3/client"
|
||||
"github.com/alibabacloud-go/tea/dara"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const (
|
||||
// aliyunRegionID 内容安全 2.0 的接入地域，SDK 据此解析出
|
||||
// green-cip.cn-shanghai.aliyuncs.com（见绿网 SDK 的 EndpointMap）。
|
||||
aliyunRegionID = "cn-shanghai"
|
||||
|
||||
// aliyunTextService 文本审核增强版的服务场景。
|
||||
//
|
||||
// 选 comment_detection（公聊评论内容检测）：官方定位「评论/弹幕/公聊/转发」，
|
||||
// 明确覆盖涉黄/涉政/涉暴等底线违规与广告导流，是本站（公开问答为主、私聊为辅）
|
||||
// 最通用的单一场景。本通道被提问/回答、投票、私聊三条链路共用，接口层不带场景信息；
|
||||
// 若要按场景细分（例如私聊改用 chat_detection），需给审核入口补 scene 参数。
|
||||
aliyunTextService = "comment_detection"
|
||||
|
||||
// aliyunContentRuneLimit 2.0 通用服务单次 content 的硬上限。
|
||||
// 官方文档：content 限定在 600 字符以内。这里按 550 切分以留出安全余量。
|
||||
aliyunContentRuneLimit = 550
|
||||
|
||||
// aliyunChunkOverlapRunes 相邻分片的重叠字符数。
|
||||
// 切分点可能正好落在一个风险词中间，两段各自都看不出问题；重叠可消除该盲区。
|
||||
// 代价是少量重复计费（文本审核标准 7.5 元/万次，可忽略）。
|
||||
aliyunChunkOverlapRunes = 50
|
||||
|
||||
aliyunReadTimeoutMs = 10000
|
||||
aliyunConnectTimeoutMs = 5000
|
||||
|
||||
// aliyunMaxRawLogBytes 错误信息里回显的原始响应长度上限（按字符计），
|
||||
// 避免异常响应体过大时把日志刷屏。
|
||||
aliyunMaxRawLogBytes = 512
|
||||
|
||||
// aliyunTextModerationAPI 归一化响应的格式标记。
|
||||
//
|
||||
// 解析器据此拒绝「非本格式」的载荷（例如迁移前内容安全 1.0 的响应），
|
||||
// 让 censor.go 的缓存回放能安全地退回一次实时调用，
|
||||
// 而不是把无法识别的历史缓存误判成「通过」。
|
||||
aliyunTextModerationAPI = "TextModeration"
|
||||
)
|
||||
|
||||
type AliyunTextCensor struct {
|
||||
@@ -25,170 +82,534 @@ func NewAliyunTextCensor(accessKey, accessKeySecret string) *AliyunTextCensor {
|
||||
}
|
||||
}
|
||||
|
||||
// 包级复用的 green.Client：审核请求高频且密钥在启动后不变，
|
||||
// 包级复用的 client：审核请求高频且密钥在启动后不变，
|
||||
// 用 sync.Once 懒加载一次并设置超时，避免每次审核都重建连接与对象。
|
||||
var (
|
||||
aliyunGreenOnce sync.Once
|
||||
aliyunGreenClient *green.Client
|
||||
aliyunGreenClient *green20220302.Client
|
||||
aliyunGreenErr error
|
||||
|
||||
// warnAliyunAccountIssueDoOnce 保证「账号侧未就绪」的告警只打一次，避免按请求刷屏。
|
||||
warnAliyunAccountIssueDoOnce sync.Once
|
||||
)
|
||||
|
||||
func (c *AliyunTextCensor) getClient() (*green.Client, error) {
|
||||
// newAliyunClient 构造内容安全 2.0 客户端。
|
||||
//
|
||||
// 不设置 Endpoint 而设置 RegionId：SDK 内置 EndpointMap 会把 cn-shanghai
|
||||
// 解析成 green-cip.cn-shanghai.aliyuncs.com。超时单位是毫秒
|
||||
// （见 darabonba-openapi 的 client.ReadTimeout → 请求参数 readTimeout）。
|
||||
func newAliyunClient(accessKey, accessKeySecret string) (*green20220302.Client, error) {
|
||||
regionID := aliyunRegionID
|
||||
readTimeout := aliyunReadTimeoutMs
|
||||
connectTimeout := aliyunConnectTimeoutMs
|
||||
|
||||
client, err := green20220302.NewClient(&openapiutil.Config{
|
||||
AccessKeyId: &accessKey,
|
||||
AccessKeySecret: &accessKeySecret,
|
||||
RegionId: ®ionID,
|
||||
ReadTimeout: &readTimeout,
|
||||
ConnectTimeout: &connectTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "new content moderation client")
|
||||
}
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func (c *AliyunTextCensor) getClient() (*green20220302.Client, error) {
|
||||
aliyunGreenOnce.Do(func() {
|
||||
aliyunGreenClient, aliyunGreenErr = green.NewClientWithAccessKey("cn-shanghai", c.accessKey, c.accessKeySecret)
|
||||
if aliyunGreenErr == nil {
|
||||
aliyunGreenClient.SetConnectTimeout(5 * time.Second)
|
||||
aliyunGreenClient.SetReadTimeout(10 * time.Second)
|
||||
}
|
||||
aliyunGreenClient, aliyunGreenErr = newAliyunClient(c.accessKey, c.accessKeySecret)
|
||||
})
|
||||
return aliyunGreenClient, aliyunGreenErr
|
||||
}
|
||||
|
||||
type AliyunTextCensorResponse struct {
|
||||
Code int `json:"code"`
|
||||
Data []struct {
|
||||
Code int `json:"code"`
|
||||
Content string `json:"content"`
|
||||
FilteredContent string `json:"filteredContent"`
|
||||
Msg string `json:"msg"`
|
||||
Results []struct {
|
||||
Details []struct {
|
||||
Contexts []struct {
|
||||
Context string `json:"context"`
|
||||
Positions []struct {
|
||||
EndPos int `json:"endPos"`
|
||||
StartPos int `json:"startPos"`
|
||||
} `json:"positions"`
|
||||
} `json:"contexts"`
|
||||
Label string `json:"label"`
|
||||
} `json:"details"`
|
||||
Label string `json:"label"`
|
||||
Rate float64 `json:"rate"`
|
||||
Scene string `json:"scene"`
|
||||
Suggestion string `json:"suggestion"`
|
||||
} `json:"results"`
|
||||
TaskId string `json:"taskId"`
|
||||
} `json:"data"`
|
||||
Msg string `json:"msg"`
|
||||
RequestId string `json:"requestId"`
|
||||
// newAliyunTextModerationRequest 构造 TextModeration 请求。
|
||||
//
|
||||
// 单独抽出便于测试断言请求形态：Service 必须是选定场景，
|
||||
// ServiceParameters 是 JSON 字符串 {"content": ...}。
|
||||
func newAliyunTextModerationRequest(text string) (*green20220302.TextModerationRequest, error) {
|
||||
serviceParameters, err := json.Marshal(map[string]string{"content": text})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "marshal service parameters")
|
||||
}
|
||||
return (&green20220302.TextModerationRequest{}).
|
||||
SetService(aliyunTextService).
|
||||
SetServiceParameters(string(serviceParameters)), nil
|
||||
}
|
||||
|
||||
func (r *AliyunTextCensorResponse) IsPass() bool {
|
||||
if len(r.Data) == 0 || len(r.Data[0].Results) == 0 {
|
||||
return false
|
||||
// splitAliyunText 把待审文本切成不超过 2.0 单次上限的分片（按 rune 切，避免切坏多字节字符）。
|
||||
//
|
||||
// 为什么必须切分：2.0 通用服务单次 content 硬上限 600 字符，而本站内容上限为
|
||||
// 提问 1000 / 回答·追问·私聊 2000 / 投票描述 5000 字，长文本必然超限。
|
||||
// 把超长文本直接发给接口会被拒，进而在 fail-closed 下变成「长内容一律发不出去」。
|
||||
//
|
||||
// 分片带 aliyunChunkOverlapRunes 的重叠，消除「风险词正好跨切分点」的漏判盲区。
|
||||
func splitAliyunText(text string) []string {
|
||||
runes := []rune(text)
|
||||
if len(runes) <= aliyunContentRuneLimit {
|
||||
return []string{text}
|
||||
}
|
||||
|
||||
// ⚠️ Right now, we allow `review` and `pass` to pass the censor.
|
||||
// 任一 scene 判定为 block 即不通过（此前只检查首个 result，存在漏审）。
|
||||
for _, result := range r.Data[0].Results {
|
||||
if result.Suggestion == "block" {
|
||||
return false
|
||||
step := aliyunContentRuneLimit - aliyunChunkOverlapRunes
|
||||
chunks := make([]string, 0, len(runes)/step+1)
|
||||
for start := 0; start < len(runes); start += step {
|
||||
end := start + aliyunContentRuneLimit
|
||||
if end > len(runes) {
|
||||
end = len(runes)
|
||||
}
|
||||
chunks = append(chunks, string(runes[start:end]))
|
||||
if end == len(runes) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return true
|
||||
return chunks
|
||||
}
|
||||
|
||||
// Censor censors text with Aliyun API.
|
||||
// https://developer.qiniu.com/censor/7260/api-text-censor
|
||||
func (c *AliyunTextCensor) Censor(_ context.Context, text string) (*TextCensorResponse, error) {
|
||||
// aliyunTextModerationReason 是 2.0 响应中 Data.Reason 的结构（该字段本身是 JSON 字符串）。
|
||||
//
|
||||
// 只声明用到的字段：riskLevel 风险级别、riskTips 子标签、riskWords 命中词。
|
||||
type aliyunTextModerationReason struct {
|
||||
RiskLevel string `json:"riskLevel"`
|
||||
RiskTips string `json:"riskTips"`
|
||||
RiskWords string `json:"riskWords"`
|
||||
}
|
||||
|
||||
// parseAliyunReason 解析 Data.Reason。
|
||||
//
|
||||
// 字段缺失或非法 JSON 时返回零值 —— 忽略解析错误是刻意的：Reason 是辅助字段，
|
||||
// 不应因为它的格式变化导致整条审核失败。风险级别拿不到时的保守处理见 aliyunRiskLevelScore。
|
||||
func parseAliyunReason(raw string) aliyunTextModerationReason {
|
||||
var reason aliyunTextModerationReason
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
return reason
|
||||
}
|
||||
_ = json.Unmarshal([]byte(raw), &reason)
|
||||
return reason
|
||||
}
|
||||
|
||||
// aliyunRiskLevelScore 把 2.0 的 riskLevel 折算成 0~1 分，供 policy.go 的置信度门槛复用。
|
||||
//
|
||||
// ⚠️ 这是有损的近似：2.0 通用服务**不返回逐标签置信度**，只返回整条内容的 riskLevel，
|
||||
// 因此同一响应内所有标签共用这一个分值（1.0 是按维度给 rate，改造后不具备该信息了）。
|
||||
// 折算后 abuseScoreFloor=0.9 的实际语义变成「标准档只拦 riskLevel=high 的辱骂」，
|
||||
// 与改造前的设计意图（标准档只拦高置信度辱骂）一致。
|
||||
func aliyunRiskLevelScore(level string) float64 {
|
||||
switch strings.ToLower(strings.TrimSpace(level)) {
|
||||
case "high":
|
||||
return 1
|
||||
case "medium":
|
||||
return 0.6
|
||||
case "low":
|
||||
return 0.3
|
||||
case "none":
|
||||
return 0
|
||||
default:
|
||||
// 标签存在却拿不到风险级别：取 1（保守）。
|
||||
// 宁可让它够到辱骂门槛被判拦截，也不要因为字段缺失把命中标签静默降权后放行。
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
// aliyunRiskLevelSuggestion 把 riskLevel 映射成审核服务自身的管控建议，
|
||||
// 供 policy.go 中「严格档采信 block 建议」的分支使用。
|
||||
func aliyunRiskLevelSuggestion(level string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(level)) {
|
||||
case "high":
|
||||
return SuggestionBlock
|
||||
case "medium":
|
||||
return SuggestionReview
|
||||
case "low":
|
||||
return SuggestionPass
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// aliyunRiskLevelRank riskLevel 的高低排序，用于多分片取最高。
|
||||
var aliyunRiskLevelRank = map[string]int{
|
||||
"none": 0,
|
||||
"low": 1,
|
||||
"medium": 2,
|
||||
"high": 3,
|
||||
}
|
||||
|
||||
// aliyunHighestRiskLevel 取多分片中最高的风险级别（high > medium > low > none）。
|
||||
// 任一分片命中高风险，整条内容即按高风险处置；未知级别按最高处理（方向保守）。
|
||||
func aliyunHighestRiskLevel(levels []string) string {
|
||||
best := ""
|
||||
bestRank := -1
|
||||
for _, level := range levels {
|
||||
normalized := strings.ToLower(strings.TrimSpace(level))
|
||||
if normalized == "" {
|
||||
continue
|
||||
}
|
||||
rank, ok := aliyunRiskLevelRank[normalized]
|
||||
if !ok {
|
||||
normalized, rank = "high", aliyunRiskLevelRank["high"]
|
||||
}
|
||||
if rank > bestRank {
|
||||
best, bestRank = normalized, rank
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// aliyunTextModerationRaw 归一化后的可落库响应。
|
||||
//
|
||||
// 它是「在线调用」与「缓存回放」两条路径的唯一输入格式：
|
||||
// Censor 把（可能多分片的）2.0 响应合并成它并序列化，随后交给 AliyunTextCensorParser；
|
||||
// censor.go 命中落库缓存时，也把这段 JSON 交给同一个 parser。
|
||||
type aliyunTextModerationRaw struct {
|
||||
// API 格式标记，取值恒为 aliyunTextModerationAPI，供解析器甄别旧格式载荷。
|
||||
API string `json:"api"`
|
||||
StatusCode int32 `json:"statusCode"`
|
||||
Code int32 `json:"code"`
|
||||
Message string `json:"message"`
|
||||
RequestIDs []string `json:"requestIds,omitempty"`
|
||||
// Labels 已过滤掉 nonLabel / normal，且已去重。
|
||||
Labels []string `json:"labels,omitempty"`
|
||||
RiskLevel string `json:"riskLevel,omitempty"`
|
||||
Hint string `json:"hint,omitempty"`
|
||||
}
|
||||
|
||||
// aliyunEmptyCleanRaw 空内容对应的归一化响应（无标签 → 通过）。见 Censor 的空内容分支。
|
||||
var aliyunEmptyCleanRaw = []byte(`{"api":"TextModeration","statusCode":200,"code":200,"message":"OK"}`)
|
||||
|
||||
// aliyunChunkResult 单个分片审核结果里我们关心的最小信息集。
|
||||
// 抽出这一层是为了让「多分片合并」这段逻辑可以脱离 SDK 与网络单独测试。
|
||||
type aliyunChunkResult struct {
|
||||
StatusCode int32
|
||||
Code int32
|
||||
Message string
|
||||
RequestID string
|
||||
Labels []string
|
||||
RiskLevel string
|
||||
Hint string
|
||||
}
|
||||
|
||||
// extractAliyunChunkResult 从 SDK 响应里提取分片结果。
|
||||
//
|
||||
// code 非 200/0 表示请求被阿里云拒绝（服务未开通、限流、参数错误等），
|
||||
// 必须报错而不是当成「未命中」—— 否则 fail-closed 会被悄悄变成 fail-open。
|
||||
func extractAliyunChunkResult(resp *green20220302.TextModerationResponse) (aliyunChunkResult, error) {
|
||||
if resp == nil || resp.Body == nil {
|
||||
return aliyunChunkResult{}, errors.New("text moderation returned an empty response body")
|
||||
}
|
||||
|
||||
result := aliyunChunkResult{
|
||||
StatusCode: dara.Int32Value(resp.StatusCode),
|
||||
Code: dara.Int32Value(resp.Body.Code),
|
||||
Message: dara.StringValue(resp.Body.Message),
|
||||
RequestID: dara.StringValue(resp.Body.RequestId),
|
||||
}
|
||||
|
||||
if result.Code != 0 && result.Code != 200 {
|
||||
return result, errors.Errorf("aliyun rejected the request: code=%d message=%q request_id=%s",
|
||||
result.Code, result.Message, result.RequestID)
|
||||
}
|
||||
if result.StatusCode != 0 && result.StatusCode/100 != 2 {
|
||||
return result, errors.Errorf("unexpected http status: status=%d message=%q request_id=%s",
|
||||
result.StatusCode, result.Message, result.RequestID)
|
||||
}
|
||||
|
||||
data := resp.Body.Data
|
||||
if data == nil {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
for _, label := range strings.Split(dara.StringValue(data.Labels), ",") {
|
||||
label = strings.TrimSpace(label)
|
||||
// nonLabel / normal 是「无风险」的占位标签，不进入明细。
|
||||
if label == "" || label == "nonLabel" || label == "normal" {
|
||||
continue
|
||||
}
|
||||
result.Labels = append(result.Labels, label)
|
||||
}
|
||||
|
||||
reason := parseAliyunReason(dara.StringValue(data.Reason))
|
||||
result.RiskLevel = reason.RiskLevel
|
||||
// Hint 优先取命中词（与 1.0 时代回显命中上下文的行为对齐），缺失时退回子标签。
|
||||
if reason.RiskWords != "" {
|
||||
result.Hint = reason.RiskWords
|
||||
} else {
|
||||
result.Hint = reason.RiskTips
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// mergeAliyunChunkResults 把多个分片的结果合并成一份归一化响应。
|
||||
//
|
||||
// 合并口径：
|
||||
// - 标签取并集（任一分片命中即视为整条内容命中）；
|
||||
// - 风险级别取最高（high > medium > low > none），未知级别按最高处理；
|
||||
// - Hint 取首个非空，与「首个命中维度决定提示文案」的既有语义保持一致。
|
||||
func mergeAliyunChunkResults(results []aliyunChunkResult) aliyunTextModerationRaw {
|
||||
merged := aliyunTextModerationRaw{
|
||||
API: aliyunTextModerationAPI,
|
||||
StatusCode: 200,
|
||||
Code: 200,
|
||||
Message: "OK",
|
||||
}
|
||||
|
||||
levels := make([]string, 0, len(results))
|
||||
seen := make(map[string]struct{}, 4)
|
||||
|
||||
for _, result := range results {
|
||||
merged.RequestIDs = append(merged.RequestIDs, result.RequestID)
|
||||
if result.RiskLevel != "" {
|
||||
levels = append(levels, result.RiskLevel)
|
||||
}
|
||||
if merged.Hint == "" {
|
||||
merged.Hint = result.Hint
|
||||
}
|
||||
for _, label := range result.Labels {
|
||||
if _, ok := seen[label]; ok {
|
||||
continue
|
||||
}
|
||||
seen[label] = struct{}{}
|
||||
merged.Labels = append(merged.Labels, label)
|
||||
}
|
||||
}
|
||||
|
||||
merged.RiskLevel = aliyunHighestRiskLevel(levels)
|
||||
return merged
|
||||
}
|
||||
|
||||
// Censor censors text with Aliyun Content Moderation 2.0 (TextModeration).
|
||||
// 接口文档：https://help.aliyun.com/practice_detail/434034
|
||||
func (c *AliyunTextCensor) Censor(ctx context.Context, text string) (*TextCensorResponse, error) {
|
||||
client, err := c.getClient()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "new client with access key")
|
||||
return nil, errors.Wrap(err, "new content moderation client")
|
||||
}
|
||||
|
||||
content, err := json.Marshal(
|
||||
map[string]interface{}{
|
||||
"scenes": []string{"antispam"},
|
||||
"tasks": []map[string]interface{}{
|
||||
{"content": text},
|
||||
},
|
||||
},
|
||||
)
|
||||
// 空内容（例如原文只是一条可信链接，已被 RemoveTrustedURL 整段移除）无可审核内容，
|
||||
// 直接按「未命中」处理：既省一次调用，也避免空 content 被判为参数错误后
|
||||
// 在 fail-closed 下把这类提交一并拒掉。
|
||||
if strings.TrimSpace(text) == "" {
|
||||
return AliyunTextCensorParser(aliyunEmptyCleanRaw)
|
||||
}
|
||||
|
||||
chunks := splitAliyunText(text)
|
||||
results := make([]aliyunChunkResult, 0, len(chunks))
|
||||
for i, chunk := range chunks {
|
||||
req, err := newAliyunTextModerationRequest(chunk)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := client.TextModeration(req)
|
||||
if err != nil {
|
||||
// 非 2xx / 签名错误 / RAM 权限不足 / 欠费停服 / 服务未开通：SDK 直接返回错误。
|
||||
warnAliyunAccountIssueOnce(ctx, err)
|
||||
return nil, errors.Wrapf(err, "text moderation (chunk %d/%d len=%d)",
|
||||
i+1, len(chunks), utf8.RuneCountInString(chunk))
|
||||
}
|
||||
|
||||
result, err := extractAliyunChunkResult(resp)
|
||||
if err != nil {
|
||||
warnAliyunAccountIssueOnce(ctx, err)
|
||||
return nil, errors.Wrapf(err, "text moderation (chunk %d/%d len=%d)",
|
||||
i+1, len(chunks), utf8.RuneCountInString(chunk))
|
||||
}
|
||||
results = append(results, result)
|
||||
}
|
||||
|
||||
raw, err := json.Marshal(mergeAliyunChunkResults(results))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "marshal content")
|
||||
return nil, errors.Wrap(err, "marshal merged response")
|
||||
}
|
||||
|
||||
req := green.CreateTextScanRequest()
|
||||
req.SetContent(content)
|
||||
resp, err := client.TextScan(req)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "text scan")
|
||||
}
|
||||
|
||||
if !resp.IsSuccess() {
|
||||
return nil, errors.New("response is not success")
|
||||
}
|
||||
|
||||
return AliyunTextCensorParser(resp.GetHttpContentBytes())
|
||||
return AliyunTextCensorParser(raw)
|
||||
}
|
||||
|
||||
func (*AliyunTextCensor) String() string {
|
||||
return "aliyun"
|
||||
}
|
||||
|
||||
// aliyunRawDigest 返回原始响应的可读摘要（超长截断），供错误诊断使用。
|
||||
func aliyunRawDigest(raw []byte) string {
|
||||
s := strings.TrimSpace(string(raw))
|
||||
if s == "" {
|
||||
return "<empty>"
|
||||
}
|
||||
runes := []rune(s)
|
||||
if len(runes) > aliyunMaxRawLogBytes {
|
||||
return string(runes[:aliyunMaxRawLogBytes]) + "...(truncated)"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// AliyunTextCensorParser 把归一化响应转成通用的 TextCensorResponse。
|
||||
//
|
||||
// 在线调用与缓存回放共用本函数（见 aliyunTextModerationRaw 的说明）。
|
||||
func AliyunTextCensorParser(raw []byte) (*TextCensorResponse, error) {
|
||||
var responseJSON AliyunTextCensorResponse
|
||||
if err := json.Unmarshal(raw, &responseJSON); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal response")
|
||||
var parsed aliyunTextModerationRaw
|
||||
if err := json.Unmarshal(raw, &parsed); err != nil {
|
||||
return nil, errors.Wrapf(err, "unmarshal response (raw=%s)", aliyunRawDigest(raw))
|
||||
}
|
||||
|
||||
if len(responseJSON.Data) == 0 {
|
||||
return nil, errors.New("response data is empty")
|
||||
// 格式标记不符：典型是迁移前内容安全 1.0 落库的缓存（形如 {"code":200,"data":[...]}）。
|
||||
// 这种载荷无法按 2.0 语义解释，必须报错，交由 censor.go 退回一次实时调用；
|
||||
// 若当成「无标签 → 通过」，历史缓存就会变成绕过审核的后门。
|
||||
if parsed.API != aliyunTextModerationAPI {
|
||||
return nil, errors.Errorf("unsupported aliyun response format (api=%q raw=%s)",
|
||||
parsed.API, aliyunRawDigest(raw))
|
||||
}
|
||||
|
||||
var hint string
|
||||
var label string
|
||||
var confidence float64
|
||||
for _, result := range responseJSON.Data[0].Results {
|
||||
if result.Label == "normal" {
|
||||
// 2.0 不返回逐标签置信度，只有整条内容的 riskLevel，所有标签共用折算后的分值。
|
||||
score := aliyunRiskLevelScore(parsed.RiskLevel)
|
||||
|
||||
// Pass 取「审核服务未给出任何风险标签」。注意它不是最终裁决 ——
|
||||
// 是否拦截由 policy.go 按箱主严格度决定。
|
||||
pass := len(parsed.Labels) == 0
|
||||
|
||||
suggestion := aliyunRiskLevelSuggestion(parsed.RiskLevel)
|
||||
// 与七牛保持一致：审核服务判为「非通过」却没给出建议时补成 block。
|
||||
// 典型场景是命中了尚未映射的新标签（既没有明细维度、也没有 riskLevel），
|
||||
// 不补的话策略层会把它当成未命中放行，与改造前的 fail-closed 行为不一致。
|
||||
if suggestion == "" && !pass {
|
||||
suggestion = SuggestionBlock
|
||||
}
|
||||
|
||||
details := make([]LabelScore, 0, len(parsed.Labels))
|
||||
var (
|
||||
forbiddenType ForbiddenType
|
||||
confidence float64
|
||||
)
|
||||
for _, label := range parsed.Labels {
|
||||
mapped := formatAliyunForbiddenType(label)
|
||||
if mapped == "" {
|
||||
// 未映射的标签不进入明细：它无法参与按维度裁决。
|
||||
// 原始标签仍完整保留在 RawResponse 中供审计。
|
||||
continue
|
||||
}
|
||||
|
||||
// Get the first context as the hint, forbidden type, confidence.
|
||||
for _, detail := range result.Details {
|
||||
for _, context := range detail.Contexts {
|
||||
hint = context.Context
|
||||
}
|
||||
}
|
||||
details = append(details, LabelScore{Label: mapped, Score: score})
|
||||
|
||||
label = result.Label
|
||||
confidence = result.Rate
|
||||
break
|
||||
// 保留既有语义：首个命中的标签决定 ForbiddenType / Confidence。
|
||||
if forbiddenType == "" {
|
||||
forbiddenType = mapped
|
||||
confidence = score
|
||||
}
|
||||
}
|
||||
|
||||
return &TextCensorResponse{
|
||||
SourceName: "aliyun",
|
||||
Pass: responseJSON.IsPass(),
|
||||
ForbiddenType: formatAliyunForbiddenType(label),
|
||||
Hint: hint,
|
||||
Confidence: confidence,
|
||||
RawResponse: raw,
|
||||
SourceName: "aliyun",
|
||||
Pass: pass,
|
||||
ForbiddenType: forbiddenType,
|
||||
Hint: parsed.Hint,
|
||||
Confidence: confidence,
|
||||
RawResponse: raw,
|
||||
Details: details,
|
||||
ProviderSuggestion: suggestion,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// formatAliyunForbiddenType 把内容安全 2.0 的标签映射为本项目的 ForbiddenType。
|
||||
//
|
||||
// 标签取值以官方《文本审核增强版标签处理建议参考》为准；括号内为官方释义与风险级别：
|
||||
//
|
||||
// political_content / regional 涉政 （高）
|
||||
// violence 暴恐 （高）
|
||||
// sexual_content / sexuality 色情 （高）
|
||||
// contraband 违禁 （高）
|
||||
// ad / PullinTraffic / ad_compliance 广告引流/广告法合规（中高）
|
||||
// profanity 辱骂 （中高）
|
||||
// religion 宗教 （中高）
|
||||
// negative_content / cyberbullying 不良内容/网络暴力 （中）
|
||||
// nonsense 无意义 （中）
|
||||
// spam 垃圾信息
|
||||
// C_customized 自定义词库命中
|
||||
//
|
||||
// 映射结果决定 policy.go 的裁决：政治/暴恐/色情/自定义属底线维度，任何档位都拦。
|
||||
// 注意 2.0 没有「灌水(flood)」维度（同类内容归入 spam），因此 ForbiddenTypeFlood
|
||||
// 不会由本通道产生，七牛仍可能返回它。
|
||||
func formatAliyunForbiddenType(typ string) ForbiddenType {
|
||||
switch typ {
|
||||
case "spam":
|
||||
return ForbiddenTypeSpam
|
||||
case "ad":
|
||||
return ForbiddenTypeAd
|
||||
case "politics":
|
||||
switch strings.ToLower(strings.TrimSpace(typ)) {
|
||||
case "political_content", "regional":
|
||||
return ForbiddenTypePolitics
|
||||
case "terrorism":
|
||||
case "violence":
|
||||
return ForbiddenTypeTerrorism
|
||||
case "abuse":
|
||||
return ForbiddenTypeAbuse
|
||||
case "porn":
|
||||
case "sexual_content", "sexuality":
|
||||
return ForbiddenTypePorn
|
||||
case "flood":
|
||||
return ForbiddenTypeFlood
|
||||
case "contraband":
|
||||
return ForbiddenTypeContraband
|
||||
case "meaningless":
|
||||
return ForbiddenTypeMeaningless
|
||||
case "harmful":
|
||||
case "ad", "pullintraffic", "ad_compliance":
|
||||
return ForbiddenTypeAd
|
||||
case "profanity":
|
||||
return ForbiddenTypeAbuse
|
||||
case "negative_content", "cyberbullying", "religion":
|
||||
return ForbiddenTypeHarmful
|
||||
case "nonsense":
|
||||
return ForbiddenTypeMeaningless
|
||||
case "spam":
|
||||
return ForbiddenTypeSpam
|
||||
case "c_customized", "customized":
|
||||
// 命中运营方在阿里云控制台配置的自定义词库（属显式规则，见 policy.go 的底线集合）。
|
||||
return ForbiddenTypeCustomized
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// aliyunAccountIssueSignals 「账号侧未就绪」的特征信号（小写匹配），覆盖结构化 code 与消息文本两条路径：
|
||||
// - 596 / "not opened"：服务未开通
|
||||
// - Forbidden：RAM 未授权
|
||||
// - InvalidAccessKeyId / SignatureDoesNotMatch：密钥错误
|
||||
var aliyunAccountIssueSignals = []string{
|
||||
"596",
|
||||
"not opened",
|
||||
"forbidden",
|
||||
"invalidaccesskeyid",
|
||||
"signaturedoesnotmatch",
|
||||
"no permission",
|
||||
}
|
||||
|
||||
// aliyunIsAccountIssue 判断一次失败是否属于「账号侧未就绪」。
|
||||
// 先看 SDK 的结构化 code（稳定），再退回错误消息文本匹配（兼容 code 缺失的情况）。
|
||||
func aliyunIsAccountIssue(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
candidates := []string{strings.ToLower(err.Error())}
|
||||
var sdkErr *dara.SDKError
|
||||
if stderrors.As(err, &sdkErr) {
|
||||
candidates = append(candidates, strings.ToLower(dara.StringValue(sdkErr.GetCode())))
|
||||
}
|
||||
|
||||
for _, candidate := range candidates {
|
||||
if candidate == "" {
|
||||
continue
|
||||
}
|
||||
for _, signal := range aliyunAccountIssueSignals {
|
||||
if strings.Contains(candidate, signal) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// warnAliyunAccountIssueOnce 在首次出现「账号侧未就绪」类失败时告警一次。
|
||||
//
|
||||
// 这类问题（服务未开通 / RAM 未授权 / 密钥无效）不会自愈，会把 fail-closed 的审核链路
|
||||
// 永久卡死：外部表现是「全站内容都发不出去」，或（auto 模式下）内容静默落到下一个通道，
|
||||
// 但日志里原本只是一条普通 ERRO，极易被当成偶发网络抖动而忽略。
|
||||
// 因此第一次遇到就打出可操作的指引，并用 sync.Once 约束为只打一次，避免按请求刷屏。
|
||||
func warnAliyunAccountIssueOnce(ctx context.Context, err error) {
|
||||
if !aliyunIsAccountIssue(err) {
|
||||
return
|
||||
}
|
||||
warnAliyunAccountIssueDoOnce.Do(func() {
|
||||
logrus.WithContext(ctx).WithError(err).
|
||||
Warn("Aliyun text moderation looks unavailable due to an account-level issue " +
|
||||
"(service not activated / RAM not authorized / invalid AccessKey). " +
|
||||
"Fix: activate 内容安全 2.0「文本审核增强版」(pay-as-you-go) at the Content Moderation " +
|
||||
"console, and grant AliyunYundunGreenWebFullAccess to the RAM user. " +
|
||||
"Note: 内容安全 1.0 requires an enterprise-verified Aliyun account and can never be " +
|
||||
"activated by an individual account (code=596 recurs forever), while 2.0 only requires " +
|
||||
"real-name verification. Until this is fixed the censor chain keeps failing closed.")
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,459 @@
|
||||
// Copyright 2022 E99p1ant. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package censor
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
|
||||
green20220302 "github.com/alibabacloud-go/green-20220302/v3/client"
|
||||
"github.com/alibabacloud-go/tea/dara"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestNewAliyunTextModerationRequestShape(t *testing.T) {
|
||||
// 2.0（TextModeration）的接口契约：Service 指明审核场景，
|
||||
// ServiceParameters 是 JSON 字符串 {"content": ...}。
|
||||
req, err := newAliyunTextModerationRequest("测试文本")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, aliyunTextService, dara.StringValue(req.Service))
|
||||
|
||||
var serviceParameters map[string]string
|
||||
assert.NoError(t, json.Unmarshal([]byte(dara.StringValue(req.ServiceParameters)), &serviceParameters))
|
||||
assert.Equal(t, "测试文本", serviceParameters["content"])
|
||||
}
|
||||
|
||||
func TestSplitAliyunText(t *testing.T) {
|
||||
// 未超限时原样返回，不做无意义的切分。
|
||||
assert.Equal(t, []string{"短文本"}, splitAliyunText("短文本"))
|
||||
|
||||
// 恰好等于上限：仍是单片。
|
||||
exact := strings.Repeat("a", aliyunContentRuneLimit)
|
||||
assert.Equal(t, []string{exact}, splitAliyunText(exact))
|
||||
|
||||
// 各档站点内容上限都要能覆盖：提问 1000 / 回答·私聊 2000 / 投票描述 5000。
|
||||
for _, length := range []int{aliyunContentRuneLimit + 1, 1000, 2000, 5000} {
|
||||
text := strings.Repeat("敏", length)
|
||||
chunks := splitAliyunText(text)
|
||||
|
||||
assert.Greater(t, len(chunks), 1, "length=%d 应当被切分", length)
|
||||
|
||||
for i, chunk := range chunks {
|
||||
// 单次不得超过 2.0 的硬上限，否则请求会被服务端拒绝。
|
||||
assert.LessOrEqual(t, utf8.RuneCountInString(chunk), aliyunContentRuneLimit,
|
||||
"length=%d chunk=%d 超过单次上限", length, i)
|
||||
// 按 rune 切分，不能切坏多字节字符。
|
||||
assert.True(t, utf8.ValidString(chunk), "length=%d chunk=%d 切坏字符", length, i)
|
||||
}
|
||||
|
||||
// 首片必须从原文开头开始。
|
||||
assert.Equal(t, string([]rune(text)[:aliyunContentRuneLimit]), chunks[0], "length=%d", length)
|
||||
// 末片必须收束到原文结尾，否则尾部内容整段漏审。
|
||||
assert.True(t, strings.HasSuffix(text, chunks[len(chunks)-1]),
|
||||
"length=%d 最后一个分片未覆盖到原文结尾", length)
|
||||
|
||||
// 相邻分片必须重叠：消除「风险词正好跨切分点」的漏判盲区。
|
||||
for i := 1; i < len(chunks); i++ {
|
||||
previous := []rune(chunks[i-1])
|
||||
current := []rune(chunks[i])
|
||||
overlap := aliyunChunkOverlapRunes
|
||||
if len(previous) < overlap {
|
||||
overlap = len(previous)
|
||||
}
|
||||
assert.Equal(t, string(previous[len(previous)-overlap:]), string(current[:overlap]),
|
||||
"length=%d 分片 %d 与上一片的重叠不符合预期", length, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAliyunRiskLevelScore(t *testing.T) {
|
||||
// 2.0 不返回逐标签置信度，只返回 riskLevel，这里折算成 0~1 供 policy.go 复用。
|
||||
assert.Equal(t, 1.0, aliyunRiskLevelScore("high"))
|
||||
assert.Equal(t, 1.0, aliyunRiskLevelScore("HIGH"))
|
||||
assert.Equal(t, 0.6, aliyunRiskLevelScore("medium"))
|
||||
assert.Equal(t, 0.3, aliyunRiskLevelScore("low"))
|
||||
assert.Equal(t, 0.0, aliyunRiskLevelScore("none"))
|
||||
// 未知/缺失级别取保守值 1：宁可够到辱骂门槛被判拦截，也不要静默降权后放行。
|
||||
assert.Equal(t, 1.0, aliyunRiskLevelScore(""))
|
||||
assert.Equal(t, 1.0, aliyunRiskLevelScore("brand-new-level"))
|
||||
}
|
||||
|
||||
func TestAliyunRiskLevelSuggestion(t *testing.T) {
|
||||
assert.Equal(t, SuggestionBlock, aliyunRiskLevelSuggestion("high"))
|
||||
assert.Equal(t, SuggestionReview, aliyunRiskLevelSuggestion("medium"))
|
||||
assert.Equal(t, SuggestionPass, aliyunRiskLevelSuggestion("low"))
|
||||
assert.Equal(t, "", aliyunRiskLevelSuggestion("none"))
|
||||
assert.Equal(t, "", aliyunRiskLevelSuggestion(""))
|
||||
}
|
||||
|
||||
func TestAliyunHighestRiskLevel(t *testing.T) {
|
||||
assert.Equal(t, "", aliyunHighestRiskLevel(nil))
|
||||
assert.Equal(t, "", aliyunHighestRiskLevel([]string{"", " "}))
|
||||
assert.Equal(t, "low", aliyunHighestRiskLevel([]string{"low"}))
|
||||
assert.Equal(t, "medium", aliyunHighestRiskLevel([]string{"low", "medium"}))
|
||||
assert.Equal(t, "high", aliyunHighestRiskLevel([]string{"low", "high", "medium"}))
|
||||
// 未知级别按最高处理（方向保守），否则新级别会把整条内容静默降级。
|
||||
assert.Equal(t, "high", aliyunHighestRiskLevel([]string{"low", "brand-new-level"}))
|
||||
}
|
||||
|
||||
func TestAliyunParserRejectsLegacyFormat(t *testing.T) {
|
||||
// 迁移前内容安全 1.0 的响应（/green/text/scan）必须被拒绝。
|
||||
// 这是关键回归护栏：若把无法识别的载荷当成「无标签 → 通过」，
|
||||
// 历史缓存就会变成绕过审核的后门。
|
||||
raw := []byte(`{
|
||||
"code": 200,
|
||||
"data": [{"code": 200, "results": [
|
||||
{"label": "porn", "rate": 99.9, "scene": "antispam", "suggestion": "block", "details": []}
|
||||
]}],
|
||||
"msg": "OK",
|
||||
"requestId": "req-legacy"
|
||||
}`)
|
||||
|
||||
_, err := AliyunTextCensorParser(raw)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "unsupported aliyun response format")
|
||||
}
|
||||
|
||||
func TestAliyunParserCleanResponsePasses(t *testing.T) {
|
||||
raw := []byte(`{"api":"TextModeration","statusCode":200,"code":200,"message":"OK","requestIds":["req-1"]}`)
|
||||
|
||||
parsed, err := AliyunTextCensorParser(raw)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, parsed.Pass)
|
||||
assert.Empty(t, parsed.Details)
|
||||
assert.Equal(t, ForbiddenType(""), parsed.ForbiddenType)
|
||||
}
|
||||
|
||||
func TestAliyunEmptyCleanRawParses(t *testing.T) {
|
||||
// 空内容分支（原文只有一条可信链接、被 RemoveTrustedURL 整段移除）也必须走同一个解析器，
|
||||
// 保证「在线调用」与「缓存回放」两条路径产出的结构完全一致。
|
||||
parsed, err := AliyunTextCensorParser(aliyunEmptyCleanRaw)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, parsed.Pass)
|
||||
assert.Empty(t, parsed.Details)
|
||||
}
|
||||
|
||||
func TestAliyunParserFeedsPolicy(t *testing.T) {
|
||||
// 端到端校验：riskLevel=medium 折算 0.6，低于标准档的辱骂门槛（0.9），
|
||||
// 因此「标准档只拦高风险辱骂」，而严格档不限置信度、照拦。
|
||||
raw := []byte(`{
|
||||
"api": "TextModeration",
|
||||
"statusCode": 200,
|
||||
"code": 200,
|
||||
"message": "OK",
|
||||
"requestIds": ["req-1"],
|
||||
"labels": ["profanity", "negative_content"],
|
||||
"riskLevel": "medium",
|
||||
"hint": "示例命中词"
|
||||
}`)
|
||||
|
||||
parsed, err := AliyunTextCensorParser(raw)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Len(t, parsed.Details, 2)
|
||||
assert.Equal(t, ForbiddenTypeAbuse, parsed.Details[0].Label)
|
||||
assert.InDelta(t, 0.6, parsed.Details[0].Score, 1e-9)
|
||||
assert.Equal(t, ForbiddenTypeHarmful, parsed.Details[1].Label)
|
||||
// 首个命中的标签决定 ForbiddenType / Confidence。
|
||||
assert.Equal(t, ForbiddenTypeAbuse, parsed.ForbiddenType)
|
||||
assert.InDelta(t, 0.6, parsed.Confidence, 1e-9)
|
||||
assert.Equal(t, "示例命中词", parsed.Hint)
|
||||
assert.Equal(t, SuggestionReview, parsed.ProviderSuggestion)
|
||||
assert.False(t, parsed.Pass)
|
||||
|
||||
decision, _ := Decide("lax", parsed)
|
||||
assert.False(t, decision.Blocked())
|
||||
|
||||
// 中等风险不足 0.9 门槛 → 标准档放行（收敛误伤）。
|
||||
decision, _ = Decide("standard", parsed)
|
||||
assert.False(t, decision.Blocked())
|
||||
|
||||
// 严格档不限置信度 → 拦截。
|
||||
decision, detail := Decide("strict", parsed)
|
||||
assert.Equal(t, DecisionBlock, decision)
|
||||
assert.NotNil(t, detail)
|
||||
assert.Equal(t, ForbiddenTypeAbuse, detail.Label)
|
||||
}
|
||||
|
||||
func TestAliyunParserMapsBaselineLabels(t *testing.T) {
|
||||
// 底线四件套在 2.0 下必须仍然映射得到（标签名与 1.0 完全不同）：
|
||||
// political_content 涉政、violence 暴恐、sexual_content 色情、C_customized 自定义词库。
|
||||
for _, tc := range []struct {
|
||||
label string
|
||||
want ForbiddenType
|
||||
}{
|
||||
{"political_content", ForbiddenTypePolitics},
|
||||
{"regional", ForbiddenTypePolitics},
|
||||
{"violence", ForbiddenTypeTerrorism},
|
||||
{"sexual_content", ForbiddenTypePorn},
|
||||
{"sexuality", ForbiddenTypePorn},
|
||||
{"C_customized", ForbiddenTypeCustomized},
|
||||
{"contraband", ForbiddenTypeContraband},
|
||||
{"ad", ForbiddenTypeAd},
|
||||
{"PullinTraffic", ForbiddenTypeAd},
|
||||
{"ad_compliance", ForbiddenTypeAd},
|
||||
{"profanity", ForbiddenTypeAbuse},
|
||||
{"negative_content", ForbiddenTypeHarmful},
|
||||
{"cyberbullying", ForbiddenTypeHarmful},
|
||||
{"religion", ForbiddenTypeHarmful},
|
||||
{"nonsense", ForbiddenTypeMeaningless},
|
||||
{"spam", ForbiddenTypeSpam},
|
||||
} {
|
||||
assert.Equal(t, tc.want, formatAliyunForbiddenType(tc.label), "label=%s", tc.label)
|
||||
|
||||
// 逐一验证：只要映射到 ForbiddenType，就必须能被策略层正常消费。
|
||||
mapped := formatAliyunForbiddenType(tc.label)
|
||||
raw, err := json.Marshal(aliyunTextModerationRaw{
|
||||
API: aliyunTextModerationAPI,
|
||||
Code: 200,
|
||||
Labels: []string{tc.label},
|
||||
RiskLevel: "low",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
parsed, err := AliyunTextCensorParser(raw)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, parsed.Details, 1)
|
||||
assert.Equal(t, mapped, parsed.Details[0].Label)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAliyunBaselineLabelsBlockAtEveryLevel(t *testing.T) {
|
||||
// 底线维度不受档位影响：即使审核服务给 low、置信度折算为 0.3，也必须拦。
|
||||
for _, label := range []string{"political_content", "violence", "sexual_content", "C_customized"} {
|
||||
raw, err := json.Marshal(aliyunTextModerationRaw{
|
||||
API: aliyunTextModerationAPI,
|
||||
Code: 200,
|
||||
Labels: []string{label},
|
||||
RiskLevel: "low",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
parsed, err := AliyunTextCensorParser(raw)
|
||||
assert.NoError(t, err)
|
||||
|
||||
for _, level := range []string{"lax", "standard", "strict"} {
|
||||
decision, detail := Decide(level, parsed)
|
||||
assert.Equal(t, DecisionBlock, decision, "label=%s level=%s", label, level)
|
||||
assert.NotNil(t, detail)
|
||||
assert.Equal(t, formatAliyunForbiddenType(label), detail.Label)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAliyunUnmappedLabelSkipped(t *testing.T) {
|
||||
// 未映射的标签不进入明细（无法参与按维度裁决），原始标签仍保留在 RawResponse 中供审计。
|
||||
raw, err := json.Marshal(aliyunTextModerationRaw{
|
||||
API: aliyunTextModerationAPI,
|
||||
Code: 200,
|
||||
Labels: []string{"brand_new_label"},
|
||||
RiskLevel: "medium",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
parsed, err := AliyunTextCensorParser(raw)
|
||||
assert.NoError(t, err)
|
||||
assert.Empty(t, parsed.Details)
|
||||
assert.Equal(t, SuggestionReview, parsed.ProviderSuggestion)
|
||||
}
|
||||
|
||||
func TestAliyunLabeledResponseWithoutRiskLevelFallsBackToBlock(t *testing.T) {
|
||||
// 与七牛保持一致：审核服务判为「非通过」却没给出建议时补成 block，
|
||||
// 否则策略层会因「没有明细维度 + 没有建议」把命中内容当成未命中放行。
|
||||
raw, err := json.Marshal(aliyunTextModerationRaw{
|
||||
API: aliyunTextModerationAPI,
|
||||
Code: 200,
|
||||
Labels: []string{"brand_new_label"},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
parsed, err := AliyunTextCensorParser(raw)
|
||||
assert.NoError(t, err)
|
||||
assert.False(t, parsed.Pass)
|
||||
assert.Equal(t, SuggestionBlock, parsed.ProviderSuggestion)
|
||||
|
||||
// 严格档采信整体 block 建议 → 拦截；宽松/标准档不采信 → 放行。
|
||||
decision, _ := Decide("strict", parsed)
|
||||
assert.Equal(t, DecisionBlock, decision)
|
||||
decision, _ = Decide("standard", parsed)
|
||||
assert.False(t, decision.Blocked())
|
||||
decision, _ = Decide("lax", parsed)
|
||||
assert.False(t, decision.Blocked())
|
||||
}
|
||||
|
||||
func TestAliyunInvalidJSONErrorCarriesRawResponse(t *testing.T) {
|
||||
// 响应不是 JSON（如网关返回 HTML 错误页）时，错误里要能看到响应原文。
|
||||
raw := []byte(`<html>502 Bad Gateway</html>`)
|
||||
|
||||
_, err := AliyunTextCensorParser(raw)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "502 Bad Gateway")
|
||||
}
|
||||
|
||||
func TestAliyunRawDigestTruncates(t *testing.T) {
|
||||
assert.Equal(t, "<empty>", aliyunRawDigest([]byte(" ")))
|
||||
|
||||
long := strings.Repeat("敏", aliyunMaxRawLogBytes+50)
|
||||
got := aliyunRawDigest([]byte(long))
|
||||
|
||||
suffix := "...(truncated)"
|
||||
assert.True(t, strings.HasSuffix(got, suffix))
|
||||
// 按字符（而非字节）截断，多字节字符不能被切成半个。
|
||||
assert.True(t, utf8.ValidString(got))
|
||||
assert.Equal(t, aliyunMaxRawLogBytes, len([]rune(strings.TrimSuffix(got, suffix))))
|
||||
}
|
||||
|
||||
// aliyunSDKResponse 构造一个 SDK 形态的 TextModeration 响应，省去每个用例重复拼结构体。
|
||||
func aliyunSDKResponse(statusCode, code int32, labels, reason string) *green20220302.TextModerationResponse {
|
||||
response := &green20220302.TextModerationResponse{
|
||||
StatusCode: dara.Int32(statusCode),
|
||||
Body: &green20220302.TextModerationResponseBody{
|
||||
Code: dara.Int32(code),
|
||||
Message: dara.String("OK"),
|
||||
RequestId: dara.String("req-1"),
|
||||
},
|
||||
}
|
||||
if labels != "" || reason != "" {
|
||||
response.Body.Data = &green20220302.TextModerationResponseBodyData{
|
||||
Labels: dara.String(labels),
|
||||
Reason: dara.String(reason),
|
||||
}
|
||||
}
|
||||
return response
|
||||
}
|
||||
|
||||
func TestExtractAliyunChunkResult(t *testing.T) {
|
||||
t.Run("成功：解析标签与 Reason", func(t *testing.T) {
|
||||
result, err := extractAliyunChunkResult(aliyunSDKResponse(200, 200, "profanity, negative_content",
|
||||
`{"riskLevel":"medium","riskTips":"辱骂_人身攻击","riskWords":"你是个傻子"}`))
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, int32(200), result.Code)
|
||||
assert.Equal(t, int32(200), result.StatusCode)
|
||||
assert.Equal(t, "req-1", result.RequestID)
|
||||
assert.Equal(t, []string{"profanity", "negative_content"}, result.Labels)
|
||||
assert.Equal(t, "medium", result.RiskLevel)
|
||||
// Hint 优先取命中词。
|
||||
assert.Equal(t, "你是个傻子", result.Hint)
|
||||
})
|
||||
|
||||
t.Run("nonLabel/normal 是占位标签，不进入明细", func(t *testing.T) {
|
||||
result, err := extractAliyunChunkResult(aliyunSDKResponse(200, 200, "nonLabel,normal,", ""))
|
||||
assert.NoError(t, err)
|
||||
assert.Empty(t, result.Labels)
|
||||
})
|
||||
|
||||
t.Run("RiskWords 缺失时 Hint 退回 RiskTips", func(t *testing.T) {
|
||||
result, err := extractAliyunChunkResult(aliyunSDKResponse(200, 200, "ad",
|
||||
`{"riskLevel":"high","riskTips":"广告_引流"}`))
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "广告_引流", result.Hint)
|
||||
})
|
||||
|
||||
t.Run("Reason 非法 JSON 不应导致整条审核失败", func(t *testing.T) {
|
||||
result, err := extractAliyunChunkResult(aliyunSDKResponse(200, 200, "ad", `not-json`))
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, []string{"ad"}, result.Labels)
|
||||
assert.Equal(t, "", result.RiskLevel)
|
||||
})
|
||||
|
||||
t.Run("code 非 200 必须报错，不能当成未命中", func(t *testing.T) {
|
||||
// 服务未开通时阿里云会返回自己的错误码。若被当成「无标签 → 通过」，
|
||||
// fail-closed 会悄悄变成 fail-open。
|
||||
_, err := extractAliyunChunkResult(aliyunSDKResponse(200, 596, "", ""))
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "596")
|
||||
})
|
||||
|
||||
t.Run("HTTP 非 2xx 必须报错", func(t *testing.T) {
|
||||
_, err := extractAliyunChunkResult(aliyunSDKResponse(502, 200, "", ""))
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "unexpected http status")
|
||||
})
|
||||
|
||||
t.Run("响应体缺失必须报错", func(t *testing.T) {
|
||||
_, err := extractAliyunChunkResult(nil)
|
||||
assert.Error(t, err)
|
||||
_, err = extractAliyunChunkResult(&green20220302.TextModerationResponse{})
|
||||
assert.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
func TestMergeAliyunChunkResults(t *testing.T) {
|
||||
merged := mergeAliyunChunkResults([]aliyunChunkResult{
|
||||
{StatusCode: 200, Code: 200, RequestID: "r1", Labels: []string{"profanity"}, RiskLevel: "low"},
|
||||
{StatusCode: 200, Code: 200, RequestID: "r2", Labels: []string{"profanity", "political_content"},
|
||||
RiskLevel: "high", Hint: "命中词"},
|
||||
})
|
||||
|
||||
assert.Equal(t, aliyunTextModerationAPI, merged.API)
|
||||
assert.Equal(t, []string{"r1", "r2"}, merged.RequestIDs)
|
||||
// 标签取并集并去重，保持出现顺序。
|
||||
assert.Equal(t, []string{"profanity", "political_content"}, merged.Labels)
|
||||
// 风险级别取最高：任一分片命中高风险，整条内容按高风险处置。
|
||||
assert.Equal(t, "high", merged.RiskLevel)
|
||||
// Hint 取首个非空。
|
||||
assert.Equal(t, "命中词", merged.Hint)
|
||||
|
||||
// 合并结果可直接被解析器消费，且高风险 + 涉政 → 任何档位都拦。
|
||||
raw, err := json.Marshal(merged)
|
||||
assert.NoError(t, err)
|
||||
|
||||
parsed, err := AliyunTextCensorParser(raw)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, parsed.Details, 2)
|
||||
for _, detail := range parsed.Details {
|
||||
assert.InDelta(t, 1.0, detail.Score, 1e-9)
|
||||
}
|
||||
decision, detail := Decide("lax", parsed)
|
||||
assert.Equal(t, DecisionBlock, decision)
|
||||
assert.NotNil(t, detail)
|
||||
assert.Equal(t, ForbiddenTypePolitics, detail.Label)
|
||||
}
|
||||
|
||||
func TestMergeAliyunChunkResultsEmpty(t *testing.T) {
|
||||
// 没有任何分片（理论上不会发生，因为空内容有单独的短路分支）时，
|
||||
// 合并结果必须仍然是「无标签」，即不产生任何命中。
|
||||
merged := mergeAliyunChunkResults(nil)
|
||||
assert.Empty(t, merged.Labels)
|
||||
assert.Equal(t, "", merged.RiskLevel)
|
||||
assert.Equal(t, aliyunTextModerationAPI, merged.API)
|
||||
}
|
||||
|
||||
func TestAliyunIsAccountIssue(t *testing.T) {
|
||||
// 账号侧未就绪的各类表现都要能识别出来 —— 它们不会自愈，必须给出可操作告警。
|
||||
for _, message := range []string{
|
||||
"code=596 You have not opened Yundun Content Moderation Service",
|
||||
"You have not opened Yundun Content Moderation Service, please go to the page",
|
||||
"Forbidden.RAM: no permission to perform this action",
|
||||
"InvalidAccessKeyId.NotFound: the access key id is invalid",
|
||||
"SignatureDoesNotMatch: signature mismatch",
|
||||
} {
|
||||
assert.True(t, aliyunIsAccountIssue(errors.New(message)), "message=%q", message)
|
||||
}
|
||||
|
||||
// 偶发/网络类问题不应被当成账号问题，否则一次性告警会被无关噪声消耗掉。
|
||||
for _, message := range []string{
|
||||
"dial tcp 1.2.3.4:443: i/o timeout",
|
||||
"EOF",
|
||||
"context deadline exceeded",
|
||||
} {
|
||||
assert.False(t, aliyunIsAccountIssue(errors.New(message)), "message=%q", message)
|
||||
}
|
||||
|
||||
assert.False(t, aliyunIsAccountIssue(nil))
|
||||
}
|
||||
|
||||
func TestAliyunIsAccountIssueFromStructuredCode(t *testing.T) {
|
||||
// SDK 的结构化 code 比消息文本更稳定：即使 message 里没有特征词也要能识别。
|
||||
sdkErr := dara.NewSDKError(map[string]interface{}{
|
||||
"code": "Forbidden",
|
||||
"message": "the request is denied",
|
||||
})
|
||||
assert.True(t, aliyunIsAccountIssue(sdkErr))
|
||||
}
|
||||
@@ -6,6 +6,8 @@ package censor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
@@ -17,8 +19,49 @@ import (
|
||||
|
||||
var (
|
||||
censorCacheNoMoreThan = 31 * 24 * time.Hour // 1 month
|
||||
|
||||
// warnNoCensorCredentialsOnce 保证「没有可用凭据」的告警只打一次，避免按请求刷屏。
|
||||
warnNoCensorCredentialsOnce sync.Once
|
||||
)
|
||||
|
||||
// censorChain 返回按配置选定的审核通道列表，顺序即优先级：
|
||||
// 靠前的通道判定「通过」即返回；未通过则继续交给后面的通道复核。
|
||||
//
|
||||
// 取值来源 [security] text_censor_source（见 conf.CensorSource*）：
|
||||
// - qiniu → 只用七牛（凭据齐备性已由 conf.Init 启动校验）
|
||||
// - aliyun → 只用阿里云（同上）
|
||||
// - 其它（含留空 / auto）→ 七牛优先、阿里云兜底，与引入该配置前完全一致
|
||||
func censorChain() []TextCensor {
|
||||
qiniu := NewQiniuTextCensor(conf.App.QiniuAccessKey, conf.App.QiniuAccessSecret)
|
||||
aliyun := NewAliyunTextCensor(conf.App.AliyunAccessKey, conf.App.AliyunAccessKeySecret)
|
||||
|
||||
switch strings.ToLower(strings.TrimSpace(conf.Security.TextCensorSource)) {
|
||||
case conf.CensorSourceQiniu:
|
||||
return []TextCensor{qiniu}
|
||||
case conf.CensorSourceAliyun:
|
||||
return []TextCensor{aliyun}
|
||||
default:
|
||||
return []TextCensor{qiniu, aliyun}
|
||||
}
|
||||
}
|
||||
|
||||
// warnIfNoCensorCredentials 在「配置里一条可用凭据都没有」时告警一次。
|
||||
//
|
||||
// 这种情况下审核调用必然失败，而调用方是 fail-closed（拒绝提交），表现为
|
||||
// 全站发不出内容/私信——现象隐蔽、只在日志可见，因此第一次触发就留下明确线索。
|
||||
// 显式选定通道但缺凭据的情况已由 conf.Init 直接挡住，这里兜的是 auto 的漏网情况。
|
||||
func warnIfNoCensorCredentials(ctx context.Context) {
|
||||
qiniuReady := conf.App.QiniuAccessKey != "" && conf.App.QiniuAccessSecret != ""
|
||||
aliyunReady := conf.App.AliyunAccessKey != "" && conf.App.AliyunAccessKeySecret != ""
|
||||
if qiniuReady || aliyunReady {
|
||||
return
|
||||
}
|
||||
warnNoCensorCredentialsOnce.Do(func() {
|
||||
logrus.WithContext(ctx).WithField("text_censor_source", conf.Security.TextCensorSource).
|
||||
Error("Text censor is enabled but no usable credentials are configured; all submissions will be rejected (fail-closed)")
|
||||
})
|
||||
}
|
||||
|
||||
// Text checks the text for sensitive content.
|
||||
// It will save the censor log to the database and invoke the callback function.
|
||||
func Text(ctx context.Context, text string) (*TextCensorResponse, error) {
|
||||
@@ -28,11 +71,8 @@ func Text(ctx context.Context, text string) (*TextCensorResponse, error) {
|
||||
|
||||
var responses []*TextCensorResponse
|
||||
|
||||
for _, censor := range []TextCensor{
|
||||
//No, we need to use qiniu's API.
|
||||
NewQiniuTextCensor(conf.App.QiniuAccessKey, conf.App.QiniuAccessSecret),
|
||||
NewAliyunTextCensor(conf.App.AliyunAccessKey, conf.App.AliyunAccessKeySecret),
|
||||
} {
|
||||
warnIfNoCensorCredentials(ctx)
|
||||
for _, censor := range censorChain() {
|
||||
sourceName := censor.String()
|
||||
|
||||
// Try to get the censor from the database log.
|
||||
@@ -46,19 +86,33 @@ func Text(ctx context.Context, text string) (*TextCensorResponse, error) {
|
||||
// We got the previous censor log cache, seems like we saved money for the API call.
|
||||
switch censorLog.SourceName {
|
||||
case "qiniu":
|
||||
// HACK: Qiniu's API response is not accurate, so we need to check the text again with aliyun API.
|
||||
// 七牛缓存命中：只有「通过」才可直接复用（省一次 API 调用）。
|
||||
// 未通过时不在此下最终结论，而是交给链上的下一个通道复核（auto 模式下即阿里云，
|
||||
// 这是引入通道配置前就有的行为）。
|
||||
// text_censor_source = qiniu 时链上只有七牛：continue 后循环结束，
|
||||
// 仍会由函数末尾返回这份七牛结果，判定不会丢失。
|
||||
response, err := QiniuTextCensorParser(raw)
|
||||
if err == nil {
|
||||
if response.Pass {
|
||||
return response, nil
|
||||
}
|
||||
|
||||
responses = append(responses, response) // If the text is not passed, we need to check it with aliyun API.
|
||||
responses = append(responses, response)
|
||||
continue
|
||||
}
|
||||
|
||||
case "aliyun":
|
||||
return AliyunTextCensorParser(raw)
|
||||
// 阿里云缓存命中：能解析就直接复用（省一次 API 调用）。
|
||||
// 解析失败必须退回一次实时调用，而不是把错误往上抛：
|
||||
// 最典型的失败源是「迁移到内容安全 2.0 之前落库的 1.0 缓存」，
|
||||
// 它无法按 2.0 语义解释；若在此返回错误，fail-closed 会让这些
|
||||
// 历史文本在缓存过期（31 天）前一直发不出去。
|
||||
response, err := AliyunTextCensorParser(raw)
|
||||
if err == nil {
|
||||
return response, nil
|
||||
}
|
||||
logrus.WithContext(ctx).WithError(err).WithField("censor_source", sourceName).
|
||||
Warn("Failed to replay cached aliyun censor result; falling back to a live call")
|
||||
|
||||
default:
|
||||
logrus.WithContext(ctx).WithField("censor_source", censorLog.SourceName).Error("Unknown censor source")
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
// Copyright 2022 E99p1ant. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package censor
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/tamakyi/TamaBox/internal/conf"
|
||||
)
|
||||
|
||||
// sourceNames 把通道列表映射成名称序列，便于断言「选了几条、哪条、顺序如何」。
|
||||
func sourceNames(chain []TextCensor) []string {
|
||||
names := make([]string, 0, len(chain))
|
||||
for _, censor := range chain {
|
||||
names = append(names, censor.String())
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// TestCensorChainSelection 覆盖 [security] text_censor_source 对审核通道链的装配结果。
|
||||
//
|
||||
// 顺序即优先级：靠前通道「通过」即返回；未通过则交给后面的通道复核。
|
||||
// 最要紧的是默认（留空 / auto）必须与引入该配置前完全一致，避免升级改变线上判定。
|
||||
func TestCensorChainSelection(t *testing.T) {
|
||||
original := conf.Security.TextCensorSource
|
||||
t.Cleanup(func() { conf.Security.TextCensorSource = original })
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
source string
|
||||
want []string
|
||||
}{
|
||||
{name: "留空：七牛优先 + 阿里云兜底（历史行为）", source: "", want: []string{"qiniu", "aliyun"}},
|
||||
{name: "auto：同留空", source: "auto", want: []string{"qiniu", "aliyun"}},
|
||||
{name: "qiniu：只走七牛", source: "qiniu", want: []string{"qiniu"}},
|
||||
{name: "aliyun：只走阿里云", source: "aliyun", want: []string{"aliyun"}},
|
||||
{name: "大小写与空白不敏感", source: " AlIyUn \t", want: []string{"aliyun"}},
|
||||
// 未知取值在启动期已被 conf.Init 直接拒绝（见 internal/conf validateTextCensorSource）；
|
||||
// 这里断言 censor 侧仍保守回退到历史行为，避免绕过 Init 的调用路径（如 CLI/tests）行为突变。
|
||||
{name: "未知取值：保守回退 auto", source: "bogus", want: []string{"qiniu", "aliyun"}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
conf.Security.TextCensorSource = tc.source
|
||||
assert.Equal(t, tc.want, sourceNames(censorChain()))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
// Copyright 2022 E99p1ant. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package censor
|
||||
|
||||
// Strictness 内容审核严格度档位。
|
||||
//
|
||||
// 背景：审核服务（本项目为七牛云文本审核）对「辱骂」等维度的召回偏激进，
|
||||
// 正常提问/对话经常被判为不通过而被整条拒绝。但我们既不能让审核服务判得更准，
|
||||
// 也不能让它判得更松（七牛文本审核不支持白名单，自定义词库需提交工单），
|
||||
// 唯一能控制的是「审核命中之后到底拦不拦」。
|
||||
//
|
||||
// 因此把「是否拦截」从审核服务自身的 Pass 判定中剥离出来，改为由本档位策略裁决，
|
||||
// 让箱主在「少误伤」与「少漏放」之间自行取舍。
|
||||
type Strictness string
|
||||
|
||||
const (
|
||||
// StrictnessLax 宽松：仅拦截底线维度，其余一律放行。误伤最小，漏放最多。
|
||||
StrictnessLax Strictness = "lax"
|
||||
// StrictnessStandard 标准：底线维度 + 高置信度的辱骂/违禁。
|
||||
StrictnessStandard Strictness = "standard"
|
||||
// StrictnessStrict 严格：底线维度 + 全部非正常维度（不限置信度）+ 采信审核服务的 block 建议。
|
||||
//
|
||||
// 这是三个档位中最严的一档，且是唯一保留「采信审核服务整体 block 建议」的档位。
|
||||
// 改造前的拦截条件是 `!Pass`（即 涉政/暴恐/色情 命中，或审核服务建议为 block），
|
||||
// 该条件可被本档完全覆盖，但本档会额外拦截全部带维度的非正常内容（如 review 的辱骂）。
|
||||
// 换言之：严格档 ⊇ 改造前的行为，是「至少与改造前一样严」，并非完全等价。
|
||||
StrictnessStrict Strictness = "strict"
|
||||
)
|
||||
|
||||
// DefaultStrictness 缺省档位。
|
||||
//
|
||||
// 取 strict 的取舍：本能力只用于「让箱主把审核调松」，从不用于「在用户未选择时把审核调松」。
|
||||
// 存量账号从未设置过该值，缺省落到最严档，保证部署本功能这一动作本身不会让任何账号
|
||||
// 的拦截面变小（合规上更保守）；想减少误伤的箱主再显式切到 标准 / 宽松。
|
||||
//
|
||||
// ⚠️ 注意方向：因为严格档比改造前更严（见 StrictnessStrict 注释），缺省为 strict 意味着
|
||||
// 部署后默认拦截面会略有扩大 —— 这是刻意的取舍（已于 2026-09-10 确认维持 strict）：
|
||||
// 合规上宁紧勿松，且「调松」应当是箱主的显式动作，部署本身不应悄悄放宽任何账号。
|
||||
// 若哪天希望默认更少误伤，把下面这行改成 StrictnessStandard 即可（一行常量）。
|
||||
const DefaultStrictness = StrictnessStrict
|
||||
|
||||
// Suggestion 审核服务自身给出的管控建议取值。
|
||||
const (
|
||||
SuggestionBlock = "block"
|
||||
SuggestionReview = "review"
|
||||
SuggestionPass = "pass"
|
||||
)
|
||||
|
||||
// baselineBlockLabels 底线拦截维度：涉政 / 暴恐 / 色情 / 自定义词库。
|
||||
//
|
||||
// 这是合规底线：任何档位下都强制生效、不受置信度门槛约束、不可关闭，
|
||||
// 也不能通过任何用户配置绕过 —— 见 Decide 中它始终排在最先、无条件返回拦截。
|
||||
//
|
||||
// ForbiddenTypeCustomized 归入底线的原因：它是运营方在审核服务控制台（阿里云）
|
||||
// 显式配置的词库命中，属「用户明写的规则」，与箱主自定义屏蔽词同一性质，
|
||||
// 不应被档位调低后放行。运营方若要放开，正确的做法是调整控制台词库，而不是调档位。
|
||||
var baselineBlockLabels = map[ForbiddenType]struct{}{
|
||||
ForbiddenTypePolitics: {},
|
||||
ForbiddenTypeTerrorism: {},
|
||||
ForbiddenTypePorn: {},
|
||||
ForbiddenTypeCustomized: {},
|
||||
}
|
||||
|
||||
// abuseScoreFloor 标准档位下「辱骂」维度生效所需的最低置信度。
|
||||
//
|
||||
// 辱骂是审核服务误伤最集中的维度（普通吐槽、语气较冲的表达常被判为辱骂），
|
||||
// 因此标准档要求高置信度才拦截；严格档则为任意置信度。
|
||||
//
|
||||
// 注：置信度的来源因服务商而异。七牛直接给 0~1 的 score；阿里云内容安全 2.0 只给
|
||||
// 整条内容的 riskLevel，折算为 1 / 0.6 / 0.3（high / medium / low），因此本阈值
|
||||
// 在阿里云下的实际语义是「标准档只拦 riskLevel=high 的辱骂」。
|
||||
const abuseScoreFloor = 0.9
|
||||
|
||||
// levelPolicy 单一档位的拦截规则。
|
||||
type levelPolicy struct {
|
||||
// blockLabels 该档位额外拦截的维度 -> 生效所需最低置信度（0 表示不限置信度）。
|
||||
// 注意：底线维度不在此表中，由 Decide 无条件前置判定。
|
||||
blockLabels map[ForbiddenType]float64
|
||||
// blockOnSuggestion 是否采信审核服务自身的 block 建议。
|
||||
//
|
||||
// 该建议不携带具体维度（可能是任何原因），无法做置信度收敛，误伤面最大，
|
||||
// 也是此前正常对话被整条拒绝的主要来源，因此仅严格档启用。
|
||||
blockOnSuggestion bool
|
||||
}
|
||||
|
||||
var strictnessPolicies = map[Strictness]levelPolicy{
|
||||
// 宽松：只留底线，其余全部放行。
|
||||
StrictnessLax: {
|
||||
blockLabels: map[ForbiddenType]float64{},
|
||||
blockOnSuggestion: false,
|
||||
},
|
||||
// 标准：底线 + 高置信度辱骂 + 违禁。
|
||||
StrictnessStandard: {
|
||||
blockLabels: map[ForbiddenType]float64{
|
||||
ForbiddenTypeAbuse: abuseScoreFloor,
|
||||
ForbiddenTypeContraband: 0,
|
||||
},
|
||||
blockOnSuggestion: false,
|
||||
},
|
||||
// 严格：底线 + 全部非正常维度（不限置信度）+ 采信审核服务 block 建议。
|
||||
StrictnessStrict: {
|
||||
blockLabels: map[ForbiddenType]float64{
|
||||
ForbiddenTypeSpam: 0,
|
||||
ForbiddenTypeAd: 0,
|
||||
ForbiddenTypeAbuse: 0,
|
||||
ForbiddenTypeFlood: 0,
|
||||
ForbiddenTypeContraband: 0,
|
||||
ForbiddenTypeMeaningless: 0,
|
||||
ForbiddenTypeHarmful: 0,
|
||||
},
|
||||
blockOnSuggestion: true,
|
||||
},
|
||||
}
|
||||
|
||||
// Decision 对单条内容的最终裁决。
|
||||
type Decision int
|
||||
|
||||
const (
|
||||
// DecisionAllow 放行。
|
||||
DecisionAllow Decision = iota
|
||||
// DecisionBlock 拒绝提交。
|
||||
DecisionBlock
|
||||
)
|
||||
|
||||
// Blocked 是否应当拒绝提交。
|
||||
func (d Decision) Blocked() bool { return d == DecisionBlock }
|
||||
|
||||
// IsValidStrictness 判断取值是否为合法档位，供后台保存前校验。
|
||||
// 非法取值一律拒绝写入，避免脏数据把账号静默切到非预期档位。
|
||||
func IsValidStrictness(s string) bool {
|
||||
switch Strictness(s) {
|
||||
case StrictnessLax, StrictnessStandard, StrictnessStrict:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// ParseStrictness 归一化严格度取值，未知/空值回退到 DefaultStrictness（保守方向，即最严）。
|
||||
// 方向刻意选保守：宁可沿用严格判定，也不要因为识别失败把账号静默放到最松档。
|
||||
func ParseStrictness(s string) Strictness {
|
||||
if !IsValidStrictness(s) {
|
||||
return DefaultStrictness
|
||||
}
|
||||
return Strictness(s)
|
||||
}
|
||||
|
||||
// Decide 按档位裁决一次内容审核结果，返回裁决结果与触发拦截的维度（未命中具体维度时为 nil）。
|
||||
//
|
||||
// 判定顺序固定为「底线维度 → 档位维度 → 审核服务建议」，且不可调换：
|
||||
// 1. 底线维度排在最前且无条件返回拦截，保证任何档位都无法把它关掉；
|
||||
// 2. 档位维度带各自的置信度门槛，用于收敛误伤最集中的维度；
|
||||
// 3. 审核服务自身的 block 建议排在最后，且仅严格档启用。
|
||||
//
|
||||
// Details 为空（例如审核服务返回了 suggestion=block 但没给出任何非 normal 维度）时，
|
||||
// 只有开启 blockOnSuggestion 的档位会拦截。
|
||||
func Decide(strictness string, r *TextCensorResponse) (Decision, *LabelScore) {
|
||||
if r == nil {
|
||||
return DecisionAllow, nil
|
||||
}
|
||||
|
||||
policy := strictnessPolicies[ParseStrictness(strictness)]
|
||||
|
||||
// 1. 底线：合规底线，任何档位强制生效、不可关闭。
|
||||
for i := range r.Details {
|
||||
detail := &r.Details[i]
|
||||
if _, ok := baselineBlockLabels[detail.Label]; ok {
|
||||
return DecisionBlock, detail
|
||||
}
|
||||
}
|
||||
|
||||
// 2. 档位维度：命中所配维度且达到该维度的置信度门槛。
|
||||
for i := range r.Details {
|
||||
detail := &r.Details[i]
|
||||
scoreFloor, ok := policy.blockLabels[detail.Label]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if scoreFloor > 0 && detail.Score < scoreFloor {
|
||||
continue
|
||||
}
|
||||
return DecisionBlock, detail
|
||||
}
|
||||
|
||||
// 3. 审核服务自身的 block 建议：仅严格档采信。
|
||||
if policy.blockOnSuggestion && r.ProviderSuggestion == SuggestionBlock {
|
||||
return DecisionBlock, nil
|
||||
}
|
||||
|
||||
return DecisionAllow, nil
|
||||
}
|
||||
|
||||
// BlockedLabels 返回指定档位最终会拦截的全部维度（含底线维度）。
|
||||
//
|
||||
// 用途：策略层的自描述 API —— 目前由 policy_test.go 用来守住「底线维度在每个档位都必须出现
|
||||
// 且不重复」这条不变量。设置页展示的拦截范围是 i18n 静态文案
|
||||
// （templates/user/moderation-strictness.html），尚未消费本函数；若将来要让 UI 直接由策略层
|
||||
// 生成文案，这里是唯一事实来源。
|
||||
func BlockedLabels(strictness string) []ForbiddenType {
|
||||
policy := strictnessPolicies[ParseStrictness(strictness)]
|
||||
|
||||
labels := make([]ForbiddenType, 0, len(baselineBlockLabels)+len(policy.blockLabels))
|
||||
for label := range baselineBlockLabels {
|
||||
labels = append(labels, label)
|
||||
}
|
||||
for label := range policy.blockLabels {
|
||||
// 底线维度已在上面加入，避免重复展示。
|
||||
if _, ok := baselineBlockLabels[label]; ok {
|
||||
continue
|
||||
}
|
||||
labels = append(labels, label)
|
||||
}
|
||||
return labels
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
// Copyright 2022 E99p1ant. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package censor
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// resp 构造一个带指定维度判定的审核结果，省去每个用例重复拼结构体。
|
||||
func resp(suggestion string, details ...LabelScore) *TextCensorResponse {
|
||||
return &TextCensorResponse{
|
||||
SourceName: "qiniu",
|
||||
Pass: suggestion != SuggestionBlock,
|
||||
ProviderSuggestion: suggestion,
|
||||
Details: details,
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecideBaselineAlwaysBlocks(t *testing.T) {
|
||||
// 底线维度（涉政 / 暴恐 / 色情 / 自定义词库）是强制项：任何档位、任何置信度都必须拦截，
|
||||
// 且不受审核服务整体建议的影响（这里刻意给 pass 建议，验证仍然拦得住）。
|
||||
baselines := []ForbiddenType{
|
||||
ForbiddenTypePolitics,
|
||||
ForbiddenTypeTerrorism,
|
||||
ForbiddenTypePorn,
|
||||
ForbiddenTypeCustomized,
|
||||
}
|
||||
levels := []string{"lax", "standard", "strict", "", "unknown-level"}
|
||||
|
||||
for _, label := range baselines {
|
||||
for _, level := range levels {
|
||||
t.Run(string(label)+"/"+level, func(t *testing.T) {
|
||||
decision, detail := Decide(level, resp(SuggestionPass, LabelScore{Label: label, Score: 0.01}))
|
||||
assert.Equal(t, DecisionBlock, decision)
|
||||
assert.NotNil(t, detail)
|
||||
assert.Equal(t, label, detail.Label)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecideByLevel(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
level string
|
||||
response *TextCensorResponse
|
||||
wantBlock bool
|
||||
wantDetail ForbiddenType
|
||||
}{
|
||||
{
|
||||
name: "宽松：高置信度辱骂也放行",
|
||||
level: "lax",
|
||||
response: resp(SuggestionReview, LabelScore{Label: ForbiddenTypeAbuse, Score: 0.99}),
|
||||
wantBlock: false,
|
||||
wantDetail: "",
|
||||
},
|
||||
{
|
||||
name: "宽松：垃圾信息放行",
|
||||
level: "lax",
|
||||
response: resp(SuggestionPass, LabelScore{Label: ForbiddenTypeSpam, Score: 0.9}),
|
||||
wantBlock: false,
|
||||
wantDetail: "",
|
||||
},
|
||||
{
|
||||
name: "宽松：不采信审核服务的整体 block 建议",
|
||||
level: "lax",
|
||||
response: resp(SuggestionBlock),
|
||||
wantBlock: false,
|
||||
wantDetail: "",
|
||||
},
|
||||
{
|
||||
name: "标准：高置信度辱骂拦截",
|
||||
level: "standard",
|
||||
response: resp(SuggestionReview, LabelScore{Label: ForbiddenTypeAbuse, Score: 0.95}),
|
||||
wantBlock: true,
|
||||
wantDetail: ForbiddenTypeAbuse,
|
||||
},
|
||||
{
|
||||
name: "标准：辱骂置信度不足 90% 放行（收敛误伤）",
|
||||
level: "standard",
|
||||
response: resp(SuggestionReview, LabelScore{Label: ForbiddenTypeAbuse, Score: 0.72}),
|
||||
wantBlock: false,
|
||||
wantDetail: "",
|
||||
},
|
||||
{
|
||||
name: "标准：辱骂恰好 90% 拦截（阈值含等号）",
|
||||
level: "standard",
|
||||
response: resp(SuggestionReview, LabelScore{Label: ForbiddenTypeAbuse, Score: 0.9}),
|
||||
wantBlock: true,
|
||||
wantDetail: ForbiddenTypeAbuse,
|
||||
},
|
||||
{
|
||||
name: "标准：违禁不限置信度",
|
||||
level: "standard",
|
||||
response: resp(SuggestionPass, LabelScore{Label: ForbiddenTypeContraband, Score: 0.3}),
|
||||
wantBlock: true,
|
||||
wantDetail: ForbiddenTypeContraband,
|
||||
},
|
||||
{
|
||||
name: "标准：垃圾信息放行",
|
||||
level: "standard",
|
||||
response: resp(SuggestionPass, LabelScore{Label: ForbiddenTypeSpam, Score: 0.99}),
|
||||
wantBlock: false,
|
||||
wantDetail: "",
|
||||
},
|
||||
{
|
||||
name: "标准：不采信整体 block 建议",
|
||||
level: "standard",
|
||||
response: resp(SuggestionBlock),
|
||||
wantBlock: false,
|
||||
wantDetail: "",
|
||||
},
|
||||
{
|
||||
name: "严格：辱骂不限置信度拦截（最严档）",
|
||||
level: "strict",
|
||||
response: resp(SuggestionReview, LabelScore{Label: ForbiddenTypeAbuse, Score: 0.2}),
|
||||
wantBlock: true,
|
||||
wantDetail: ForbiddenTypeAbuse,
|
||||
},
|
||||
{
|
||||
name: "严格：无意义内容拦截",
|
||||
level: "strict",
|
||||
response: resp(SuggestionReview, LabelScore{Label: ForbiddenTypeMeaningless, Score: 0.5}),
|
||||
wantBlock: true,
|
||||
wantDetail: ForbiddenTypeMeaningless,
|
||||
},
|
||||
{
|
||||
name: "严格：采信整体 block 建议（无明细维度）",
|
||||
level: "strict",
|
||||
response: resp(SuggestionBlock),
|
||||
wantBlock: true,
|
||||
wantDetail: "",
|
||||
},
|
||||
{
|
||||
name: "严格：仅 review 且无维度命中则放行",
|
||||
level: "strict",
|
||||
response: resp(SuggestionReview),
|
||||
wantBlock: false,
|
||||
wantDetail: "",
|
||||
},
|
||||
{
|
||||
name: "全部正常文本放行",
|
||||
level: "strict",
|
||||
response: resp(SuggestionPass),
|
||||
wantBlock: false,
|
||||
wantDetail: "",
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
decision, detail := Decide(tc.level, tc.response)
|
||||
assert.Equal(t, tc.wantBlock, decision.Blocked())
|
||||
|
||||
if tc.wantDetail == "" {
|
||||
assert.Nil(t, detail)
|
||||
return
|
||||
}
|
||||
assert.NotNil(t, detail)
|
||||
assert.Equal(t, tc.wantDetail, detail.Label)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecideNilResponse(t *testing.T) {
|
||||
// 空结果（审核未返回）不应被当成命中，避免把「拿不到结论」误判为违规。
|
||||
decision, detail := Decide("strict", nil)
|
||||
assert.Equal(t, DecisionAllow, decision)
|
||||
assert.Nil(t, detail)
|
||||
}
|
||||
|
||||
func TestDecideUnknownLevelFallsBackToStrict(t *testing.T) {
|
||||
// 脏数据 / 未识别取值必须回退「严格」，方向保守：
|
||||
// 宁可沿用升级前的严格判定，也不要因为认不出档位就静默放宽。
|
||||
unknown := resp(SuggestionPass, LabelScore{Label: ForbiddenTypeSpam, Score: 0.9})
|
||||
decision, _ := Decide("", unknown)
|
||||
assert.Equal(t, DecisionBlock, decision)
|
||||
decision, _ = Decide("bogus", unknown)
|
||||
assert.Equal(t, DecisionBlock, decision)
|
||||
}
|
||||
|
||||
func TestStrictnessHelpers(t *testing.T) {
|
||||
assert.True(t, IsValidStrictness("lax"))
|
||||
assert.True(t, IsValidStrictness("standard"))
|
||||
assert.True(t, IsValidStrictness("strict"))
|
||||
assert.False(t, IsValidStrictness(""))
|
||||
assert.False(t, IsValidStrictness("STRICT"))
|
||||
assert.False(t, IsValidStrictness("off"))
|
||||
|
||||
assert.Equal(t, StrictnessStrict, ParseStrictness(""))
|
||||
assert.Equal(t, StrictnessStrict, ParseStrictness("nope"))
|
||||
assert.Equal(t, StrictnessLax, ParseStrictness("lax"))
|
||||
assert.Equal(t, StrictnessStandard, ParseStrictness("standard"))
|
||||
}
|
||||
|
||||
func TestQiniuParserFeedsPolicy(t *testing.T) {
|
||||
// 策略层完全依赖 parser 填好的 Details / ProviderSuggestion，这里做一次端到端校验：
|
||||
// 审核服务给出 review（自身判定为放行）但带 abuse 维度，严格档仍应拦得住。
|
||||
raw := []byte(`{
|
||||
"code": 200,
|
||||
"message": "ok",
|
||||
"result": {
|
||||
"suggestion": "review",
|
||||
"scenes": {
|
||||
"antispam": {
|
||||
"suggestion": "review",
|
||||
"details": [
|
||||
{"label": "normal", "score": 0.02, "contexts": []},
|
||||
{"label": "abuse", "score": 0.94, "contexts": [{"context": "示例片段", "positions": [{"startPos": 0, "endPos": 4}]}]}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}`)
|
||||
|
||||
parsed, err := QiniuTextCensorParser(raw)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// normal 维度必须被过滤掉，只保留真正命中的维度。
|
||||
assert.Len(t, parsed.Details, 1)
|
||||
assert.Equal(t, ForbiddenTypeAbuse, parsed.Details[0].Label)
|
||||
assert.Equal(t, 0.94, parsed.Details[0].Score)
|
||||
assert.Equal(t, SuggestionReview, parsed.ProviderSuggestion)
|
||||
assert.Equal(t, "示例片段", parsed.Hint)
|
||||
|
||||
// 审核服务自己判 review（Pass=true），但档位策略仍可覆盖它。
|
||||
assert.True(t, parsed.Pass)
|
||||
|
||||
decision, detail := Decide("strict", parsed)
|
||||
assert.Equal(t, DecisionBlock, decision)
|
||||
assert.NotNil(t, detail)
|
||||
assert.Equal(t, ForbiddenTypeAbuse, detail.Label)
|
||||
|
||||
decision, _ = Decide("standard", parsed)
|
||||
assert.Equal(t, DecisionBlock, decision)
|
||||
|
||||
decision, _ = Decide("lax", parsed)
|
||||
assert.False(t, decision.Blocked())
|
||||
}
|
||||
|
||||
func TestQiniuParserProviderSuggestionFallback(t *testing.T) {
|
||||
// 外层 suggestion 缺失时回退到 antispam 场景内的建议，避免整体建议丢失后
|
||||
// 严格档的「采信整体 block」分支失效。
|
||||
raw := []byte(`{"code":200,"result":{"scenes":{"antispam":{"suggestion":"block","details":[]}}}}`)
|
||||
|
||||
parsed, err := QiniuTextCensorParser(raw)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, SuggestionBlock, parsed.ProviderSuggestion)
|
||||
assert.Empty(t, parsed.Details)
|
||||
|
||||
// 严格档：无具体维度，但整体建议为 block → 拦截。
|
||||
decision, detail := Decide("strict", parsed)
|
||||
assert.Equal(t, DecisionBlock, decision)
|
||||
assert.Nil(t, detail)
|
||||
|
||||
// 标准档不采信整体建议 → 放行（这是「少误伤」的核心差异）。
|
||||
decision, _ = Decide("standard", parsed)
|
||||
assert.False(t, decision.Blocked())
|
||||
}
|
||||
|
||||
func TestBlockedLabelsIncludesBaselineAtEveryLevel(t *testing.T) {
|
||||
// 供设置页展示拦截范围：底线维度必须出现在每一个档位的清单里。
|
||||
for _, level := range []string{"lax", "standard", "strict"} {
|
||||
labels := BlockedLabels(level)
|
||||
assert.Contains(t, labels, ForbiddenTypePolitics, "level=%s", level)
|
||||
assert.Contains(t, labels, ForbiddenTypeTerrorism, "level=%s", level)
|
||||
assert.Contains(t, labels, ForbiddenTypePorn, "level=%s", level)
|
||||
assert.Contains(t, labels, ForbiddenTypeCustomized, "level=%s", level)
|
||||
|
||||
// 不重复展示底线维度。
|
||||
seen := map[ForbiddenType]int{}
|
||||
for _, label := range labels {
|
||||
seen[label]++
|
||||
}
|
||||
for label, count := range seen {
|
||||
assert.Equal(t, 1, count, "label %s 重复出现", label)
|
||||
}
|
||||
}
|
||||
|
||||
// 宽松档只含底线四个维度（涉政 / 暴恐 / 色情 / 自定义词库）。
|
||||
assert.Len(t, BlockedLabels("lax"), 4)
|
||||
assert.Contains(t, BlockedLabels("standard"), ForbiddenTypeAbuse)
|
||||
assert.NotContains(t, BlockedLabels("standard"), ForbiddenTypeSpam)
|
||||
assert.Contains(t, BlockedLabels("strict"), ForbiddenTypeSpam)
|
||||
}
|
||||
@@ -128,27 +128,52 @@ func QiniuTextCensorParser(raw []byte) (*TextCensorResponse, error) {
|
||||
var hint string
|
||||
var detailKey string
|
||||
var confidence float64
|
||||
for _, detail := range responseJSON.Result.Scenes.Antispam.Details {
|
||||
if detail.Label == "normal" {
|
||||
// 收集全部非 normal 维度的判定明细，供严格度策略按维度裁决（见 policy.go）。
|
||||
// 只取首个维度的做法会丢掉其余维度信息，无法做到「只拦高置信度的辱骂」。
|
||||
details := make([]LabelScore, 0, len(responseJSON.Result.Scenes.Antispam.Details))
|
||||
for _, d := range responseJSON.Result.Scenes.Antispam.Details {
|
||||
if d.Label == "normal" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Get the first context as the hint, forbidden type, confidence.
|
||||
for _, context := range detail.Contexts {
|
||||
hint = context.Context
|
||||
details = append(details, LabelScore{
|
||||
Label: formatQiniuForbiddenType(d.Label),
|
||||
Score: d.Score,
|
||||
})
|
||||
|
||||
// 保留原有语义：首个非 normal 维度决定 ForbiddenType / Hint / Confidence。
|
||||
if detailKey == "" {
|
||||
for _, context := range d.Contexts {
|
||||
hint = context.Context
|
||||
}
|
||||
detailKey = d.Label
|
||||
confidence = d.Score
|
||||
}
|
||||
detailKey = detail.Label
|
||||
confidence = detail.Score
|
||||
break
|
||||
}
|
||||
|
||||
pass := responseJSON.IsPass()
|
||||
|
||||
// 整体建议优先取外层，缺失时回退到 antispam 场景的建议。
|
||||
suggestion := responseJSON.Result.Suggestion
|
||||
if suggestion == "" {
|
||||
suggestion = responseJSON.Result.Scenes.Antispam.Suggestion
|
||||
}
|
||||
// 建议字段整体缺失、但服务自身判定为不放行时，把整体建议补成 block：
|
||||
// 否则策略层会因「没有明细维度 + 没有建议」把它当成未命中而放行，
|
||||
// 与改造前的 fail-closed 行为不一致。
|
||||
if suggestion == "" && !pass {
|
||||
suggestion = SuggestionBlock
|
||||
}
|
||||
|
||||
return &TextCensorResponse{
|
||||
SourceName: "qiniu",
|
||||
Pass: responseJSON.IsPass(),
|
||||
ForbiddenType: formatQiniuForbiddenType(detailKey),
|
||||
Hint: hint,
|
||||
Confidence: confidence,
|
||||
RawResponse: raw,
|
||||
SourceName: "qiniu",
|
||||
Pass: pass,
|
||||
ForbiddenType: formatQiniuForbiddenType(detailKey),
|
||||
Hint: hint,
|
||||
Confidence: confidence,
|
||||
RawResponse: raw,
|
||||
Details: details,
|
||||
ProviderSuggestion: suggestion,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,10 @@ const (
|
||||
ForbiddenTypeContraband ForbiddenType = "contraband"
|
||||
ForbiddenTypeMeaningless ForbiddenType = "meaningless"
|
||||
ForbiddenTypeHarmful ForbiddenType = "harmful"
|
||||
// ForbiddenTypeCustomized 命中审核服务侧的自定义词库（阿里云控制台配置，
|
||||
// 七牛不支持服务端词库因此不会返回该维度）。属运营方显式配置的规则，
|
||||
// 在 policy.go 中被归入底线集合，任何档位都不放行。
|
||||
ForbiddenTypeCustomized ForbiddenType = "customized"
|
||||
)
|
||||
|
||||
func (f ForbiddenType) String() string {
|
||||
@@ -38,9 +42,25 @@ func (f ForbiddenType) String() string {
|
||||
ForbiddenTypeContraband: "违禁",
|
||||
ForbiddenTypeMeaningless: "无意义",
|
||||
ForbiddenTypeHarmful: "不良场景",
|
||||
ForbiddenTypeCustomized: "自定义词库",
|
||||
}[f]
|
||||
}
|
||||
|
||||
// LabelScore 审核服务针对单个维度（label）给出的判定与置信度。
|
||||
//
|
||||
// 各服务商都会返回「多维度 + 每个维度一个分值」的结果，过去只取首个非 normal 维度，
|
||||
// 丢掉了其余维度的信息，导致无法按维度做差异化处置（例如只拦高置信度的辱骂）。
|
||||
//
|
||||
// ⚠️ Score 已统一归一化到 0~1，与提供方无关：七牛本身即 0~1。
|
||||
// 阿里云内容安全 2.0 不返回逐标签置信度，只返回整条内容的 riskLevel，
|
||||
// 由 AliyunTextCensorParser 折算为 0~1（high=1 / medium=0.6 / low=0.3）后由该响应的
|
||||
// 所有标签共用 —— 这是有损的近似，详见 AliyunTextCensorParser 与 aliyunRiskLevelScore 的注释。
|
||||
// policy.go 的阈值因此对两家服务商都成立（对阿里云的语义为「按风险级别分级」）。
|
||||
type LabelScore struct {
|
||||
Label ForbiddenType `json:"label"`
|
||||
Score float64 `json:"score"`
|
||||
}
|
||||
|
||||
type TextCensorResponse struct {
|
||||
SourceName string `json:"source_name"`
|
||||
Pass bool `json:"pass"`
|
||||
@@ -48,6 +68,12 @@ type TextCensorResponse struct {
|
||||
Hint string `json:"hint"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
RawResponse json.RawMessage `json:"raw_response"`
|
||||
// Details 全部非 normal 维度的判定明细，供严格度策略按维度裁决。
|
||||
// 仅在回放缓存结果时为 nil（见 censor.go），此时 Pass 已足以表达判定。
|
||||
Details []LabelScore `json:"details,omitempty"`
|
||||
// ProviderSuggestion 审核服务自身给出的管控建议（block / review / pass）。
|
||||
// 它不携带具体维度，无法做置信度收敛，因此只在严格档被采信（见 policy.go）。
|
||||
ProviderSuggestion string `json:"provider_suggestion,omitempty"`
|
||||
}
|
||||
|
||||
func (r *TextCensorResponse) ToJSON() []byte {
|
||||
@@ -55,10 +81,16 @@ func (r *TextCensorResponse) ToJSON() []byte {
|
||||
return jsonBytes
|
||||
}
|
||||
|
||||
func (r *TextCensorResponse) ErrorMessage() string {
|
||||
// ErrorMessageFor 生成针对指定命中维度的拒绝提示。
|
||||
//
|
||||
// 与已移除的 ErrorMessage 的区别：后者固定使用首个非 normal 维度，而实际触发拦截的
|
||||
// 可能是另一个维度（策略层按档位裁决），两者不一致时用户会看到对不上的原因。
|
||||
// 因此拦截时统一用本方法，按真正命中的维度出文案。
|
||||
// detail 为 nil（即由审核服务的整体 block 建议触发）时退化为不带维度的基础文案。
|
||||
func (r *TextCensorResponse) ErrorMessageFor(detail *LabelScore) string {
|
||||
errorMessage := "内容安全检查不通过"
|
||||
if r.ForbiddenType != "" {
|
||||
errorMessage += fmt.Sprintf(" [%s]", r.ForbiddenType.String())
|
||||
if detail != nil && detail.Label != "" {
|
||||
errorMessage += fmt.Sprintf(" [%s]", detail.Label.String())
|
||||
}
|
||||
if r.Hint != "" {
|
||||
errorMessage += fmt.Sprintf("，相关标签：%q", r.Hint)
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
// Copyright 2022 E99p1ant. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package moderation 提供跨场景统一的内容审核守卫。
|
||||
//
|
||||
// 存在意义：提问、投票、私聊三条链路都要对用户输入做内容审核，过去各自调用
|
||||
// censor.Text 并自行解释结果，判定口径一旦调整就要改多处，极易遗漏。
|
||||
// 本包收敛为唯一入口：调审核 → 按箱主「审核严格度」裁决 → 返回是否放行。
|
||||
package moderation
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/tamakyi/TamaBox/internal/conf"
|
||||
"github.com/tamakyi/TamaBox/internal/i18n"
|
||||
"github.com/tamakyi/TamaBox/internal/security/censor"
|
||||
)
|
||||
|
||||
// StrictnessFor 解析账号最终生效的严格度档位：
|
||||
// - 账号显式设置了档位（lax / standard / strict）→ 用该档位；
|
||||
// - 账号未设置（空值，存量账号的默认形态）→ 跟随站点默认
|
||||
// [security] default_moderation_strictness；
|
||||
// - 站点默认同样为空或非法 → 回退 censor.DefaultStrictness（严格）。
|
||||
//
|
||||
// 回退方向刻意取「严格」：配置缺失或非法时宁可从严，也不要因为识别失败
|
||||
// 把全站静默切到最松档（那等于在配置出错时悄悄放大漏放面）。
|
||||
func StrictnessFor(raw string) censor.Strictness {
|
||||
if raw != "" {
|
||||
return censor.ParseStrictness(raw)
|
||||
}
|
||||
return censor.ParseStrictness(conf.Security.DefaultModerationStrictness)
|
||||
}
|
||||
|
||||
// Check 执行一次内容审核，并按箱主严格度裁决是否放行。
|
||||
//
|
||||
// 这是提问 / 投票 / 私聊三类场景**唯一**应当使用的审核入口。
|
||||
//
|
||||
// 返回：
|
||||
// - resp：审核明细，调用方用于落 content_censor_metadata 等审计字段；裁决失败时可能为 nil。
|
||||
// - err != nil：应当拒绝本次提交，错误信息已按 lang 本地化，可直接展示给用户。
|
||||
//
|
||||
// 审核服务不可用（超时/报错）时仍为 fail-closed 拒绝提交：这是引入本能力之前就存在的
|
||||
// 行为，不在本次改造范围内（相关可用性问题另见 outputs/llm-moderation-feasibility 评审）。
|
||||
func Check(ctx context.Context, strictness censor.Strictness, lang, text string) (*censor.TextCensorResponse, error) {
|
||||
resp, err := censor.Text(ctx, text)
|
||||
if err != nil {
|
||||
logrus.WithContext(ctx).WithError(err).
|
||||
WithField("strictness", string(strictness)).
|
||||
Error("Failed to censor text")
|
||||
return nil, errors.New(i18n.T(lang, "censor.unavailable"))
|
||||
}
|
||||
if resp == nil {
|
||||
return nil, errors.New(i18n.T(lang, "censor.unavailable"))
|
||||
}
|
||||
|
||||
decision, detail := censor.Decide(string(strictness), resp)
|
||||
if decision.Blocked() {
|
||||
logrus.WithContext(ctx).
|
||||
WithField("strictness", string(strictness)).
|
||||
WithField("source", resp.SourceName).
|
||||
WithField("forbidden_type", resp.ForbiddenType).
|
||||
Info("Text blocked by moderation strictness policy")
|
||||
return resp, errors.New(resp.ErrorMessageFor(detail))
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
Reference in New Issue
Block a user