tamakyi/tamabox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: baseline snapshot before moderation-strictness feature

Browse Source 
Repo .git was damaged (refs/ missing, index truncated, object store near-empty).
Repaired via git init (rebuilt refs), stale index preserved as .git/index.corrupt-20260910.bak,
then full worktree committed as a rollback baseline.
This commit is contained in:
tamakyi
2026-09-10 10:56:27 +08:00
commit e3e59e7c69
296 changed files with 67458 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/security/autoban/autoban.go
+298
View File
@@ -0,0 +1,298 @@
// Package autoban 提供 fail2ban 风格的自动临时封禁：检测批量扫描/探测特征后
// 临时封禁来源 IP，并可回调发送告警。纯标准库实现，便于独立测试与复用。
package autoban
import (
"strings"
"sync"
"time"
)
// 探测事件类型
const (
KindNone = 0 // 无探测特征
KindSensitive = 1 // 命中敏感路径
KindNotFound = 2 // 404 响应（未命中敏感路径）
)
// 默认阈值/窗口（Options 未配置或为 0 时兜底）。
const (
DefaultWindowSeconds = 60
DefaultDurationMinutes = 15
DefaultThresholdSensitive = 10
DefaultThreshold404 = 20
pathSampleMax = 6
)
// Options 自动封禁配置。
type Options struct {
Enabled bool
// TrackNotFound 是否把 KindNotFound（未注册路径的 404）计入封禁统计。
// 默认 false：正常业务路径一律放行，仅敏感扫描特征路径参与计数封禁，
// 避免共享出口下正常访问被 404 探测连坐误伤（详见 conf.Security.BanTrackNotFound）。
TrackNotFound bool
ThresholdSensitive int // 窗口内敏感路径命中次数
Threshold404 int // 窗口内 404 次数
Window time.Duration // 统计窗口
Duration time.Duration // 封禁时长
AlertEmail string // 告警收件人（留空不发送）
SendAlert func(ip string, paths []string, durationMinutes int) error // 告警发送函数（nil 则不发送）
}
// sensitivePaths 常见后台/配置/漏洞探测路径特征（忽略大小写，完整段前缀匹配）。
// 注意：前缀匹配抓不到 //xxx/wp-includes/wlwmanifest.xml 这类"目录枚举变体"（敏感段
// 出现在路径中部），需要与 sensitiveFragments（任意位置子串）配合使用。
var sensitivePaths = []string{
"/admin", "/wp-admin", "/wp-login.php", "/wp-content", "/wordpress",
"/administrator", "/manager", "/cpanel", "/plesk",
"/.env", "/.git", "/.aws", "/.ssh", "/.svn", "/.hg", "/.htaccess",
"/config", "/configuration", "/backup", "/database", "/db",
"/actuator", "/swagger", "/api-docs", "/openapi", "/v2/api-docs",
"/phpmyadmin", "/pma", "/mysql", "/server-status", "/server-info",
"/shell", "/webshell", "/cmd", "/eval", "/hack", "/shell.php",
"/login.php", "/admin.php", "/index.php", "/console", "/vendor",
}
// sensitiveFragments 任意位置命中即算敏感的"强指纹"（忽略大小写，子串匹配）。
// 与 sensitivePaths 互补：捕获 /<枚举前缀>/wp-includes/wlwmanifest.xml、//wp/xmlrpc.php
// 等前缀匹配失效的变体。仅收录业务 URL 空间（/、/_/、/user、/api/v1、/vote、/explore、
// /i18n、/static…）绝不可能出现的 WordPress 特征，确保不误伤正常路径。
var sensitiveFragments = []string{
"/wp-includes/", "/wp-json/", "/wp-cron.php", "/wp-trackback.php",
"/wp-admin", "/wp-content", "/wp-login.php", "/wlwmanifest.xml",
"/xmlrpc.php", "/wordpress",
}
type probeEvent struct {
at time.Time
kind int
}
type probeStat struct {
events []probeEvent
sensitive int
notFound int
paths []string // 命中路径样本（告警用）
}
// Banner 自动封禁器。方法并发安全。
type Banner struct {
mu sync.Mutex
probes map[string]*probeStat
banned map[string]time.Time
alerted map[string]bool
thresholdSensitive int
threshold404 int
trackNotFound bool
window time.Duration
duration time.Duration
alertEmail string
sendAlert func(ip string, paths []string, durationMinutes int) error
enabled bool
}
// New 按配置创建封禁器并注册全局清理。
func New(opts Options) *Banner {
window := opts.Window
if window <= 0 {
window = DefaultWindowSeconds * time.Second
}
duration := opts.Duration
if duration <= 0 {
duration = DefaultDurationMinutes * time.Minute
}
sensitive := opts.ThresholdSensitive
if sensitive <= 0 {
sensitive = DefaultThresholdSensitive
}
notFound := opts.Threshold404
if notFound <= 0 {
notFound = DefaultThreshold404
}
b := &Banner{
probes: make(map[string]*probeStat),
banned: make(map[string]time.Time),
alerted: make(map[string]bool),
thresholdSensitive: sensitive,
threshold404: notFound,
trackNotFound: opts.TrackNotFound,
window: window,
duration: duration,
alertEmail: strings.TrimSpace(opts.AlertEmail),
sendAlert: opts.SendAlert,
enabled: opts.Enabled,
}
register(b)
return b
}
// Record 记录一次探测事件，达到阈值时封禁 IP 并触发一次告警。
func (b *Banner) Record(ip, path string, kind int) {
if !b.enabled || ip == "" || kind == KindNone {
return
}
// 404 计数默认关闭（正常业务路径一律放行，仅敏感扫描特征参与封禁）：
// 只有显式开启 TrackNotFound 时才把未注册路径的 404 计入统计。
if kind == KindNotFound && !b.trackNotFound {
return
}
b.mu.Lock()
defer b.mu.Unlock()
now := time.Now()
cutoff := now.Add(-b.window)
st := b.probes[ip]
if st == nil {
st = &probeStat{}
b.probes[ip] = st
}
// 滚动清理过期事件并重算计数
valid := st.events[:0]
sensitive, notFound := 0, 0
for _, e := range st.events {
if e.at.After(cutoff) {
valid = append(valid, e)
if e.kind == KindSensitive {
sensitive++
} else if e.kind == KindNotFound {
notFound++
}
}
}
valid = append(valid, probeEvent{at: now, kind: kind})
if kind == KindSensitive {
sensitive++
} else {
notFound++
}
st.events = valid
st.sensitive = sensitive
st.notFound = notFound
if len(st.paths) < pathSampleMax && path != "" {
st.paths = append(st.paths, path)
}
if sensitive < b.thresholdSensitive && notFound < b.threshold404 {
return
}
// 已达阈值：封禁（已在封禁中则跳过）
if until, ok := b.banned[ip]; ok && until.After(now) {
return
}
until := now.Add(b.duration)
b.banned[ip] = until
// 告警（每 IP 仅一次）
if b.alertEmail != "" && b.sendAlert != nil && !b.alerted[ip] {
b.alerted[ip] = true
sample := append([]string{}, st.paths...)
dur := int(b.duration.Minutes())
go func() {
_ = b.sendAlert(ip, sample, dur)
}()
}
}
// IsBanned 判断 IP 是否处于封禁期；到期自动解除。
func (b *Banner) IsBanned(ip string) bool {
if !b.enabled || ip == "" {
return false
}
b.mu.Lock()
defer b.mu.Unlock()
if until, ok := b.banned[ip]; ok {
if until.After(time.Now()) {
return true
}
delete(b.banned, ip)
}
return false
}
// IsSensitivePath 判断路径是否命中敏感探测特征（忽略大小写）。
// 命中条件：① 与 sensitivePaths 完整段前缀匹配（/admin、/.env 等）；
// ② 包含 sensitiveFragments 强指纹子串（wp-includes/xmlrpc.php 等 WordPress 探测面，
//
// 覆盖 // 双斜杠与目录枚举变体）。
func IsSensitivePath(p string) bool {
lower := strings.ToLower(p)
for _, s := range sensitivePaths {
if lower == s || strings.HasPrefix(lower, s+"/") {
return true
}
}
for _, f := range sensitiveFragments {
if strings.Contains(lower, f) {
return true
}
}
return false
}
// ---------- 全局注册与清理 ----------
var (
bannersMu sync.Mutex
banners = make(map[*Banner]struct{})
bannersOnce sync.Once
)
func register(b *Banner) {
bannersMu.Lock()
banners[b] = struct{}{}
bannersMu.Unlock()
bannersOnce.Do(func() {
go func() {
for {
time.Sleep(2 * time.Minute)
bannersMu.Lock()
for b := range banners {
b.cleanup()
}
bannersMu.Unlock()
}
}()
})
}
// cleanup 定期清理过期探测记录与已到期封禁（防止内存膨胀）。
func (b *Banner) cleanup() {
b.mu.Lock()
defer b.mu.Unlock()
now := time.Now()
cutoff := now.Add(-b.window)
for ip, st := range b.probes {
if len(st.events) == 0 || st.events[len(st.events)-1].at.Before(cutoff) {
delete(b.probes, ip)
continue
}
valid := st.events[:0]
sensitive, notFound := 0, 0
for _, e := range st.events {
if e.at.After(cutoff) {
valid = append(valid, e)
if e.kind == KindSensitive {
sensitive++
} else if e.kind == KindNotFound {
notFound++
}
}
}
st.events = valid
st.sensitive = sensitive
st.notFound = notFound
}
for ip, until := range b.banned {
if !until.After(now) {
delete(b.banned, ip)
}
}
}
internal/security/autoban/autoban_test.go
+157
View File
@@ -0,0 +1,157 @@
package autoban
import (
"sync/atomic"
"testing"
"time"
)
func newTestBanner(thresholdSensitive, threshold404 int, window, duration time.Duration, alertEmail string, alertCalls *int64) *Banner {
return New(Options{
Enabled: true,
TrackNotFound: true, // 测试 helper 显式开启 404 跟踪，覆盖两种计数的既有用例
ThresholdSensitive: thresholdSensitive,
Threshold404: threshold404,
Window: window,
Duration: duration,
AlertEmail: alertEmail,
SendAlert: func(ip string, paths []string, dur int) error {
if alertCalls != nil {
atomic.AddInt64(alertCalls, 1)
}
return nil
},
})
}
func TestSensitiveThreshold(t *testing.T) {
b := newTestBanner(2, 5, time.Minute, time.Minute, "", nil)
b.Record("1.2.3.4", "/admin", KindSensitive)
if b.IsBanned("1.2.3.4") {
t.Fatal("should not be banned before reaching threshold")
}
b.Record("1.2.3.4", "/.env", KindSensitive)
if !b.IsBanned("1.2.3.4") {
t.Fatal("expected ban after reaching sensitive threshold")
}
}
func TestNotFoundThreshold(t *testing.T) {
b := newTestBanner(10, 3, time.Minute, time.Minute, "", nil)
for i := 0; i < 3; i++ {
b.Record("5.6.7.8", "/random-"+string(rune('a'+i)), KindNotFound)
}
if !b.IsBanned("5.6.7.8") {
t.Fatal("expected ban after reaching 404 threshold")
}
}
func TestNotFoundTrackDisabledByDefault(t *testing.T) {
// 默认（TrackNotFound=false）下 404 不参与统计：正常业务路径一律放行，
// 仅敏感扫描路径仍计数封禁——防止共享出口下正常访问被 404 探测连坐误伤。
b := New(Options{Enabled: true, ThresholdSensitive: 1, Threshold404: 1, Window: time.Minute, Duration: time.Minute})
for i := 0; i < 5; i++ {
b.Record("7.7.7.7", "/random-"+string(rune('a'+i)), KindNotFound)
}
if b.IsBanned("7.7.7.7") {
t.Fatal("404 events must not accumulate when TrackNotFound is false")
}
b.Record("7.7.7.7", "/.env", KindSensitive)
if !b.IsBanned("7.7.7.7") {
t.Fatal("sensitive events must still ban even when 404 tracking is off")
}
}
func TestWindowExpiry(t *testing.T) {
b := newTestBanner(2, 5, 50*time.Millisecond, time.Minute, "", nil)
b.Record("1.1.1.1", "/admin", KindSensitive)
time.Sleep(80 * time.Millisecond) // 事件滑出窗口
b.Record("1.1.1.1", "/.env", KindSensitive)
if b.IsBanned("1.1.1.1") {
t.Fatal("events outside the window must not accumulate")
}
}
func TestDurationExpiry(t *testing.T) {
b := newTestBanner(1, 5, time.Minute, 60*time.Millisecond, "", nil)
b.Record("2.2.2.2", "/admin", KindSensitive)
if !b.IsBanned("2.2.2.2") {
t.Fatal("expected ban")
}
time.Sleep(100 * time.Millisecond) // 封禁到期
if b.IsBanned("2.2.2.2") {
t.Fatal("ban should expire after duration")
}
}
func TestAlertOncePerIP(t *testing.T) {
var calls int64
b := newTestBanner(1, 5, time.Minute, time.Minute, "[email protected]", &calls)
b.Record("9.9.9.9", "/wp-admin", KindSensitive)
b.Record("9.9.9.9", "/.git", KindSensitive) // 已在封禁中，不应重复告警
b.Record("9.9.9.9", "/config", KindSensitive)
// sendAlert 异步调用，轮询等待最多 500ms
deadline := time.Now().Add(500 * time.Millisecond)
for atomic.LoadInt64(&calls) == 0 && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
}
if got := atomic.LoadInt64(&calls); got != 1 {
t.Fatalf("expected exactly 1 alert, got %d", got)
}
}
func TestDisabled(t *testing.T) {
b := New(Options{Enabled: false})
b.Record("3.3.3.3", "/admin", KindSensitive)
if b.IsBanned("3.3.3.3") {
t.Fatal("disabled auto-ban must not ban")
}
}
func TestDefaults(t *testing.T) {
b := New(Options{Enabled: true}) // 全部走默认值
if b.thresholdSensitive != DefaultThresholdSensitive || b.threshold404 != DefaultThreshold404 {
t.Fatalf("default thresholds wrong: %d/%d", b.thresholdSensitive, b.threshold404)
}
if b.window != DefaultWindowSeconds*time.Second || b.duration != DefaultDurationMinutes*time.Minute {
t.Fatalf("default window/duration wrong: %v/%v", b.window, b.duration)
}
if b.trackNotFound {
t.Fatal("404 tracking must be disabled by default")
}
}
func TestIsSensitivePath(t *testing.T) {
cases := []struct {
path string
want bool
}{
{"/admin", true},
{"/admin/panel", true},
{"/WP-ADMIN", true},
{"/.env", true},
{"/user/questions", false},
{"/vote/abc123", false},
{"/administratorx", false}, // 前缀必须完整段匹配，避免误伤
// 目录枚举 / 双斜杠变体（wlwmanifest 扫描）：前缀匹配漏网，片段匹配兜底
{"//wp/wp-includes/wlwmanifest.xml", true},
{"//news/wp-includes/wlwmanifest.xml", true},
{"//2018/wp-includes/wlwmanifest.xml", true},
{"//shop/wp-includes/wlwmanifest.xml", true},
{"//test/xmlrpc.php", true},
{"/blog/wp-json/", true},
{"/wp-content/uploads/evil.php", true},
// 业务路径不得被片段匹配误伤
{"/_/tamakyi", false},
{"/_/admin6", false},
{"/api/v1/question/abc123", false},
{"/explore/tags/golang", false},
{"/i18n/zh-CN", false},
}
for _, c := range cases {
if got := IsSensitivePath(c.path); got != c.want {
t.Errorf("IsSensitivePath(%q) = %v, want %v", c.path, got, c.want)
}
}
}
internal/security/censor/aliyun.go
+194
View File
@@ -0,0 +1,194 @@
// Copyright 2022 E99p1ant. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package censor
import (
"context"
"encoding/json"
"sync"
"time"
"github.com/aliyun/alibaba-cloud-sdk-go/services/green"
"github.com/pkg/errors"
)
type AliyunTextCensor struct {
accessKey, accessKeySecret string
}
func NewAliyunTextCensor(accessKey, accessKeySecret string) *AliyunTextCensor {
return &AliyunTextCensor{
accessKey: accessKey,
accessKeySecret: accessKeySecret,
}
}
// 包级复用的 green.Client：审核请求高频且密钥在启动后不变，
// 用 sync.Once 懒加载一次并设置超时，避免每次审核都重建连接与对象。
var (
aliyunGreenOnce sync.Once
aliyunGreenClient *green.Client
aliyunGreenErr error
)
func (c *AliyunTextCensor) getClient() (*green.Client, error) {
aliyunGreenOnce.Do(func() {
aliyunGreenClient, aliyunGreenErr = green.NewClientWithAccessKey("cn-shanghai", c.accessKey, c.accessKeySecret)
if aliyunGreenErr == nil {
aliyunGreenClient.SetConnectTimeout(5 * time.Second)
aliyunGreenClient.SetReadTimeout(10 * time.Second)
}
})
return aliyunGreenClient, aliyunGreenErr
}
type AliyunTextCensorResponse struct {
Code int `json:"code"`
Data []struct {
Code int `json:"code"`
Content string `json:"content"`
FilteredContent string `json:"filteredContent"`
Msg string `json:"msg"`
Results []struct {
Details []struct {
Contexts []struct {
Context string `json:"context"`
Positions []struct {
EndPos int `json:"endPos"`
StartPos int `json:"startPos"`
} `json:"positions"`
} `json:"contexts"`
Label string `json:"label"`
} `json:"details"`
Label string `json:"label"`
Rate float64 `json:"rate"`
Scene string `json:"scene"`
Suggestion string `json:"suggestion"`
} `json:"results"`
TaskId string `json:"taskId"`
} `json:"data"`
Msg string `json:"msg"`
RequestId string `json:"requestId"`
}
func (r *AliyunTextCensorResponse) IsPass() bool {
if len(r.Data) == 0 || len(r.Data[0].Results) == 0 {
return false
}
// ⚠️ Right now, we allow `review` and `pass` to pass the censor.
// 任一 scene 判定为 block 即不通过（此前只检查首个 result，存在漏审）。
for _, result := range r.Data[0].Results {
if result.Suggestion == "block" {
return false
}
}
return true
}
// Censor censors text with Aliyun API.
// https://developer.qiniu.com/censor/7260/api-text-censor
func (c *AliyunTextCensor) Censor(_ context.Context, text string) (*TextCensorResponse, error) {
client, err := c.getClient()
if err != nil {
return nil, errors.Wrap(err, "new client with access key")
}
content, err := json.Marshal(
map[string]interface{}{
"scenes": []string{"antispam"},
"tasks": []map[string]interface{}{
{"content": text},
},
},
)
if err != nil {
return nil, errors.Wrap(err, "marshal content")
}
req := green.CreateTextScanRequest()
req.SetContent(content)
resp, err := client.TextScan(req)
if err != nil {
return nil, errors.Wrap(err, "text scan")
}
if !resp.IsSuccess() {
return nil, errors.New("response is not success")
}
return AliyunTextCensorParser(resp.GetHttpContentBytes())
}
func (*AliyunTextCensor) String() string {
return "aliyun"
}
func AliyunTextCensorParser(raw []byte) (*TextCensorResponse, error) {
var responseJSON AliyunTextCensorResponse
if err := json.Unmarshal(raw, &responseJSON); err != nil {
return nil, errors.Wrap(err, "unmarshal response")
}
if len(responseJSON.Data) == 0 {
return nil, errors.New("response data is empty")
}
var hint string
var label string
var confidence float64
for _, result := range responseJSON.Data[0].Results {
if result.Label == "normal" {
continue
}
// Get the first context as the hint, forbidden type, confidence.
for _, detail := range result.Details {
for _, context := range detail.Contexts {
hint = context.Context
}
}
label = result.Label
confidence = result.Rate
break
}
return &TextCensorResponse{
SourceName: "aliyun",
Pass: responseJSON.IsPass(),
ForbiddenType: formatAliyunForbiddenType(label),
Hint: hint,
Confidence: confidence,
RawResponse: raw,
}, nil
}
func formatAliyunForbiddenType(typ string) ForbiddenType {
switch typ {
case "spam":
return ForbiddenTypeSpam
case "ad":
return ForbiddenTypeAd
case "politics":
return ForbiddenTypePolitics
case "terrorism":
return ForbiddenTypeTerrorism
case "abuse":
return ForbiddenTypeAbuse
case "porn":
return ForbiddenTypePorn
case "flood":
return ForbiddenTypeFlood
case "contraband":
return ForbiddenTypeContraband
case "meaningless":
return ForbiddenTypeMeaningless
case "harmful":
return ForbiddenTypeHarmful
default:
return ""
}
}
internal/security/censor/censor.go
+96
View File
@@ -0,0 +1,96 @@
// Copyright 2022 E99p1ant. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package censor
import (
"context"
"time"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/tamakyi/TamaBox/internal/conf"
"github.com/tamakyi/TamaBox/internal/db"
)
var (
censorCacheNoMoreThan = 31 * 24 * time.Hour // 1 month
)
// Text checks the text for sensitive content.
// It will save the censor log to the database and invoke the callback function.
func Text(ctx context.Context, text string) (*TextCensorResponse, error) {
if !conf.Security.EnableTextCensor {
return &TextCensorResponse{Pass: true}, nil
}
var responses []*TextCensorResponse
for _, censor := range []TextCensor{
//No, we need to use qiniu's API.
NewQiniuTextCensor(conf.App.QiniuAccessKey, conf.App.QiniuAccessSecret),
NewAliyunTextCensor(conf.App.AliyunAccessKey, conf.App.AliyunAccessKeySecret),
} {
sourceName := censor.String()
// Try to get the censor from the database log.
censorLog, err := db.CensorLogs.GetByText(ctx, sourceName, text, time.Now().Add(-censorCacheNoMoreThan))
if err != nil {
if !errors.Is(err, db.ErrCensorLogsNotFound) {
logrus.WithContext(ctx).WithError(err).Error("Failed to get censor log")
}
} else {
raw := censorLog.RawResponse
// We got the previous censor log cache, seems like we saved money for the API call.
switch censorLog.SourceName {
case "qiniu":
// HACK: Qiniu's API response is not accurate, so we need to check the text again with aliyun API.
response, err := QiniuTextCensorParser(raw)
if err == nil {
if response.Pass {
return response, nil
}
responses = append(responses, response) // If the text is not passed, we need to check it with aliyun API.
continue
}
case "aliyun":
return AliyunTextCensorParser(raw)
default:
logrus.WithContext(ctx).WithField("censor_source", censorLog.SourceName).Error("Unknown censor source")
}
}
// response, err := censor.Censor(ctx, text)
censorText := RemoveTrustedURL(text)
response, err := censor.Censor(ctx, censorText)
if err != nil {
logrus.WithContext(ctx).WithError(err).WithField("censor_source", sourceName).Error("Failed to censor text")
continue
}
responses = append(responses, response)
// Save the censor log to the database.
if err := db.CensorLogs.Create(ctx, db.CreateCensorLogOptions{
SourceName: sourceName,
Input: text,
Pass: response.Pass,
RawResponse: response.RawResponse,
}); err != nil {
logrus.WithContext(ctx).WithError(err).Error("Failed to save censor log")
}
if response.Pass {
return response, nil
}
}
if len(responses) == 0 {
return nil, errors.New("no censor source available")
}
return responses[len(responses)-1], nil
}
internal/security/censor/qiniu.go
+178
View File
@@ -0,0 +1,178 @@
// Copyright 2022 E99p1ant. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package censor
import (
"bytes"
"context"
"encoding/json"
"io"
"net/http"
"time"
"github.com/pkg/errors"
"github.com/qiniu/go-sdk/v7/auth"
"github.com/samber/lo"
)
type QiniuTextCensor struct {
accessKey, accessSecret string
}
func NewQiniuTextCensor(accessKey, accessSecret string) *QiniuTextCensor {
return &QiniuTextCensor{
accessKey: accessKey,
accessSecret: accessSecret,
}
}
type QiniuTextCensorResponse struct {
Code int `json:"code"`
Message string `json:"message"`
Result struct {
Suggestion string `json:"suggestion"`
Scenes struct {
Antispam struct {
Suggestion string `json:"suggestion"`
Details []struct {
Label string `json:"label"`
Score float64 `json:"score"`
Contexts []struct {
Context string `json:"context"`
Positions []struct {
StartPos int `json:"startPos"`
EndPos int `json:"endPos"`
} `json:"positions"`
} `json:"contexts"`
} `json:"details"`
} `json:"antispam"`
} `json:"scenes"`
} `json:"result"`
}
func (r *QiniuTextCensorResponse) IsPass() bool {
for _, detail := range r.Result.Scenes.Antispam.Details {
label := detail.Label
// These are absolutely not allowed to display to users.
blockedLabels := []string{"politics", "terrorism", "porn"}
if lo.Contains(blockedLabels, label) {
return false
}
}
// ⚠️ Right now, we allow `review` and `pass` to pass the censor.
return r.Result.Scenes.Antispam.Suggestion != "block"
}
// Censor censors text with Qiniu API.
// https://developer.qiniu.com/censor/7260/api-text-censor
func (c *QiniuTextCensor) Censor(ctx context.Context, text string) (*TextCensorResponse, error) {
var bodyBuffer bytes.Buffer
if err := json.NewEncoder(&bodyBuffer).Encode(map[string]interface{}{
"data": map[string]interface{}{
"text": text,
},
"params": map[string]interface{}{
"scenes": []string{"antispam"},
},
}); err != nil {
return nil, errors.Wrap(err, "encode request body")
}
request, err := http.NewRequest(http.MethodPost, "https://ai.qiniuapi.com/v3/text/censor", &bodyBuffer)
if err != nil {
return nil, errors.Wrap(err, "new request")
}
request = request.WithContext(ctx)
credentials := auth.New(c.accessKey, c.accessSecret)
token, err := credentials.SignRequestV2(request)
if err != nil {
return nil, errors.Wrap(err, "sign request")
}
request.Header.Set("Authorization", "Qiniu "+token)
// 带超时：避免七牛审核服务无响应时请求永久挂起，拖垮提问/回答流程
client := http.Client{Timeout: 10 * time.Second}
resp, err := client.Do(request)
if err != nil {
return nil, errors.Wrap(err, "do request")
}
defer func() { _ = resp.Body.Close() }()
bodyBytes, err := io.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrap(err, "read body")
}
if resp.StatusCode/100 != 2 {
return nil, errors.Errorf("unexpected error code: %d, response body: %s", resp.StatusCode, string(bodyBytes))
}
return QiniuTextCensorParser(bodyBytes)
}
func (*QiniuTextCensor) String() string {
return "qiniu"
}
func QiniuTextCensorParser(raw []byte) (*TextCensorResponse, error) {
var responseJSON QiniuTextCensorResponse
if err := json.Unmarshal(raw, &responseJSON); err != nil {
return nil, errors.Wrap(err, "decode response body")
}
var hint string
var detailKey string
var confidence float64
for _, detail := range responseJSON.Result.Scenes.Antispam.Details {
if detail.Label == "normal" {
continue
}
// Get the first context as the hint, forbidden type, confidence.
for _, context := range detail.Contexts {
hint = context.Context
}
detailKey = detail.Label
confidence = detail.Score
break
}
return &TextCensorResponse{
SourceName: "qiniu",
Pass: responseJSON.IsPass(),
ForbiddenType: formatQiniuForbiddenType(detailKey),
Hint: hint,
Confidence: confidence,
RawResponse: raw,
}, nil
}
func formatQiniuForbiddenType(typ string) ForbiddenType {
switch typ {
case "spam":
return ForbiddenTypeSpam
case "ad":
return ForbiddenTypeAd
case "politics":
return ForbiddenTypePolitics
case "terrorism":
return ForbiddenTypeTerrorism
case "abuse":
return ForbiddenTypeAbuse
case "porn":
return ForbiddenTypePorn
case "flood":
return ForbiddenTypeFlood
case "contraband":
return ForbiddenTypeContraband
case "meaningless":
return ForbiddenTypeMeaningless
default:
return ""
}
}
internal/security/censor/text.go
+121
View File
@@ -0,0 +1,121 @@
// Copyright 2022 E99p1ant. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package censor
import (
"context"
"encoding/json"
"fmt"
"strings"
)
type ForbiddenType string
const (
ForbiddenTypeSpam ForbiddenType = "spam"
ForbiddenTypeAd ForbiddenType = "ad"
ForbiddenTypePolitics ForbiddenType = "politics"
ForbiddenTypeTerrorism ForbiddenType = "terrorism"
ForbiddenTypeAbuse ForbiddenType = "abuse"
ForbiddenTypePorn ForbiddenType = "porn"
ForbiddenTypeFlood ForbiddenType = "flood"
ForbiddenTypeContraband ForbiddenType = "contraband"
ForbiddenTypeMeaningless ForbiddenType = "meaningless"
ForbiddenTypeHarmful ForbiddenType = "harmful"
)
func (f ForbiddenType) String() string {
return map[ForbiddenType]string{
ForbiddenTypeSpam: "含垃圾信息",
ForbiddenTypeAd: "广告",
ForbiddenTypePolitics: "涉政",
ForbiddenTypeTerrorism: "暴恐",
ForbiddenTypeAbuse: "辱骂",
ForbiddenTypePorn: "色情",
ForbiddenTypeFlood: "灌水",
ForbiddenTypeContraband: "违禁",
ForbiddenTypeMeaningless: "无意义",
ForbiddenTypeHarmful: "不良场景",
}[f]
}
type TextCensorResponse struct {
SourceName string `json:"source_name"`
Pass bool `json:"pass"`
ForbiddenType ForbiddenType `json:"forbidden_type"`
Hint string `json:"hint"`
Confidence float64 `json:"confidence"`
RawResponse json.RawMessage `json:"raw_response"`
}
func (r *TextCensorResponse) ToJSON() []byte {
jsonBytes, _ := json.Marshal(r)
return jsonBytes
}
func (r *TextCensorResponse) ErrorMessage() string {
errorMessage := "内容安全检查不通过"
if r.ForbiddenType != "" {
errorMessage += fmt.Sprintf(" [%s]", r.ForbiddenType.String())
}
if r.Hint != "" {
errorMessage += fmt.Sprintf("，相关标签：%q", r.Hint)
}
return errorMessage
}
type TextCensor interface {
Censor(ctx context.Context, text string) (*TextCensorResponse, error)
String() string
}
var TrustedHosts = []string{
"xhslink.com", // 小红书
"weibo.com", // 微博
"b23.tv", // Bilibili
"acfun.cn",
"aixifan.com",
"github.com",
// 自有域名：旧域 tama.guru 保留至 2026-12 到期前的过渡期结束，
// 期间用户分享的旧链接仍需视为可信（不被判为广告）；新域 shiroko.one 为主域。
"tama.guru",
"shiroko.one",
"337845818.best",
"x.com",
"bilibili.com",
}
// RemoveTrustedURL removes the trusted URL from the text,
// in order to bypass the URL censoring.
// 仅当可信域名后紧跟路径分隔符（/ ? #）或位于字符串结尾时才视为可信，
// 避免 github.com.evil.tld 被朴素替换误判为 github.com 之子域而漏审。
func RemoveTrustedURL(str string) string {
for _, host := range TrustedHosts {
for _, protocol := range []string{"http://", "https://"} {
needle := protocol + host
var b strings.Builder
rest := str
for {
idx := strings.Index(rest, needle)
if idx < 0 {
b.WriteString(rest)
break
}
b.WriteString(rest[:idx])
after := rest[idx+len(needle):]
if len(after) == 0 || after[0] == '/' || after[0] == '?' || after[0] == '#' {
// 可信域名边界：跳过该前缀，从域名之后继续（域名本身被移除，不进入审核）。
rest = after
} else {
// 非可信域名边界（如 github.com.evil）：保留前缀，从其后继续扫描。
b.WriteString(needle)
rest = after
}
}
str = b.String()
}
}
return str
}
internal/security/censor/text_test.go
+29
View File
@@ -0,0 +1,29 @@
// Copyright 2025 E99p1ant. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package censor
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestRemoveTrustedURL(t *testing.T) {
for _, tc := range []struct {
name string
text string
want string
}{
{name: "empty string", text: "", want: ""},
{name: "normal", text: "你喜欢看什么书？", want: "你喜欢看什么书？"},
{name: "xiaohongshu", text: `59 茄酱发布了一篇小红书笔记，快来看吧！ 😆 jk14SBf2G75FTW2 😆 http://xhslink.com/a/WYaQ6Tjn86Hab，复制本条信息，打开【小红书】App查看精彩内容！`, want: `59 茄酱发布了一篇小红书笔记，快来看吧！ 😆 jk14SBf2G75FTW2 😆 /a/WYaQ6Tjn86Hab，复制本条信息，打开【小红书】App查看精彩内容！`},
{name: "weibo", text: `TK 发微博啦~ https://weibo.com/1401527553/5144129806795225 真好看！`, want: `TK 发微博啦~ /1401527553/5144129806795225 真好看！`},
} {
t.Run(tc.name, func(t *testing.T) {
got := RemoveTrustedURL(tc.text)
assert.Equal(t, tc.want, got)
})
}
}
internal/security/ipinfo/ipinfo.go
+84
View File
@@ -0,0 +1,84 @@
// Package ipinfo 识别 IP 的网络归属（CDN / 云厂商 / WAF）。
// 基于 projectdiscovery/cdncheck 的离线签名（编译期嵌入，上游每周更新指纹），
// 判定过程零外部网络请求，适合 1C1G 小型服务器与告警链路使用。
// 当前用于安全告警邮件：展示扫描来源的网络归属，帮助管理员判断告警严重性。
package ipinfo
import (
"fmt"
"net"
"strings"
"sync"
"github.com/projectdiscovery/cdncheck"
)
var (
clientOnce sync.Once
client *cdncheck.Client
)
// getClient 懒加载判定客户端（签名数据编译期嵌入，New 无错误返回）。
func getClient() *cdncheck.Client {
clientOnce.Do(func() {
client = cdncheck.New()
})
return client
}
// Lookup 判定 IP 的网络归属。
// 返回：
// - label：展示用标签，如 "Cloudflare（CDN）"、"亚马逊（云厂商）"；
// 未命中返回 "未识别（普通网络）"，输入非法或判定出错返回 "识别失败"。
// - proxied：IP 是否属于 CDN/WAF/代理网络（用于提示"封禁可能无法拦截真实来源"）。
//
// 任何内部错误都被折叠为展示标签，不向调用方传播——告警链路不应因归属识别失败而中断。
func Lookup(ipStr string) (label string, proxied bool) {
ip := net.ParseIP(strings.TrimSpace(ipStr))
if ip == nil {
return "识别失败（IP 格式异常）", false
}
matched, value, itemType, err := getClient().Check(ip)
if err != nil {
return "识别失败", false
}
if !matched || value == "" {
return "未识别（普通网络）", false
}
return fmt.Sprintf("%s（%s）", title(value), typeLabel(itemType)), true
}
// typeLabel 将 cdncheck 的 itemType 翻译为中文展示。
func typeLabel(itemType string) string {
switch itemType {
case "cdn":
return "CDN"
case "waf":
return "WAF"
case "cloud":
return "云厂商"
default:
return "未知类型"
}
}
// acronymProviders 需要保持全大写的提供商缩写（cdncheck 返回全小写，如 aws → AWS）。
var acronymProviders = map[string]bool{
"aws": true, "gcp": true, "oci": true, "gds": true, "ovh": true,
}
// title 将提供商名转为展示形式：缩写全大写（aws → AWS），其余首字母大写
// （cloudflare → Cloudflare）。
func title(s string) string {
if s == "" {
return s
}
lower := strings.ToLower(s)
if acronymProviders[lower] {
return strings.ToUpper(lower)
}
r := []rune(lower)
return strings.ToUpper(string(r[0])) + string(r[1:])
}
internal/security/ipinfo/ipinfo_test.go
+27
View File
@@ -0,0 +1,27 @@
package ipinfo
import "testing"
// TestLookupInvalidIP 非法输入必须折叠为展示标签且不 panic，proxied 恒为 false。
func TestLookupInvalidIP(t *testing.T) {
for _, in := range []string{"", "not-an-ip", "999.1.1.1", " "} {
label, proxied := Lookup(in)
if proxied {
t.Errorf("Lookup(%q) proxied = true, want false", in)
}
if label == "" {
t.Errorf("Lookup(%q) label is empty, want a fallback label", in)
}
}
}
// TestLookupReturnsLabel 合法 IP 应返回非空标签且不 panic（不锁定具体归属，
// 避免上游签名快照更新导致测试抖动）。
func TestLookupReturnsLabel(t *testing.T) {
for _, in := range []string{"1.1.1.1", "8.8.8.8", "2606:4700:4700::1111"} {
label, _ := Lookup(in)
if label == "" {
t.Errorf("Lookup(%q) label is empty", in)
}
}
}