口令失败记录加上硬上限,并且不再傻等到满 delay
审查发现这张表的容量可以被外部放大:IP 来自 r.RemoteAddr,是请求方能左右的输入。
1. 原来为什么会被写满
清理逻辑只在 len > 1000 时才触发一次,而且只删「10 分钟没动静」的。
换着来源持续撞口令时,每条记录的 last 都是刚刚更新的,于是「虽然超了
1000 条但一条都不符合删除条件」,表现就是表不停往上涨。
实测:3000 个不同来源各失败一次之后,表里 3000 条,一条没删。
2. 现在怎么收敛
- 加硬上限 authFailMaxEntries(1 万条)。到上限立刻扫一次,
只靠「每 256 条扫一次」的话两个周期之间涌进来的会把表顶破。
- 扫表先删过期项,还多就按「最久没动静」排序淘汰。
- 淘汰时留一成余量(淘汰到九成),这样满表之后还能再收一千来个
新来源才需要排下一次,不会变成「每来一个就排一遍全表」。
- 同一来源的失败次数封顶。延迟早就封顶了(2 秒),数字继续涨没有意义。
3. 延迟等待改成可以被断开
原来在 handler 里直接 time.Sleep(最多 2 秒)。并发撞口令时每个请求各占一个
goroutine 干等两秒,很容易把服务拖住。现在用 timer + r.Context():
客户端一断开立刻返回,不再占着连接。
已验证:
- 新增 5 个用例:
TestAuthFailTableHasHardCap(2 万个新来源撞完,表必须被压在 1 万以内)、
TestAuthFailExpiredEntriesGetSwept(过期项扫干净)、
TestAuthFailSweepsPeriodically(定期扫是自动的,不用调用方记着)、
TestAuthFailCountCapped(同来源次数封顶、延迟封顶)、
TestAwaitAuthDelayStopsOnCancel(上游取消后立刻返回,不能傻等满 2 秒)。
- 反向验证:把淘汰目标临时改成 authFailMaxEntries*1000(等于没上限),
TestAuthFailTableHasHardCap 立刻报「应压在 10000 以内,实际 20000」。
- gofmt / go vet / go build / GOOS=linux go build / go test / go test -race 全过。
This commit is contained in:
@@ -2460,7 +2460,8 @@ func checkAuthToken(r *http.Request) bool {
|
|||||||
// 口令错误次数的记录,给反复尝试的请求加一点延迟
|
// 口令错误次数的记录,给反复尝试的请求加一点延迟
|
||||||
var authFail struct {
|
var authFail struct {
|
||||||
sync.Mutex
|
sync.Mutex
|
||||||
m map[string]authFailEntry
|
m map[string]authFailEntry
|
||||||
|
sinceSweep int // 距上次扫表又记了多少条
|
||||||
}
|
}
|
||||||
|
|
||||||
type authFailEntry struct {
|
type authFailEntry struct {
|
||||||
@@ -2471,6 +2472,62 @@ type authFailEntry struct {
|
|||||||
// 口令错误时最多延迟这么久,避免拖垮服务器
|
// 口令错误时最多延迟这么久,避免拖垮服务器
|
||||||
const authMaxDelay = 2 * time.Second
|
const authMaxDelay = 2 * time.Second
|
||||||
|
|
||||||
|
const (
|
||||||
|
// 表里最多留这么多条。IP 是外部给的,必须有个硬上限:只靠过期清理的话,
|
||||||
|
// 每换一个新来源就多一条,清理追不上写入,最后把内存写满
|
||||||
|
authFailMaxEntries = 10000
|
||||||
|
// 超过这么久没再失败,这条记录就当没人用了
|
||||||
|
authFailExpire = 10 * time.Minute
|
||||||
|
// 同一个来源最多记到这个次数:再往上涨延迟也是封顶的,没必要让数字一直加
|
||||||
|
authFailMaxCount = 1000
|
||||||
|
// 每记这么多条扫一次表。扫一次要遍历(必要时还要排序),不必每次都做
|
||||||
|
authFailSweepEvery = 256
|
||||||
|
)
|
||||||
|
|
||||||
|
// 扫一遍口令错误记录:先删过期项,还多就把「最久没动静」的淘汰掉。
|
||||||
|
// 淘汰时留出一成余量,这样满了之后还能再收一千来个新来源才需要重排下一次全表,
|
||||||
|
// 不至于每来一个就把整张表排一遍(那反倒成了另一条拖慢服务的路子)。
|
||||||
|
func sweepAuthFail(now time.Time) {
|
||||||
|
for k, v := range authFail.m {
|
||||||
|
if now.Sub(v.last) > authFailExpire {
|
||||||
|
delete(authFail.m, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
target := authFailMaxEntries * 9 / 10
|
||||||
|
over := len(authFail.m) - target
|
||||||
|
if over <= 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
type record struct {
|
||||||
|
ip string
|
||||||
|
last time.Time
|
||||||
|
}
|
||||||
|
all := make([]record, 0, len(authFail.m))
|
||||||
|
for k, v := range authFail.m {
|
||||||
|
all = append(all, record{k, v.last})
|
||||||
|
}
|
||||||
|
sort.Slice(all, func(i, j int) bool { return all[i].last.Before(all[j].last) })
|
||||||
|
for i := 0; i < over && i < len(all); i++ {
|
||||||
|
delete(authFail.m, all[i].ip)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 等待口令错误的惩罚延迟。客户端一断开就立刻返回,
|
||||||
|
// 否则并发撞口令时一堆 goroutine 各占着一条连接干等两秒,很容易把服务拖住
|
||||||
|
func awaitAuthDelay(ctx context.Context, d time.Duration) bool {
|
||||||
|
if d <= 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
timer := time.NewTimer(d)
|
||||||
|
defer timer.Stop()
|
||||||
|
select {
|
||||||
|
case <-timer.C:
|
||||||
|
return true
|
||||||
|
case <-ctx.Done():
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// 取请求的来源 IP
|
// 取请求的来源 IP
|
||||||
func clientIP(r *http.Request) string {
|
func clientIP(r *http.Request) string {
|
||||||
if host, _, err := net.SplitHostPort(r.RemoteAddr); err == nil {
|
if host, _, err := net.SplitHostPort(r.RemoteAddr); err == nil {
|
||||||
@@ -2486,17 +2543,23 @@ func authFailDelay(ip string) time.Duration {
|
|||||||
if authFail.m == nil {
|
if authFail.m == nil {
|
||||||
authFail.m = make(map[string]authFailEntry)
|
authFail.m = make(map[string]authFailEntry)
|
||||||
}
|
}
|
||||||
// 记录太多时清掉过期的,避免大量不同来源的请求把内存撑满
|
now := time.Now()
|
||||||
if len(authFail.m) > 1000 {
|
// 到上限就立刻扫一次。只靠定期扫的话,两个周期之间涌入的新来源
|
||||||
for k, v := range authFail.m {
|
// 会把表顶破上限(IP 是外部给的,写入速度不受我们控制)
|
||||||
if time.Since(v.last) > 10*time.Minute {
|
if len(authFail.m) >= authFailMaxEntries {
|
||||||
delete(authFail.m, k)
|
authFail.sinceSweep = 0
|
||||||
}
|
sweepAuthFail(now)
|
||||||
}
|
}
|
||||||
|
authFail.sinceSweep++
|
||||||
|
if authFail.sinceSweep >= authFailSweepEvery {
|
||||||
|
authFail.sinceSweep = 0
|
||||||
|
sweepAuthFail(now)
|
||||||
}
|
}
|
||||||
e := authFail.m[ip]
|
e := authFail.m[ip]
|
||||||
e.count++
|
if e.count < authFailMaxCount {
|
||||||
e.last = time.Now()
|
e.count++
|
||||||
|
}
|
||||||
|
e.last = now
|
||||||
authFail.m[ip] = e
|
authFail.m[ip] = e
|
||||||
d := time.Duration(e.count) * 100 * time.Millisecond
|
d := time.Duration(e.count) * 100 * time.Millisecond
|
||||||
if d > authMaxDelay {
|
if d > authMaxDelay {
|
||||||
@@ -2527,7 +2590,10 @@ func webAuth(next http.Handler) http.Handler {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
if d := authFailDelay(ip); d > 0 {
|
if d := authFailDelay(ip); d > 0 {
|
||||||
time.Sleep(d)
|
// 客户端已经断开就不用再等了,直接收工
|
||||||
|
if !awaitAuthDelay(r.Context(), d) {
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||||
w.WriteHeader(http.StatusUnauthorized)
|
w.WriteHeader(http.StatusUnauthorized)
|
||||||
|
|||||||
+148
@@ -2,10 +2,13 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"net/url"
|
"net/url"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// 测试期间换一个管理口令
|
// 测试期间换一个管理口令
|
||||||
@@ -155,3 +158,148 @@ func TestAuthTokenCandidatesOrder(t *testing.T) {
|
|||||||
t.Fatalf("非 Bearer 的 Authorization 不该当候选:%q", got)
|
t.Fatalf("非 Bearer 的 Authorization 不该当候选:%q", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ==================== 口令错误记录的容量 ====================
|
||||||
|
// IP 是外部给的:只靠「过期清理」挡不住换着地址撞口令的情况,
|
||||||
|
// 必须要有硬上限。这些用例锁住这一点。
|
||||||
|
|
||||||
|
// 换着来源持续撞口令,表必须被压在上限以内,不能一直涨
|
||||||
|
func TestAuthFailTableHasHardCap(t *testing.T) {
|
||||||
|
saved, savedSweep := authFail.m, authFail.sinceSweep
|
||||||
|
t.Cleanup(func() {
|
||||||
|
authFail.Lock()
|
||||||
|
authFail.m, authFail.sinceSweep = saved, savedSweep
|
||||||
|
authFail.Unlock()
|
||||||
|
})
|
||||||
|
authFail.Lock()
|
||||||
|
authFail.m, authFail.sinceSweep = nil, 0
|
||||||
|
authFail.Unlock()
|
||||||
|
|
||||||
|
// 每个都是新来源、且都是「刚刚」失败过 —— 过期清理一条都删不掉的情况
|
||||||
|
for i := 0; i < authFailMaxEntries*2; i++ {
|
||||||
|
authFailDelay(fmt.Sprintf("198.51.100.%d", i))
|
||||||
|
}
|
||||||
|
got := 0
|
||||||
|
authFail.Lock()
|
||||||
|
got = len(authFail.m)
|
||||||
|
authFail.Unlock()
|
||||||
|
if got > authFailMaxEntries {
|
||||||
|
t.Errorf("记录数应被压在上限 %d 以内,实际 %d(换 IP 撞口令会把内存写满)",
|
||||||
|
authFailMaxEntries, got)
|
||||||
|
}
|
||||||
|
t.Logf(" %d 个新来源撞完,表内留下 %d 条(上限 %d)", authFailMaxEntries*2, got, authFailMaxEntries)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 超过过期时间的记录要被扫掉,不是在表里一直躺着
|
||||||
|
func TestAuthFailExpiredEntriesGetSwept(t *testing.T) {
|
||||||
|
saved, savedSweep := authFail.m, authFail.sinceSweep
|
||||||
|
t.Cleanup(func() {
|
||||||
|
authFail.Lock()
|
||||||
|
authFail.m, authFail.sinceSweep = saved, savedSweep
|
||||||
|
authFail.Unlock()
|
||||||
|
})
|
||||||
|
authFail.Lock()
|
||||||
|
authFail.m, authFail.sinceSweep = nil, 0
|
||||||
|
authFail.Unlock()
|
||||||
|
|
||||||
|
for i := 0; i < authFailSweepEvery+5; i++ {
|
||||||
|
authFailDelay(fmt.Sprintf("203.0.113.%d", i))
|
||||||
|
}
|
||||||
|
authFail.Lock()
|
||||||
|
before := len(authFail.m)
|
||||||
|
// 把记录的最后时间推到过期之前
|
||||||
|
for k, v := range authFail.m {
|
||||||
|
authFail.m[k] = authFailEntry{count: v.count, last: time.Now().Add(-2 * authFailExpire)}
|
||||||
|
}
|
||||||
|
authFail.Unlock()
|
||||||
|
|
||||||
|
sweepAuthFail(time.Now())
|
||||||
|
authFail.Lock()
|
||||||
|
after := len(authFail.m)
|
||||||
|
authFail.Unlock()
|
||||||
|
if after != 0 {
|
||||||
|
t.Errorf("已经全部过期的记录应被扫干净,扫之前 %d 条,扫之后 %d 条", before, after)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 扫表必须是自动发生的:不要求调用方记着手动清理,
|
||||||
|
// 混在里面的过期项应该在若干次记录之后自己消失
|
||||||
|
func TestAuthFailSweepsPeriodically(t *testing.T) {
|
||||||
|
saved, savedSweep := authFail.m, authFail.sinceSweep
|
||||||
|
t.Cleanup(func() {
|
||||||
|
authFail.Lock()
|
||||||
|
authFail.m, authFail.sinceSweep = saved, savedSweep
|
||||||
|
authFail.Unlock()
|
||||||
|
})
|
||||||
|
authFail.Lock()
|
||||||
|
authFail.m, authFail.sinceSweep = nil, 0
|
||||||
|
authFail.Unlock()
|
||||||
|
|
||||||
|
// 直接往里塞一条早就过期了的记录(绕过 authFailDelay,模拟「以前留下的」)
|
||||||
|
authFail.Lock()
|
||||||
|
if authFail.m == nil {
|
||||||
|
authFail.m = make(map[string]authFailEntry)
|
||||||
|
}
|
||||||
|
authFail.m["203.0.113.99"] = authFailEntry{count: 1, last: time.Now().Add(-2 * authFailExpire)}
|
||||||
|
authFail.Unlock()
|
||||||
|
|
||||||
|
// 之后只要还在有人失败,过一阵子它就应该被清掉
|
||||||
|
for i := 0; i < authFailSweepEvery+1; i++ {
|
||||||
|
authFailDelay(fmt.Sprintf("198.51.100.%d", i))
|
||||||
|
}
|
||||||
|
|
||||||
|
authFail.Lock()
|
||||||
|
_, still := authFail.m["203.0.113.99"]
|
||||||
|
authFail.Unlock()
|
||||||
|
if still {
|
||||||
|
t.Errorf("记了 %d 次之后,过期记录应当已经被自动扫掉", authFailSweepEvery+1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 同一个来源的计数不能无限涨(临界点是延迟早已封顶,再涨只是让数字溢出)
|
||||||
|
func TestAuthFailCountCapped(t *testing.T) {
|
||||||
|
saved, savedSweep := authFail.m, authFail.sinceSweep
|
||||||
|
t.Cleanup(func() {
|
||||||
|
authFail.Lock()
|
||||||
|
authFail.m, authFail.sinceSweep = saved, savedSweep
|
||||||
|
authFail.Unlock()
|
||||||
|
})
|
||||||
|
authFail.Lock()
|
||||||
|
authFail.m, authFail.sinceSweep = nil, 0
|
||||||
|
authFail.Unlock()
|
||||||
|
|
||||||
|
for i := 0; i < authFailMaxCount+50; i++ {
|
||||||
|
authFailDelay("192.0.2.9")
|
||||||
|
}
|
||||||
|
authFail.Lock()
|
||||||
|
c := authFail.m["192.0.2.9"].count
|
||||||
|
authFail.Unlock()
|
||||||
|
if c > authFailMaxCount {
|
||||||
|
t.Errorf("同一来源的次数应封顶在 %d,实际 %d", authFailMaxCount, c)
|
||||||
|
}
|
||||||
|
if d := authFailDelay("192.0.2.9"); d != authMaxDelay {
|
||||||
|
t.Errorf("打满之后延迟应封顶在 %v,实际 %v", authMaxDelay, d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 客户端断开时延迟等待必须立刻返回,不能傻等满 2 秒
|
||||||
|
func TestAwaitAuthDelayStopsOnCancel(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
go func() {
|
||||||
|
time.Sleep(80 * time.Millisecond)
|
||||||
|
cancel()
|
||||||
|
}()
|
||||||
|
start := time.Now()
|
||||||
|
waited := awaitAuthDelay(ctx, 2*time.Second)
|
||||||
|
used := time.Since(start)
|
||||||
|
if waited {
|
||||||
|
t.Error("上游取消时 awaitAuthDelay 应返回 false")
|
||||||
|
}
|
||||||
|
if used > time.Second {
|
||||||
|
t.Errorf("上游取消后应立刻返回,实际等了 %v", used)
|
||||||
|
}
|
||||||
|
// 正常等待不受影响
|
||||||
|
if !awaitAuthDelay(context.Background(), 10*time.Millisecond) {
|
||||||
|
t.Error("没被取消时应当正常等完再返回 true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user