可靠性: 完善 R2 备份恢复与对象生命周期
This commit is contained in:
tamakyi
@@ -0,0 +1,270 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/tamakyi/TamaBox/internal/conf"
|
||||
"github.com/tamakyi/TamaBox/internal/db"
|
||||
)
|
||||
|
||||
type fakeR2OutboxFailure struct {
|
||||
nextRetry time.Time
|
||||
lastError string
|
||||
}
|
||||
|
||||
type fakeR2OutboxStore struct {
|
||||
tasks []*db.R2DeleteTask
|
||||
references map[string]bool
|
||||
referenceErrs map[string]error
|
||||
referenceCalls []string
|
||||
deleted map[uint]bool
|
||||
deleteErr error
|
||||
failures map[uint]fakeR2OutboxFailure
|
||||
markFailedErr error
|
||||
}
|
||||
|
||||
func newFakeR2OutboxStore(tasks ...*db.R2DeleteTask) *fakeR2OutboxStore {
|
||||
return &fakeR2OutboxStore{
|
||||
tasks: tasks,
|
||||
references: make(map[string]bool),
|
||||
referenceErrs: make(map[string]error),
|
||||
deleted: make(map[uint]bool),
|
||||
failures: make(map[uint]fakeR2OutboxFailure),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *fakeR2OutboxStore) ListDue(ctx context.Context, limit int) ([]*db.R2DeleteTask, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tasks := s.tasks
|
||||
if limit >= 0 && len(tasks) > limit {
|
||||
tasks = tasks[:limit]
|
||||
}
|
||||
return append([]*db.R2DeleteTask(nil), tasks...), nil
|
||||
}
|
||||
|
||||
func (s *fakeR2OutboxStore) Delete(id uint) error {
|
||||
if s.deleteErr != nil {
|
||||
return s.deleteErr
|
||||
}
|
||||
s.deleted[id] = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *fakeR2OutboxStore) MarkFailed(id uint, nextRetry time.Time, lastError string) error {
|
||||
if s.markFailedErr != nil {
|
||||
return s.markFailedErr
|
||||
}
|
||||
s.failures[id] = fakeR2OutboxFailure{nextRetry: nextRetry, lastError: lastError}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *fakeR2OutboxStore) HasReferences(ctx context.Context, _, key string) (bool, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
s.referenceCalls = append(s.referenceCalls, key)
|
||||
return s.references[key], s.referenceErrs[key]
|
||||
}
|
||||
|
||||
type fakeR2DeleteCall struct {
|
||||
bucket string
|
||||
key string
|
||||
}
|
||||
|
||||
type fakeR2ObjectDeleter struct {
|
||||
calls []fakeR2DeleteCall
|
||||
errs map[string]error
|
||||
onDelete func(context.Context, string, string) error
|
||||
}
|
||||
|
||||
func (d *fakeR2ObjectDeleter) DeleteObject(ctx context.Context, bucket, key string) error {
|
||||
d.calls = append(d.calls, fakeR2DeleteCall{bucket: bucket, key: key})
|
||||
if d.onDelete != nil {
|
||||
return d.onDelete(ctx, bucket, key)
|
||||
}
|
||||
return d.errs[key]
|
||||
}
|
||||
|
||||
type fakeR2HTTPStatusError int
|
||||
|
||||
func (e fakeR2HTTPStatusError) Error() string {
|
||||
return fmt.Sprintf("R2 returned HTTP %d", int(e))
|
||||
}
|
||||
|
||||
func (e fakeR2HTTPStatusError) HTTPStatusCode() int {
|
||||
return int(e)
|
||||
}
|
||||
|
||||
func setR2OutboxTestConfig(t *testing.T) {
|
||||
t.Helper()
|
||||
old := conf.Upload
|
||||
conf.Upload.ImageEndpoint = "https://r2.example.test"
|
||||
conf.Upload.ImageAccessID = "test-access-id"
|
||||
conf.Upload.ImageAccessSecret = "test-secret"
|
||||
conf.Upload.ImageBucket = "configured-bucket"
|
||||
conf.Upload.ImageBucketCDNHost = "cdn.example.test"
|
||||
t.Cleanup(func() {
|
||||
conf.Upload = old
|
||||
})
|
||||
}
|
||||
|
||||
func TestProcessR2OutboxSuccessAndNotFound(t *testing.T) {
|
||||
setR2OutboxTestConfig(t)
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
deleteErr error
|
||||
}{
|
||||
{name: "success"},
|
||||
{name: "not found is success", deleteErr: fakeR2HTTPStatusError(404)},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
first := &db.R2DeleteTask{ID: 1, Bucket: conf.Upload.ImageBucket, Key: "first.webp"}
|
||||
second := &db.R2DeleteTask{ID: 2, Bucket: conf.Upload.ImageBucket, Key: "second.webp"}
|
||||
store := newFakeR2OutboxStore(first, second)
|
||||
deleter := &fakeR2ObjectDeleter{errs: map[string]error{"second.webp": tt.deleteErr}}
|
||||
|
||||
result, err := processR2Outbox(context.Background(), store, deleter, 100)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, result.Listed)
|
||||
require.Equal(t, 2, result.Removed)
|
||||
require.Equal(t, 2, result.ObjectsDeleted)
|
||||
require.Zero(t, result.Failed)
|
||||
require.Zero(t, result.Remaining)
|
||||
require.Equal(t, []fakeR2DeleteCall{
|
||||
{bucket: conf.Upload.ImageBucket, key: "first.webp"},
|
||||
{bucket: conf.Upload.ImageBucket, key: "second.webp"},
|
||||
}, deleter.calls, "任务必须按 ListDue 顺序串行处理")
|
||||
require.True(t, store.deleted[first.ID])
|
||||
require.True(t, store.deleted[second.ID])
|
||||
require.Empty(t, store.failures)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessR2OutboxReferencedTaskDeletesNoObject(t *testing.T) {
|
||||
setR2OutboxTestConfig(t)
|
||||
task := &db.R2DeleteTask{ID: 11, Bucket: conf.Upload.ImageBucket, Key: "shared.webp"}
|
||||
store := newFakeR2OutboxStore(task)
|
||||
store.references[task.Key] = true
|
||||
deleter := &fakeR2ObjectDeleter{}
|
||||
|
||||
result, err := processR2Outbox(context.Background(), store, deleter, 100)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, deleter.calls, "仍有引用时绝不能调用 R2 删除")
|
||||
require.True(t, store.deleted[task.ID], "过期任务本身应删除，最后引用移除时会重新入队")
|
||||
require.Equal(t, 1, result.ReferenceOnly)
|
||||
require.Equal(t, 1, result.Removed)
|
||||
require.Zero(t, result.ObjectsDeleted)
|
||||
require.Zero(t, result.Remaining)
|
||||
}
|
||||
|
||||
func TestProcessR2OutboxReferenceErrorSchedulesRetry(t *testing.T) {
|
||||
setR2OutboxTestConfig(t)
|
||||
task := &db.R2DeleteTask{ID: 21, Bucket: conf.Upload.ImageBucket, Key: "unknown.webp"}
|
||||
store := newFakeR2OutboxStore(task)
|
||||
store.referenceErrs[task.Key] = errors.New("reference query failed")
|
||||
deleter := &fakeR2ObjectDeleter{}
|
||||
before := time.Now()
|
||||
|
||||
result, err := processR2Outbox(context.Background(), store, deleter, 100)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, deleter.calls)
|
||||
require.False(t, store.deleted[task.ID])
|
||||
require.Contains(t, store.failures, task.ID)
|
||||
failure := store.failures[task.ID]
|
||||
require.Equal(t, "reference query failed", failure.lastError)
|
||||
require.WithinDuration(t, before.Add(time.Minute), failure.nextRetry, 2*time.Second)
|
||||
require.Equal(t, 1, result.Failed)
|
||||
require.Equal(t, 1, result.Remaining)
|
||||
}
|
||||
|
||||
func TestProcessR2OutboxDeleteFailureUsesBackoffAndBoundsLastError(t *testing.T) {
|
||||
setR2OutboxTestConfig(t)
|
||||
task := &db.R2DeleteTask{ID: 31, Bucket: conf.Upload.ImageBucket, Key: "retry.webp", Attempts: 2}
|
||||
store := newFakeR2OutboxStore(task)
|
||||
longErr := fmt.Errorf("delete failed endpoint=%s credential=%s %s",
|
||||
conf.Upload.ImageEndpoint,
|
||||
conf.Upload.ImageAccessSecret,
|
||||
strings.Repeat("x", 1100),
|
||||
)
|
||||
deleter := &fakeR2ObjectDeleter{errs: map[string]error{task.Key: longErr}}
|
||||
before := time.Now()
|
||||
|
||||
result, err := processR2Outbox(context.Background(), store, deleter, 100)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, deleter.calls, 1)
|
||||
require.False(t, store.deleted[task.ID])
|
||||
require.Contains(t, store.failures, task.ID)
|
||||
failure := store.failures[task.ID]
|
||||
require.WithinDuration(t, before.Add(4*time.Minute), failure.nextRetry, 2*time.Second)
|
||||
require.LessOrEqual(t, len(failure.lastError), r2OutboxMaxLastErrorSize)
|
||||
require.NotContains(t, failure.lastError, conf.Upload.ImageEndpoint)
|
||||
require.NotContains(t, failure.lastError, conf.Upload.ImageAccessSecret)
|
||||
require.Equal(t, 1, result.Failed)
|
||||
require.Equal(t, 1, result.Remaining)
|
||||
}
|
||||
|
||||
func TestProcessR2OutboxBucketMismatchIsRetainedWithoutDeletion(t *testing.T) {
|
||||
setR2OutboxTestConfig(t)
|
||||
task := &db.R2DeleteTask{ID: 41, Bucket: "different-bucket", Key: "foreign.webp"}
|
||||
store := newFakeR2OutboxStore(task)
|
||||
deleter := &fakeR2ObjectDeleter{}
|
||||
|
||||
result, err := processR2Outbox(context.Background(), store, deleter, 100)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, store.referenceCalls, "bucket 不匹配时不应查询当前 bucket 的引用")
|
||||
require.Empty(t, deleter.calls)
|
||||
require.Empty(t, store.failures)
|
||||
require.False(t, store.deleted[task.ID])
|
||||
require.Equal(t, 1, result.Skipped)
|
||||
require.Zero(t, result.Failed)
|
||||
require.Equal(t, 1, result.Remaining)
|
||||
}
|
||||
|
||||
func TestProcessR2OutboxContextCancellationStopsWithoutMarkFailed(t *testing.T) {
|
||||
setR2OutboxTestConfig(t)
|
||||
first := &db.R2DeleteTask{ID: 51, Bucket: conf.Upload.ImageBucket, Key: "first.webp"}
|
||||
second := &db.R2DeleteTask{ID: 52, Bucket: conf.Upload.ImageBucket, Key: "second.webp"}
|
||||
store := newFakeR2OutboxStore(first, second)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
deleter := &fakeR2ObjectDeleter{
|
||||
onDelete: func(ctx context.Context, _, _ string) error {
|
||||
cancel()
|
||||
return ctx.Err()
|
||||
},
|
||||
}
|
||||
|
||||
result, err := processR2Outbox(ctx, store, deleter, 100)
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
require.Len(t, deleter.calls, 1, "取消后不得继续处理第二个任务")
|
||||
require.Empty(t, store.failures, "优雅关闭导致的取消不应增加任务失败次数")
|
||||
require.Empty(t, store.deleted)
|
||||
require.Zero(t, result.Failed)
|
||||
require.Equal(t, 2, result.Remaining)
|
||||
}
|
||||
|
||||
func TestR2OutboxBackoffCapsAtOneHour(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
attempts int
|
||||
expected time.Duration
|
||||
}{
|
||||
{attempts: -1, expected: time.Minute},
|
||||
{attempts: 0, expected: time.Minute},
|
||||
{attempts: 1, expected: 2 * time.Minute},
|
||||
{attempts: 2, expected: 4 * time.Minute},
|
||||
{attempts: 5, expected: 32 * time.Minute},
|
||||
{attempts: 6, expected: time.Hour},
|
||||
{attempts: 100, expected: time.Hour},
|
||||
} {
|
||||
require.Equal(t, tt.expected, r2OutboxBackoff(tt.attempts))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user