271 lines
8.7 KiB
Go
271 lines
8.7 KiB
Go
package cmd
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/tamakyi/TamaBox/internal/conf"
|
|
"github.com/tamakyi/TamaBox/internal/db"
|
|
)
|
|
|
|
type fakeR2OutboxFailure struct {
|
|
nextRetry time.Time
|
|
lastError string
|
|
}
|
|
|
|
type fakeR2OutboxStore struct {
|
|
tasks []*db.R2DeleteTask
|
|
references map[string]bool
|
|
referenceErrs map[string]error
|
|
referenceCalls []string
|
|
deleted map[uint]bool
|
|
deleteErr error
|
|
failures map[uint]fakeR2OutboxFailure
|
|
markFailedErr error
|
|
}
|
|
|
|
func newFakeR2OutboxStore(tasks ...*db.R2DeleteTask) *fakeR2OutboxStore {
|
|
return &fakeR2OutboxStore{
|
|
tasks: tasks,
|
|
references: make(map[string]bool),
|
|
referenceErrs: make(map[string]error),
|
|
deleted: make(map[uint]bool),
|
|
failures: make(map[uint]fakeR2OutboxFailure),
|
|
}
|
|
}
|
|
|
|
func (s *fakeR2OutboxStore) ListDue(ctx context.Context, limit int) ([]*db.R2DeleteTask, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
tasks := s.tasks
|
|
if limit >= 0 && len(tasks) > limit {
|
|
tasks = tasks[:limit]
|
|
}
|
|
return append([]*db.R2DeleteTask(nil), tasks...), nil
|
|
}
|
|
|
|
func (s *fakeR2OutboxStore) Delete(id uint) error {
|
|
if s.deleteErr != nil {
|
|
return s.deleteErr
|
|
}
|
|
s.deleted[id] = true
|
|
return nil
|
|
}
|
|
|
|
func (s *fakeR2OutboxStore) MarkFailed(id uint, nextRetry time.Time, lastError string) error {
|
|
if s.markFailedErr != nil {
|
|
return s.markFailedErr
|
|
}
|
|
s.failures[id] = fakeR2OutboxFailure{nextRetry: nextRetry, lastError: lastError}
|
|
return nil
|
|
}
|
|
|
|
func (s *fakeR2OutboxStore) HasReferences(ctx context.Context, _, key string) (bool, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return false, err
|
|
}
|
|
s.referenceCalls = append(s.referenceCalls, key)
|
|
return s.references[key], s.referenceErrs[key]
|
|
}
|
|
|
|
type fakeR2DeleteCall struct {
|
|
bucket string
|
|
key string
|
|
}
|
|
|
|
type fakeR2ObjectDeleter struct {
|
|
calls []fakeR2DeleteCall
|
|
errs map[string]error
|
|
onDelete func(context.Context, string, string) error
|
|
}
|
|
|
|
func (d *fakeR2ObjectDeleter) DeleteObject(ctx context.Context, bucket, key string) error {
|
|
d.calls = append(d.calls, fakeR2DeleteCall{bucket: bucket, key: key})
|
|
if d.onDelete != nil {
|
|
return d.onDelete(ctx, bucket, key)
|
|
}
|
|
return d.errs[key]
|
|
}
|
|
|
|
type fakeR2HTTPStatusError int
|
|
|
|
func (e fakeR2HTTPStatusError) Error() string {
|
|
return fmt.Sprintf("R2 returned HTTP %d", int(e))
|
|
}
|
|
|
|
func (e fakeR2HTTPStatusError) HTTPStatusCode() int {
|
|
return int(e)
|
|
}
|
|
|
|
func setR2OutboxTestConfig(t *testing.T) {
|
|
t.Helper()
|
|
old := conf.Upload
|
|
conf.Upload.ImageEndpoint = "https://r2.example.test"
|
|
conf.Upload.ImageAccessID = "test-access-id"
|
|
conf.Upload.ImageAccessSecret = "test-secret"
|
|
conf.Upload.ImageBucket = "configured-bucket"
|
|
conf.Upload.ImageBucketCDNHost = "cdn.example.test"
|
|
t.Cleanup(func() {
|
|
conf.Upload = old
|
|
})
|
|
}
|
|
|
|
func TestProcessR2OutboxSuccessAndNotFound(t *testing.T) {
|
|
setR2OutboxTestConfig(t)
|
|
for _, tt := range []struct {
|
|
name string
|
|
deleteErr error
|
|
}{
|
|
{name: "success"},
|
|
{name: "not found is success", deleteErr: fakeR2HTTPStatusError(404)},
|
|
} {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
first := &db.R2DeleteTask{ID: 1, Bucket: conf.Upload.ImageBucket, Key: "first.webp"}
|
|
second := &db.R2DeleteTask{ID: 2, Bucket: conf.Upload.ImageBucket, Key: "second.webp"}
|
|
store := newFakeR2OutboxStore(first, second)
|
|
deleter := &fakeR2ObjectDeleter{errs: map[string]error{"second.webp": tt.deleteErr}}
|
|
|
|
result, err := processR2Outbox(context.Background(), store, deleter, 100)
|
|
require.NoError(t, err)
|
|
require.Equal(t, 2, result.Listed)
|
|
require.Equal(t, 2, result.Removed)
|
|
require.Equal(t, 2, result.ObjectsDeleted)
|
|
require.Zero(t, result.Failed)
|
|
require.Zero(t, result.Remaining)
|
|
require.Equal(t, []fakeR2DeleteCall{
|
|
{bucket: conf.Upload.ImageBucket, key: "first.webp"},
|
|
{bucket: conf.Upload.ImageBucket, key: "second.webp"},
|
|
}, deleter.calls, "任务必须按 ListDue 顺序串行处理")
|
|
require.True(t, store.deleted[first.ID])
|
|
require.True(t, store.deleted[second.ID])
|
|
require.Empty(t, store.failures)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestProcessR2OutboxReferencedTaskDeletesNoObject(t *testing.T) {
|
|
setR2OutboxTestConfig(t)
|
|
task := &db.R2DeleteTask{ID: 11, Bucket: conf.Upload.ImageBucket, Key: "shared.webp"}
|
|
store := newFakeR2OutboxStore(task)
|
|
store.references[task.Key] = true
|
|
deleter := &fakeR2ObjectDeleter{}
|
|
|
|
result, err := processR2Outbox(context.Background(), store, deleter, 100)
|
|
require.NoError(t, err)
|
|
require.Empty(t, deleter.calls, "仍有引用时绝不能调用 R2 删除")
|
|
require.True(t, store.deleted[task.ID], "过期任务本身应删除，最后引用移除时会重新入队")
|
|
require.Equal(t, 1, result.ReferenceOnly)
|
|
require.Equal(t, 1, result.Removed)
|
|
require.Zero(t, result.ObjectsDeleted)
|
|
require.Zero(t, result.Remaining)
|
|
}
|
|
|
|
func TestProcessR2OutboxReferenceErrorSchedulesRetry(t *testing.T) {
|
|
setR2OutboxTestConfig(t)
|
|
task := &db.R2DeleteTask{ID: 21, Bucket: conf.Upload.ImageBucket, Key: "unknown.webp"}
|
|
store := newFakeR2OutboxStore(task)
|
|
store.referenceErrs[task.Key] = errors.New("reference query failed")
|
|
deleter := &fakeR2ObjectDeleter{}
|
|
before := time.Now()
|
|
|
|
result, err := processR2Outbox(context.Background(), store, deleter, 100)
|
|
require.NoError(t, err)
|
|
require.Empty(t, deleter.calls)
|
|
require.False(t, store.deleted[task.ID])
|
|
require.Contains(t, store.failures, task.ID)
|
|
failure := store.failures[task.ID]
|
|
require.Equal(t, "reference query failed", failure.lastError)
|
|
require.WithinDuration(t, before.Add(time.Minute), failure.nextRetry, 2*time.Second)
|
|
require.Equal(t, 1, result.Failed)
|
|
require.Equal(t, 1, result.Remaining)
|
|
}
|
|
|
|
func TestProcessR2OutboxDeleteFailureUsesBackoffAndBoundsLastError(t *testing.T) {
|
|
setR2OutboxTestConfig(t)
|
|
task := &db.R2DeleteTask{ID: 31, Bucket: conf.Upload.ImageBucket, Key: "retry.webp", Attempts: 2}
|
|
store := newFakeR2OutboxStore(task)
|
|
longErr := fmt.Errorf("delete failed endpoint=%s credential=%s %s",
|
|
conf.Upload.ImageEndpoint,
|
|
conf.Upload.ImageAccessSecret,
|
|
strings.Repeat("x", 1100),
|
|
)
|
|
deleter := &fakeR2ObjectDeleter{errs: map[string]error{task.Key: longErr}}
|
|
before := time.Now()
|
|
|
|
result, err := processR2Outbox(context.Background(), store, deleter, 100)
|
|
require.NoError(t, err)
|
|
require.Len(t, deleter.calls, 1)
|
|
require.False(t, store.deleted[task.ID])
|
|
require.Contains(t, store.failures, task.ID)
|
|
failure := store.failures[task.ID]
|
|
require.WithinDuration(t, before.Add(4*time.Minute), failure.nextRetry, 2*time.Second)
|
|
require.LessOrEqual(t, len(failure.lastError), r2OutboxMaxLastErrorSize)
|
|
require.NotContains(t, failure.lastError, conf.Upload.ImageEndpoint)
|
|
require.NotContains(t, failure.lastError, conf.Upload.ImageAccessSecret)
|
|
require.Equal(t, 1, result.Failed)
|
|
require.Equal(t, 1, result.Remaining)
|
|
}
|
|
|
|
func TestProcessR2OutboxBucketMismatchIsRetainedWithoutDeletion(t *testing.T) {
|
|
setR2OutboxTestConfig(t)
|
|
task := &db.R2DeleteTask{ID: 41, Bucket: "different-bucket", Key: "foreign.webp"}
|
|
store := newFakeR2OutboxStore(task)
|
|
deleter := &fakeR2ObjectDeleter{}
|
|
|
|
result, err := processR2Outbox(context.Background(), store, deleter, 100)
|
|
require.NoError(t, err)
|
|
require.Empty(t, store.referenceCalls, "bucket 不匹配时不应查询当前 bucket 的引用")
|
|
require.Empty(t, deleter.calls)
|
|
require.Empty(t, store.failures)
|
|
require.False(t, store.deleted[task.ID])
|
|
require.Equal(t, 1, result.Skipped)
|
|
require.Zero(t, result.Failed)
|
|
require.Equal(t, 1, result.Remaining)
|
|
}
|
|
|
|
func TestProcessR2OutboxContextCancellationStopsWithoutMarkFailed(t *testing.T) {
|
|
setR2OutboxTestConfig(t)
|
|
first := &db.R2DeleteTask{ID: 51, Bucket: conf.Upload.ImageBucket, Key: "first.webp"}
|
|
second := &db.R2DeleteTask{ID: 52, Bucket: conf.Upload.ImageBucket, Key: "second.webp"}
|
|
store := newFakeR2OutboxStore(first, second)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
deleter := &fakeR2ObjectDeleter{
|
|
onDelete: func(ctx context.Context, _, _ string) error {
|
|
cancel()
|
|
return ctx.Err()
|
|
},
|
|
}
|
|
|
|
result, err := processR2Outbox(ctx, store, deleter, 100)
|
|
require.ErrorIs(t, err, context.Canceled)
|
|
require.Len(t, deleter.calls, 1, "取消后不得继续处理第二个任务")
|
|
require.Empty(t, store.failures, "优雅关闭导致的取消不应增加任务失败次数")
|
|
require.Empty(t, store.deleted)
|
|
require.Zero(t, result.Failed)
|
|
require.Equal(t, 2, result.Remaining)
|
|
}
|
|
|
|
func TestR2OutboxBackoffCapsAtOneHour(t *testing.T) {
|
|
for _, tt := range []struct {
|
|
attempts int
|
|
expected time.Duration
|
|
}{
|
|
{attempts: -1, expected: time.Minute},
|
|
{attempts: 0, expected: time.Minute},
|
|
{attempts: 1, expected: 2 * time.Minute},
|
|
{attempts: 2, expected: 4 * time.Minute},
|
|
{attempts: 5, expected: 32 * time.Minute},
|
|
{attempts: 6, expected: time.Hour},
|
|
{attempts: 100, expected: time.Hour},
|
|
} {
|
|
require.Equal(t, tt.expected, r2OutboxBackoff(tt.attempts))
|
|
}
|
|
}
|