可靠性: 完善 R2 备份恢复与对象生命周期

2026-11-05 08:41:55 +08:00
parent 3e7719f200
commit 937e688cfa
32 changed files with 4151 additions and 231 deletions
@@ -0,0 +1,141 @@
+package db
+
+import (
+	"context"
+	"crypto/sha256"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/tamakyi/TamaBox/internal/conf"
+)
+
+func setR2TestConfig(t *testing.T) {
+	t.Helper()
+	oldBucket := conf.Upload.ImageBucket
+	oldCDNHost := conf.Upload.ImageBucketCDNHost
+	conf.Upload.ImageBucket = "test-r2-bucket"
+	conf.Upload.ImageBucketCDNHost = "cdn.example.test/assets"
+	oldDefaults := []string{conf.Upload.DefaultAvatarURL, conf.Upload.DefaultBackground, conf.Upload.DefaultBackgroundImageURL}
+	conf.Upload.DefaultAvatarURL = "https://cdn.example.test/assets/default-avatar.webp"
+	conf.Upload.DefaultBackground = "https://cdn.example.test/assets/default-background.webp"
+	conf.Upload.DefaultBackgroundImageURL = "https://cdn.example.test/assets/default-qr.webp"
+	t.Cleanup(func() {
+		conf.Upload.DefaultAvatarURL, conf.Upload.DefaultBackground, conf.Upload.DefaultBackgroundImageURL = oldDefaults[0], oldDefaults[1], oldDefaults[2]
+		conf.Upload.ImageBucket = oldBucket
+		conf.Upload.ImageBucketCDNHost = oldCDNHost
+	})
+}
+
+func TestR2DeleteTaskStore(t *testing.T) {
+	setR2TestConfig(t)
+	conn, _ := newTestDB(t)
+	ctx := context.Background()
+	store := NewR2DeleteTaskStore(conn)
+
+	require.NoError(t, store.Enqueue(ctx, "", []string{"ignored.webp"}))
+	require.NoError(t, store.Enqueue(ctx, conf.Upload.ImageBucket, nil))
+	require.NoError(t, store.Enqueue(ctx, conf.Upload.ImageBucket, []string{
+		"2026/11/shared.webp", "2026/11/shared.webp", "2026/11/other.webp", "",
+	}))
+
+	tasks, err := store.ListDue(ctx, 10)
+	require.NoError(t, err)
+	require.Len(t, tasks, 2)
+	byKey := make(map[string]*R2DeleteTask, len(tasks))
+	for _, task := range tasks {
+		byKey[task.Key] = task
+		require.Equal(t, conf.Upload.ImageBucket, task.Bucket)
+		require.Len(t, task.KeyHash, 64)
+	}
+	shared := byKey["2026/11/shared.webp"]
+	other := byKey["2026/11/other.webp"]
+	require.NotNil(t, shared)
+	require.NotNil(t, other)
+	require.Equal(t, fmt.Sprintf("%x", sha256.Sum256([]byte(shared.Key))), shared.KeyHash)
+
+	future := time.Now().Add(time.Hour).Truncate(time.Microsecond)
+	require.NoError(t, store.MarkFailed(shared.ID, future, "temporary failure"))
+	due, err := store.ListDue(ctx, 10)
+	require.NoError(t, err)
+	require.Len(t, due, 1)
+	require.Equal(t, other.ID, due[0].ID)
+
+	require.NoError(t, store.Enqueue(ctx, conf.Upload.ImageBucket, []string{shared.Key, shared.Key}))
+	var requeued R2DeleteTask
+	require.NoError(t, conn.WithContext(ctx).First(&requeued, shared.ID).Error)
+	require.Equal(t, shared.ID, requeued.ID)
+	require.Zero(t, requeued.Attempts)
+	require.Empty(t, requeued.LastError)
+	require.WithinDuration(t, time.Now(), requeued.NextRetryAt, 5*time.Second)
+
+	// 同一个 key 可在不同 bucket 各有一条任务，但同一 bucket 内仍保持唯一。
+	require.NoError(t, store.Enqueue(ctx, "another-bucket", []string{shared.Key}))
+	var count int64
+	require.NoError(t, conn.WithContext(ctx).Model(&R2DeleteTask{}).Count(&count).Error)
+	require.Equal(t, int64(3), count)
+
+	require.NoError(t, store.Delete(other.ID))
+	require.NoError(t, conn.WithContext(ctx).Model(&R2DeleteTask{}).Count(&count).Error)
+	require.Equal(t, int64(2), count)
+	empty, err := store.ListDue(ctx, 0)
+	require.NoError(t, err)
+	require.NotNil(t, empty)
+	require.Empty(t, empty)
+}
+
+func TestR2DeleteTaskHasReferences(t *testing.T) {
+	setR2TestConfig(t)
+	conn, _ := newTestDB(t)
+	ctx := context.Background()
+	store := NewR2DeleteTaskStore(conn)
+	bucket := conf.Upload.ImageBucket
+
+	upload := &UploadImage{Key: "upload.webp"}
+	require.NoError(t, conn.WithContext(ctx).Create(upload).Error)
+	require.NoError(t, conn.WithContext(ctx).Delete(upload).Error)
+	referenced, err := store.HasReferences(ctx, bucket, upload.Key)
+	require.NoError(t, err)
+	require.True(t, referenced, "软删除的 upload_images 仍必须通过 Unscoped 阻止对象删除")
+
+	message := &MessageImage{MessageID: 1, ObjectKey: "message.webp", URL: "https://cdn.example.test/assets/message.webp"}
+	require.NoError(t, conn.WithContext(ctx).Create(message).Error)
+	require.NoError(t, conn.WithContext(ctx).Delete(message).Error)
+	referenced, err = store.HasReferences(ctx, bucket, message.ObjectKey)
+	require.NoError(t, err)
+	require.True(t, referenced, "软删除的 message_images 仍必须通过 Unscoped 阻止对象删除")
+
+	legacyMessage := &MessageImage{MessageID: 2, URL: "https://cdn.example.test/assets/legacy-message.webp"}
+	legacyUpload := &UploadImage{Key: "https://cdn.example.test/assets/legacy-upload.webp"}
+	external := &MessageImage{MessageID: 3, URL: "https://external.example.net/external.webp"}
+	require.NoError(t, conn.WithContext(ctx).Create(legacyMessage).Error)
+	require.NoError(t, conn.WithContext(ctx).Create(legacyUpload).Error)
+	require.NoError(t, conn.WithContext(ctx).Create(external).Error)
+
+	for _, key := range []string{"legacy-message.webp", "legacy-upload.webp"} {
+		referenced, err = store.HasReferences(ctx, bucket, key)
+		require.NoError(t, err)
+		require.True(t, referenced, key)
+	}
+	referenced, err = store.HasReferences(ctx, bucket, "external.webp")
+	require.NoError(t, err)
+	require.False(t, referenced)
+	referenced, err = store.HasReferences(ctx, bucket, "missing.webp")
+	require.NoError(t, err)
+	require.False(t, referenced)
+
+	profileUser := &User{Avatar: "https://cdn.example.test/assets/profile-avatar.webp", Background: "profile-background.webp", BackgroundImage: "https://external.example.net/external-profile.webp"}
+	require.NoError(t, conn.WithContext(ctx).Create(profileUser).Error)
+	for _, key := range []string{"profile-avatar.webp", "profile-background.webp", "default-avatar.webp", "default-background.webp", "default-qr.webp"} {
+		referenced, err = store.HasReferences(ctx, bucket, key)
+		require.NoError(t, err)
+		require.True(t, referenced, key)
+	}
+
+	// 第二个引用查询失败时必须显式返回错误，不能误报为无引用。
+	require.NoError(t, conn.Migrator().DropTable(&MessageImage{}))
+	_, err = store.HasReferences(ctx, bucket, "missing-again.webp")
+	require.Error(t, err)
+}