可靠性: 完善 R2 备份恢复与对象生命周期

This commit is contained in:
tamakyi
2026-11-05 08:41:55 +08:00
parent 3e7719f200
commit 937e688cfa
32 changed files with 4151 additions and 231 deletions
+2 -1
View File
@@ -36,7 +36,7 @@ func Ping(ctx context.Context) error {
var AllTables = []interface{}{
&User{}, &Question{}, &FollowUp{}, &CensorLog{}, &UploadImage{}, &UploadImageQuestion{},
&Conversation{}, &Message{}, &MessageImage{}, &UserBlock{},
&Conversation{}, &Message{}, &MessageImage{}, &UserBlock{}, &R2DeleteTask{},
// 匿名投票
&Vote{}, &VoteOption{}, &VoteFormField{}, &VoteSubmission{},
// 功能开关(提问 / 私聊 / 投票 的开关、定时、定量收集)
@@ -132,6 +132,7 @@ func Init(typ, dsn string) (*gorm.DB, error) {
Conversations = NewConversationStore(db)
Messages = NewMessageStore(db)
MessageImages = NewMessageImageStore(db)
R2DeleteTasks = NewR2DeleteTaskStore(db)
Votes = NewVotesStore(db)
VoteOptions = NewVoteOptionsStore(db)
VoteFormFields = NewVoteFormFieldsStore(db)
+3
View File
@@ -26,6 +26,7 @@ type MessageImage struct {
gorm.Model
MessageID uint `gorm:"index" json:"message_id"`
URL string `gorm:"size:500" json:"url"`
ObjectKey string `gorm:"type:text" json:"-"`
FileName string `gorm:"size:200" json:"file_name"`
FileSize int64 `json:"file_size"`
}
@@ -33,6 +34,7 @@ type MessageImage struct {
type CreateMessageImageOptions struct {
MessageID uint
URL string
ObjectKey string
FileName string
FileSize int64
}
@@ -41,6 +43,7 @@ func (db *messageImages) Create(ctx context.Context, opts CreateMessageImageOpti
img := &MessageImage{
MessageID: opts.MessageID,
URL: opts.URL,
ObjectKey: opts.ObjectKey,
FileName: opts.FileName,
FileSize: opts.FileSize,
}
+39
View File
@@ -0,0 +1,39 @@
package db
import (
"context"
"encoding/json"
"testing"
"github.com/stretchr/testify/require"
)
func TestMessageImageCreatePersistsObjectKey(t *testing.T) {
conn, _ := newTestDB(t)
ctx := context.Background()
store := NewMessageImageStore(conn)
created, err := store.Create(ctx, CreateMessageImageOptions{
MessageID: 42,
URL: "https://cdn.example.test/assets/2026/11/private.webp",
ObjectKey: "2026/11/private.webp",
FileName: "private.webp",
FileSize: 1234,
})
require.NoError(t, err)
require.Equal(t, "2026/11/private.webp", created.ObjectKey)
var persisted MessageImage
require.NoError(t, conn.WithContext(ctx).First(&persisted, created.ID).Error)
require.Equal(t, created.ObjectKey, persisted.ObjectKey)
byMessage, err := store.GetByMessageIDs(ctx, []uint{42})
require.NoError(t, err)
require.Len(t, byMessage[42], 1)
require.Equal(t, created.ObjectKey, byMessage[42][0].ObjectKey)
encoded, err := json.Marshal(&persisted)
require.NoError(t, err)
require.NotContains(t, string(encoded), "ObjectKey")
require.NotContains(t, string(encoded), "object_key")
}
+277
View File
@@ -0,0 +1,277 @@
package db
import (
"context"
"crypto/sha256"
"encoding/hex"
"strings"
"time"
"github.com/pkg/errors"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"github.com/tamakyi/TamaBox/internal/conf"
"github.com/tamakyi/TamaBox/internal/dbutil"
"github.com/tamakyi/TamaBox/internal/storage"
)
var R2DeleteTasks R2DeleteTaskStore
var _ R2DeleteTaskStore = (*r2DeleteTasks)(nil)
// R2DeleteTask 是 R2 对象删除的持久化 outbox 任务。
// 数据库事务只负责登记任务,实际对象删除由 worker 在事务外执行。
type R2DeleteTask struct {
ID uint `gorm:"primaryKey"`
Bucket string `gorm:"size:255;not null;uniqueIndex:uk_r2_delete_tasks_bucket_key_hash,priority:1"`
Key string `gorm:"type:text;not null"`
KeyHash string `gorm:"size:64;not null;uniqueIndex:uk_r2_delete_tasks_bucket_key_hash,priority:2"`
Attempts int `gorm:"not null;default:0"`
NextRetryAt time.Time `gorm:"not null;index:idx_r2_delete_tasks_next_retry_at"`
LastError string `gorm:"type:text"`
CreatedAt time.Time
UpdatedAt time.Time
}
type R2DeleteTaskStore interface {
Enqueue(ctx context.Context, bucket string, keys []string) error
EnqueueValues(ctx context.Context, bucket string, values []string) error
ListDue(ctx context.Context, limit int) ([]*R2DeleteTask, error)
Delete(id uint) error
MarkFailed(id uint, nextRetry time.Time, lastError string) error
// HasReferences 仅供 worker 对当前配置的 bucket 调用。
// 它只查询数据库,不执行网络请求;查询或旧 URL 解析失败时必须返回错误。
HasReferences(ctx context.Context, bucket, key string) (bool, error)
}
func NewR2DeleteTaskStore(db *gorm.DB) R2DeleteTaskStore {
return &r2DeleteTasks{DB: db}
}
type r2DeleteTasks struct {
*gorm.DB
}
// EnqueueR2DeleteTasks 在调用方事务中登记对象删除任务。
// 相同 (bucket,key) 重复入队会重置重试状态,使新事务对应的删除请求立即可见。
func EnqueueR2DeleteTasks(tx *gorm.DB, bucket string, keys []string) error {
if bucket == "" || len(keys) == 0 {
return nil
}
now := time.Now()
tasks := make([]*R2DeleteTask, 0, len(keys))
seen := make(map[string]struct{}, len(keys))
for _, key := range keys {
if key == "" {
continue
}
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
sum := sha256.Sum256([]byte(key))
tasks = append(tasks, &R2DeleteTask{
Bucket: bucket,
Key: key,
KeyHash: hex.EncodeToString(sum[:]),
NextRetryAt: now,
})
}
if len(tasks) == 0 {
return nil
}
if err := tx.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "bucket"}, {Name: "key_hash"}},
DoUpdates: clause.Assignments(map[string]interface{}{
"attempts": 0,
"next_retry_at": now,
"last_error": "",
"updated_at": now,
}),
}).Create(&tasks).Error; err != nil {
return errors.Wrap(err, "enqueue R2 delete tasks")
}
return nil
}
func (db *r2DeleteTasks) Enqueue(ctx context.Context, bucket string, keys []string) error {
return EnqueueR2DeleteTasks(db.WithContext(ctx), bucket, keys)
}
func (db *r2DeleteTasks) EnqueueValues(ctx context.Context, bucket string, values []string) error {
return enqueueR2DeleteTasksForValues(db.WithContext(ctx), bucket, values)
}
func (db *r2DeleteTasks) ListDue(ctx context.Context, limit int) ([]*R2DeleteTask, error) {
tasks := make([]*R2DeleteTask, 0)
if limit <= 0 {
return tasks, nil
}
if err := db.WithContext(ctx).
Where("next_retry_at <= ?", time.Now()).
Order("next_retry_at ASC, id ASC").
Limit(limit).
Find(&tasks).Error; err != nil {
return nil, errors.Wrap(err, "list due R2 delete tasks")
}
return tasks, nil
}
func (db *r2DeleteTasks) Delete(id uint) error {
if err := db.DB.Delete(&R2DeleteTask{}, id).Error; err != nil {
return errors.Wrap(err, "delete R2 delete task")
}
return nil
}
func (db *r2DeleteTasks) MarkFailed(id uint, nextRetry time.Time, lastError string) error {
if err := db.Model(&R2DeleteTask{}).Where("id = ?", id).Updates(map[string]interface{}{
"attempts": gorm.Expr("attempts + 1"),
"next_retry_at": nextRetry,
"last_error": lastError,
"updated_at": time.Now(),
}).Error; err != nil {
return errors.Wrap(err, "mark R2 delete task failed")
}
return nil
}
func (db *r2DeleteTasks) HasReferences(ctx context.Context, bucket, key string) (bool, error) {
if bucket == "" || key == "" {
return false, nil
}
defaultKey, owned, err := r2ObjectKeyFromValue(key)
if err != nil {
return false, err
}
if owned && isDefaultProfileObject(defaultKey) {
return true, nil
}
quotedKey := dbutil.QuoteIdentifier(db.Dialector.Name(), "key")
var count int64
if err := db.WithContext(ctx).Unscoped().Model(&UploadImage{}).
Where(quotedKey+" = ?", key).Count(&count).Error; err != nil {
return false, errors.Wrap(err, "check upload image references")
}
if count > 0 {
return true, nil
}
count = 0
if err := db.WithContext(ctx).Unscoped().Model(&MessageImage{}).
Where("object_key = ? OR url = ?", key, key).Count(&count).Error; err != nil {
return false, errors.Wrap(err, "check message image references")
}
if count > 0 {
return true, nil
}
var uploadURLs []string
if err := db.WithContext(ctx).Unscoped().Model(&UploadImage{}).
Where(quotedKey+" LIKE ? OR "+quotedKey+" LIKE ?", "%://%", "//%").
Pluck(quotedKey, &uploadURLs).Error; err != nil {
return false, errors.Wrap(err, "list legacy upload image URLs")
}
referenced, err := hasLegacyCDNReference(uploadURLs, key)
if err != nil || referenced {
return referenced, err
}
var messageURLs []string
if err := db.WithContext(ctx).Unscoped().Model(&MessageImage{}).
Where("url LIKE ? OR url LIKE ?", "%://%", "//%").
Pluck("url", &messageURLs).Error; err != nil {
return false, errors.Wrap(err, "list legacy message image URLs")
}
referenced, err = hasLegacyCDNReference(messageURLs, key)
if err != nil || referenced {
return referenced, err
}
var profileUsers []*User
if err := db.WithContext(ctx).Unscoped().Model(&User{}).
Select("avatar", "background", "background_image").
Where("avatar = ? OR background = ? OR background_image = ? OR avatar LIKE ? OR background LIKE ? OR background_image LIKE ?", key, key, key, "%://%", "%://%", "%://%").
Find(&profileUsers).Error; err != nil {
return false, errors.Wrap(err, "list profile image references")
}
for _, user := range profileUsers {
referenced, err = hasLegacyCDNReference([]string{user.Avatar, user.Background, user.BackgroundImage}, key)
if err != nil || referenced {
return referenced, err
}
}
return false, nil
}
func isDefaultProfileObject(key string) bool {
for _, value := range []string{conf.Upload.DefaultAvatarURL, conf.Upload.DefaultBackground, conf.Upload.DefaultBackgroundImageURL} {
defaultKey, owned, err := r2ObjectKeyFromValue(value)
if err == nil && owned && defaultKey == key {
return true
}
}
return false
}
func hasLegacyCDNReference(values []string, targetKey string) (bool, error) {
for _, value := range values {
key, owned, err := r2ObjectKeyFromValue(value)
if err != nil {
return false, err
}
if owned && key == targetKey {
return true, nil
}
}
return false, nil
}
// r2ObjectKeyFromValue 接受当前对象 key 或旧 CDN URL。
// 外部 URL 不属于当前 CDN,安全返回 owned=false;畸形 URL 返回错误以避免误判无引用。
func r2ObjectKeyFromValue(value string) (key string, owned bool, err error) {
if value == "" {
return "", false, nil
}
if !strings.Contains(value, "://") && !strings.HasPrefix(value, "//") {
return value, true, nil
}
key, owned, err = storage.ObjectKeyFromCDNURL(value, conf.Upload.ImageBucketCDNHost)
if err != nil {
return "", false, errors.Wrap(err, "parse legacy CDN URL")
}
if !owned || key == "" {
return "", false, nil
}
return key, true, nil
}
func enqueueR2DeleteTasksForValues(tx *gorm.DB, bucket string, values []string, excludedKeys ...string) error {
if bucket == "" || len(values) == 0 {
return nil
}
excluded := make(map[string]struct{}, len(excludedKeys))
for _, key := range excludedKeys {
excluded[key] = struct{}{}
}
keys := make([]string, 0, len(values))
for _, value := range values {
key, owned, err := r2ObjectKeyFromValue(value)
if err != nil {
return err
}
if !owned || isDefaultProfileObject(key) {
continue
}
if _, skip := excluded[key]; skip {
continue
}
keys = append(keys, key)
}
return EnqueueR2DeleteTasks(tx, bucket, keys)
}
+141
View File
@@ -0,0 +1,141 @@
package db
import (
"context"
"crypto/sha256"
"fmt"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/tamakyi/TamaBox/internal/conf"
)
func setR2TestConfig(t *testing.T) {
t.Helper()
oldBucket := conf.Upload.ImageBucket
oldCDNHost := conf.Upload.ImageBucketCDNHost
conf.Upload.ImageBucket = "test-r2-bucket"
conf.Upload.ImageBucketCDNHost = "cdn.example.test/assets"
oldDefaults := []string{conf.Upload.DefaultAvatarURL, conf.Upload.DefaultBackground, conf.Upload.DefaultBackgroundImageURL}
conf.Upload.DefaultAvatarURL = "https://cdn.example.test/assets/default-avatar.webp"
conf.Upload.DefaultBackground = "https://cdn.example.test/assets/default-background.webp"
conf.Upload.DefaultBackgroundImageURL = "https://cdn.example.test/assets/default-qr.webp"
t.Cleanup(func() {
conf.Upload.DefaultAvatarURL, conf.Upload.DefaultBackground, conf.Upload.DefaultBackgroundImageURL = oldDefaults[0], oldDefaults[1], oldDefaults[2]
conf.Upload.ImageBucket = oldBucket
conf.Upload.ImageBucketCDNHost = oldCDNHost
})
}
func TestR2DeleteTaskStore(t *testing.T) {
setR2TestConfig(t)
conn, _ := newTestDB(t)
ctx := context.Background()
store := NewR2DeleteTaskStore(conn)
require.NoError(t, store.Enqueue(ctx, "", []string{"ignored.webp"}))
require.NoError(t, store.Enqueue(ctx, conf.Upload.ImageBucket, nil))
require.NoError(t, store.Enqueue(ctx, conf.Upload.ImageBucket, []string{
"2026/11/shared.webp", "2026/11/shared.webp", "2026/11/other.webp", "",
}))
tasks, err := store.ListDue(ctx, 10)
require.NoError(t, err)
require.Len(t, tasks, 2)
byKey := make(map[string]*R2DeleteTask, len(tasks))
for _, task := range tasks {
byKey[task.Key] = task
require.Equal(t, conf.Upload.ImageBucket, task.Bucket)
require.Len(t, task.KeyHash, 64)
}
shared := byKey["2026/11/shared.webp"]
other := byKey["2026/11/other.webp"]
require.NotNil(t, shared)
require.NotNil(t, other)
require.Equal(t, fmt.Sprintf("%x", sha256.Sum256([]byte(shared.Key))), shared.KeyHash)
future := time.Now().Add(time.Hour).Truncate(time.Microsecond)
require.NoError(t, store.MarkFailed(shared.ID, future, "temporary failure"))
due, err := store.ListDue(ctx, 10)
require.NoError(t, err)
require.Len(t, due, 1)
require.Equal(t, other.ID, due[0].ID)
require.NoError(t, store.Enqueue(ctx, conf.Upload.ImageBucket, []string{shared.Key, shared.Key}))
var requeued R2DeleteTask
require.NoError(t, conn.WithContext(ctx).First(&requeued, shared.ID).Error)
require.Equal(t, shared.ID, requeued.ID)
require.Zero(t, requeued.Attempts)
require.Empty(t, requeued.LastError)
require.WithinDuration(t, time.Now(), requeued.NextRetryAt, 5*time.Second)
// 同一个 key 可在不同 bucket 各有一条任务,但同一 bucket 内仍保持唯一。
require.NoError(t, store.Enqueue(ctx, "another-bucket", []string{shared.Key}))
var count int64
require.NoError(t, conn.WithContext(ctx).Model(&R2DeleteTask{}).Count(&count).Error)
require.Equal(t, int64(3), count)
require.NoError(t, store.Delete(other.ID))
require.NoError(t, conn.WithContext(ctx).Model(&R2DeleteTask{}).Count(&count).Error)
require.Equal(t, int64(2), count)
empty, err := store.ListDue(ctx, 0)
require.NoError(t, err)
require.NotNil(t, empty)
require.Empty(t, empty)
}
func TestR2DeleteTaskHasReferences(t *testing.T) {
setR2TestConfig(t)
conn, _ := newTestDB(t)
ctx := context.Background()
store := NewR2DeleteTaskStore(conn)
bucket := conf.Upload.ImageBucket
upload := &UploadImage{Key: "upload.webp"}
require.NoError(t, conn.WithContext(ctx).Create(upload).Error)
require.NoError(t, conn.WithContext(ctx).Delete(upload).Error)
referenced, err := store.HasReferences(ctx, bucket, upload.Key)
require.NoError(t, err)
require.True(t, referenced, "软删除的 upload_images 仍必须通过 Unscoped 阻止对象删除")
message := &MessageImage{MessageID: 1, ObjectKey: "message.webp", URL: "https://cdn.example.test/assets/message.webp"}
require.NoError(t, conn.WithContext(ctx).Create(message).Error)
require.NoError(t, conn.WithContext(ctx).Delete(message).Error)
referenced, err = store.HasReferences(ctx, bucket, message.ObjectKey)
require.NoError(t, err)
require.True(t, referenced, "软删除的 message_images 仍必须通过 Unscoped 阻止对象删除")
legacyMessage := &MessageImage{MessageID: 2, URL: "https://cdn.example.test/assets/legacy-message.webp"}
legacyUpload := &UploadImage{Key: "https://cdn.example.test/assets/legacy-upload.webp"}
external := &MessageImage{MessageID: 3, URL: "https://external.example.net/external.webp"}
require.NoError(t, conn.WithContext(ctx).Create(legacyMessage).Error)
require.NoError(t, conn.WithContext(ctx).Create(legacyUpload).Error)
require.NoError(t, conn.WithContext(ctx).Create(external).Error)
for _, key := range []string{"legacy-message.webp", "legacy-upload.webp"} {
referenced, err = store.HasReferences(ctx, bucket, key)
require.NoError(t, err)
require.True(t, referenced, key)
}
referenced, err = store.HasReferences(ctx, bucket, "external.webp")
require.NoError(t, err)
require.False(t, referenced)
referenced, err = store.HasReferences(ctx, bucket, "missing.webp")
require.NoError(t, err)
require.False(t, referenced)
profileUser := &User{Avatar: "https://cdn.example.test/assets/profile-avatar.webp", Background: "profile-background.webp", BackgroundImage: "https://external.example.net/external-profile.webp"}
require.NoError(t, conn.WithContext(ctx).Create(profileUser).Error)
for _, key := range []string{"profile-avatar.webp", "profile-background.webp", "default-avatar.webp", "default-background.webp", "default-qr.webp"} {
referenced, err = store.HasReferences(ctx, bucket, key)
require.NoError(t, err)
require.True(t, referenced, key)
}
// 第二个引用查询失败时必须显式返回错误,不能误报为无引用。
require.NoError(t, conn.Migrator().DropTable(&MessageImage{}))
_, err = store.HasReferences(ctx, bucket, "missing-again.webp")
require.Error(t, err)
}
+60 -16
View File
@@ -10,6 +10,8 @@ import (
"github.com/pkg/errors"
"github.com/samber/lo"
"gorm.io/gorm"
"github.com/tamakyi/TamaBox/internal/conf"
)
var UploadImages UploadImagesStore
@@ -23,12 +25,8 @@ type UploadImagesStore interface {
// 用于账号导出等场景,将 N 次单题查询合并为少量 IN 查询,避免 N+1。
GetByQuestionIDs(ctx context.Context, questionIDs []uint) (map[uint][]*UploadImage, error)
GetByTypeQuestionID(ctx context.Context, typ UploadImageQuestionType, questionID uint) ([]*UploadImage, error)
// DeleteByQuestionID 删除某条提问时级联清理其全部上传图片记录与关联行。
//
// 用 Unscoped 真删:图片元数据(name/md5/key)随提问一并真删,与 questions.DeleteByID
// 对主表的真删口径一致。软删会留下带 deleted_at 的图片行,且关联行的 question_id
// 指向已不存在的提问。注意这里只清数据库记录,对象存储里的实际文件不在本函数删除
// (图片可能被多条记录复用,避免误删共享资源)。
// DeleteByQuestionID 先删除当前题目的关联,再物理删除不再被任何题目引用的主体。
// 实际删除的主体的对象 key 与元数据删除在同一事务中登记 R2 outbox。
DeleteByQuestionID(ctx context.Context, questionID uint) error
}
@@ -87,9 +85,24 @@ func (db *uploadImages) Create(ctx context.Context, opts CreateUploadImageOption
}
if opts.IsDeletingPrevious {
if err := tx.WithContext(ctx).Where("type = ? AND question_id = ?", opts.Type, opts.QuestionID).Delete(&UploadImageQuestion{}).Error; err != nil {
var previousLinks []*UploadImageQuestion
if err := tx.WithContext(ctx).
Where("type = ? AND question_id = ?", opts.Type, opts.QuestionID).
Find(&previousLinks).Error; err != nil {
return errors.Wrap(err, "get previous image question links")
}
previousImageIDs := lo.Map(previousLinks, func(item *UploadImageQuestion, _ int) uint {
return item.UploadImageID
})
if err := tx.WithContext(ctx).
Where("type = ? AND question_id = ?", opts.Type, opts.QuestionID).
Delete(&UploadImageQuestion{}).Error; err != nil {
return errors.Wrap(err, "delete previous image question link")
}
// 新图已经创建但尚未关联;即使它与旧图使用相同 key,也绝不加入删除队列。
if err := deleteUnreferencedUploadImages(ctx, tx, previousImageIDs, opts.Key); err != nil {
return err
}
}
if err := tx.WithContext(ctx).Create(&UploadImageQuestion{
Type: opts.Type,
@@ -167,10 +180,44 @@ func (db *uploadImages) GetByTypeQuestionID(ctx context.Context, typ UploadImage
return db.getBy(ctx, "type = ? AND question_id = ?", typ, questionID)
}
// DeleteByQuestionID 删除某条提问时级联清理其全部上传图片记录(软删除,与提问保持一致)。
// 注意:question_id 列在关联表 upload_image_questions 上,不在 upload_images 表上,
// 需先查关联行拿到 imageID 列表,再软删图片主体并删除关联行(同一事务内完成)。
// 注:仅清理数据库记录,对象存储中的实际文件不在此处删除(避免误删复用资源)。
// unreferencedUploadImagePredicate 是所有 UploadImage 删除入口共用的共享保护谓词。
// 调用方必须先解除当前作用域的关联,再使用该谓词收集并删除真正的孤儿主体。
const unreferencedUploadImagePredicate = `NOT EXISTS (
SELECT 1 FROM upload_image_questions AS shared_links
WHERE shared_links.upload_image_id = upload_images.id
)`
func deleteUnreferencedUploadImages(ctx context.Context, tx *gorm.DB, imageIDs []uint, excludedKeys ...string) error {
if len(imageIDs) == 0 {
return nil
}
var images []*UploadImage
if err := tx.WithContext(ctx).Unscoped().
Where("id IN ?", imageIDs).
Where(unreferencedUploadImagePredicate).
Find(&images).Error; err != nil {
return errors.Wrap(err, "get unreferenced upload images")
}
if len(images) == 0 {
return nil
}
ids := lo.Map(images, func(image *UploadImage, _ int) uint { return image.ID })
if err := tx.WithContext(ctx).Unscoped().
Where("id IN ?", ids).
Where(unreferencedUploadImagePredicate).
Delete(&UploadImage{}).Error; err != nil {
return errors.Wrap(err, "delete unreferenced upload images")
}
values := lo.Map(images, func(image *UploadImage, _ int) string { return image.Key })
if err := enqueueR2DeleteTasksForValues(tx.WithContext(ctx), conf.Upload.ImageBucket, values, excludedKeys...); err != nil {
return errors.Wrap(err, "enqueue deleted upload images")
}
return nil
}
func (db *uploadImages) DeleteByQuestionID(ctx context.Context, questionID uint) error {
return db.Transaction(func(tx *gorm.DB) error {
var links []*UploadImageQuestion
@@ -184,13 +231,10 @@ func (db *uploadImages) DeleteByQuestionID(ctx context.Context, questionID uint)
imageIDs := lo.Map(links, func(item *UploadImageQuestion, _ int) uint {
return item.UploadImageID
})
if err := tx.WithContext(ctx).Where("id IN ?", imageIDs).Unscoped().Delete(&UploadImage{}).Error; err != nil {
return errors.Wrap(err, "delete upload images by question id")
}
// 关联表 UploadImageQuestion 无 DeletedAt,Delete 本就是真删,无需 Unscoped。
// 关联表无 DeletedAt,Delete 本就是真删;先解除当前题目关联。
if err := tx.WithContext(ctx).Where("question_id = ?", questionID).Delete(&UploadImageQuestion{}).Error; err != nil {
return errors.Wrap(err, "delete image question links")
}
return nil
return deleteUnreferencedUploadImages(ctx, tx, imageIDs)
})
}
+95
View File
@@ -0,0 +1,95 @@
package db
import (
"context"
"testing"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
)
func uploadImageCount(t *testing.T, conn *gorm.DB, id uint) int64 {
t.Helper()
var count int64
require.NoError(t, conn.Unscoped().Model(&UploadImage{}).Where("id = ?", id).Count(&count).Error)
return count
}
func TestUploadImagesDeleteByQuestionIDOutbox(t *testing.T) {
setR2TestConfig(t)
conn, _ := newTestDB(t)
ctx := context.Background()
images := NewUploadImagesStore(conn)
tasks := NewR2DeleteTaskStore(conn)
first, err := images.Create(ctx, CreateUploadImageOptions{QuestionID: 1, Key: "unique.webp"})
require.NoError(t, err)
duplicate, err := images.Create(ctx, CreateUploadImageOptions{QuestionID: 1, Key: "unique.webp"})
require.NoError(t, err)
shared, err := images.Create(ctx, CreateUploadImageOptions{QuestionID: 1, Key: "shared.webp"})
require.NoError(t, err)
require.NoError(t, conn.WithContext(ctx).Create(&UploadImageQuestion{
UploadImageID: shared.ID,
QuestionID: 2,
}).Error)
require.NoError(t, images.DeleteByQuestionID(ctx, 1))
require.Zero(t, uploadImageCount(t, conn, first.ID))
require.Zero(t, uploadImageCount(t, conn, duplicate.ID))
require.Equal(t, int64(1), uploadImageCount(t, conn, shared.ID))
due, err := tasks.ListDue(ctx, 10)
require.NoError(t, err)
require.Len(t, due, 1, "重复 key 必须去重,共享主体不得入队")
require.Equal(t, "unique.webp", due[0].Key)
require.NoError(t, images.DeleteByQuestionID(ctx, 2))
require.Zero(t, uploadImageCount(t, conn, shared.ID))
due, err = tasks.ListDue(ctx, 10)
require.NoError(t, err)
require.Len(t, due, 2)
require.ElementsMatch(t, []string{"unique.webp", "shared.webp"}, []string{due[0].Key, due[1].Key})
}
func TestUploadImagesReplaceDeletesOnlyOrphanedPrevious(t *testing.T) {
setR2TestConfig(t)
conn, _ := newTestDB(t)
ctx := context.Background()
images := NewUploadImagesStore(conn)
tasks := NewR2DeleteTaskStore(conn)
old, err := images.Create(ctx, CreateUploadImageOptions{
Type: UploadImageQuestionTypeAnswer,
QuestionID: 10,
Key: "old.webp",
})
require.NoError(t, err)
shared, err := images.Create(ctx, CreateUploadImageOptions{
Type: UploadImageQuestionTypeAnswer,
QuestionID: 10,
Key: "shared-old.webp",
})
require.NoError(t, err)
require.NoError(t, conn.WithContext(ctx).Create(&UploadImageQuestion{
Type: UploadImageQuestionTypeAnswer,
UploadImageID: shared.ID,
QuestionID: 11,
}).Error)
newImage, err := images.Create(ctx, CreateUploadImageOptions{
Type: UploadImageQuestionTypeAnswer,
QuestionID: 10,
Key: "new.webp",
IsDeletingPrevious: true,
})
require.NoError(t, err)
require.Zero(t, uploadImageCount(t, conn, old.ID))
require.Equal(t, int64(1), uploadImageCount(t, conn, shared.ID))
require.Equal(t, int64(1), uploadImageCount(t, conn, newImage.ID))
due, err := tasks.ListDue(ctx, 10)
require.NoError(t, err)
require.Len(t, due, 1)
require.Equal(t, "old.webp", due[0].Key)
require.NotEqual(t, newImage.Key, due[0].Key)
}
+66 -23
View File
@@ -386,10 +386,25 @@ func (db *users) Update(ctx context.Context, id uint, opts UpdateUserOptions) er
"Language": opts.Language,
}
if err := db.WithContext(ctx).Model(&User{}).Where("id = ?", id).Updates(updates).Error; err != nil {
return errors.Wrap(err, "update user")
}
return nil
return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var current User
if err := tx.WithContext(ctx).Select("id", "avatar", "background", "background_image").First(&current, id).Error; err != nil {
return errors.Wrap(err, "get current profile images")
}
if err := tx.WithContext(ctx).Model(&User{}).Where("id = ?", id).Updates(updates).Error; err != nil {
return errors.Wrap(err, "update user")
}
oldValues := make([]string, 0, 3)
for _, pair := range [][2]string{{current.Avatar, opts.Avatar}, {current.Background, opts.Background}, {current.BackgroundImage, opts.BackgroundImage}} {
if pair[0] != pair[1] {
oldValues = append(oldValues, pair[0])
}
}
if err := enqueueR2DeleteTasksForValues(tx.WithContext(ctx), conf.Upload.ImageBucket, oldValues, opts.Avatar, opts.Background, opts.BackgroundImage); err != nil {
return errors.Wrap(err, "enqueue replaced profile images")
}
return nil
})
}
// UpdateHarassmentType 仅更新骚扰防护类型(none / register_only)。
@@ -1021,6 +1036,19 @@ func (db *users) BanUserAndCleanup(ctx context.Context, userID uint) error {
// 直接复用会把上一条语句的 Where 条件串到下一条上(曾把 messages 的条件串进 follow_ups,
// 报「字段 conversation_id 不存在」)。
func purgeUserContent(ctx context.Context, tx *gorm.DB, userID uint) error {
var profile User
if err := tx.WithContext(ctx).Select("id", "avatar", "background", "background_image").First(&profile, userID).Error; err != nil {
return errors.Wrap(err, "get profile images of user")
}
if err := enqueueR2DeleteTasksForValues(tx.WithContext(ctx), conf.Upload.ImageBucket, []string{profile.Avatar, profile.Background, profile.BackgroundImage}); err != nil {
return errors.Wrap(err, "enqueue profile images of user")
}
if err := tx.WithContext(ctx).Model(&User{}).Where("id = ?", userID).Updates(map[string]interface{}{
"avatar": "", "background": "", "background_image": "",
}).Error; err != nil {
return errors.Wrap(err, "clear profile images of user")
}
// 追问:属于该用户提问下的追问
if err := tx.WithContext(ctx).Unscoped().
Where("question_id IN (SELECT id FROM questions WHERE user_id = ?)", userID).
@@ -1029,11 +1057,31 @@ func purgeUserContent(ctx context.Context, tx *gorm.DB, userID uint) error {
}
// 消息图片必须先于消息删除,否则消息子查询将无法再匹配待删图片。
// 新数据优先使用物化 ObjectKey,旧数据回退到 URL;外部 URL 不会进入 R2 outbox。
messageImageScope := "message_id IN (SELECT id FROM messages WHERE conversation_id IN (SELECT id FROM conversations WHERE owner_id = ? OR asker_id = ?))"
var messageImages []*MessageImage
if err := tx.WithContext(ctx).Unscoped().
Where("message_id IN (SELECT id FROM messages WHERE conversation_id IN (SELECT id FROM conversations WHERE owner_id = ? OR asker_id = ?))", userID, userID).
Where(messageImageScope, userID, userID).
Find(&messageImages).Error; err != nil {
return errors.Wrap(err, "get message images of user")
}
messageImageValues := make([]string, 0, len(messageImages))
for _, image := range messageImages {
value := image.ObjectKey
if value == "" {
value = image.URL
}
messageImageValues = append(messageImageValues, value)
}
if err := tx.WithContext(ctx).Unscoped().
Where(messageImageScope, userID, userID).
Delete(&MessageImage{}).Error; err != nil {
return errors.Wrap(err, "delete message images of user")
}
if err := enqueueR2DeleteTasksForValues(tx.WithContext(ctx), conf.Upload.ImageBucket, messageImageValues); err != nil {
return errors.Wrap(err, "enqueue deleted message images")
}
// 消息:属于该用户作为 owner 或 asker 的会话
if err := tx.WithContext(ctx).Unscoped().
Where("conversation_id IN (SELECT id FROM conversations WHERE owner_id = ? OR asker_id = ?)", userID, userID).
@@ -1048,30 +1096,25 @@ func purgeUserContent(ctx context.Context, tx *gorm.DB, userID uint) error {
return errors.Wrap(err, "delete conversations of user")
}
// 提问图片元数据:删除本用户上传的孤立图片,以及仅链接到本用户提问的图片。
// 仍被其他提问引用的图片必须保留;这里只清数据库元数据,不操作对象存储。
if err := tx.WithContext(ctx).Unscoped().
Where(`(
uploader_user_id = ?
OR upload_images.id IN (
SELECT upload_image_id FROM upload_image_questions
WHERE question_id IN (SELECT id FROM questions WHERE user_id = ?)
)
) AND NOT EXISTS (
SELECT 1 FROM upload_image_questions AS shared_links
WHERE shared_links.upload_image_id = upload_images.id
AND shared_links.question_id NOT IN (SELECT id FROM questions WHERE user_id = ?)
)`, userID, userID, userID).
Delete(&UploadImage{}).Error; err != nil {
return errors.Wrap(err, "delete upload images of user")
// 先确定本用户上传或仅与当前作用域有关的候选主体,再解除本用户题目的关联。
// 最终是否可删除由与 UploadImages.DeleteByQuestionID 完全相同的共享保护谓词决定。
var uploadImageIDs []uint
if err := tx.WithContext(ctx).Unscoped().Model(&UploadImage{}).
Where(`uploader_user_id = ? OR upload_images.id IN (
SELECT upload_image_id FROM upload_image_questions
WHERE question_id IN (SELECT id FROM questions WHERE user_id = ?)
)`, userID, userID).
Pluck("upload_images.id", &uploadImageIDs).Error; err != nil {
return errors.Wrap(err, "get upload images of user")
}
// 提问-图片关联
if err := tx.WithContext(ctx).Unscoped().
Where("question_id IN (SELECT id FROM questions WHERE user_id = ?)", userID).
Delete(&UploadImageQuestion{}).Error; err != nil {
return errors.Wrap(err, "delete upload image links of user")
}
if err := deleteUnreferencedUploadImages(ctx, tx, uploadImageIDs); err != nil {
return errors.Wrap(err, "delete upload images of user")
}
// 提问
if err := tx.WithContext(ctx).Unscoped().
+142
View File
@@ -0,0 +1,142 @@
package db
import (
"context"
"testing"
"github.com/stretchr/testify/require"
"github.com/tamakyi/TamaBox/internal/conf"
)
func createOutboxTestUser(t *testing.T, ctx context.Context, store UsersStore, email, domain string) *User {
t.Helper()
require.NoError(t, store.Create(ctx, CreateUserOptions{
Name: domain,
Password: "test-only-password",
Email: email,
Domain: domain,
}))
user, err := store.GetByEmail(ctx, email)
require.NoError(t, err)
return user
}
func TestPurgeUserContentEnqueuesOwnedR2Objects(t *testing.T) {
setR2TestConfig(t)
conn, _ := newTestDB(t)
ctx := context.Background()
users := NewUsersStore(conn)
owner := createOutboxTestUser(t, ctx, users, "[email protected]", "outbox-owner")
other := createOutboxTestUser(t, ctx, users, "[email protected]", "outbox-other")
require.NoError(t, conn.Model(owner).Updates(map[string]interface{}{
"avatar": "https://cdn.example.test/assets/profile-owned.webp", "background": "https://cdn.example.test/assets/default-background.webp", "background_image": "https://cdn.example.test/assets/profile-shared.webp",
}).Error)
require.NoError(t, conn.Model(other).Update("background_image", "https://cdn.example.test/assets/profile-shared.webp").Error)
questions := NewQuestionsStore(conn)
question, err := questions.Create(ctx, CreateQuestionOptions{UserID: owner.ID, Content: "owned"})
require.NoError(t, err)
otherQuestion, err := questions.Create(ctx, CreateQuestionOptions{UserID: other.ID, Content: "other"})
require.NoError(t, err)
images := NewUploadImagesStore(conn)
owned, err := images.Create(ctx, CreateUploadImageOptions{QuestionID: question.ID, UploaderUserID: owner.ID, Key: "owned.webp"})
require.NoError(t, err)
_, err = images.Create(ctx, CreateUploadImageOptions{QuestionID: question.ID, UploaderUserID: owner.ID, Key: "owned.webp"})
require.NoError(t, err)
legacy, err := images.Create(ctx, CreateUploadImageOptions{
QuestionID: question.ID, UploaderUserID: owner.ID, Key: "https://cdn.example.test/assets/legacy.webp",
})
require.NoError(t, err)
external, err := images.Create(ctx, CreateUploadImageOptions{
QuestionID: question.ID, UploaderUserID: owner.ID, Key: "https://external.example.net/external.webp",
})
require.NoError(t, err)
shared, err := images.Create(ctx, CreateUploadImageOptions{QuestionID: question.ID, UploaderUserID: owner.ID, Key: "shared.webp"})
require.NoError(t, err)
require.NoError(t, conn.WithContext(ctx).Create(&UploadImageQuestion{
UploadImageID: shared.ID, QuestionID: otherQuestion.ID,
}).Error)
conversation, message, err := NewConversationStore(conn).CreateWithFirstMessage(ctx, CreateConversationOptions{
OwnerID: owner.ID, AskerID: other.ID, MaxMessages: 20,
}, other.ID, "private")
require.NoError(t, err)
require.NotNil(t, conversation)
messageImages := NewMessageImageStore(conn)
for _, opts := range []CreateMessageImageOptions{
{MessageID: message.ID, ObjectKey: "message-key.webp", URL: "https://cdn.example.test/assets/message-key.webp"},
{MessageID: message.ID, ObjectKey: "owned.webp", URL: "https://cdn.example.test/assets/owned.webp"},
{MessageID: message.ID, URL: "https://cdn.example.test/assets/message-legacy.webp"},
{MessageID: message.ID, URL: "https://external.example.net/message-external.webp"},
} {
_, err = messageImages.Create(ctx, opts)
require.NoError(t, err)
}
require.NoError(t, users.Deactivate(ctx, owner.ID))
require.Zero(t, uploadImageCount(t, conn, owned.ID))
require.Zero(t, uploadImageCount(t, conn, legacy.ID))
require.Zero(t, uploadImageCount(t, conn, external.ID))
require.Equal(t, int64(1), uploadImageCount(t, conn, shared.ID))
var sharedLinks int64
require.NoError(t, conn.WithContext(ctx).Model(&UploadImageQuestion{}).
Where("upload_image_id = ? AND question_id = ?", shared.ID, otherQuestion.ID).Count(&sharedLinks).Error)
require.Equal(t, int64(1), sharedLinks)
due, err := NewR2DeleteTaskStore(conn).ListDue(ctx, 20)
require.NoError(t, err)
keys := make([]string, 0, len(due))
for _, task := range due {
keys = append(keys, task.Key)
}
require.ElementsMatch(t, []string{
"owned.webp", "legacy.webp", "message-key.webp", "message-legacy.webp", "profile-owned.webp", "profile-shared.webp",
}, keys)
require.NotContains(t, keys, "shared.webp")
require.NotContains(t, keys, "external.webp")
require.NotContains(t, keys, "message-external.webp")
require.NotContains(t, keys, "default-background.webp")
var deactivated User
require.NoError(t, conn.Unscoped().First(&deactivated, owner.ID).Error)
require.Empty(t, deactivated.Avatar)
require.Empty(t, deactivated.Background)
require.Empty(t, deactivated.BackgroundImage)
other, err = users.GetByID(ctx, other.ID)
require.NoError(t, err)
require.Equal(t, "https://cdn.example.test/assets/profile-shared.webp", other.BackgroundImage)
referenced, err := NewR2DeleteTaskStore(conn).HasReferences(ctx, conf.Upload.ImageBucket, "profile-shared.webp")
require.NoError(t, err)
require.True(t, referenced)
referenced, err = NewR2DeleteTaskStore(conn).HasReferences(ctx, conf.Upload.ImageBucket, "default-background.webp")
require.NoError(t, err)
require.True(t, referenced)
}
func TestPurgeUserContentOutboxFailureRollsBack(t *testing.T) {
setR2TestConfig(t)
conn, _ := newTestDB(t)
ctx := context.Background()
users := NewUsersStore(conn)
owner := createOutboxTestUser(t, ctx, users, "[email protected]", "outbox-rollback")
question, err := NewQuestionsStore(conn).Create(ctx, CreateQuestionOptions{UserID: owner.ID, Content: "rollback"})
require.NoError(t, err)
image, err := NewUploadImagesStore(conn).Create(ctx, CreateUploadImageOptions{
QuestionID: question.ID, UploaderUserID: owner.ID, Key: "rollback.webp",
})
require.NoError(t, err)
require.NoError(t, conn.Migrator().DropTable(&R2DeleteTask{}))
require.Error(t, users.Deactivate(ctx, owner.ID))
require.Equal(t, int64(1), uploadImageCount(t, conn, image.ID))
var links int64
require.NoError(t, conn.WithContext(ctx).Model(&UploadImageQuestion{}).
Where("upload_image_id = ? AND question_id = ?", image.ID, question.ID).Count(&links).Error)
require.Equal(t, int64(1), links)
var questions int64
require.NoError(t, conn.WithContext(ctx).Unscoped().Model(&Question{}).Where("id = ?", question.ID).Count(&questions).Error)
require.Equal(t, int64(1), questions)
var user User
require.NoError(t, conn.WithContext(ctx).First(&user, owner.ID).Error)
require.False(t, user.DeletedAt.Valid)
}