可靠性: 完善 R2 备份恢复与对象生命周期

This commit is contained in:
tamakyi
2026-11-05 08:41:55 +08:00
parent 3e7719f200
commit 937e688cfa
32 changed files with 4151 additions and 231 deletions
+603
View File
@@ -0,0 +1,603 @@
package cmd
import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"testing"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/stretchr/testify/require"
)
type fakeBackupObject struct {
body []byte
etag string
contentType string
cacheControl string
interruptedAttempts int
wrongLengthAttempts int
}
type fakeBackupS3 struct {
pages []*s3.ListObjectsV2Output
listInputs []*s3.ListObjectsV2Input
objects map[string]fakeBackupObject
getCalls map[string]int
getOrder []string
}
func (f *fakeBackupS3) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) {
f.listInputs = append(f.listInputs, input)
index := len(f.listInputs) - 1
if index >= len(f.pages) {
return nil, fmt.Errorf("unexpected ListObjectsV2 call %d", index+1)
}
return f.pages[index], nil
}
func (f *fakeBackupS3) GetObject(_ context.Context, input *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
key := aws.ToString(input.Key)
if f.getCalls == nil {
f.getCalls = make(map[string]int)
}
f.getCalls[key]++
f.getOrder = append(f.getOrder, key)
object, exists := f.objects[key]
if !exists {
return nil, fmt.Errorf("unexpected GetObject key %q", key)
}
var body io.ReadCloser
if f.getCalls[key] <= object.interruptedAttempts {
body = io.NopCloser(&interruptedReader{data: append([]byte(nil), object.body...)})
} else {
body = io.NopCloser(strings.NewReader(string(object.body)))
}
contentLength := int64(len(object.body))
if f.getCalls[key] <= object.wrongLengthAttempts {
contentLength++
}
return &s3.GetObjectOutput{
Body: body,
ContentLength: aws.Int64(contentLength),
ETag: aws.String(object.etag),
ContentType: aws.String(object.contentType),
CacheControl: aws.String(object.cacheControl),
}, nil
}
type interruptedReader struct {
data []byte
}
func (r *interruptedReader) Read(p []byte) (int, error) {
if len(r.data) == 0 {
return 0, io.ErrUnexpectedEOF
}
n := copy(p, r.data)
r.data = r.data[n:]
if len(r.data) == 0 {
return n, io.ErrUnexpectedEOF
}
return n, nil
}
func backupListPage(keys []string, truncated bool, nextToken string) *s3.ListObjectsV2Output {
objects := make([]types.Object, 0, len(keys))
for _, key := range keys {
objects = append(objects, types.Object{
Key: aws.String(key),
Size: aws.Int64(999999), // 故意不可信;manifest 必须记录实际下载字节数。
ETag: aws.String("list-etag-must-not-be-used"),
})
}
output := &s3.ListObjectsV2Output{Contents: objects, IsTruncated: aws.Bool(truncated)}
if nextToken != "" {
output.NextContinuationToken = aws.String(nextToken)
}
return output
}
func TestBackupImagesWithClientCoversAllKeysAndPages(t *testing.T) {
keys := []string{
"2026/11/04/root.webp",
"picture/2026/11/04/picture.webp",
"tamabox/2025/12/historical.jpg",
"static/site.css",
}
objects := make(map[string]fakeBackupObject, len(keys))
for index, key := range keys {
objects[key] = fakeBackupObject{
body: []byte(fmt.Sprintf("actual-body-%d", index)),
etag: fmt.Sprintf("get-etag-%d", index),
contentType: []string{"image/webp", "image/webp", "image/jpeg", "text/css"}[index],
cacheControl: fmt.Sprintf("public, max-age=%d", index+60),
}
}
client := &fakeBackupS3{
pages: []*s3.ListObjectsV2Output{
backupListPage(keys[:2], true, "page-2"),
backupListPage(keys[2:], false, ""),
},
objects: objects,
}
outDir := t.TempDir()
require.NoError(t, backupImagesWithClient(context.Background(), client, "configured-bucket", outDir, ""))
require.Len(t, client.listInputs, 2)
for index, input := range client.listInputs {
require.Equal(t, "configured-bucket", aws.ToString(input.Bucket))
require.Equal(t, "", aws.ToString(input.Prefix), "生产默认必须列举整个 bucket")
if index == 0 {
require.Nil(t, input.ContinuationToken)
} else {
require.Equal(t, "page-2", aws.ToString(input.ContinuationToken))
}
}
require.ElementsMatch(t, keys, client.getOrder)
manifestBytes, err := os.ReadFile(filepath.Join(outDir, "images-manifest.json"))
require.NoError(t, err)
var manifest []imageManifestEntry
require.NoError(t, json.Unmarshal(manifestBytes, &manifest))
require.Len(t, manifest, len(keys))
for index, entry := range manifest {
require.Equal(t, keys[index], entry.Key)
object := objects[entry.Key]
require.Equal(t, int64(len(object.body)), entry.Size)
digest := sha256.Sum256(object.body)
require.Equal(t, fmt.Sprintf("%x", digest), entry.SHA256)
require.Equal(t, object.etag, entry.ETag)
require.Equal(t, object.contentType, entry.ContentType)
require.Equal(t, object.cacheControl, entry.CacheControl)
path, err := localImagePath(filepath.Join(outDir, "images"), entry.Key)
require.NoError(t, err)
content, err := os.ReadFile(path)
require.NoError(t, err)
require.Equal(t, object.body, content)
}
}
func TestBackupImagesWithClientSupportsIsolatedPrefixAndEmptyManifest(t *testing.T) {
t.Run("prefix", func(t *testing.T) {
key := "tamabox-backup-test/unit/source/root.webp"
client := &fakeBackupS3{
pages: []*s3.ListObjectsV2Output{backupListPage([]string{key}, false, "")},
objects: map[string]fakeBackupObject{
key: {body: []byte("isolated"), contentType: "image/webp"},
},
}
require.NoError(t, backupImagesWithClient(context.Background(), client, "bucket", t.TempDir(), "tamabox-backup-test/unit/source/"))
require.Len(t, client.listInputs, 1)
require.Equal(t, "tamabox-backup-test/unit/source/", aws.ToString(client.listInputs[0].Prefix))
require.Equal(t, []string{key}, client.getOrder)
})
t.Run("empty array", func(t *testing.T) {
client := &fakeBackupS3{pages: []*s3.ListObjectsV2Output{backupListPage(nil, false, "")}}
outDir := t.TempDir()
require.NoError(t, backupImagesWithClient(context.Background(), client, "bucket", outDir, ""))
content, err := os.ReadFile(filepath.Join(outDir, "images-manifest.json"))
require.NoError(t, err)
require.Equal(t, "[]", strings.TrimSpace(string(content)))
})
}
func TestBackupRejectsPathAttacksBeforeAnyDownload(t *testing.T) {
tests := []struct {
name string
key *string
}{
{name: "nil key", key: nil},
{name: "empty key", key: aws.String("")},
{name: "absolute POSIX", key: aws.String("/outside")},
{name: "absolute Windows", key: aws.String("C:/outside")},
{name: "volume relative", key: aws.String("C:outside")},
{name: "parent", key: aws.String("../outside")},
{name: "nested parent", key: aws.String("picture/../outside")},
{name: "backslash", key: aws.String(`picture\outside`)},
{name: "NUL", key: aws.String("picture/\x00outside")},
{name: "empty segment", key: aws.String("picture//outside")},
{name: "leading dot segment", key: aws.String("./outside")},
{name: "inner dot segment", key: aws.String("picture/./outside")},
{name: "directory", key: aws.String("picture/")},
{name: "dot only", key: aws.String(".")},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
page := backupListPage(nil, false, "")
page.Contents = []types.Object{{Key: tt.key}}
client := &fakeBackupS3{pages: []*s3.ListObjectsV2Output{page}}
err := backupImagesWithClient(context.Background(), client, "bucket", t.TempDir(), "")
require.Error(t, err)
require.Empty(t, client.getOrder, "非法 key 必须在任何下载前被拒绝")
})
}
}
func TestBackupPreflightsAllPagesAndLocalCollisionsBeforeDownload(t *testing.T) {
t.Run("invalid key on later page", func(t *testing.T) {
client := &fakeBackupS3{
pages: []*s3.ListObjectsV2Output{
backupListPage([]string{"picture/valid.webp"}, true, "next"),
backupListPage([]string{"../escape"}, false, ""),
},
objects: map[string]fakeBackupObject{"picture/valid.webp": {body: []byte("valid")}},
}
require.Error(t, backupImagesWithClient(context.Background(), client, "bucket", t.TempDir(), ""))
require.Len(t, client.listInputs, 2)
require.Empty(t, client.getOrder)
})
for _, keys := range [][]string{{"a", "a/b"}, {"a/b", "a"}, {"same", "same"}} {
t.Run(fmt.Sprintf("collision %q", keys), func(t *testing.T) {
client := &fakeBackupS3{pages: []*s3.ListObjectsV2Output{backupListPage(keys, false, "")}}
require.Error(t, backupImagesWithClient(context.Background(), client, "bucket", t.TempDir(), ""))
require.Empty(t, client.getOrder)
})
}
t.Run("existing file blocks planned directory", func(t *testing.T) {
outDir := t.TempDir()
require.NoError(t, os.MkdirAll(filepath.Join(outDir, "images"), 0o700))
require.NoError(t, os.WriteFile(filepath.Join(outDir, "images", "a"), []byte("existing"), 0o600))
client := &fakeBackupS3{pages: []*s3.ListObjectsV2Output{backupListPage([]string{"a/b"}, false, "")}}
require.Error(t, backupImagesWithClient(context.Background(), client, "bucket", outDir, ""))
require.Empty(t, client.getOrder)
})
t.Run("existing directory blocks planned file", func(t *testing.T) {
outDir := t.TempDir()
require.NoError(t, os.MkdirAll(filepath.Join(outDir, "images", "a"), 0o700))
client := &fakeBackupS3{pages: []*s3.ListObjectsV2Output{backupListPage([]string{"a"}, false, "")}}
require.Error(t, backupImagesWithClient(context.Background(), client, "bucket", outDir, ""))
require.Empty(t, client.getOrder)
})
}
func TestBackupRetriesInterruptedStreamsAndLeavesNoPart(t *testing.T) {
t.Run("eventual success", func(t *testing.T) {
key := "picture/retry.webp"
body := []byte("complete-object-after-retries")
client := &fakeBackupS3{
pages: []*s3.ListObjectsV2Output{backupListPage([]string{key}, false, "")},
objects: map[string]fakeBackupObject{
key: {body: body, interruptedAttempts: 2, etag: "successful-get-etag"},
},
}
outDir := t.TempDir()
require.NoError(t, backupImagesWithClient(context.Background(), client, "bucket", outDir, ""))
require.Equal(t, 3, client.getCalls[key])
content, err := os.ReadFile(filepath.Join(outDir, "images", "picture", "retry.webp"))
require.NoError(t, err)
require.Equal(t, body, content)
requireNoPartFiles(t, filepath.Join(outDir, "images", "picture"))
})
t.Run("short body relative to GET content length", func(t *testing.T) {
key := "short-body.webp"
client := &fakeBackupS3{
pages: []*s3.ListObjectsV2Output{backupListPage([]string{key}, false, "")},
objects: map[string]fakeBackupObject{
key: {body: []byte("complete-on-second-attempt"), wrongLengthAttempts: 1},
},
}
outDir := t.TempDir()
require.NoError(t, backupImagesWithClient(context.Background(), client, "bucket", outDir, ""))
require.Equal(t, 2, client.getCalls[key])
requireNoPartFiles(t, filepath.Join(outDir, "images"))
})
t.Run("final failure", func(t *testing.T) {
key := "retry.txt"
client := &fakeBackupS3{
pages: []*s3.ListObjectsV2Output{backupListPage([]string{key}, false, "")},
objects: map[string]fakeBackupObject{
key: {body: []byte("never-complete"), interruptedAttempts: 3},
},
}
outDir := t.TempDir()
err := backupImagesWithClient(context.Background(), client, "bucket", outDir, "")
require.Error(t, err)
require.Contains(t, err.Error(), "after 3 attempts")
require.Equal(t, 3, client.getCalls[key])
require.NoFileExists(t, filepath.Join(outDir, "images", key))
requireNoPartFiles(t, filepath.Join(outDir, "images"))
require.NoFileExists(t, filepath.Join(outDir, "images-manifest.json"))
})
}
type capturedRestorePut struct {
key string
body []byte
contentType string
cacheControl string
size int64
}
type fakeRestoreStore struct {
existing map[string]bool
headErrors map[string]error
putFailures map[string]error
headCalls []string
puts []capturedRestorePut
}
func (f *fakeRestoreStore) HeadObject(_ context.Context, input *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) {
key := aws.ToString(input.Key)
f.headCalls = append(f.headCalls, key)
if err := f.headErrors[key]; err != nil {
return nil, err
}
if f.existing[key] {
return &s3.HeadObjectOutput{}, nil
}
return nil, &types.NotFound{}
}
func (f *fakeRestoreStore) PutObject(_ context.Context, input *s3.PutObjectInput) error {
body, err := io.ReadAll(input.Body)
if err != nil {
return err
}
key := aws.ToString(input.Key)
f.puts = append(f.puts, capturedRestorePut{
key: key,
body: body,
contentType: aws.ToString(input.ContentType),
cacheControl: aws.ToString(input.CacheControl),
size: aws.ToInt64(input.ContentLength),
})
return f.putFailures[key]
}
type restoreFixtureObject struct {
key string
body []byte
storedBody []byte
manifestSize *int64
manifestSHA string
contentType string
cacheControl string
}
func createRestoreFixture(t *testing.T, objects []restoreFixtureObject) string {
t.Helper()
dir := t.TempDir()
manifest := make([]imageManifestEntry, 0, len(objects))
for _, object := range objects {
storedBody := object.storedBody
if storedBody == nil {
storedBody = object.body
}
path := filepath.Join(dir, "images", filepath.FromSlash(object.key))
require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o700))
require.NoError(t, os.WriteFile(path, storedBody, 0o600))
size := int64(len(object.body))
if object.manifestSize != nil {
size = *object.manifestSize
}
digest := sha256.Sum256(object.body)
hash := fmt.Sprintf("%x", digest)
if object.manifestSHA != "" {
hash = object.manifestSHA
}
manifest = append(manifest, imageManifestEntry{
Key: object.key,
Size: size,
SHA256: hash,
ETag: "source-get-etag",
ContentType: object.contentType,
CacheControl: object.cacheControl,
})
}
data, err := json.MarshalIndent(manifest, "", " ")
require.NoError(t, err)
require.NoError(t, os.WriteFile(filepath.Join(dir, "images-manifest.json"), data, 0o600))
return dir
}
func TestRestoreImagesPreservesKeysContentAndMetadataWithTargetPrefix(t *testing.T) {
objects := []restoreFixtureObject{
{key: "2026/11/04/root.webp", body: []byte("root-body"), contentType: "image/webp", cacheControl: "public, max-age=60"},
{key: "picture/2026/11/04/pic.webp", body: []byte("picture-body"), contentType: "image/webp", cacheControl: "public, max-age=120"},
{key: "static/site.css", body: []byte("body{}"), contentType: "text/css", cacheControl: "no-cache"},
}
input := createRestoreFixture(t, objects)
store := &fakeRestoreStore{}
err := restoreImagesWithStore(context.Background(), store, "bucket", restoreImagesOptions{
Input: input,
Yes: true,
TargetPrefix: "isolated-restore", // 不带斜杠也应规范化为恰好一个路径分隔符。
})
require.NoError(t, err)
require.Len(t, store.headCalls, len(objects))
require.Len(t, store.puts, len(objects))
byKey := make(map[string]capturedRestorePut, len(store.puts))
for _, put := range store.puts {
byKey[put.key] = put
}
for _, object := range objects {
target := "isolated-restore/" + object.key
put := byKey[target]
require.Equal(t, object.body, put.body)
require.Equal(t, object.contentType, put.contentType)
require.Equal(t, object.cacheControl, put.cacheControl)
require.Equal(t, int64(len(object.body)), put.size)
}
}
func TestRestoreImagesDefaultRefusesOverwriteWithZeroPut(t *testing.T) {
objects := []restoreFixtureObject{
{key: "first.webp", body: []byte("first")},
{key: "second.webp", body: []byte("second")},
}
store := &fakeRestoreStore{existing: map[string]bool{"second.webp": true}}
err := restoreImagesWithStore(context.Background(), store, "bucket", restoreImagesOptions{
Input: createRestoreFixture(t, objects),
Yes: true,
})
require.Error(t, err)
require.Contains(t, err.Error(), "no objects were uploaded")
require.Empty(t, store.puts)
}
func TestRestoreImagesOverwriteAndDryRun(t *testing.T) {
t.Run("overwrite", func(t *testing.T) {
store := &fakeRestoreStore{existing: map[string]bool{"picture.webp": true}}
err := restoreImagesWithStore(context.Background(), store, "bucket", restoreImagesOptions{
Input: createRestoreFixture(t, []restoreFixtureObject{{key: "picture.webp", body: []byte("replacement")}}),
Yes: true,
Overwrite: true,
})
require.NoError(t, err)
require.Empty(t, store.headCalls, "显式 overwrite 不需要默认防覆盖 Head")
require.Len(t, store.puts, 1)
require.Equal(t, []byte("replacement"), store.puts[0].body)
})
t.Run("dry run without yes", func(t *testing.T) {
store := &fakeRestoreStore{}
err := restoreImagesWithStore(context.Background(), store, "bucket", restoreImagesOptions{
Input: createRestoreFixture(t, []restoreFixtureObject{{key: "picture.webp", body: []byte("dry-run")}}),
DryRun: true,
TargetPrefix: "drill/",
})
require.NoError(t, err)
require.Equal(t, []string{"drill/picture.webp"}, store.headCalls)
require.Empty(t, store.puts)
})
t.Run("actual write requires yes", func(t *testing.T) {
store := &fakeRestoreStore{}
err := restoreImagesWithStore(context.Background(), store, "bucket", restoreImagesOptions{
Input: createRestoreFixture(t, []restoreFixtureObject{{key: "picture.webp", body: []byte("write")}}),
})
require.Error(t, err)
require.Contains(t, err.Error(), "--yes")
require.Empty(t, store.headCalls)
require.Empty(t, store.puts)
})
}
func TestRestoreRejectsTamperedFilesBeforeAnyRemoteCall(t *testing.T) {
wrongSize := int64(999)
tests := []struct {
name string
second restoreFixtureObject
}{
{
name: "SHA256 tamper with unchanged size",
second: restoreFixtureObject{
key: "second.webp", body: []byte("second"), storedBody: []byte("SECOND"),
},
},
{
name: "size tamper",
second: restoreFixtureObject{
key: "second.webp", body: []byte("second"), manifestSize: &wrongSize,
},
},
{
name: "manifest hash tamper",
second: restoreFixtureObject{
key: "second.webp", body: []byte("second"), manifestSHA: strings.Repeat("0", 64),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
store := &fakeRestoreStore{}
input := createRestoreFixture(t, []restoreFixtureObject{
{key: "first.webp", body: []byte("first")},
tt.second,
})
err := restoreImagesWithStore(context.Background(), store, "bucket", restoreImagesOptions{Input: input, Yes: true})
require.Error(t, err)
require.Empty(t, store.headCalls, "全部本地文件应在任何远端操作前完成校验")
require.Empty(t, store.puts, "首个有效文件也不能在后续文件校验失败前上传")
})
}
}
func TestReadImageManifestRejectsInvalidStructure(t *testing.T) {
validHash := strings.Repeat("a", 64)
tests := []struct {
name string
json string
}{
{name: "null", json: `null`},
{name: "trailing JSON", json: `[] {}`},
{name: "trailing garbage", json: `[] garbage`},
{name: "duplicate key", json: fmt.Sprintf(`[{"key":"same","size":0,"sha256":%q},{"key":"same","size":0,"sha256":%q}]`, validHash, validHash)},
{name: "short hash", json: `[{"key":"image","size":1,"sha256":"abc"}]`},
{name: "non hex hash", json: fmt.Sprintf(`[{"key":"image","size":1,"sha256":%q}]`, strings.Repeat("z", 64))},
{name: "negative size", json: fmt.Sprintf(`[{"key":"image","size":-1,"sha256":%q}]`, validHash)},
{name: "path traversal", json: fmt.Sprintf(`[{"key":"../image","size":1,"sha256":%q}]`, validHash)},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
path := filepath.Join(t.TempDir(), "images-manifest.json")
// 非数组用例直接保留原 JSON;其余用例同样应被解析层拒绝。
require.NoError(t, os.WriteFile(path, []byte(tt.json), 0o600))
_, err := readImageManifest(path)
require.Error(t, err)
})
}
}
func TestRestoreRejectsSymlinkBeforeRemoteCalls(t *testing.T) {
input := createRestoreFixture(t, []restoreFixtureObject{{key: "picture.webp", body: []byte("content")}})
source := filepath.Join(input, "images", "picture.webp")
outside := filepath.Join(t.TempDir(), "outside.webp")
require.NoError(t, os.WriteFile(outside, []byte("content"), 0o600))
require.NoError(t, os.Remove(source))
if err := os.Symlink(outside, source); err != nil {
t.Skipf("当前平台无法创建 symlink:%v", err)
}
store := &fakeRestoreStore{}
err := restoreImagesWithStore(context.Background(), store, "bucket", restoreImagesOptions{Input: input, DryRun: true})
require.Error(t, err)
require.Empty(t, store.headCalls)
require.Empty(t, store.puts)
}
func TestRestoreReportsPartialFailureWithoutClaimingTransaction(t *testing.T) {
store := &fakeRestoreStore{putFailures: map[string]error{"second.webp": errors.New("injected put failure")}}
err := restoreImagesWithStore(context.Background(), store, "bucket", restoreImagesOptions{
Input: createRestoreFixture(t, []restoreFixtureObject{
{key: "first.webp", body: []byte("first")},
{key: "second.webp", body: []byte("second")},
}),
Yes: true,
})
require.Error(t, err)
require.Contains(t, err.Error(), "1 successful uploads")
require.Contains(t, err.Error(), "not transactional")
require.Len(t, store.puts, 2)
}
func TestRestoreRejectsUnsafeTargetPrefixBeforeRemoteCalls(t *testing.T) {
store := &fakeRestoreStore{}
err := restoreImagesWithStore(context.Background(), store, "bucket", restoreImagesOptions{
Input: createRestoreFixture(t, []restoreFixtureObject{{key: "picture.webp", body: []byte("content")}}),
Yes: true,
TargetPrefix: "../outside/",
})
require.Error(t, err)
require.Empty(t, store.headCalls)
require.Empty(t, store.puts)
}