可靠性: 完善 R2 备份恢复与对象生命周期
This commit is contained in:
tamakyi
+412
-143
@@ -6,40 +6,46 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/urfave/cli/v2"
|
||||
|
||||
"github.com/tamakyi/TamaBox/internal/conf"
|
||||
"github.com/tamakyi/TamaBox/internal/storage"
|
||||
)
|
||||
|
||||
const (
|
||||
imageObjectAttemptTimeout = 60 * time.Second
|
||||
imageObjectMaxAttempts = 3
|
||||
imageObjectRetryBackoff = 100 * time.Millisecond
|
||||
)
|
||||
|
||||
// Backup 全量备份数据库与对象存储（Cloudflare R2）图片到本地目录。
|
||||
//
|
||||
// 为什么必须同时备份图片：站点的所有图片（头像 / 背景 / 二维码 logo / 提问图 / 回答图）
|
||||
// 都以 picture/YYYY/MM/DD/<hex>（新）或 tamabox/YYYY/MM/<hex>.<ext>（历史）的形式存在
|
||||
// R2 上，数据库里只存 key（upload_images.key）或 URL（users.avatar/background 里的完整
|
||||
// CDN 地址）。只备份数据库的话，恢复后这些引用指向的图片若已不在 R2（bucket 误删 /
|
||||
// 账号丢失 / 图片被清理），页面就全是裂图。因此备份必须把 R2 上的图片一并落盘。
|
||||
// 都以根日期 key、picture/ 或 tamabox/ 等形式存在 R2 上，数据库里只存 key 或 CDN URL。
|
||||
// 只备份数据库的话，恢复后这些引用指向的图片若已不在 R2，页面就全是裂图。
|
||||
//
|
||||
// 输出目录结构：
|
||||
//
|
||||
// <output>/
|
||||
// db-<20060102-150405>.sql # pg_dump 全量导出（schema + data，可用 psql 恢复）
|
||||
// images/<key> # R2 对象，保持原 key 路径
|
||||
// images-manifest.json # key → md5 / size 清单，用于恢复后完整性校验
|
||||
// images-manifest.json # key → 实际大小 / SHA-256 / GET 元数据
|
||||
//
|
||||
// 用法：TamaBox backup --output /var/backups/tamabox
|
||||
//
|
||||
@@ -47,7 +53,7 @@ import (
|
||||
// TamaBox backup --output . --images-only # 只备份图片
|
||||
var Backup = &cli.Command{
|
||||
Name: "backup",
|
||||
Usage: "Back up the database and R2 images to a local directory",
|
||||
Usage: "Back up the database and all R2 objects to a local directory",
|
||||
Action: runBackup,
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
@@ -67,6 +73,9 @@ var Backup = &cli.Command{
|
||||
}
|
||||
|
||||
func runBackup(ctx *cli.Context) error {
|
||||
if ctx.Bool("db-only") && ctx.Bool("images-only") {
|
||||
return errors.New("--db-only and --images-only cannot be used together")
|
||||
}
|
||||
if err := conf.Init(); err != nil {
|
||||
return errors.Wrap(err, "load configuration")
|
||||
}
|
||||
@@ -127,190 +136,450 @@ func backupDatabase(ctx context.Context, outDir string) error {
|
||||
)
|
||||
cmd.Env = append(os.Environ(), "PGPASSWORD="+conf.Database.Password)
|
||||
|
||||
f, err := os.OpenFile(outFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "create backup file %q", outFile)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
var dumpErr strings.Builder
|
||||
cmd.Stdout = f
|
||||
cmd.Stderr = &dumpErr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return errors.Wrapf(err, "pg_dump failed: %s", strings.TrimSpace(dumpErr.String()))
|
||||
if err := writeFileAtomically(outFile, func(w io.Writer) error {
|
||||
cmd.Stdout = w
|
||||
cmd.Stderr = &dumpErr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return errors.Wrapf(err, "pg_dump failed: %s", strings.TrimSpace(dumpErr.String()))
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logrus.WithField("file", outFile).Info("数据库已备份")
|
||||
return nil
|
||||
}
|
||||
|
||||
// imageManifestEntry 图片备份清单里的一条：key → 大小 → md5（用于恢复后校验完整性）。
|
||||
type imageManifestEntry struct {
|
||||
Key string `json:"key"`
|
||||
Size int64 `json:"size"`
|
||||
MD5 string `json:"md5,omitempty"`
|
||||
// writeFileAtomically 保留数据库、manifest 与对象下载共用的原子写修复：
|
||||
// 只有完整写入、Sync 并关闭成功后才替换目标，失败时清理临时 .part 文件。
|
||||
func writeFileAtomically(outFile string, write func(io.Writer) error) error {
|
||||
f, err := os.CreateTemp(filepath.Dir(outFile), "."+filepath.Base(outFile)+"-*.part")
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "create temporary file for %q", outFile)
|
||||
}
|
||||
tmpFile := f.Name()
|
||||
defer os.Remove(tmpFile)
|
||||
defer f.Close()
|
||||
|
||||
if err := write(f); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := f.Sync(); err != nil {
|
||||
return errors.Wrapf(err, "sync temporary file for %q", outFile)
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
return errors.Wrapf(err, "close temporary file for %q", outFile)
|
||||
}
|
||||
if err := os.Rename(tmpFile, outFile); err != nil {
|
||||
return errors.Wrapf(err, "finalize %q", outFile)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// backupImages 从 R2 下载 tamabox 站的全部图片对象到本地，并生成 manifest。
|
||||
//
|
||||
// R2 里 tamabox 站图片的 key 都带 `tamabox/` 前缀（如 tamabox/2025/12/<ts>.jpg），
|
||||
// 这是「bucket 用 path-style 访问时 List 返回的 key 带 bucket 名作前缀」这一 R2 行为。
|
||||
// 因此只需列出 `tamabox/` 前缀即可覆盖全部图片。blog/ 是同一 R2 account 下另一个站点
|
||||
// 的内容，不在 tamabox 的备份范围。
|
||||
// imageManifestEntry 记录实际下载内容的大小和 SHA-256，以及 GET 返回的原始元数据。
|
||||
// ETag 只是服务商元数据，不能当作内容 MD5 或完整性校验的替代品。
|
||||
type imageManifestEntry struct {
|
||||
Key string `json:"key"`
|
||||
Size int64 `json:"size"`
|
||||
SHA256 string `json:"sha256"`
|
||||
ETag string `json:"etag,omitempty"`
|
||||
ContentType string `json:"content_type,omitempty"`
|
||||
CacheControl string `json:"cache_control,omitempty"`
|
||||
}
|
||||
|
||||
type backupS3API interface {
|
||||
ListObjectsV2(context.Context, *s3.ListObjectsV2Input, ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
|
||||
GetObject(context.Context, *s3.GetObjectInput, ...func(*s3.Options)) (*s3.GetObjectOutput, error)
|
||||
}
|
||||
|
||||
type backupDownloadPlan struct {
|
||||
key string
|
||||
destination string
|
||||
}
|
||||
|
||||
type downloadedImageObject struct {
|
||||
size int64
|
||||
sha256 string
|
||||
etag string
|
||||
contentType string
|
||||
cacheControl string
|
||||
}
|
||||
|
||||
// backupImages 默认使用空前缀列举配置 bucket 的全部对象，而不是假定图片都在 tamabox/ 下。
|
||||
func backupImages(ctx context.Context, outDir string) error {
|
||||
client, err := backupS3Client()
|
||||
client, err := backupS3Client(ctx)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "get R2 client for backup")
|
||||
}
|
||||
return backupImagesWithClient(ctx, client, conf.Upload.ImageBucket, outDir, "")
|
||||
}
|
||||
|
||||
imageDir := filepath.Join(outDir, "images")
|
||||
if err := os.MkdirAll(imageDir, 0o700); err != nil {
|
||||
return errors.Wrap(err, "create images directory")
|
||||
}
|
||||
|
||||
var manifest []imageManifestEntry
|
||||
var count int64
|
||||
var totalBytes int64
|
||||
|
||||
// 见函数注释：tamabox/ 前缀即覆盖全部图片（含新旧 key 格式）。
|
||||
if err := backupPrefix(ctx, client, imageDir, "tamabox/", &manifest, &count, &totalBytes); err != nil {
|
||||
// backupImagesWithClient 允许测试和隔离演练显式指定 prefix。
|
||||
// 所有分页 List、key 校验和本地路径预检都在第一次 GetObject 之前完成。
|
||||
func backupImagesWithClient(ctx context.Context, client backupS3API, bucket, outDir, prefix string) error {
|
||||
keys, err := listAllBackupKeys(ctx, client, bucket, prefix)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 写 manifest：记录 key → 大小 → md5，恢复后可用于校验完整性。
|
||||
mf, err := os.Create(filepath.Join(outDir, "images-manifest.json"))
|
||||
imageDir := filepath.Join(outDir, "images")
|
||||
plans, err := planBackupDownloads(imageDir, keys)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "create manifest file")
|
||||
return err
|
||||
}
|
||||
defer mf.Close()
|
||||
enc := json.NewEncoder(mf)
|
||||
enc.SetIndent("", " ")
|
||||
if err := enc.Encode(manifest); err != nil {
|
||||
return errors.Wrap(err, "write manifest")
|
||||
|
||||
manifest := make([]imageManifestEntry, 0, len(plans))
|
||||
var totalBytes int64
|
||||
for _, plan := range plans {
|
||||
if err := ensureBackupDestinationDirectory(imageDir, plan.destination); err != nil {
|
||||
return err
|
||||
}
|
||||
downloaded, err := downloadObject(ctx, client, bucket, plan.key, plan.destination)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifest = append(manifest, imageManifestEntry{
|
||||
Key: plan.key,
|
||||
Size: downloaded.size,
|
||||
SHA256: downloaded.sha256,
|
||||
ETag: downloaded.etag,
|
||||
ContentType: downloaded.contentType,
|
||||
CacheControl: downloaded.cacheControl,
|
||||
})
|
||||
totalBytes += downloaded.size
|
||||
}
|
||||
|
||||
manifestFile := filepath.Join(outDir, "images-manifest.json")
|
||||
if err := writeFileAtomically(manifestFile, func(w io.Writer) error {
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetIndent("", " ")
|
||||
if err := enc.Encode(manifest); err != nil {
|
||||
return errors.Wrap(err, "write manifest")
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logrus.WithFields(logrus.Fields{
|
||||
"objects": count,
|
||||
"objects": len(manifest),
|
||||
"bytes": totalBytes,
|
||||
"dir": imageDir,
|
||||
"prefix": prefix,
|
||||
}).Info("图片已备份")
|
||||
return nil
|
||||
}
|
||||
|
||||
// backupPrefix 分页列出并下载指定前缀下的全部对象。
|
||||
func backupPrefix(ctx context.Context, client *s3.Client, imageDir, prefix string, manifest *[]imageManifestEntry, count, totalBytes *int64) error {
|
||||
paginator := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(conf.Upload.ImageBucket),
|
||||
Prefix: aws.String(prefix),
|
||||
})
|
||||
for paginator.HasMorePages() {
|
||||
page, err := paginator.NextPage(ctx)
|
||||
func listAllBackupKeys(ctx context.Context, client backupS3API, bucket, prefix string) ([]string, error) {
|
||||
keys := make([]string, 0)
|
||||
var continuationToken *string
|
||||
seenTokens := make(map[string]struct{})
|
||||
|
||||
for {
|
||||
page, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucket),
|
||||
Prefix: aws.String(prefix),
|
||||
ContinuationToken: continuationToken,
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "list R2 objects prefix %q", prefix)
|
||||
return nil, errors.Wrapf(err, "list R2 objects prefix %q", prefix)
|
||||
}
|
||||
if page == nil {
|
||||
return nil, errors.New("list R2 objects returned a nil page")
|
||||
}
|
||||
for _, object := range page.Contents {
|
||||
// nil key 与空 key 一样是无效输入；统一转成空串，由完整 List 后的预检拒绝。
|
||||
keys = append(keys, aws.ToString(object.Key))
|
||||
}
|
||||
for _, obj := range page.Contents {
|
||||
if obj.Key == nil {
|
||||
continue
|
||||
}
|
||||
key := *obj.Key
|
||||
|
||||
// 落盘路径 = imageDir + "/" + key，保持对象 key 结构（含 tamabox/ 前缀），
|
||||
// 恢复时可直接用同一 key 回传 R2。
|
||||
dst := filepath.Join(imageDir, filepath.FromSlash(key))
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o700); err != nil {
|
||||
return errors.Wrapf(err, "create dir for %q", key)
|
||||
}
|
||||
if err := downloadObject(ctx, client, conf.Upload.ImageBucket, key, dst); err != nil {
|
||||
return err
|
||||
}
|
||||
if !aws.ToBool(page.IsTruncated) {
|
||||
return keys, nil
|
||||
}
|
||||
next := aws.ToString(page.NextContinuationToken)
|
||||
if next == "" {
|
||||
return nil, errors.New("truncated R2 list response has no continuation token")
|
||||
}
|
||||
if _, exists := seenTokens[next]; exists {
|
||||
return nil, errors.Errorf("R2 list continuation token %q repeated", next)
|
||||
}
|
||||
seenTokens[next] = struct{}{}
|
||||
continuationToken = aws.String(next)
|
||||
}
|
||||
}
|
||||
|
||||
size := int64(0)
|
||||
if obj.Size != nil {
|
||||
size = *obj.Size
|
||||
}
|
||||
md5 := ""
|
||||
if obj.ETag != nil {
|
||||
md5 = strings.Trim(*obj.ETag, `"`)
|
||||
}
|
||||
*manifest = append(*manifest, imageManifestEntry{Key: key, Size: size, MD5: md5})
|
||||
func planBackupDownloads(imageDir string, keys []string) ([]backupDownloadPlan, error) {
|
||||
plans := make([]backupDownloadPlan, 0, len(keys))
|
||||
files := make(map[string]string, len(keys))
|
||||
directories := make(map[string]string)
|
||||
|
||||
*count++
|
||||
*totalBytes += size
|
||||
root, err := filepath.Abs(imageDir)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "resolve images directory")
|
||||
}
|
||||
directories[canonicalLocalPath(root)] = ""
|
||||
|
||||
for _, key := range keys {
|
||||
destination, err := localImagePath(root, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
canonicalDestination := canonicalLocalPath(destination)
|
||||
if previous, exists := files[canonicalDestination]; exists {
|
||||
return nil, errors.Errorf("objects %q and %q map to the same local file", previous, key)
|
||||
}
|
||||
if previous, exists := directories[canonicalDestination]; exists {
|
||||
return nil, errors.Errorf("file/directory collision between objects %q and %q", previous, key)
|
||||
}
|
||||
|
||||
for parent := filepath.Dir(destination); pathWithinDirectory(root, parent); parent = filepath.Dir(parent) {
|
||||
canonicalParent := canonicalLocalPath(parent)
|
||||
if previous, exists := files[canonicalParent]; exists {
|
||||
return nil, errors.Errorf("file/directory collision between objects %q and %q", previous, key)
|
||||
}
|
||||
if _, exists := directories[canonicalParent]; !exists {
|
||||
directories[canonicalParent] = key
|
||||
}
|
||||
if parent == root {
|
||||
break
|
||||
}
|
||||
}
|
||||
files[canonicalDestination] = key
|
||||
plans = append(plans, backupDownloadPlan{key: key, destination: destination})
|
||||
}
|
||||
|
||||
if err := prepareBackupImageRoot(root); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, plan := range plans {
|
||||
if err := preflightBackupLocalPath(root, plan.destination); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return plans, nil
|
||||
}
|
||||
|
||||
func validateObjectKey(key string) error {
|
||||
if key == "" {
|
||||
return errors.New("R2 object key must not be empty")
|
||||
}
|
||||
if strings.ContainsRune(key, 0) {
|
||||
return errors.Errorf("R2 object key %q contains NUL", key)
|
||||
}
|
||||
if strings.Contains(key, `\`) {
|
||||
return errors.Errorf("R2 object key %q contains a backslash", key)
|
||||
}
|
||||
if strings.HasSuffix(key, "/") {
|
||||
return errors.Errorf("R2 object key %q is a directory key", key)
|
||||
}
|
||||
if strings.HasPrefix(key, "/") || filepath.IsAbs(filepath.FromSlash(key)) || filepath.VolumeName(filepath.FromSlash(key)) != "" ||
|
||||
(len(key) >= 2 && ((key[0] >= 'A' && key[0] <= 'Z') || (key[0] >= 'a' && key[0] <= 'z')) && key[1] == ':') {
|
||||
return errors.Errorf("R2 object key %q is an absolute or volume-qualified path", key)
|
||||
}
|
||||
for _, segment := range strings.Split(key, "/") {
|
||||
if segment == "" || segment == "." || segment == ".." {
|
||||
return errors.Errorf("R2 object key %q contains an empty or dot path segment", key)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// backupS3Client 构造用于备份的 R2 客户端。
|
||||
//
|
||||
// 不能直接复用 storage.GetS3Client 做 List：那个客户端把 bucket 路径拼进了 endpoint
|
||||
// （image_endpoint 形如 https://<account>.r2.cloudflarestorage.com/<bucket>），
|
||||
// 它对 GetObject/PutObject 恰好可用（R2 容错），但 ListObjectsV2 会把 URL 拼成
|
||||
// /<bucket>/<bucket>?list-type=2，R2 找不到这个 key 前缀而返回 404 NoSuchKey。
|
||||
//
|
||||
// 这里剥离 endpoint 的 bucket 后缀，得到纯 account endpoint，并启用 path-style：
|
||||
// List/Get 都用 Bucket 参数 + 完整 key（List 返回的 key 自带 bucket 名作前缀，
|
||||
// 如 tamabox/2025/12/<ts>.jpg），这是 R2 上唯一稳定可用的姿势。
|
||||
func backupS3Client() (*s3.Client, error) {
|
||||
endpoint := strings.TrimRight(conf.Upload.ImageEndpoint, "/")
|
||||
// 剥离最后一个路径段（bucket 名）。endpoint 形如
|
||||
// https://<account>.r2.cloudflarestorage.com/<bucket>，去掉 /<bucket>。
|
||||
if i := strings.LastIndex(endpoint, "/"); i > len("https://") {
|
||||
endpoint = endpoint[:i]
|
||||
func localImagePath(imageDir, key string) (string, error) {
|
||||
if err := validateObjectKey(key); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
return aws.Endpoint{
|
||||
URL: endpoint,
|
||||
HostnameImmutable: true,
|
||||
Source: aws.EndpointSourceCustom,
|
||||
}, nil
|
||||
})
|
||||
cfg, err := config.LoadDefaultConfig(context.Background(),
|
||||
config.WithEndpointResolverWithOptions(resolver),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
|
||||
conf.Upload.ImageAccessID,
|
||||
conf.Upload.ImageAccessSecret,
|
||||
"",
|
||||
)),
|
||||
config.WithRegion("auto"),
|
||||
)
|
||||
root, err := filepath.Abs(imageDir)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "load R2 config")
|
||||
return "", errors.Wrap(err, "resolve images directory")
|
||||
}
|
||||
return s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.UsePathStyle = true
|
||||
}), nil
|
||||
destination := filepath.Join(root, filepath.FromSlash(key))
|
||||
if !pathWithinDirectory(root, destination) || destination == root {
|
||||
return "", errors.Errorf("R2 object key %q escapes the images directory", key)
|
||||
}
|
||||
return destination, nil
|
||||
}
|
||||
|
||||
// downloadObject 下载单个对象到本地文件，失败时删除半成品避免留下损坏文件。
|
||||
func downloadObject(ctx context.Context, client *s3.Client, bucket, key, dst string) error {
|
||||
func pathWithinDirectory(root, candidate string) bool {
|
||||
rel, err := filepath.Rel(root, candidate)
|
||||
if err != nil || filepath.IsAbs(rel) {
|
||||
return false
|
||||
}
|
||||
return rel == "." || rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))
|
||||
}
|
||||
|
||||
func canonicalLocalPath(path string) string {
|
||||
cleaned := filepath.Clean(path)
|
||||
if runtime.GOOS == "windows" {
|
||||
return strings.ToLower(cleaned)
|
||||
}
|
||||
return cleaned
|
||||
}
|
||||
|
||||
func prepareBackupImageRoot(imageDir string) error {
|
||||
info, err := os.Lstat(imageDir)
|
||||
if os.IsNotExist(err) {
|
||||
if err := os.MkdirAll(imageDir, 0o700); err != nil {
|
||||
return errors.Wrap(err, "create images directory")
|
||||
}
|
||||
info, err = os.Lstat(imageDir)
|
||||
}
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "inspect images directory")
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
|
||||
return errors.Errorf("images path %q must be a real directory, not a symlink or file", imageDir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func preflightBackupLocalPath(imageDir, destination string) error {
|
||||
info, err := os.Lstat(destination)
|
||||
if err == nil && (info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular()) {
|
||||
return errors.Errorf("backup destination %q is not a regular file", destination)
|
||||
}
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return errors.Wrapf(err, "inspect backup destination %q", destination)
|
||||
}
|
||||
|
||||
for parent := filepath.Dir(destination); parent != imageDir; parent = filepath.Dir(parent) {
|
||||
if !pathWithinDirectory(imageDir, parent) {
|
||||
return errors.Errorf("backup destination %q escapes images directory", destination)
|
||||
}
|
||||
info, err := os.Lstat(parent)
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "inspect backup parent %q", parent)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
|
||||
return errors.Errorf("backup parent %q is not a real directory", parent)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureBackupDestinationDirectory(imageDir, destination string) error {
|
||||
root, err := filepath.Abs(imageDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
parent := filepath.Dir(destination)
|
||||
rel, err := filepath.Rel(root, parent)
|
||||
if err != nil || !pathWithinDirectory(root, parent) {
|
||||
return errors.Errorf("invalid backup destination %q", destination)
|
||||
}
|
||||
if rel == "." {
|
||||
return nil
|
||||
}
|
||||
|
||||
current := root
|
||||
for _, segment := range strings.Split(rel, string(filepath.Separator)) {
|
||||
current = filepath.Join(current, segment)
|
||||
info, err := os.Lstat(current)
|
||||
if os.IsNotExist(err) {
|
||||
if err := os.Mkdir(current, 0o700); err != nil && !os.IsExist(err) {
|
||||
return errors.Wrapf(err, "create backup directory %q", current)
|
||||
}
|
||||
info, err = os.Lstat(current)
|
||||
}
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "inspect backup directory %q", current)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
|
||||
return errors.Errorf("backup parent %q is not a real directory", current)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// backupS3Client 与上传、恢复共用 storage.NewS3Client 的 endpoint 语义：
|
||||
// endpoint 可不带路径或只带 /<bucket>，请求始终以 Bucket 参数和原始完整 key 寻址。
|
||||
// List 返回什么 key 就下载什么 key，不额外添加或剥离 bucket 前缀。
|
||||
func backupS3Client(ctx context.Context) (*s3.Client, error) {
|
||||
return storage.NewS3Client(
|
||||
ctx,
|
||||
conf.Upload.ImageEndpoint,
|
||||
conf.Upload.ImageBucket,
|
||||
conf.Upload.ImageAccessID,
|
||||
conf.Upload.ImageAccessSecret,
|
||||
)
|
||||
}
|
||||
|
||||
// downloadObject 原子下载单个对象，并返回实际落盘内容的 SHA-256 和 GET 元数据。
|
||||
// 每次尝试都有独立 60 秒 context；流中断会从头重试整个对象，最多 3 次。
|
||||
func downloadObject(ctx context.Context, client backupS3API, bucket, key, destination string) (downloadedImageObject, error) {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < imageObjectMaxAttempts; attempt++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return downloadedImageObject{}, err
|
||||
}
|
||||
attemptCtx, cancel := context.WithTimeout(ctx, imageObjectAttemptTimeout)
|
||||
downloaded, err := downloadObjectOnce(attemptCtx, client, bucket, key, destination)
|
||||
cancel()
|
||||
if err == nil {
|
||||
return downloaded, nil
|
||||
}
|
||||
lastErr = err
|
||||
if ctx.Err() != nil {
|
||||
return downloadedImageObject{}, ctx.Err()
|
||||
}
|
||||
if attempt == imageObjectMaxAttempts-1 {
|
||||
break
|
||||
}
|
||||
if err := waitForImageObjectRetry(ctx, imageObjectRetryBackoff<<attempt); err != nil {
|
||||
return downloadedImageObject{}, err
|
||||
}
|
||||
}
|
||||
return downloadedImageObject{}, errors.Wrapf(lastErr, "download %q after %d attempts", key, imageObjectMaxAttempts)
|
||||
}
|
||||
|
||||
func downloadObjectOnce(ctx context.Context, client backupS3API, bucket, key, destination string) (downloadedImageObject, error) {
|
||||
out, err := client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "get R2 object %q", key)
|
||||
return downloadedImageObject{}, errors.Wrapf(err, "get R2 object %q", key)
|
||||
}
|
||||
if out == nil || out.Body == nil {
|
||||
return downloadedImageObject{}, errors.Errorf("get R2 object %q returned an empty response body", key)
|
||||
}
|
||||
defer out.Body.Close()
|
||||
|
||||
tmp := dst + ".part"
|
||||
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "create %q", tmp)
|
||||
hash := sha256.New()
|
||||
var size int64
|
||||
writeErr := writeFileAtomically(destination, func(w io.Writer) error {
|
||||
var err error
|
||||
size, err = io.Copy(io.MultiWriter(w, hash), out.Body)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "download %q", key)
|
||||
}
|
||||
if out.ContentLength != nil && size != aws.ToInt64(out.ContentLength) {
|
||||
return errors.Wrapf(io.ErrUnexpectedEOF, "download %q: GET content length is %d, received %d", key, aws.ToInt64(out.ContentLength), size)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
_ = out.Body.Close()
|
||||
if writeErr != nil {
|
||||
return downloadedImageObject{}, writeErr
|
||||
}
|
||||
return downloadedImageObject{
|
||||
size: size,
|
||||
sha256: hex.EncodeToString(hash.Sum(nil)),
|
||||
etag: aws.ToString(out.ETag),
|
||||
contentType: aws.ToString(out.ContentType),
|
||||
cacheControl: aws.ToString(out.CacheControl),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func waitForImageObjectRetry(ctx context.Context, delay time.Duration) error {
|
||||
timer := time.NewTimer(delay)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
if _, err := io.Copy(f, out.Body); err != nil {
|
||||
f.Close()
|
||||
os.Remove(tmp)
|
||||
return errors.Wrapf(err, "download %q", key)
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
os.Remove(tmp)
|
||||
return errors.Wrapf(err, "close %q", tmp)
|
||||
}
|
||||
if err := os.Rename(tmp, dst); err != nil {
|
||||
os.Remove(tmp)
|
||||
return errors.Wrapf(err, "finalize %q", dst)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user