生成器可检测与管理远端主题素材（列表 / 差异 / 覆盖 / 删除）
「传上去」之后还得知道「传得对不对」：同一 slug 传过三次、换过 slug 旧图还在、 重新导出后远端仍是旧版，这些在页面上都看不出来。 - serve.py：s3_request 支持 query；新增 ListObjectsV2（含分页）与 DeleteObject， 端点 /api/s3/list、/api/s3/delete。 - index.html：远端素材清单按对象键名跟本地比 —— 一致 / 旧图（同名但大小不同）/ 多余（当前主题已不引用）/ 缺失。「补齐差异」只传缺的与旧图，「删除多余」批量删， 单行可 ↻ 覆盖或 ✕ 删除；删除走 confirm 二次确认。改过 slug 会提示重新检测。 - 修：list 返回完整键而 upload/delete 收相对键，不剥前缀会删到不存在的键 （S3 回 204，看起来删成功其实没删）。加 relKey() 统一。 - check_sigv4.py 补带 query 的用例（ListObjectsV2 三参数、乱序 token）→ 8/8； selftest_upload.py 补列表/分页/幂等删除 → 20 项全过。
This commit is contained in:
tamakyi
+119
-7
@@ -70,7 +70,26 @@ def _quote_key(key: str) -> str:
|
||||
return urllib.parse.quote(key, safe="/-_.~")
|
||||
|
||||
|
||||
def sign_v4(cfg, method, canonical_path, payload_hash, amz_date, host):
|
||||
def _quote_query(s):
|
||||
"""query 的编码规则比 path 更严：`/` 也要编码成 %2F，只有 A-Za-z0-9-_.~ 原样保留。"""
|
||||
return urllib.parse.quote(str(s), safe="-_.~")
|
||||
|
||||
|
||||
def canonical_query(params):
|
||||
"""按 AWS 规则拼 canonical query：先按键排序，同名多值再按值排序，逐项编码。
|
||||
|
||||
ListObjectsV2 的 `list-type` / `prefix` / `continuation-token` 全走这里。
|
||||
顺序错、或漏了编码都是 SignatureDoesNotMatch —— 而 S3 不会告诉你差在哪。
|
||||
"""
|
||||
pairs = []
|
||||
for k, v in (params or {}).items():
|
||||
for one in (v if isinstance(v, (list, tuple)) else [v]):
|
||||
pairs.append((_quote_query(k), _quote_query(one)))
|
||||
pairs.sort()
|
||||
return "&".join("%s=%s" % kv for kv in pairs)
|
||||
|
||||
|
||||
def sign_v4(cfg, method, canonical_path, payload_hash, amz_date, host, query=""):
|
||||
region = cfg.region
|
||||
date = amz_date[:8]
|
||||
signed_headers = "host;x-amz-content-sha256;x-amz-date"
|
||||
@@ -82,7 +101,7 @@ def sign_v4(cfg, method, canonical_path, payload_hash, amz_date, host):
|
||||
# CanonicalHeaders 自身每行以 \n 结尾，与 SignedHeaders 之间因此有一个空行，
|
||||
# 这是规范要求的形态，不是笔误。
|
||||
canonical_request = "\n".join([
|
||||
method, canonical_path, "", canonical_headers, signed_headers, payload_hash,
|
||||
method, canonical_path, query or "", canonical_headers, signed_headers, payload_hash,
|
||||
])
|
||||
scope = date + "/" + region + "/s3/aws4_request"
|
||||
string_to_sign = "\n".join([
|
||||
@@ -178,8 +197,9 @@ def _now():
|
||||
return datetime.datetime.now(datetime.timezone.utc)
|
||||
|
||||
|
||||
def s3_request(cfg, method, key="", body=b"", content_type=None, cache_control=None, timeout=25):
|
||||
"""发一个签名请求。key 为空表示桶级操作（HEAD bucket）。
|
||||
def s3_request(cfg, method, key="", body=b"", content_type=None, cache_control=None,
|
||||
query=None, timeout=25):
|
||||
"""发一个签名请求。key 为空表示桶级操作（HEAD bucket / ListObjectsV2）。
|
||||
|
||||
返回 (status, headers, body)；4xx/5xx 不抛异常，交由调用方判断 —— 测试连接
|
||||
需要把「403 是权限问题」和「连不上是网络问题」区分开讲。
|
||||
@@ -188,11 +208,13 @@ def s3_request(cfg, method, key="", body=b"", content_type=None, cache_control=N
|
||||
path = cfg.object_path(key)
|
||||
else:
|
||||
path = ("/" + cfg.bucket) if cfg.path_style else "/"
|
||||
qs = canonical_query(query)
|
||||
payload_hash = _sha256(body) if body else EMPTY_SHA256
|
||||
amz_date = _now().strftime("%Y%m%dT%H%M%SZ")
|
||||
auth = sign_v4(cfg, method, path, payload_hash, amz_date, cfg.host)
|
||||
auth = sign_v4(cfg, method, path, payload_hash, amz_date, cfg.host, qs)
|
||||
|
||||
req = urllib.request.Request(cfg.url(path), data=body if body else None, method=method)
|
||||
url = cfg.url(path) + (("?" + qs) if qs else "")
|
||||
req = urllib.request.Request(url, data=body if body else None, method=method)
|
||||
req.add_header("x-amz-content-sha256", payload_hash)
|
||||
req.add_header("x-amz-date", amz_date)
|
||||
req.add_header("Authorization", auth)
|
||||
@@ -317,6 +339,92 @@ def upload_files(cfg, files):
|
||||
return {"ok": not failed, "files": results, "failed": len(failed), "total": len(results)}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 远端清单 / 删除
|
||||
# ---------------------------------------------------------------------------
|
||||
def _local(tag):
|
||||
"""去掉 XML 命名空间。各家 S3 兼容实现都带 `http://s3.amazonaws.com/doc/2006-03-01/`，
|
||||
个别（老 MinIO / 某些网关）又不带 —— 只比 localname 两边都能认。"""
|
||||
return tag.rsplit("}", 1)[-1]
|
||||
|
||||
|
||||
def parse_list_xml(body):
|
||||
"""解析 ListObjectsV2 响应。返回 (对象数组, 下一页 token 或 None)。"""
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
root = ET.fromstring(body)
|
||||
objects, token, truncated = [], None, False
|
||||
for node in root:
|
||||
name = _local(node.tag)
|
||||
if name == "Contents":
|
||||
row = {}
|
||||
for f in node:
|
||||
row[_local(f.tag)] = f.text
|
||||
objects.append({
|
||||
"key": row.get("Key") or "",
|
||||
"size": int(row.get("Size") or 0),
|
||||
"modified": row.get("LastModified") or "",
|
||||
"etag": (row.get("ETag") or "").strip('"'),
|
||||
})
|
||||
elif name == "NextContinuationToken":
|
||||
token = node.text
|
||||
elif name == "IsTruncated":
|
||||
truncated = (node.text or "").strip().lower() == "true"
|
||||
return objects, (token if truncated else None)
|
||||
|
||||
|
||||
def list_objects(cfg, prefix="", max_pages=20, max_keys=1000):
|
||||
"""列 `<cfg.prefix>/<prefix>` 下所有对象。分页上限只是防呆，正常一个主题不到 100 个。"""
|
||||
key_prefix = full_key(cfg, prefix) if str(prefix or "").strip() else (cfg.prefix + "/" if cfg.prefix else "")
|
||||
|
||||
objects, token, pages = [], None, 0
|
||||
while pages < max_pages:
|
||||
pages += 1
|
||||
query = {"list-type": "2", "max-keys": str(max_keys)}
|
||||
if key_prefix:
|
||||
query["prefix"] = key_prefix
|
||||
if token:
|
||||
query["continuation-token"] = token
|
||||
status, _, body = s3_request(cfg, "GET", query=query)
|
||||
if status >= 400:
|
||||
code, msg = _parse_s3_error(body)
|
||||
raise S3Error("ListObjects", status, code or "HTTP%d" % status, msg)
|
||||
page, token = parse_list_xml(body)
|
||||
objects.extend(page)
|
||||
if not token:
|
||||
break
|
||||
return {"ok": True, "prefix": key_prefix, "objects": objects, "truncated": bool(token)}
|
||||
|
||||
|
||||
def delete_objects(cfg, keys):
|
||||
"""逐个 DeleteObject。
|
||||
|
||||
没用批量 POST ?delete：那要额外算 Content-MD5、拼 XML body，而一次最多几十个键，
|
||||
省下的往返抵不上多一条签名路径的风险。S3 的 DELETE 本身幂等（不存在的键也回 204）。
|
||||
"""
|
||||
if not keys:
|
||||
raise ValueError("没有要删除的对象")
|
||||
if len(keys) > 500:
|
||||
raise ValueError("一次最多删 500 个对象")
|
||||
|
||||
results = []
|
||||
for k in keys:
|
||||
try:
|
||||
key = full_key(cfg, k)
|
||||
status, _, body = s3_request(cfg, "DELETE", key)
|
||||
if status >= 400 and status != 404:
|
||||
code, msg = _parse_s3_error(body)
|
||||
results.append({"key": key, "ok": False, "status": status,
|
||||
"code": code or "HTTP%d" % status, "error": msg})
|
||||
else:
|
||||
results.append({"key": key, "ok": True})
|
||||
except Exception as e:
|
||||
results.append({"key": str(k), "ok": False, "error": str(e)})
|
||||
|
||||
failed = [r for r in results if not r["ok"]]
|
||||
return {"ok": not failed, "files": results, "failed": len(failed), "total": len(results)}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTTP 层
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -359,7 +467,7 @@ class Handler(http.server.SimpleHTTPRequestHandler):
|
||||
|
||||
def do_POST(self):
|
||||
route = self.path.split("?")[0]
|
||||
if route not in ("/api/s3/test", "/api/s3/upload"):
|
||||
if route not in ("/api/s3/test", "/api/s3/upload", "/api/s3/list", "/api/s3/delete"):
|
||||
return self._json({"ok": False, "error": "未知接口 %s" % route}, 404)
|
||||
try:
|
||||
payload = self._read_json()
|
||||
@@ -374,6 +482,10 @@ class Handler(http.server.SimpleHTTPRequestHandler):
|
||||
try:
|
||||
if route == "/api/s3/test":
|
||||
return self._json(test_connection(cfg))
|
||||
if route == "/api/s3/list":
|
||||
return self._json(list_objects(cfg, payload.get("prefix") or ""))
|
||||
if route == "/api/s3/delete":
|
||||
return self._json(delete_objects(cfg, payload.get("keys") or []))
|
||||
return self._json(upload_files(cfg, payload.get("files") or []))
|
||||
except S3Error as e:
|
||||
return self._json({
|
||||
|
||||
Reference in New Issue
Block a user