tamakyi
9c961b3c05
素材位置改成两选一（本地随二进制 / 自行填 Endpoint 的 S3 兼容存储）， R2、MinIO、B2、OSS-COS 的 S3 端点都只差 Endpoint/Region/路径风格。 - serve.py：手写 SigV4（PUT/GET/DELETE/HEAD），新增 /api/s3/test 与 /api/s3/upload。测试连接依次验连通、写权限、读回一致、匿名可读， 失败把 S3 的 <Code> 翻成可执行的建议；上传带 immutable 缓存头。 - index.html：新增「素材托管」面板（模式切换 + 7 个字段 + 测试/上传按钮）， 上传集合与导出共用 usedAssetIds()；成功后自动回填「素材前缀」。 密钥只进 sessionStorage 与内存，不进草稿/工程 json/导出 zip。 - 修：usedAssetIds 返回 Set，误用 length/map 导致「还没有素材」。 - 修：字段填完 700ms 内刷新会丢配置 → 失焦与模式切换时立即落盘。 - 自检：check_sigv4.py 与 aws-sdk-go-v2 signer 逐字节对拍（6/6）， selftest_upload.py 起假 S3 跑通测试连接→上传→读回（16 项）， mock_s3.py 供手动点面板用。
423 lines
17 KiB
Python
423 lines
17 KiB
Python
#!/usr/bin/env python3
|
||
"""主题生成器的本地预览服务 + 素材上传代理。
|
||
|
||
为什么必须起服务：生成器不再自己画一个"仿制页面"，而是把项目真实渲染出来的
|
||
页面（outputs/preview-*.html）放进 iframe 当画板，再往里注入正在编辑的主题 CSS。
|
||
file:// 下浏览器不允许父页面访问 iframe 的 DOM（即使同目录），预览就只能退回示意图。
|
||
起一个本地 HTTP 服务后父子同源，拖拽、注入、实时改令牌都能正常工作。
|
||
|
||
为什么上传也走这个服务：浏览器直连对象存储要配 CORS、还要把密钥交给页面里的
|
||
fetch，各家 S3 兼容实现的 CORS 配置界面又都不一样。放到本机 Python 里代发，
|
||
生成器只跟 127.0.0.1 说话：不碰 CORS，密钥不落盘、不进工程文件、不进导出 zip。
|
||
|
||
用法（在项目根目录执行任意一种）：
|
||
|
||
python dev/theme-builder/serve.py
|
||
python -m http.server 8777 # 只有预览，没有上传能力
|
||
|
||
然后打开 http://localhost:8777/dev/theme-builder/
|
||
|
||
这个脚本只读文件，不写任何东西，也不改项目配置。
|
||
"""
|
||
|
||
import argparse
|
||
import base64
|
||
import datetime
|
||
import functools
|
||
import hashlib
|
||
import hmac
|
||
import http.server
|
||
import json
|
||
import os
|
||
import re
|
||
import socketserver
|
||
import sys
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
|
||
# dev/theme-builder/serve.py → 上两级就是项目根
|
||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||
|
||
# 单次请求体上限：一次上传十几张 1600px 的 WebP，base64 后约 1.4 倍体积，
|
||
# 64MB 足够，同时挡住误连到公网时的内存放大。
|
||
MAX_BODY_BYTES = 64 * 1024 * 1024
|
||
|
||
EMPTY_SHA256 = hashlib.sha256(b"").hexdigest()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# SigV4 签名
|
||
# ---------------------------------------------------------------------------
|
||
# 不用 boto3：这个脚本的定位是「零依赖、拷走就能跑」，装一个 SDK 只为签四个请求
|
||
# 不划算。下面是 AWS 文档里 SigV4 for S3 的最小实现，覆盖 PUT / GET / DELETE / HEAD。
|
||
|
||
def _hmac(key: bytes, msg: str) -> bytes:
|
||
return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest()
|
||
|
||
|
||
def _sha256(data: bytes) -> str:
|
||
return hashlib.sha256(data).hexdigest()
|
||
|
||
|
||
def _quote_key(key: str) -> str:
|
||
"""按 AWS 规则编码对象键：保留 `/`，其余非 unreserved 字符全部百分号编码。
|
||
|
||
用 quote(safe='/-_.~') 而不是逐段 quote：两者等价，但后者容易漏掉空段
|
||
（`a//b`）导致的路径塌缩。S3 的签名 URI 必须与请求行完全一致，差一个字符
|
||
就是 SignatureDoesNotMatch。
|
||
"""
|
||
return urllib.parse.quote(key, safe="/-_.~")
|
||
|
||
|
||
def sign_v4(cfg, method, canonical_path, payload_hash, amz_date, host):
|
||
region = cfg.region
|
||
date = amz_date[:8]
|
||
signed_headers = "host;x-amz-content-sha256;x-amz-date"
|
||
canonical_headers = (
|
||
"host:" + host + "\n"
|
||
"x-amz-content-sha256:" + payload_hash + "\n"
|
||
"x-amz-date:" + amz_date + "\n"
|
||
)
|
||
# CanonicalHeaders 自身每行以 \n 结尾，与 SignedHeaders 之间因此有一个空行，
|
||
# 这是规范要求的形态，不是笔误。
|
||
canonical_request = "\n".join([
|
||
method, canonical_path, "", canonical_headers, signed_headers, payload_hash,
|
||
])
|
||
scope = date + "/" + region + "/s3/aws4_request"
|
||
string_to_sign = "\n".join([
|
||
"AWS4-HMAC-SHA256", amz_date, scope, _sha256(canonical_request.encode("utf-8")),
|
||
])
|
||
|
||
k = _hmac(("AWS4" + cfg.secret_key).encode("utf-8"), date)
|
||
k = _hmac(k, region)
|
||
k = _hmac(k, "s3")
|
||
k = _hmac(k, "aws4_request")
|
||
signature = hmac.new(k, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest()
|
||
|
||
return (
|
||
"AWS4-HMAC-SHA256 Credential=" + cfg.access_key + "/" + scope
|
||
+ ", SignedHeaders=" + signed_headers + ", Signature=" + signature
|
||
)
|
||
|
||
|
||
class S3Error(Exception):
|
||
def __init__(self, step, status, code, message):
|
||
Exception.__init__(self, message or code or ("HTTP %s" % status))
|
||
self.step = step
|
||
self.status = status
|
||
self.code = code
|
||
self.message = message or ""
|
||
|
||
|
||
# 常见错误 → 人话。签名失败是最常见的，原因几乎总是密钥/region/时钟三者之一。
|
||
S3_HINTS = {
|
||
"SignatureDoesNotMatch": "签名不匹配：核对 Secret Key 是否复制完整、Region 是否与桶一致（R2 填 auto），以及本机时间偏差是否超过 15 分钟。",
|
||
"InvalidAccessKeyId": "Access Key 无效：核对 ID，或该密钥没有这个桶的权限。",
|
||
"AccessDenied": "权限不足：这个密钥没有写权限，或桶策略禁止了该操作。",
|
||
"NoSuchBucket": "桶不存在：核对桶名与 Endpoint 是否属于同一账号/区域。",
|
||
"PermanentRedirect": "Endpoint 与桶所在区域不匹配：换对该区域的 Endpoint（或改用虚拟主机风格）。",
|
||
"AuthorizationHeaderMalformed": "Region 写错了：填桶实际所在区域（R2 用 auto，MinIO 常为 us-east-1）。",
|
||
"InvalidBucketName": "桶名不合法：S3 桶名只允许小写字母、数字、点和中划线。",
|
||
"RequestTimeTooSkewed": "本机时间与服务器偏差过大：校准系统时间后重试。",
|
||
}
|
||
|
||
|
||
def _parse_s3_error(body):
|
||
text = body[:2000].decode("utf-8", "replace")
|
||
code = re.search(r"<Code>([^<]+)</Code>", text)
|
||
msg = re.search(r"<Message>([^<]+)</Message>", text)
|
||
return (code.group(1).strip() if code else ""), (msg.group(1).strip() if msg else text.strip()[:200])
|
||
|
||
|
||
class S3Config:
|
||
"""一次上传用到的全部参数。只活在内存里，不落盘。"""
|
||
|
||
def __init__(self, d):
|
||
ep = str(d.get("endpoint") or "").strip()
|
||
if not ep:
|
||
raise ValueError("Endpoint 没填")
|
||
if "://" not in ep:
|
||
ep = "https://" + ep
|
||
u = urllib.parse.urlsplit(ep)
|
||
if u.scheme not in ("http", "https"):
|
||
raise ValueError("Endpoint 只支持 http / https")
|
||
if not u.hostname:
|
||
raise ValueError("Endpoint 里没有主机名")
|
||
# 路径部分一律忽略：Endpoint 只表达「服务在哪」，桶和键由字段决定。
|
||
self.scheme = u.scheme
|
||
self.port = u.port
|
||
self.host_only = u.hostname
|
||
self.region = str(d.get("region") or "").strip() or "us-east-1"
|
||
self.bucket = str(d.get("bucket") or "").strip()
|
||
if not self.bucket:
|
||
raise ValueError("桶名没填")
|
||
self.prefix = str(d.get("prefix") or "").strip().strip("/")
|
||
self.access_key = str(d.get("accessKey") or "").strip()
|
||
self.secret_key = str(d.get("secretKey") or "").strip()
|
||
if not self.access_key or not self.secret_key:
|
||
raise ValueError("Access Key / Secret Key 没填")
|
||
self.path_style = bool(d.get("pathStyle", True))
|
||
|
||
@property
|
||
def host(self):
|
||
if self.path_style:
|
||
return self.host_only if not self.port else "%s:%d" % (self.host_only, self.port)
|
||
return "%s.%s" % (self.bucket, self.host_only) + (":%d" % self.port if self.port else "")
|
||
|
||
def object_path(self, key):
|
||
"""S3 的签名 URI 必须与请求行逐字一致，所以两者共用这一个函数。"""
|
||
path = ("/" + self.bucket + "/" + key) if self.path_style else ("/" + key)
|
||
return _quote_key(path)
|
||
|
||
def url(self, path):
|
||
return "%s://%s%s" % (self.scheme, self.host, path)
|
||
|
||
|
||
def _now():
|
||
return datetime.datetime.now(datetime.timezone.utc)
|
||
|
||
|
||
def s3_request(cfg, method, key="", body=b"", content_type=None, cache_control=None, timeout=25):
|
||
"""发一个签名请求。key 为空表示桶级操作（HEAD bucket）。
|
||
|
||
返回 (status, headers, body)；4xx/5xx 不抛异常，交由调用方判断 —— 测试连接
|
||
需要把「403 是权限问题」和「连不上是网络问题」区分开讲。
|
||
"""
|
||
if key:
|
||
path = cfg.object_path(key)
|
||
else:
|
||
path = ("/" + cfg.bucket) if cfg.path_style else "/"
|
||
payload_hash = _sha256(body) if body else EMPTY_SHA256
|
||
amz_date = _now().strftime("%Y%m%dT%H%M%SZ")
|
||
auth = sign_v4(cfg, method, path, payload_hash, amz_date, cfg.host)
|
||
|
||
req = urllib.request.Request(cfg.url(path), data=body if body else None, method=method)
|
||
req.add_header("x-amz-content-sha256", payload_hash)
|
||
req.add_header("x-amz-date", amz_date)
|
||
req.add_header("Authorization", auth)
|
||
if content_type:
|
||
req.add_header("Content-Type", content_type)
|
||
if cache_control:
|
||
req.add_header("Cache-Control", cache_control)
|
||
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||
return r.status, dict(r.headers), r.read()
|
||
except urllib.error.HTTPError as e:
|
||
return e.code, dict(e.headers), e.read()
|
||
except Exception as e: # URLError / timeout / SSL 错误
|
||
raise S3Error("connect", 0, "NetworkError", "%s（%s）" % (e, cfg.host))
|
||
|
||
|
||
def full_key(cfg, key):
|
||
key = str(key or "").strip().lstrip("/")
|
||
if not key:
|
||
raise ValueError("对象键为空")
|
||
if ".." in key.split("/") or "\\" in key:
|
||
raise ValueError("对象键不能包含 .. 或反斜杠：%s" % key)
|
||
return (cfg.prefix + "/" + key) if cfg.prefix else key
|
||
|
||
|
||
def probe_key(cfg):
|
||
stamp = _now().strftime("%Y%m%dT%H%M%S")
|
||
return full_key(cfg, ".tb-probe-%s.txt" % stamp)
|
||
|
||
|
||
def _public_read(url, timeout=10):
|
||
"""匿名 GET 一次，判断「传上去的图能不能直接被浏览器读到」。
|
||
|
||
这一步不是必须的（很多人会用自定义域/CDN），但它能把最常见的坑当场暴露：
|
||
桶是私有的，上传全成功、页面里全是 403。
|
||
"""
|
||
try:
|
||
with urllib.request.urlopen(url, timeout=timeout) as r:
|
||
return r.status == 200
|
||
except urllib.error.HTTPError as e:
|
||
return e.code == 200
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def test_connection(cfg):
|
||
steps = []
|
||
|
||
status, _, body = s3_request(cfg, "HEAD")
|
||
if status >= 400:
|
||
code, msg = _parse_s3_error(body)
|
||
if not code:
|
||
# HEAD 响应按规范没有 body，S3 的错误码拿不到。只能按状态码归因，
|
||
# 否则前端只能显示一句「HTTP 403」，用户完全不知道该改哪。
|
||
code = {403: "AccessDenied", 404: "NoSuchBucket",
|
||
301: "PermanentRedirect", 307: "PermanentRedirect",
|
||
400: "BadRequest"}.get(status, "HTTP%d" % status)
|
||
msg = "桶级探测失败（HEAD 无响应体，按状态码判断）"
|
||
raise S3Error("HeadBucket", status, code, msg)
|
||
steps.append({"name": "连接与凭据", "ok": True, "detail": "桶可访问（HEAD %s）" % cfg.bucket})
|
||
|
||
key = probe_key(cfg)
|
||
payload = b"TamaBox theme builder probe\n"
|
||
status, _, body = s3_request(cfg, "PUT", key, payload, "text/plain; charset=utf-8", "no-store")
|
||
if status >= 400:
|
||
code, msg = _parse_s3_error(body)
|
||
raise S3Error("PutObject", status, code, msg)
|
||
steps.append({"name": "写入权限", "ok": True, "detail": "已写入探针 %s" % key})
|
||
|
||
status, _, body = s3_request(cfg, "GET", key)
|
||
ok = status == 200 and body == payload
|
||
steps.append({
|
||
"name": "读回校验", "ok": ok,
|
||
"detail": "读回一致" if ok else "读回失败（HTTP %s）" % status,
|
||
})
|
||
|
||
public = _public_read(cfg.url(cfg.object_path(key)))
|
||
steps.append({
|
||
"name": "匿名可读", "ok": True,
|
||
"detail": "该路径可直接匿名访问" if public
|
||
else "匿名访问被拒 —— 桶没开公开读时正常，靠自定义域或 CDN 分发即可",
|
||
"warn": not public,
|
||
})
|
||
|
||
s3_request(cfg, "DELETE", key)
|
||
return {"ok": True, "steps": steps, "key": key}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 上传
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def upload_files(cfg, files):
|
||
if not files:
|
||
raise ValueError("没有要上传的素材")
|
||
if len(files) > 200:
|
||
raise ValueError("一次最多 200 个文件")
|
||
|
||
results = []
|
||
for f in files:
|
||
name = str(f.get("name") or f.get("key") or "?")
|
||
try:
|
||
key = full_key(cfg, f.get("key") or f.get("name"))
|
||
data = base64.b64decode(f.get("data") or "", validate=False)
|
||
if not data:
|
||
raise ValueError("内容为空")
|
||
ctype = str(f.get("contentType") or "application/octet-stream")
|
||
# 素材键里带内容版本（导出时由 id 决定），内容不会原地变化 → immutable。
|
||
status, _, body = s3_request(
|
||
cfg, "PUT", key, data, ctype, "public, max-age=31536000, immutable")
|
||
if status >= 400:
|
||
code, msg = _parse_s3_error(body)
|
||
results.append({"key": key, "name": name, "ok": False,
|
||
"status": status, "code": code, "error": msg})
|
||
else:
|
||
results.append({"key": key, "name": name, "ok": True, "bytes": len(data)})
|
||
except Exception as e:
|
||
results.append({"key": name, "name": name, "ok": False, "error": str(e)})
|
||
|
||
failed = [r for r in results if not r["ok"]]
|
||
return {"ok": not failed, "files": results, "failed": len(failed), "total": len(results)}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# HTTP 层
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class Handler(http.server.SimpleHTTPRequestHandler):
|
||
server_version = "TamaBoxThemeBuilder/1"
|
||
|
||
def end_headers(self):
|
||
# 预览页是刚由 go test 生成的，禁止缓存，否则改了 CSS 看不到变化
|
||
self.send_header("Cache-Control", "no-store, must-revalidate")
|
||
super().end_headers()
|
||
|
||
def log_message(self, fmt, *args):
|
||
# 只报 4xx/5xx，正常请求不刷屏
|
||
status = args[1] if len(args) > 1 else ""
|
||
if str(status).startswith(("4", "5")):
|
||
sys.stderr.write(" %s\n" % (fmt % args))
|
||
|
||
def _json(self, payload, status=200):
|
||
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||
self.send_response(status)
|
||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||
self.send_header("Content-Length", str(len(data)))
|
||
self.end_headers()
|
||
self.wfile.write(data)
|
||
|
||
def _read_json(self):
|
||
n = int(self.headers.get("Content-Length") or 0)
|
||
if n <= 0:
|
||
raise ValueError("请求体为空")
|
||
if n > MAX_BODY_BYTES:
|
||
raise ValueError("请求体过大（%d 字节），一次别传太多素材" % n)
|
||
return json.loads(self.rfile.read(n).decode("utf-8"))
|
||
|
||
def do_GET(self):
|
||
if self.path.split("?")[0] == "/api/s3/ping":
|
||
# 生成器靠它判断「当前是不是 serve.py 起的」——python -m http.server 没有这个端点。
|
||
return self._json({"ok": True, "upload": True})
|
||
return super().do_GET()
|
||
|
||
def do_POST(self):
|
||
route = self.path.split("?")[0]
|
||
if route not in ("/api/s3/test", "/api/s3/upload"):
|
||
return self._json({"ok": False, "error": "未知接口 %s" % route}, 404)
|
||
try:
|
||
payload = self._read_json()
|
||
except Exception as e:
|
||
return self._json({"ok": False, "error": "请求解析失败：%s" % e})
|
||
|
||
try:
|
||
cfg = S3Config(payload.get("config") or {})
|
||
except ValueError as e:
|
||
return self._json({"ok": False, "error": str(e)})
|
||
|
||
try:
|
||
if route == "/api/s3/test":
|
||
return self._json(test_connection(cfg))
|
||
return self._json(upload_files(cfg, payload.get("files") or []))
|
||
except S3Error as e:
|
||
return self._json({
|
||
"ok": False, "step": e.step, "status": e.status, "code": e.code,
|
||
"error": e.message, "hint": S3_HINTS.get(e.code, ""),
|
||
})
|
||
except ValueError as e:
|
||
return self._json({"ok": False, "error": str(e)})
|
||
except Exception as e: # 兜底：别把本地服务弄崩，把异常原样告诉页面
|
||
return self._json({"ok": False, "error": "%s: %s" % (type(e).__name__, e)})
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser(description="TamaBox 主题生成器的本地预览服务")
|
||
ap.add_argument("--port", type=int, default=8777, help="端口，默认 8777")
|
||
ap.add_argument("--host", default="127.0.0.1", help="监听地址，默认 127.0.0.1")
|
||
args = ap.parse_args()
|
||
|
||
if not os.path.isdir(os.path.join(PROJECT_ROOT, "outputs")):
|
||
sys.stderr.write(
|
||
"还没生成预览快照。先在项目根执行：\n"
|
||
" TEST_PREVIEW=1 go test ./templates -run TestGenPreview\n"
|
||
)
|
||
|
||
if args.host not in ("127.0.0.1", "localhost", "::1"):
|
||
sys.stderr.write(
|
||
"⚠️ 监听在 %s：上传接口没有鉴权，会代任何能访问到它的人向对象存储写文件。\n"
|
||
" 只在完全可信的内网这么做。\n" % args.host
|
||
)
|
||
|
||
handler = functools.partial(Handler, directory=PROJECT_ROOT)
|
||
socketserver.TCPServer.allow_reuse_address = True
|
||
with socketserver.TCPServer((args.host, args.port), handler) as httpd:
|
||
url = "http://%s:%d/dev/theme-builder/" % (args.host, args.port)
|
||
print("项目根：%s" % PROJECT_ROOT)
|
||
print("生成器：%s" % url)
|
||
print("上传接口：/api/s3/test /api/s3/upload（密钥只在内存，不落盘）")
|
||
print("Ctrl+C 停止")
|
||
try:
|
||
httpd.serve_forever()
|
||
except KeyboardInterrupt:
|
||
print("\n已停止")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|