feat: 数据库备份支持S3兼容对象存储(SigV4纯标准库实现,无配置时兼容原本地备份)

This commit is contained in:
gouki
2026-08-05 11:57:17 +00:00
parent f1cdcf716c
commit 7f1268f508
3 changed files with 179 additions and 6 deletions
+28
View File
@@ -29,3 +29,31 @@ SMTP_TO=
# ---- 续费提醒阈值(天)---- # ---- 续费提醒阈值(天)----
RENEWAL_THRESHOLD_DAYS=30 RENEWAL_THRESHOLD_DAYS=30
# ---- 监控数据保留策略(自动清理,天数)----
METRICS_RETENTION_DAYS=30
SECURITY_RETENTION_DAYS=90
EVENT_LOG_RETENTION_DAYS=180
# 自动清理间隔(小时),0 表示禁用后台自动清理
CLEANUP_INTERVAL_HOURS=24
# ---- S3 兼容对象存储备份(可选,未配置则仅本地备份)----
# 备份文件通过 AWS SigV4 签名上传,兼容 Cloudflare R2 / MinIO / 阿里云 OSS / 腾讯云 COS 等
# R2 示例:S3_ENDPOINT_URL=https://<account-id>.r2.cloudflarestorage.comS3_REGION=auto
# OSS 示例:S3_ENDPOINT_URL=https://oss-cn-hangzhou.aliyuncs.comS3_REGION=oss-cn-hangzhou
# COS 示例:S3_ENDPOINT_URL=https://cos.ap-shanghai.myqcloud.comS3_REGION=ap-shanghai
S3_ENDPOINT_URL=
S3_ACCESS_KEY=
S3_SECRET_KEY=
S3_BUCKET=
# 远端对象前缀(默认 vps-manager,对象结构:<prefix>/assets|metrics/<文件名>
S3_PREFIX=vps-manager
# 区域(R2 为 autoMinIO 随意,OSS/COS 填各自区域)
S3_REGION=auto
# ---- Agent 上报频率限制 ----
# 同一资产两次上报的最小间隔(秒),0 表示不限制
AGENT_REPORT_MIN_INTERVAL=30
# ---- CORS 允许来源(逗号分隔)----
CORS_ORIGINS=http://127.0.0.1:8000,http://localhost:8000,https://dify.taile5765c.ts.net
+145 -6
View File
@@ -1,9 +1,14 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# 数据库分库备份脚本 # 数据库分库备份脚本(本地 + S3 兼容对象存储)
# 用法:backup.sh [assets|metrics|all] # 用法:backup.sh [assets|metrics|all]
# assets -> 备份 assets.db,保留 14 份(建议每日) # assets -> 备份 assets.db,保留 14 份(建议每日)
# metrics -> 备份 metrics.db,保留 4 份(建议每周) # metrics -> 备份 metrics.db,保留 4 份(建议每周)
# all -> 两者都备份 # all -> 两者都备份
#
# S3 上传(可选):配置 .env 中 S3_* 项后,备份文件自动上传到
# 任意 S3 兼容对象存储(Cloudflare R2 / MinIO / 阿里云 OSS / 腾讯云 COS 等),
# 远端保留份数与本地一致。未配置 S3 时仅本地备份(向后兼容)。
# 实现为纯 Python 标准库(AWS SigV4 签名),零额外依赖。
set -euo pipefail set -euo pipefail
APP_DIR="${APP_DIR:-/opt/vps-manager}" APP_DIR="${APP_DIR:-/opt/vps-manager}"
@@ -12,9 +17,16 @@ cd "$APP_DIR"
TARGET="${1:-all}" TARGET="${1:-all}"
python3 - "$TARGET" <<'PYEOF' python3 - "$TARGET" <<'PYEOF'
import hashlib
import hmac
import os
import sqlite3 import sqlite3
import sys import sys
from datetime import datetime import urllib.error
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
target = sys.argv[1] target = sys.argv[1]
@@ -22,13 +34,14 @@ data_dir = Path("data")
backup_root = data_dir / "backups" backup_root = data_dir / "backups"
ts = datetime.now().strftime("%Y%m%d_%H%M%S") ts = datetime.now().strftime("%Y%m%d_%H%M%S")
# ---------------- 本地备份 ----------------
def backup(db_name: str, keep: int) -> None: def backup(db_name: str, keep: int) -> Path | None:
src = data_dir / db_name src = data_dir / db_name
stem = db_name.replace(".db", "") stem = db_name.replace(".db", "")
if not src.exists(): if not src.exists():
print(f"[skip] {db_name} 不存在") print(f"[skip] {db_name} 不存在")
return return None
out_dir = backup_root / stem out_dir = backup_root / stem
out_dir.mkdir(parents=True, exist_ok=True) out_dir.mkdir(parents=True, exist_ok=True)
dst = out_dir / f"{stem}_{ts}.db" dst = out_dir / f"{stem}_{ts}.db"
@@ -47,10 +60,136 @@ def backup(db_name: str, keep: int) -> None:
for old in files[keep:]: for old in files[keep:]:
old.unlink() old.unlink()
print(f"[ok] {db_name} -> {dst}(保留 {keep} 份)") print(f"[ok] {db_name} -> {dst}(保留 {keep} 份)")
return dst
# ---------------- S3 上传(纯标准库 SigV4 ----------------
def load_env(path: Path) -> dict:
"""极简 .env 解析(KEY=VALUE,忽略注释/空行),os.environ 优先"""
env = {}
if path.exists():
for line in path.read_text().splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
k, v = line.split("=", 1)
env[k.strip()] = v.strip().strip('"').strip("'")
for k, v in os.environ.items():
if v:
env[k] = v
return env
def _sign(key: bytes, msg: str) -> bytes:
return hmac.new(key, msg.encode(), hashlib.sha256).digest()
def _sig_key(secret: str, date: str, region: str, service: str) -> bytes:
k = _sign(f"AWS4{secret}".encode(), date)
k = _sign(k, region)
k = _sign(k, service)
return _sign(k, "aws4_request")
def _s3_request(endpoint: str, access: str, secret: str, region: str,
method: str, bucket: str, key: str, query: str,
body: bytes, content_type: str = "application/octet-stream") -> bytes:
"""S3 API 请求(path-style),返回响应体"""
now = datetime.now(timezone.utc)
amz_date = now.strftime("%Y%m%dT%H%M%SZ")
date = now.strftime("%Y%m%d")
payload_hash = hashlib.sha256(body).hexdigest()
host = urllib.parse.urlparse(endpoint).netloc
q = urllib.parse.quote(key, safe="/")
canonical_uri = f"/{bucket}/{q}"
canonical_query = query # 已按 RFC3986 编码
canonical_headers = (
f"host:{host}\n"
f"x-amz-content-sha256:{payload_hash}\n"
f"x-amz-date:{amz_date}\n"
)
signed_headers = "host;x-amz-content-sha256;x-amz-date"
canonical_request = "\n".join(
[method, canonical_uri, canonical_query, canonical_headers, signed_headers, payload_hash]
)
scope = f"{date}/{region}/s3/aws4_request"
string_to_sign = "\n".join(
["AWS4-HMAC-SHA256", amz_date, scope,
hashlib.sha256(canonical_request.encode()).hexdigest()]
)
signature = hmac.new(
_sig_key(secret, date, region, "s3"), string_to_sign.encode(), hashlib.sha256
).hexdigest()
auth = (
f"AWS4-HMAC-SHA256 Credential={access}/{scope}, "
f"SignedHeaders={signed_headers}, Signature={signature}"
)
url = f"{endpoint.rstrip('/')}/{bucket}/{q}"
if query:
url += "?" + query
req = urllib.request.Request(
url, data=body, method=method,
headers={
"Authorization": auth,
"x-amz-content-sha256": payload_hash,
"x-amz-date": amz_date,
"Content-Type": content_type,
},
)
try:
with urllib.request.urlopen(req, timeout=90) as resp:
return resp.read()
except urllib.error.HTTPError as e:
raise RuntimeError(f"S3 {method} {key} 失败: HTTP {e.code} {e.read().decode(errors='replace')[:300]}")
def s3_upload(env: dict, dst: Path, stem: str, keep: int) -> None:
"""上传单个备份文件到 S3,并清理远端过期备份(保留 keep 份)"""
endpoint = env.get("S3_ENDPOINT_URL", "")
access = env.get("S3_ACCESS_KEY", "")
secret = env.get("S3_SECRET_KEY", "")
bucket = env.get("S3_BUCKET", "")
if not (endpoint and access and secret and bucket):
print("[s3] 未配置 S3_*ENDPOINT/ACCESS_KEY/SECRET_KEY/BUCKET),跳过远端备份")
return
region = env.get("S3_REGION", "auto")
prefix = env.get("S3_PREFIX", "vps-manager").rstrip("/")
key = f"{prefix}/{stem}/{dst.name}"
body = dst.read_bytes()
_s3_request(endpoint, access, secret, region, "PUT", bucket, key, "", body)
print(f"[s3] {dst.name} -> {endpoint}/{bucket}/{key}")
# 列出同目录下已有备份,按 key 排序(时间戳命名即字典序),清理过期对象
# 注意:prefix 必须带参数名且值按 RFC3986 编码(含 / -> %2F),否则会列出全桶并误删其他目录
list_query = "list-type=2&prefix=" + urllib.parse.quote(f"{prefix}/{stem}/", safe="")
resp = _s3_request(endpoint, access, secret, region, "GET", bucket, "", list_query, b"")
root = ET.fromstring(resp)
ns = {"s3": "http://s3.amazonaws.com/doc/2006-03-01/"}
keys = [e.text for e in root.findall(".//s3:Key", ns) if e.text]
keys.sort(reverse=True)
for old_key in keys[keep:]:
_s3_request(endpoint, access, secret, region, "DELETE", bucket, old_key, "", b"")
print(f"[s3] 清理过期备份: {old_key}")
def run(db_name: str, keep: int) -> None:
dst = backup(db_name, keep)
if dst is None:
return
try:
env = load_env(Path(".env"))
s3_upload(env, dst, db_name.replace(".db", ""), keep)
except Exception as e:
# 本地备份已成功;S3 失败需要显式暴露(systemd 会标记 failed
print(f"[s3][error] {e}", file=sys.stderr)
sys.exit(1)
if target in ("assets", "all"): if target in ("assets", "all"):
backup("assets.db", 14) run("assets.db", 14)
if target in ("metrics", "all"): if target in ("metrics", "all"):
backup("metrics.db", 4) run("metrics.db", 4)
PYEOF PYEOF
+6
View File
@@ -54,6 +54,12 @@ SMTP_PASSWORD=
SMTP_FROM= SMTP_FROM=
SMTP_TO= SMTP_TO=
RENEWAL_THRESHOLD_DAYS=30 RENEWAL_THRESHOLD_DAYS=30
S3_ENDPOINT_URL=
S3_ACCESS_KEY=
S3_SECRET_KEY=
S3_BUCKET=
S3_PREFIX=vps-manager
S3_REGION=auto
EOF EOF
chmod 600 .env chmod 600 .env
echo " 已生成 .envMASTER_KEY 自动生成,请妥善保管)" echo " 已生成 .envMASTER_KEY 自动生成,请妥善保管)"