feat: 数据库备份支持S3兼容对象存储(SigV4纯标准库实现,无配置时兼容原本地备份)
This commit is contained in:
@@ -29,3 +29,31 @@ SMTP_TO=
|
|||||||
|
|
||||||
# ---- 续费提醒阈值(天)----
|
# ---- 续费提醒阈值(天)----
|
||||||
RENEWAL_THRESHOLD_DAYS=30
|
RENEWAL_THRESHOLD_DAYS=30
|
||||||
|
|
||||||
|
# ---- 监控数据保留策略(自动清理,天数)----
|
||||||
|
METRICS_RETENTION_DAYS=30
|
||||||
|
SECURITY_RETENTION_DAYS=90
|
||||||
|
EVENT_LOG_RETENTION_DAYS=180
|
||||||
|
# 自动清理间隔(小时),0 表示禁用后台自动清理
|
||||||
|
CLEANUP_INTERVAL_HOURS=24
|
||||||
|
|
||||||
|
# ---- S3 兼容对象存储备份(可选,未配置则仅本地备份)----
|
||||||
|
# 备份文件通过 AWS SigV4 签名上传,兼容 Cloudflare R2 / MinIO / 阿里云 OSS / 腾讯云 COS 等
|
||||||
|
# R2 示例:S3_ENDPOINT_URL=https://<account-id>.r2.cloudflarestorage.com,S3_REGION=auto
|
||||||
|
# OSS 示例:S3_ENDPOINT_URL=https://oss-cn-hangzhou.aliyuncs.com,S3_REGION=oss-cn-hangzhou
|
||||||
|
# COS 示例:S3_ENDPOINT_URL=https://cos.ap-shanghai.myqcloud.com,S3_REGION=ap-shanghai
|
||||||
|
S3_ENDPOINT_URL=
|
||||||
|
S3_ACCESS_KEY=
|
||||||
|
S3_SECRET_KEY=
|
||||||
|
S3_BUCKET=
|
||||||
|
# 远端对象前缀(默认 vps-manager,对象结构:<prefix>/assets|metrics/<文件名>)
|
||||||
|
S3_PREFIX=vps-manager
|
||||||
|
# 区域(R2 为 auto,MinIO 随意,OSS/COS 填各自区域)
|
||||||
|
S3_REGION=auto
|
||||||
|
|
||||||
|
# ---- Agent 上报频率限制 ----
|
||||||
|
# 同一资产两次上报的最小间隔(秒),0 表示不限制
|
||||||
|
AGENT_REPORT_MIN_INTERVAL=30
|
||||||
|
|
||||||
|
# ---- CORS 允许来源(逗号分隔)----
|
||||||
|
CORS_ORIGINS=http://127.0.0.1:8000,http://localhost:8000,https://dify.taile5765c.ts.net
|
||||||
|
|||||||
+145
-6
@@ -1,9 +1,14 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# 数据库分库备份脚本
|
# 数据库分库备份脚本(本地 + S3 兼容对象存储)
|
||||||
# 用法:backup.sh [assets|metrics|all]
|
# 用法:backup.sh [assets|metrics|all]
|
||||||
# assets -> 备份 assets.db,保留 14 份(建议每日)
|
# assets -> 备份 assets.db,保留 14 份(建议每日)
|
||||||
# metrics -> 备份 metrics.db,保留 4 份(建议每周)
|
# metrics -> 备份 metrics.db,保留 4 份(建议每周)
|
||||||
# all -> 两者都备份
|
# all -> 两者都备份
|
||||||
|
#
|
||||||
|
# S3 上传(可选):配置 .env 中 S3_* 项后,备份文件自动上传到
|
||||||
|
# 任意 S3 兼容对象存储(Cloudflare R2 / MinIO / 阿里云 OSS / 腾讯云 COS 等),
|
||||||
|
# 远端保留份数与本地一致。未配置 S3 时仅本地备份(向后兼容)。
|
||||||
|
# 实现为纯 Python 标准库(AWS SigV4 签名),零额外依赖。
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
APP_DIR="${APP_DIR:-/opt/vps-manager}"
|
APP_DIR="${APP_DIR:-/opt/vps-manager}"
|
||||||
@@ -12,9 +17,16 @@ cd "$APP_DIR"
|
|||||||
TARGET="${1:-all}"
|
TARGET="${1:-all}"
|
||||||
|
|
||||||
python3 - "$TARGET" <<'PYEOF'
|
python3 - "$TARGET" <<'PYEOF'
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
import os
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import sys
|
import sys
|
||||||
from datetime import datetime
|
import urllib.error
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
import xml.etree.ElementTree as ET
|
||||||
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
target = sys.argv[1]
|
target = sys.argv[1]
|
||||||
@@ -22,13 +34,14 @@ data_dir = Path("data")
|
|||||||
backup_root = data_dir / "backups"
|
backup_root = data_dir / "backups"
|
||||||
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||||
|
|
||||||
|
# ---------------- 本地备份 ----------------
|
||||||
|
|
||||||
def backup(db_name: str, keep: int) -> None:
|
def backup(db_name: str, keep: int) -> Path | None:
|
||||||
src = data_dir / db_name
|
src = data_dir / db_name
|
||||||
stem = db_name.replace(".db", "")
|
stem = db_name.replace(".db", "")
|
||||||
if not src.exists():
|
if not src.exists():
|
||||||
print(f"[skip] {db_name} 不存在")
|
print(f"[skip] {db_name} 不存在")
|
||||||
return
|
return None
|
||||||
out_dir = backup_root / stem
|
out_dir = backup_root / stem
|
||||||
out_dir.mkdir(parents=True, exist_ok=True)
|
out_dir.mkdir(parents=True, exist_ok=True)
|
||||||
dst = out_dir / f"{stem}_{ts}.db"
|
dst = out_dir / f"{stem}_{ts}.db"
|
||||||
@@ -47,10 +60,136 @@ def backup(db_name: str, keep: int) -> None:
|
|||||||
for old in files[keep:]:
|
for old in files[keep:]:
|
||||||
old.unlink()
|
old.unlink()
|
||||||
print(f"[ok] {db_name} -> {dst}(保留 {keep} 份)")
|
print(f"[ok] {db_name} -> {dst}(保留 {keep} 份)")
|
||||||
|
return dst
|
||||||
|
|
||||||
|
# ---------------- S3 上传(纯标准库 SigV4) ----------------
|
||||||
|
|
||||||
|
def load_env(path: Path) -> dict:
|
||||||
|
"""极简 .env 解析(KEY=VALUE,忽略注释/空行),os.environ 优先"""
|
||||||
|
env = {}
|
||||||
|
if path.exists():
|
||||||
|
for line in path.read_text().splitlines():
|
||||||
|
line = line.strip()
|
||||||
|
if not line or line.startswith("#") or "=" not in line:
|
||||||
|
continue
|
||||||
|
k, v = line.split("=", 1)
|
||||||
|
env[k.strip()] = v.strip().strip('"').strip("'")
|
||||||
|
for k, v in os.environ.items():
|
||||||
|
if v:
|
||||||
|
env[k] = v
|
||||||
|
return env
|
||||||
|
|
||||||
|
|
||||||
|
def _sign(key: bytes, msg: str) -> bytes:
|
||||||
|
return hmac.new(key, msg.encode(), hashlib.sha256).digest()
|
||||||
|
|
||||||
|
|
||||||
|
def _sig_key(secret: str, date: str, region: str, service: str) -> bytes:
|
||||||
|
k = _sign(f"AWS4{secret}".encode(), date)
|
||||||
|
k = _sign(k, region)
|
||||||
|
k = _sign(k, service)
|
||||||
|
return _sign(k, "aws4_request")
|
||||||
|
|
||||||
|
|
||||||
|
def _s3_request(endpoint: str, access: str, secret: str, region: str,
|
||||||
|
method: str, bucket: str, key: str, query: str,
|
||||||
|
body: bytes, content_type: str = "application/octet-stream") -> bytes:
|
||||||
|
"""S3 API 请求(path-style),返回响应体"""
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
amz_date = now.strftime("%Y%m%dT%H%M%SZ")
|
||||||
|
date = now.strftime("%Y%m%d")
|
||||||
|
payload_hash = hashlib.sha256(body).hexdigest()
|
||||||
|
host = urllib.parse.urlparse(endpoint).netloc
|
||||||
|
q = urllib.parse.quote(key, safe="/")
|
||||||
|
|
||||||
|
canonical_uri = f"/{bucket}/{q}"
|
||||||
|
canonical_query = query # 已按 RFC3986 编码
|
||||||
|
canonical_headers = (
|
||||||
|
f"host:{host}\n"
|
||||||
|
f"x-amz-content-sha256:{payload_hash}\n"
|
||||||
|
f"x-amz-date:{amz_date}\n"
|
||||||
|
)
|
||||||
|
signed_headers = "host;x-amz-content-sha256;x-amz-date"
|
||||||
|
canonical_request = "\n".join(
|
||||||
|
[method, canonical_uri, canonical_query, canonical_headers, signed_headers, payload_hash]
|
||||||
|
)
|
||||||
|
scope = f"{date}/{region}/s3/aws4_request"
|
||||||
|
string_to_sign = "\n".join(
|
||||||
|
["AWS4-HMAC-SHA256", amz_date, scope,
|
||||||
|
hashlib.sha256(canonical_request.encode()).hexdigest()]
|
||||||
|
)
|
||||||
|
signature = hmac.new(
|
||||||
|
_sig_key(secret, date, region, "s3"), string_to_sign.encode(), hashlib.sha256
|
||||||
|
).hexdigest()
|
||||||
|
auth = (
|
||||||
|
f"AWS4-HMAC-SHA256 Credential={access}/{scope}, "
|
||||||
|
f"SignedHeaders={signed_headers}, Signature={signature}"
|
||||||
|
)
|
||||||
|
|
||||||
|
url = f"{endpoint.rstrip('/')}/{bucket}/{q}"
|
||||||
|
if query:
|
||||||
|
url += "?" + query
|
||||||
|
req = urllib.request.Request(
|
||||||
|
url, data=body, method=method,
|
||||||
|
headers={
|
||||||
|
"Authorization": auth,
|
||||||
|
"x-amz-content-sha256": payload_hash,
|
||||||
|
"x-amz-date": amz_date,
|
||||||
|
"Content-Type": content_type,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=90) as resp:
|
||||||
|
return resp.read()
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
raise RuntimeError(f"S3 {method} {key} 失败: HTTP {e.code} {e.read().decode(errors='replace')[:300]}")
|
||||||
|
|
||||||
|
|
||||||
|
def s3_upload(env: dict, dst: Path, stem: str, keep: int) -> None:
|
||||||
|
"""上传单个备份文件到 S3,并清理远端过期备份(保留 keep 份)"""
|
||||||
|
endpoint = env.get("S3_ENDPOINT_URL", "")
|
||||||
|
access = env.get("S3_ACCESS_KEY", "")
|
||||||
|
secret = env.get("S3_SECRET_KEY", "")
|
||||||
|
bucket = env.get("S3_BUCKET", "")
|
||||||
|
if not (endpoint and access and secret and bucket):
|
||||||
|
print("[s3] 未配置 S3_*(ENDPOINT/ACCESS_KEY/SECRET_KEY/BUCKET),跳过远端备份")
|
||||||
|
return
|
||||||
|
region = env.get("S3_REGION", "auto")
|
||||||
|
prefix = env.get("S3_PREFIX", "vps-manager").rstrip("/")
|
||||||
|
key = f"{prefix}/{stem}/{dst.name}"
|
||||||
|
|
||||||
|
body = dst.read_bytes()
|
||||||
|
_s3_request(endpoint, access, secret, region, "PUT", bucket, key, "", body)
|
||||||
|
print(f"[s3] {dst.name} -> {endpoint}/{bucket}/{key}")
|
||||||
|
|
||||||
|
# 列出同目录下已有备份,按 key 排序(时间戳命名即字典序),清理过期对象
|
||||||
|
# 注意:prefix 必须带参数名且值按 RFC3986 编码(含 / -> %2F),否则会列出全桶并误删其他目录
|
||||||
|
list_query = "list-type=2&prefix=" + urllib.parse.quote(f"{prefix}/{stem}/", safe="")
|
||||||
|
resp = _s3_request(endpoint, access, secret, region, "GET", bucket, "", list_query, b"")
|
||||||
|
root = ET.fromstring(resp)
|
||||||
|
ns = {"s3": "http://s3.amazonaws.com/doc/2006-03-01/"}
|
||||||
|
keys = [e.text for e in root.findall(".//s3:Key", ns) if e.text]
|
||||||
|
keys.sort(reverse=True)
|
||||||
|
for old_key in keys[keep:]:
|
||||||
|
_s3_request(endpoint, access, secret, region, "DELETE", bucket, old_key, "", b"")
|
||||||
|
print(f"[s3] 清理过期备份: {old_key}")
|
||||||
|
|
||||||
|
|
||||||
|
def run(db_name: str, keep: int) -> None:
|
||||||
|
dst = backup(db_name, keep)
|
||||||
|
if dst is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
env = load_env(Path(".env"))
|
||||||
|
s3_upload(env, dst, db_name.replace(".db", ""), keep)
|
||||||
|
except Exception as e:
|
||||||
|
# 本地备份已成功;S3 失败需要显式暴露(systemd 会标记 failed)
|
||||||
|
print(f"[s3][error] {e}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
if target in ("assets", "all"):
|
if target in ("assets", "all"):
|
||||||
backup("assets.db", 14)
|
run("assets.db", 14)
|
||||||
if target in ("metrics", "all"):
|
if target in ("metrics", "all"):
|
||||||
backup("metrics.db", 4)
|
run("metrics.db", 4)
|
||||||
PYEOF
|
PYEOF
|
||||||
|
|||||||
@@ -54,6 +54,12 @@ SMTP_PASSWORD=
|
|||||||
SMTP_FROM=
|
SMTP_FROM=
|
||||||
SMTP_TO=
|
SMTP_TO=
|
||||||
RENEWAL_THRESHOLD_DAYS=30
|
RENEWAL_THRESHOLD_DAYS=30
|
||||||
|
S3_ENDPOINT_URL=
|
||||||
|
S3_ACCESS_KEY=
|
||||||
|
S3_SECRET_KEY=
|
||||||
|
S3_BUCKET=
|
||||||
|
S3_PREFIX=vps-manager
|
||||||
|
S3_REGION=auto
|
||||||
EOF
|
EOF
|
||||||
chmod 600 .env
|
chmod 600 .env
|
||||||
echo " 已生成 .env(MASTER_KEY 自动生成,请妥善保管)"
|
echo " 已生成 .env(MASTER_KEY 自动生成,请妥善保管)"
|
||||||
|
|||||||
Reference in New Issue
Block a user