diff --git a/.env.example b/.env.example index 1fca2c3..b41dc32 100644 --- a/.env.example +++ b/.env.example @@ -29,3 +29,31 @@ SMTP_TO= # ---- 续费提醒阈值(天)---- RENEWAL_THRESHOLD_DAYS=30 + +# ---- 监控数据保留策略(自动清理,天数)---- +METRICS_RETENTION_DAYS=30 +SECURITY_RETENTION_DAYS=90 +EVENT_LOG_RETENTION_DAYS=180 +# 自动清理间隔(小时),0 表示禁用后台自动清理 +CLEANUP_INTERVAL_HOURS=24 + +# ---- S3 兼容对象存储备份(可选,未配置则仅本地备份)---- +# 备份文件通过 AWS SigV4 签名上传,兼容 Cloudflare R2 / MinIO / 阿里云 OSS / 腾讯云 COS 等 +# R2 示例:S3_ENDPOINT_URL=https://.r2.cloudflarestorage.com,S3_REGION=auto +# OSS 示例:S3_ENDPOINT_URL=https://oss-cn-hangzhou.aliyuncs.com,S3_REGION=oss-cn-hangzhou +# COS 示例:S3_ENDPOINT_URL=https://cos.ap-shanghai.myqcloud.com,S3_REGION=ap-shanghai +S3_ENDPOINT_URL= +S3_ACCESS_KEY= +S3_SECRET_KEY= +S3_BUCKET= +# 远端对象前缀(默认 vps-manager,对象结构:/assets|metrics/<文件名>) +S3_PREFIX=vps-manager +# 区域(R2 为 auto,MinIO 随意,OSS/COS 填各自区域) +S3_REGION=auto + +# ---- Agent 上报频率限制 ---- +# 同一资产两次上报的最小间隔(秒),0 表示不限制 +AGENT_REPORT_MIN_INTERVAL=30 + +# ---- CORS 允许来源(逗号分隔)---- +CORS_ORIGINS=http://127.0.0.1:8000,http://localhost:8000,https://dify.taile5765c.ts.net diff --git a/deploy/backup.sh b/deploy/backup.sh index 33fa085..075b195 100755 --- a/deploy/backup.sh +++ b/deploy/backup.sh @@ -1,9 +1,14 @@ #!/usr/bin/env bash -# 数据库分库备份脚本 +# 数据库分库备份脚本(本地 + S3 兼容对象存储) # 用法:backup.sh [assets|metrics|all] # assets -> 备份 assets.db,保留 14 份(建议每日) # metrics -> 备份 metrics.db,保留 4 份(建议每周) # all -> 两者都备份 +# +# S3 上传(可选):配置 .env 中 S3_* 项后,备份文件自动上传到 +# 任意 S3 兼容对象存储(Cloudflare R2 / MinIO / 阿里云 OSS / 腾讯云 COS 等), +# 远端保留份数与本地一致。未配置 S3 时仅本地备份(向后兼容)。 +# 实现为纯 Python 标准库(AWS SigV4 签名),零额外依赖。 set -euo pipefail APP_DIR="${APP_DIR:-/opt/vps-manager}" @@ -12,9 +17,16 @@ cd "$APP_DIR" TARGET="${1:-all}" python3 - "$TARGET" <<'PYEOF' +import hashlib +import hmac +import os import sqlite3 import sys -from datetime import datetime +import urllib.error +import urllib.parse +import urllib.request +import xml.etree.ElementTree as ET +from datetime import datetime, timezone from pathlib import Path target = sys.argv[1] @@ -22,13 +34,14 @@ data_dir = Path("data") backup_root = data_dir / "backups" ts = datetime.now().strftime("%Y%m%d_%H%M%S") +# ---------------- 本地备份 ---------------- -def backup(db_name: str, keep: int) -> None: +def backup(db_name: str, keep: int) -> Path | None: src = data_dir / db_name stem = db_name.replace(".db", "") if not src.exists(): print(f"[skip] {db_name} 不存在") - return + return None out_dir = backup_root / stem out_dir.mkdir(parents=True, exist_ok=True) dst = out_dir / f"{stem}_{ts}.db" @@ -47,10 +60,136 @@ def backup(db_name: str, keep: int) -> None: for old in files[keep:]: old.unlink() print(f"[ok] {db_name} -> {dst}(保留 {keep} 份)") + return dst + +# ---------------- S3 上传(纯标准库 SigV4) ---------------- + +def load_env(path: Path) -> dict: + """极简 .env 解析(KEY=VALUE,忽略注释/空行),os.environ 优先""" + env = {} + if path.exists(): + for line in path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + env[k.strip()] = v.strip().strip('"').strip("'") + for k, v in os.environ.items(): + if v: + env[k] = v + return env + + +def _sign(key: bytes, msg: str) -> bytes: + return hmac.new(key, msg.encode(), hashlib.sha256).digest() + + +def _sig_key(secret: str, date: str, region: str, service: str) -> bytes: + k = _sign(f"AWS4{secret}".encode(), date) + k = _sign(k, region) + k = _sign(k, service) + return _sign(k, "aws4_request") + + +def _s3_request(endpoint: str, access: str, secret: str, region: str, + method: str, bucket: str, key: str, query: str, + body: bytes, content_type: str = "application/octet-stream") -> bytes: + """S3 API 请求(path-style),返回响应体""" + now = datetime.now(timezone.utc) + amz_date = now.strftime("%Y%m%dT%H%M%SZ") + date = now.strftime("%Y%m%d") + payload_hash = hashlib.sha256(body).hexdigest() + host = urllib.parse.urlparse(endpoint).netloc + q = urllib.parse.quote(key, safe="/") + + canonical_uri = f"/{bucket}/{q}" + canonical_query = query # 已按 RFC3986 编码 + canonical_headers = ( + f"host:{host}\n" + f"x-amz-content-sha256:{payload_hash}\n" + f"x-amz-date:{amz_date}\n" + ) + signed_headers = "host;x-amz-content-sha256;x-amz-date" + canonical_request = "\n".join( + [method, canonical_uri, canonical_query, canonical_headers, signed_headers, payload_hash] + ) + scope = f"{date}/{region}/s3/aws4_request" + string_to_sign = "\n".join( + ["AWS4-HMAC-SHA256", amz_date, scope, + hashlib.sha256(canonical_request.encode()).hexdigest()] + ) + signature = hmac.new( + _sig_key(secret, date, region, "s3"), string_to_sign.encode(), hashlib.sha256 + ).hexdigest() + auth = ( + f"AWS4-HMAC-SHA256 Credential={access}/{scope}, " + f"SignedHeaders={signed_headers}, Signature={signature}" + ) + + url = f"{endpoint.rstrip('/')}/{bucket}/{q}" + if query: + url += "?" + query + req = urllib.request.Request( + url, data=body, method=method, + headers={ + "Authorization": auth, + "x-amz-content-sha256": payload_hash, + "x-amz-date": amz_date, + "Content-Type": content_type, + }, + ) + try: + with urllib.request.urlopen(req, timeout=90) as resp: + return resp.read() + except urllib.error.HTTPError as e: + raise RuntimeError(f"S3 {method} {key} 失败: HTTP {e.code} {e.read().decode(errors='replace')[:300]}") + + +def s3_upload(env: dict, dst: Path, stem: str, keep: int) -> None: + """上传单个备份文件到 S3,并清理远端过期备份(保留 keep 份)""" + endpoint = env.get("S3_ENDPOINT_URL", "") + access = env.get("S3_ACCESS_KEY", "") + secret = env.get("S3_SECRET_KEY", "") + bucket = env.get("S3_BUCKET", "") + if not (endpoint and access and secret and bucket): + print("[s3] 未配置 S3_*(ENDPOINT/ACCESS_KEY/SECRET_KEY/BUCKET),跳过远端备份") + return + region = env.get("S3_REGION", "auto") + prefix = env.get("S3_PREFIX", "vps-manager").rstrip("/") + key = f"{prefix}/{stem}/{dst.name}" + + body = dst.read_bytes() + _s3_request(endpoint, access, secret, region, "PUT", bucket, key, "", body) + print(f"[s3] {dst.name} -> {endpoint}/{bucket}/{key}") + + # 列出同目录下已有备份,按 key 排序(时间戳命名即字典序),清理过期对象 + # 注意:prefix 必须带参数名且值按 RFC3986 编码(含 / -> %2F),否则会列出全桶并误删其他目录 + list_query = "list-type=2&prefix=" + urllib.parse.quote(f"{prefix}/{stem}/", safe="") + resp = _s3_request(endpoint, access, secret, region, "GET", bucket, "", list_query, b"") + root = ET.fromstring(resp) + ns = {"s3": "http://s3.amazonaws.com/doc/2006-03-01/"} + keys = [e.text for e in root.findall(".//s3:Key", ns) if e.text] + keys.sort(reverse=True) + for old_key in keys[keep:]: + _s3_request(endpoint, access, secret, region, "DELETE", bucket, old_key, "", b"") + print(f"[s3] 清理过期备份: {old_key}") + + +def run(db_name: str, keep: int) -> None: + dst = backup(db_name, keep) + if dst is None: + return + try: + env = load_env(Path(".env")) + s3_upload(env, dst, db_name.replace(".db", ""), keep) + except Exception as e: + # 本地备份已成功;S3 失败需要显式暴露(systemd 会标记 failed) + print(f"[s3][error] {e}", file=sys.stderr) + sys.exit(1) if target in ("assets", "all"): - backup("assets.db", 14) + run("assets.db", 14) if target in ("metrics", "all"): - backup("metrics.db", 4) + run("metrics.db", 4) PYEOF diff --git a/deploy/setup.sh b/deploy/setup.sh index 47c0e40..f1ca938 100755 --- a/deploy/setup.sh +++ b/deploy/setup.sh @@ -54,6 +54,12 @@ SMTP_PASSWORD= SMTP_FROM= SMTP_TO= RENEWAL_THRESHOLD_DAYS=30 +S3_ENDPOINT_URL= +S3_ACCESS_KEY= +S3_SECRET_KEY= +S3_BUCKET= +S3_PREFIX=vps-manager +S3_REGION=auto EOF chmod 600 .env echo " 已生成 .env(MASTER_KEY 自动生成,请妥善保管)"