fix+perf: SQLite连接busy_timeout修复并发锁/静态版本号缓存/删除资产批量清理/安全检查最新记录子查询修正/AI余额与SSL探测并发化/密钥常量时间比较

This commit is contained in:
gouki
2026-08-05 22:49:01 +00:00
parent 4f855ba0da
commit 4dba505d82
8 changed files with 121 additions and 58 deletions
+8 -3
View File
@@ -283,14 +283,19 @@ def delete_asset(session: Session, asset_id: int) -> None:
def _cleanup_metrics_for_asset(asset_id: int) -> None:
"""清理 metrics.db 中该资产的 MetricPoint/ServerInfo/SecurityCheck/EventLog"""
"""清理 metrics.db 中该资产的 MetricPoint/ServerInfo/SecurityCheck/EventLog
使用批量 DELETE(而非逐行加载后删除):监控时序数据可能上万行,
逐行删除会全部载入内存且产生数万次 ORM 操作。
"""
from sqlmodel import delete
from app.database import metrics_engine
from app.models.monitor import EventLog, MetricPoint, SecurityCheck, ServerInfo
with Session(metrics_engine) as ms:
for model in (MetricPoint, ServerInfo, SecurityCheck, EventLog):
for row in ms.exec(select(model).where(model.asset_id == asset_id)).all():
ms.delete(row)
ms.exec(delete(model).where(model.asset_id == asset_id))
ms.commit()
+18 -11
View File
@@ -15,19 +15,26 @@ DEFAULT_WEIGHT = 20
def latest_checks(session: Session, asset_id: int) -> list:
"""取每个检查项的最新一条"""
stmt = (
select(SecurityCheck)
"""取每个检查项的最新一条
用 GROUP BY max(ts) 子查询精确取每项最新记录:若用 limit(50) 后去重,
当某个检查项连续上报超过 50 次时会把其他项的最新记录挤出,导致评分失真。
"""
latest_ts = (
select(
SecurityCheck.check_item,
func.max(SecurityCheck.ts).label("max_ts"),
)
.where(SecurityCheck.asset_id == asset_id)
.order_by(SecurityCheck.ts.desc())
.limit(50)
.group_by(SecurityCheck.check_item)
.subquery()
)
checks = session.exec(stmt).all()
latest = {}
for check in checks:
if check.check_item not in latest:
latest[check.check_item] = check
return list(latest.values())
stmt = select(SecurityCheck).where(SecurityCheck.asset_id == asset_id).join(
latest_ts,
(SecurityCheck.check_item == latest_ts.c.check_item)
& (SecurityCheck.ts == latest_ts.c.max_ts),
)
return list(session.exec(stmt).all())
def compute_security_score(checks: list):
+26 -2
View File
@@ -224,12 +224,36 @@ def delete_site_cert(session: Session, cert_id: int) -> None:
def check_all_site_certs(session: Session) -> dict:
"""全量刷新所有站点证书,返回统计与异常清单"""
"""全量刷新所有站点证书,返回统计与异常清单
探测为纯网络 IO(单次最长 8s),用线程池并发探测后统一写库:
串行时 N 个站点最坏耗时 N×8s,并发后接近单站点耗时;
写库集中在主线程一次 commit(原来逐条 commit 产生 N 次事务)。
"""
from concurrent.futures import ThreadPoolExecutor
certs = session.exec(select(SiteCert)).all()
stats = {"total": len(certs), "ok": 0, "error": 0, "expiring": 0, "expired": 0}
problems = []
if not certs:
return {"stats": stats, "problems": problems}
# 并发探测(不碰数据库,线程安全)
with ThreadPoolExecutor(max_workers=min(8, len(certs))) as pool:
futures = {pool.submit(probe_site_cert, c.hostname, c.port): c for c in certs}
for future in futures:
cert = futures[future]
try:
_apply_probe(cert, future.result())
except Exception as e: # noqa: BLE001
cert.status = "error"
cert.error = str(e)[:200]
cert.last_checked_at = utcnow()
session.add(cert)
session.commit()
for cert in certs:
check_one(session, cert)
session.refresh(cert)
if cert.status == "error":
stats["error"] += 1
problems.append({"hostname": cert.hostname, "detail": cert.error})
+31 -13
View File
@@ -299,26 +299,44 @@ def refresh_ai_balance(session: Session, asset_id: int) -> dict:
}
def sync_all_ai_balances(session: Session) -> dict:
"""遍历所有 AI 账号资产,逐个刷新余额(单个失败不中断整体)"""
def sync_all_ai_balances(session: Session, max_workers: int = 5) -> dict:
"""并发刷新所有 AI 账号余额(单个失败不中断整体)
每个账号需调用外部 API(单次最长 30s),串行时总耗时随账号数线性增长;
改为线程池并发后显著提速。注意:SQLite Session 不能跨线程共享,
每个 worker 使用独立 Session(WAL 模式下多连接读写安全)。
"""
from concurrent.futures import ThreadPoolExecutor
from app.database import assets_engine
ai_assets = session.exec(
select(Asset).where(Asset.asset_type == AssetType.AI_AGENT)
).all()
asset_ids = [(a.id, a.name) for a in ai_assets]
def _refresh_one(asset_id: int):
with Session(assets_engine) as worker_session:
refresh_ai_balance(worker_session, asset_id)
success = 0
failed = 0
errors = []
for asset in ai_assets:
try:
refresh_ai_balance(session, asset.id)
success += 1
except HTTPException as e:
failed += 1
errors.append(f"{asset.name}: {e.detail}")
except Exception as e: # noqa: BLE001
failed += 1
errors.append(f"{asset.name}: {e}")
with ThreadPoolExecutor(max_workers=max_workers) as pool:
futures = {pool.submit(_refresh_one, aid): name for aid, name in asset_ids}
for future in futures:
name = futures[future]
try:
future.result()
success += 1
except HTTPException as e:
failed += 1
errors.append(f"{name}: {e.detail}")
except Exception as e: # noqa: BLE001
failed += 1
errors.append(f"{name}: {e}")
return {
"total": len(ai_assets),
"total": len(asset_ids),
"success": success,
"failed": failed,
"errors": errors,