fix+perf: SQLite连接busy_timeout修复并发锁/静态版本号缓存/删除资产批量清理/安全检查最新记录子查询修正/AI余额与SSL探测并发化/密钥常量时间比较
This commit is contained in:
@@ -283,14 +283,19 @@ def delete_asset(session: Session, asset_id: int) -> None:
|
||||
|
||||
|
||||
def _cleanup_metrics_for_asset(asset_id: int) -> None:
|
||||
"""清理 metrics.db 中该资产的 MetricPoint/ServerInfo/SecurityCheck/EventLog"""
|
||||
"""清理 metrics.db 中该资产的 MetricPoint/ServerInfo/SecurityCheck/EventLog
|
||||
|
||||
使用批量 DELETE(而非逐行加载后删除):监控时序数据可能上万行,
|
||||
逐行删除会全部载入内存且产生数万次 ORM 操作。
|
||||
"""
|
||||
from sqlmodel import delete
|
||||
|
||||
from app.database import metrics_engine
|
||||
from app.models.monitor import EventLog, MetricPoint, SecurityCheck, ServerInfo
|
||||
|
||||
with Session(metrics_engine) as ms:
|
||||
for model in (MetricPoint, ServerInfo, SecurityCheck, EventLog):
|
||||
for row in ms.exec(select(model).where(model.asset_id == asset_id)).all():
|
||||
ms.delete(row)
|
||||
ms.exec(delete(model).where(model.asset_id == asset_id))
|
||||
ms.commit()
|
||||
|
||||
|
||||
|
||||
@@ -15,19 +15,26 @@ DEFAULT_WEIGHT = 20
|
||||
|
||||
|
||||
def latest_checks(session: Session, asset_id: int) -> list:
|
||||
"""取每个检查项的最新一条"""
|
||||
stmt = (
|
||||
select(SecurityCheck)
|
||||
"""取每个检查项的最新一条
|
||||
|
||||
用 GROUP BY max(ts) 子查询精确取每项最新记录:若用 limit(50) 后去重,
|
||||
当某个检查项连续上报超过 50 次时会把其他项的最新记录挤出,导致评分失真。
|
||||
"""
|
||||
latest_ts = (
|
||||
select(
|
||||
SecurityCheck.check_item,
|
||||
func.max(SecurityCheck.ts).label("max_ts"),
|
||||
)
|
||||
.where(SecurityCheck.asset_id == asset_id)
|
||||
.order_by(SecurityCheck.ts.desc())
|
||||
.limit(50)
|
||||
.group_by(SecurityCheck.check_item)
|
||||
.subquery()
|
||||
)
|
||||
checks = session.exec(stmt).all()
|
||||
latest = {}
|
||||
for check in checks:
|
||||
if check.check_item not in latest:
|
||||
latest[check.check_item] = check
|
||||
return list(latest.values())
|
||||
stmt = select(SecurityCheck).where(SecurityCheck.asset_id == asset_id).join(
|
||||
latest_ts,
|
||||
(SecurityCheck.check_item == latest_ts.c.check_item)
|
||||
& (SecurityCheck.ts == latest_ts.c.max_ts),
|
||||
)
|
||||
return list(session.exec(stmt).all())
|
||||
|
||||
|
||||
def compute_security_score(checks: list):
|
||||
|
||||
@@ -224,12 +224,36 @@ def delete_site_cert(session: Session, cert_id: int) -> None:
|
||||
|
||||
|
||||
def check_all_site_certs(session: Session) -> dict:
|
||||
"""全量刷新所有站点证书,返回统计与异常清单"""
|
||||
"""全量刷新所有站点证书,返回统计与异常清单
|
||||
|
||||
探测为纯网络 IO(单次最长 8s),用线程池并发探测后统一写库:
|
||||
串行时 N 个站点最坏耗时 N×8s,并发后接近单站点耗时;
|
||||
写库集中在主线程一次 commit(原来逐条 commit 产生 N 次事务)。
|
||||
"""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
certs = session.exec(select(SiteCert)).all()
|
||||
stats = {"total": len(certs), "ok": 0, "error": 0, "expiring": 0, "expired": 0}
|
||||
problems = []
|
||||
if not certs:
|
||||
return {"stats": stats, "problems": problems}
|
||||
|
||||
# 并发探测(不碰数据库,线程安全)
|
||||
with ThreadPoolExecutor(max_workers=min(8, len(certs))) as pool:
|
||||
futures = {pool.submit(probe_site_cert, c.hostname, c.port): c for c in certs}
|
||||
for future in futures:
|
||||
cert = futures[future]
|
||||
try:
|
||||
_apply_probe(cert, future.result())
|
||||
except Exception as e: # noqa: BLE001
|
||||
cert.status = "error"
|
||||
cert.error = str(e)[:200]
|
||||
cert.last_checked_at = utcnow()
|
||||
session.add(cert)
|
||||
session.commit()
|
||||
|
||||
for cert in certs:
|
||||
check_one(session, cert)
|
||||
session.refresh(cert)
|
||||
if cert.status == "error":
|
||||
stats["error"] += 1
|
||||
problems.append({"hostname": cert.hostname, "detail": cert.error})
|
||||
|
||||
@@ -299,26 +299,44 @@ def refresh_ai_balance(session: Session, asset_id: int) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def sync_all_ai_balances(session: Session) -> dict:
|
||||
"""遍历所有 AI 账号资产,逐个刷新余额(单个失败不中断整体)"""
|
||||
def sync_all_ai_balances(session: Session, max_workers: int = 5) -> dict:
|
||||
"""并发刷新所有 AI 账号余额(单个失败不中断整体)
|
||||
|
||||
每个账号需调用外部 API(单次最长 30s),串行时总耗时随账号数线性增长;
|
||||
改为线程池并发后显著提速。注意:SQLite Session 不能跨线程共享,
|
||||
每个 worker 使用独立 Session(WAL 模式下多连接读写安全)。
|
||||
"""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from app.database import assets_engine
|
||||
|
||||
ai_assets = session.exec(
|
||||
select(Asset).where(Asset.asset_type == AssetType.AI_AGENT)
|
||||
).all()
|
||||
asset_ids = [(a.id, a.name) for a in ai_assets]
|
||||
|
||||
def _refresh_one(asset_id: int):
|
||||
with Session(assets_engine) as worker_session:
|
||||
refresh_ai_balance(worker_session, asset_id)
|
||||
|
||||
success = 0
|
||||
failed = 0
|
||||
errors = []
|
||||
for asset in ai_assets:
|
||||
try:
|
||||
refresh_ai_balance(session, asset.id)
|
||||
success += 1
|
||||
except HTTPException as e:
|
||||
failed += 1
|
||||
errors.append(f"{asset.name}: {e.detail}")
|
||||
except Exception as e: # noqa: BLE001
|
||||
failed += 1
|
||||
errors.append(f"{asset.name}: {e}")
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as pool:
|
||||
futures = {pool.submit(_refresh_one, aid): name for aid, name in asset_ids}
|
||||
for future in futures:
|
||||
name = futures[future]
|
||||
try:
|
||||
future.result()
|
||||
success += 1
|
||||
except HTTPException as e:
|
||||
failed += 1
|
||||
errors.append(f"{name}: {e.detail}")
|
||||
except Exception as e: # noqa: BLE001
|
||||
failed += 1
|
||||
errors.append(f"{name}: {e}")
|
||||
return {
|
||||
"total": len(ai_assets),
|
||||
"total": len(asset_ids),
|
||||
"success": success,
|
||||
"failed": failed,
|
||||
"errors": errors,
|
||||
|
||||
Reference in New Issue
Block a user