ops: add production observability and recovery
This commit is contained in:
hailin
330
backend/apps/system/blueprint_readiness.py
Normal file
330
backend/apps/system/blueprint_readiness.py
Normal file
@@ -0,0 +1,330 @@
|
||||
"""聚合阶段 2～6 的真实案例、外部服务和恢复证据，生成统一蓝图退出报告。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from django.db import DatabaseError
|
||||
|
||||
from apps.tickets.evaluation import build_stage_2_acceptance_report
|
||||
from apps.tickets.support_evaluation import build_stage_3_acceptance_report
|
||||
|
||||
SENSITIVE_KEY_PARTS = ("password", "secret", "token", "credential")
|
||||
|
||||
EXTERNAL_GATE_SPECS: dict[str, dict[str, Any]] = {
|
||||
"stage3_retrieval": {
|
||||
"title": "阶段 3：PostgreSQL 混合检索与重排",
|
||||
"required_true": ["thresholds_met"],
|
||||
"required_nonempty": [
|
||||
"embedding_deployment",
|
||||
"rerank_deployment",
|
||||
"metrics",
|
||||
"hnsw",
|
||||
],
|
||||
"required_positive": ["sample_count"],
|
||||
"required_values": {"environment": "postgresql"},
|
||||
},
|
||||
"stage4_distillation": {
|
||||
"title": "阶段 4：真实模型知识蒸馏闭环",
|
||||
"required_true": [
|
||||
"independent_review",
|
||||
"anonymization_review_passed",
|
||||
"published",
|
||||
"retrieved_after_publish",
|
||||
],
|
||||
"required_positive": ["sample_count"],
|
||||
"required_sets": {
|
||||
"services": {"postgresql", "embedding", "rerank", "text_model"},
|
||||
},
|
||||
},
|
||||
"stage4_documents": {
|
||||
"title": "阶段 4：真实文档安全导入闭环",
|
||||
"required_true": ["failure_recovery_verified"],
|
||||
"required_sets": {
|
||||
"formats": {"PDF", "TXT", "LOG", "DOCX", "XLSX", "PPTX"},
|
||||
"services": {"minio", "clamav", "postgresql", "embedding"},
|
||||
},
|
||||
},
|
||||
"stage6_channel": {
|
||||
"title": "阶段 6：真实企业渠道",
|
||||
"required_true": [
|
||||
"signature_verified",
|
||||
"inbound_verified",
|
||||
"outbound_verified",
|
||||
"receipt_verified",
|
||||
"idempotency_verified",
|
||||
],
|
||||
"required_nonempty": ["provider"],
|
||||
},
|
||||
"stage6_identity": {
|
||||
"title": "阶段 6：真实企业身份与目录",
|
||||
"required_true": [
|
||||
"oidc_login_verified",
|
||||
"group_claim_verified",
|
||||
"failure_fallback_verified",
|
||||
"ldap_directory_sync_verified",
|
||||
],
|
||||
"required_nonempty": ["oidc_provider", "ldap_directory"],
|
||||
},
|
||||
"stage6_production_ops": {
|
||||
"title": "阶段 6：生产告警、调度与异地备份",
|
||||
"required_true": [
|
||||
"alerts_verified",
|
||||
"backup_schedule_installed",
|
||||
"offsite_encrypted_copy_verified",
|
||||
],
|
||||
"required_nonempty": ["alert_receiver", "backup_schedule", "offsite_storage"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def build_blueprint_readiness_report(
|
||||
*,
|
||||
include_synthetic: bool = False,
|
||||
external_evidence: dict[str, Any] | None = None,
|
||||
recovery_evidence: dict[str, Any] | None = None,
|
||||
stage2_report: dict[str, Any] | None = None,
|
||||
stage3_report: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""生成九门槛退出报告；只有全部真实证据通过时才返回 ``exit_ready``。"""
|
||||
|
||||
stage2_report = (
|
||||
stage2_report
|
||||
if stage2_report is not None
|
||||
else _safe_evaluation_report(
|
||||
build_stage_2_acceptance_report,
|
||||
"阶段 2",
|
||||
include_synthetic,
|
||||
)
|
||||
)
|
||||
stage3_report = (
|
||||
stage3_report
|
||||
if stage3_report is not None
|
||||
else _safe_evaluation_report(
|
||||
build_stage_3_acceptance_report,
|
||||
"阶段 3",
|
||||
include_synthetic,
|
||||
)
|
||||
)
|
||||
gates = [
|
||||
_evaluation_gate(
|
||||
"stage2_intake",
|
||||
"阶段 2：脱敏真实案例智能建单",
|
||||
stage2_report,
|
||||
),
|
||||
_evaluation_gate(
|
||||
"stage3_support",
|
||||
"阶段 3：脱敏真实案例支持闭环",
|
||||
stage3_report,
|
||||
),
|
||||
]
|
||||
|
||||
manifest_reasons = _manifest_reasons(external_evidence)
|
||||
evidence_gates = (
|
||||
external_evidence.get("gates", {})
|
||||
if isinstance(external_evidence, dict)
|
||||
and isinstance(external_evidence.get("gates"), dict)
|
||||
else {}
|
||||
)
|
||||
for code, spec in EXTERNAL_GATE_SPECS.items():
|
||||
record = evidence_gates.get(code)
|
||||
reasons = (
|
||||
[*manifest_reasons]
|
||||
if manifest_reasons
|
||||
else _external_gate_reasons(code, record, spec)
|
||||
)
|
||||
gates.append(
|
||||
{
|
||||
"code": code,
|
||||
"title": spec["title"],
|
||||
"status": "passed" if not reasons else "not_ready",
|
||||
"source": "external_evidence_manifest",
|
||||
"reasons": reasons,
|
||||
"evidence": record if isinstance(record, dict) else {},
|
||||
}
|
||||
)
|
||||
|
||||
recovery_reasons = _recovery_reasons(recovery_evidence)
|
||||
gates.append(
|
||||
{
|
||||
"code": "stage6_recovery",
|
||||
"title": "阶段 6：隔离一致性恢复",
|
||||
"status": "passed" if not recovery_reasons else "not_ready",
|
||||
"source": "recovery_evidence",
|
||||
"reasons": recovery_reasons,
|
||||
"evidence": recovery_evidence if isinstance(recovery_evidence, dict) else {},
|
||||
}
|
||||
)
|
||||
|
||||
passed_count = sum(gate["status"] == "passed" for gate in gates)
|
||||
exit_ready = passed_count == len(gates)
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"generated_at": datetime.now(tz=UTC).isoformat(),
|
||||
"status": "passed" if exit_ready else "not_ready",
|
||||
"exit_ready": exit_ready,
|
||||
"summary": {
|
||||
"passed_gate_count": passed_count,
|
||||
"total_gate_count": len(gates),
|
||||
"remaining_gate_count": len(gates) - passed_count,
|
||||
},
|
||||
"gates": gates,
|
||||
}
|
||||
|
||||
|
||||
def _safe_evaluation_report(builder, stage: str, include_synthetic: bool) -> dict[str, Any]:
|
||||
"""将目标数据库未迁移或不可用转换为明确的未就绪结果。"""
|
||||
|
||||
try:
|
||||
return builder(include_synthetic=include_synthetic)
|
||||
except DatabaseError:
|
||||
return {
|
||||
"status": "not_ready",
|
||||
"exit_ready": False,
|
||||
"metrics": {},
|
||||
"reasons": [
|
||||
{
|
||||
"code": "database_schema_unavailable",
|
||||
"message": f"{stage} 验收数据不可读取，请先在目标环境执行最新数据库迁移",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _evaluation_gate(code: str, title: str, report: dict[str, Any]) -> dict[str, Any]:
|
||||
reasons = [
|
||||
str(reason.get("message") or reason.get("code") or "验收条件未满足")
|
||||
for reason in report.get("reasons", [])
|
||||
if isinstance(reason, dict)
|
||||
]
|
||||
if not report.get("exit_ready") and not reasons:
|
||||
reasons.append(f"{title}尚无足够真实案例证据")
|
||||
return {
|
||||
"code": code,
|
||||
"title": title,
|
||||
"status": "passed" if report.get("exit_ready") else "not_ready",
|
||||
"source": "database_evaluation",
|
||||
"reasons": reasons,
|
||||
"evidence": report,
|
||||
}
|
||||
|
||||
|
||||
def _manifest_reasons(payload: dict[str, Any] | None) -> list[str]:
|
||||
if payload is None:
|
||||
return ["未提供外部验收证据清单"]
|
||||
if not isinstance(payload, dict):
|
||||
return ["外部验收证据必须是 JSON 对象"]
|
||||
if payload.get("schema_version") != 1:
|
||||
return ["外部验收证据 schema_version 必须为 1"]
|
||||
if _find_sensitive_keys(payload):
|
||||
return ["外部验收证据不得包含密码、密钥、令牌或凭据字段"]
|
||||
if not isinstance(payload.get("gates"), dict):
|
||||
return ["外部验收证据缺少 gates 对象"]
|
||||
return []
|
||||
|
||||
|
||||
def _external_gate_reasons(code: str, record: Any, spec: dict[str, Any]) -> list[str]:
|
||||
if not isinstance(record, dict):
|
||||
return ["缺少该退出条件的外部验收记录"]
|
||||
reasons = []
|
||||
if record.get("status") != "passed":
|
||||
reasons.append("验收状态尚未标记为 passed")
|
||||
if not _valid_timestamp(record.get("verified_at")):
|
||||
reasons.append("verified_at 必须是带时区的 ISO 8601 时间")
|
||||
if not str(record.get("reviewer") or "").strip():
|
||||
reasons.append("缺少验收人 reviewer")
|
||||
artifacts = record.get("artifact_refs")
|
||||
if not isinstance(artifacts, list) or not artifacts or not all(
|
||||
isinstance(item, str) and item.strip() for item in artifacts
|
||||
):
|
||||
reasons.append("artifact_refs 必须包含至少一个非空证据引用")
|
||||
for field in spec.get("required_true", []):
|
||||
if record.get(field) is not True:
|
||||
reasons.append(f"{field} 必须为 true")
|
||||
for field in spec.get("required_nonempty", []):
|
||||
if record.get(field) in (None, "", [], {}):
|
||||
reasons.append(f"{field} 不能为空")
|
||||
for field in spec.get("required_positive", []):
|
||||
value = record.get(field)
|
||||
if not isinstance(value, (int, float)) or isinstance(value, bool) or value <= 0:
|
||||
reasons.append(f"{field} 必须大于 0")
|
||||
for field, expected in spec.get("required_values", {}).items():
|
||||
if record.get(field) != expected:
|
||||
reasons.append(f"{field} 必须为 {expected}")
|
||||
for field, expected_items in spec.get("required_sets", {}).items():
|
||||
value = record.get(field)
|
||||
actual = set(value) if isinstance(value, list) else set()
|
||||
missing = sorted(expected_items - actual)
|
||||
if missing:
|
||||
reasons.append(f"{field} 缺少: {', '.join(missing)}")
|
||||
if record.get("status") == "passed" and code == "stage3_retrieval":
|
||||
reasons.extend(_retrieval_metric_reasons(record))
|
||||
return reasons
|
||||
|
||||
|
||||
def _retrieval_metric_reasons(record: dict[str, Any]) -> list[str]:
|
||||
metrics = record.get("metrics")
|
||||
hnsw = record.get("hnsw")
|
||||
reasons = []
|
||||
if not isinstance(metrics, dict):
|
||||
return ["metrics 必须记录 recall_at_5 和 p95_latency_ms"]
|
||||
recall = metrics.get("recall_at_5")
|
||||
if not isinstance(recall, (int, float)) or isinstance(recall, bool) or not 0 <= recall <= 1:
|
||||
reasons.append("metrics.recall_at_5 必须位于 0 到 1")
|
||||
latency = metrics.get("p95_latency_ms")
|
||||
if not isinstance(latency, (int, float)) or isinstance(latency, bool) or latency <= 0:
|
||||
reasons.append("metrics.p95_latency_ms 必须大于 0")
|
||||
if not isinstance(hnsw, dict) or any(
|
||||
not isinstance(hnsw.get(key), int) or hnsw[key] <= 0
|
||||
for key in ("m", "ef_construction", "ef_search")
|
||||
):
|
||||
reasons.append("hnsw 必须记录正整数 m、ef_construction 和 ef_search")
|
||||
return reasons
|
||||
|
||||
|
||||
def _recovery_reasons(payload: dict[str, Any] | None) -> list[str]:
|
||||
if not isinstance(payload, dict):
|
||||
return ["缺少机器可读的隔离恢复演练证据"]
|
||||
reasons = []
|
||||
expected = {
|
||||
"status": "passed",
|
||||
"database_smoke": "ticket_and_attachment_restored",
|
||||
"object_storage_smoke": "marker_restored",
|
||||
"django_check": "passed",
|
||||
}
|
||||
for field, value in expected.items():
|
||||
if payload.get(field) != value:
|
||||
reasons.append(f"{field} 必须为 {value}")
|
||||
if not str(payload.get("backup_id") or "").strip():
|
||||
reasons.append("缺少 backup_id")
|
||||
duration = payload.get("duration_seconds")
|
||||
if not isinstance(duration, (int, float)) or isinstance(duration, bool) or duration <= 0:
|
||||
reasons.append("duration_seconds 必须大于 0")
|
||||
return reasons
|
||||
|
||||
|
||||
def _valid_timestamp(value: Any) -> bool:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
return False
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return False
|
||||
return parsed.tzinfo is not None
|
||||
|
||||
|
||||
def _find_sensitive_keys(value: Any, path: str = "") -> list[str]:
|
||||
"""递归定位可能承载密码、密钥、令牌或凭据的字段路径。"""
|
||||
|
||||
found = []
|
||||
if isinstance(value, dict):
|
||||
for key, item in value.items():
|
||||
next_path = f"{path}.{key}" if path else str(key)
|
||||
if any(part in str(key).lower() for part in SENSITIVE_KEY_PARTS):
|
||||
found.append(next_path)
|
||||
found.extend(_find_sensitive_keys(item, next_path))
|
||||
elif isinstance(value, list):
|
||||
for index, item in enumerate(value):
|
||||
found.extend(_find_sensitive_keys(item, f"{path}[{index}]"))
|
||||
return found
|
||||
Reference in New Issue
Block a user