ops: add production observability and recovery
This commit is contained in:
hailin
@@ -1,4 +1,19 @@
|
||||
from ninja import Router
|
||||
import hmac
|
||||
from datetime import timedelta
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.cache import cache
|
||||
from django.db import connection
|
||||
from django.db.models import Count
|
||||
from django.http import HttpResponse
|
||||
from django.utils import timezone
|
||||
from ninja import Router, Status
|
||||
|
||||
from apps.accounts.api import require_permission
|
||||
from apps.agents.models import AgentRun
|
||||
from apps.channels.models import ChannelMessage
|
||||
from apps.tools.models import ToolCall
|
||||
from apps.workitems.models import WorkItem
|
||||
|
||||
router = Router(tags=["system"])
|
||||
|
||||
@@ -6,3 +21,114 @@ router = Router(tags=["system"])
|
||||
@router.get("/health")
|
||||
def health(request):
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.get("/health/live", auth=None)
|
||||
def liveness(request):
|
||||
return {"status": "alive"}
|
||||
|
||||
|
||||
@router.get("/health/ready", response={200: dict, 503: dict}, auth=None)
|
||||
def readiness(request):
|
||||
checks = {}
|
||||
try:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("SELECT 1")
|
||||
checks["database"] = cursor.fetchone()[0] == 1
|
||||
except Exception:
|
||||
checks["database"] = False
|
||||
try:
|
||||
cache_key = "system-readiness"
|
||||
cache.set(cache_key, "ready", timeout=5)
|
||||
checks["cache"] = cache.get(cache_key) == "ready"
|
||||
cache.delete(cache_key)
|
||||
except Exception:
|
||||
checks["cache"] = False
|
||||
ready = all(checks.values())
|
||||
return Status(
|
||||
200 if ready else 503,
|
||||
{
|
||||
"status": "ready" if ready else "not_ready",
|
||||
"checks": checks,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def require_metrics_access(request) -> None:
|
||||
expected = settings.METRICS_TOKEN
|
||||
provided = request.headers.get("X-Metrics-Token", "")
|
||||
authorization = request.headers.get("Authorization", "")
|
||||
if not provided and authorization.startswith("Bearer "):
|
||||
provided = authorization.removeprefix("Bearer ").strip()
|
||||
if expected and hmac.compare_digest(expected, provided):
|
||||
return
|
||||
require_permission(request, "system.audit.view")
|
||||
|
||||
|
||||
def metric_lines(name: str, queryset) -> list[str]:
|
||||
return [
|
||||
f'{name}{{status="{row["status"]}"}} {row["total"]}'
|
||||
for row in queryset.values("status").annotate(total=Count("id")).order_by("status")
|
||||
]
|
||||
|
||||
|
||||
@router.get("/metrics")
|
||||
def metrics(request):
|
||||
require_metrics_access(request)
|
||||
now = timezone.now()
|
||||
recent_cutoff = now - timedelta(minutes=15)
|
||||
stale_cutoff = now - timedelta(minutes=10)
|
||||
lines = [
|
||||
"# HELP ai_ticket_channel_messages_total Channel messages by delivery status.",
|
||||
"# TYPE ai_ticket_channel_messages_total gauge",
|
||||
*metric_lines("ai_ticket_channel_messages_total", ChannelMessage.objects.all()),
|
||||
"# HELP ai_ticket_agent_runs_total Agent runs by status.",
|
||||
"# TYPE ai_ticket_agent_runs_total gauge",
|
||||
*metric_lines("ai_ticket_agent_runs_total", AgentRun.objects.all()),
|
||||
"# HELP ai_ticket_tool_calls_total Tool calls by status.",
|
||||
"# TYPE ai_ticket_tool_calls_total gauge",
|
||||
*metric_lines("ai_ticket_tool_calls_total", ToolCall.objects.all()),
|
||||
"# HELP ai_ticket_work_items_active Active work items.",
|
||||
"# TYPE ai_ticket_work_items_active gauge",
|
||||
f'ai_ticket_work_items_active {WorkItem.objects.filter(status="active").count()}',
|
||||
(
|
||||
"# HELP ai_ticket_channel_messages_failed_recent "
|
||||
"Failed channel messages updated in the last 15 minutes."
|
||||
),
|
||||
"# TYPE ai_ticket_channel_messages_failed_recent gauge",
|
||||
(
|
||||
"ai_ticket_channel_messages_failed_recent "
|
||||
f"{ChannelMessage.objects.filter(
|
||||
status='failed',
|
||||
updated_at__gte=recent_cutoff,
|
||||
).count()}"
|
||||
),
|
||||
"# HELP ai_ticket_channel_messages_stuck Channel messages queued for more than 10 minutes.",
|
||||
"# TYPE ai_ticket_channel_messages_stuck gauge",
|
||||
(
|
||||
"ai_ticket_channel_messages_stuck "
|
||||
f'{ChannelMessage.objects.filter(status="queued", created_at__lt=stale_cutoff).count()}'
|
||||
),
|
||||
"# HELP ai_ticket_agent_runs_failed_recent Agent runs failed in the last 15 minutes.",
|
||||
"# TYPE ai_ticket_agent_runs_failed_recent gauge",
|
||||
(
|
||||
"ai_ticket_agent_runs_failed_recent "
|
||||
f"{AgentRun.objects.filter(
|
||||
status='failed',
|
||||
finished_at__gte=recent_cutoff,
|
||||
).count()}"
|
||||
),
|
||||
"# HELP ai_ticket_tool_calls_failed_recent Tool calls failed in the last 15 minutes.",
|
||||
"# TYPE ai_ticket_tool_calls_failed_recent gauge",
|
||||
(
|
||||
"ai_ticket_tool_calls_failed_recent "
|
||||
f"{ToolCall.objects.filter(
|
||||
status='failed',
|
||||
finished_at__gte=recent_cutoff,
|
||||
).count()}"
|
||||
),
|
||||
]
|
||||
return HttpResponse(
|
||||
"\n".join(lines) + "\n",
|
||||
content_type="text/plain; version=0.0.4; charset=utf-8",
|
||||
)
|
||||
|
||||
330
backend/apps/system/blueprint_readiness.py
Normal file
330
backend/apps/system/blueprint_readiness.py
Normal file
@@ -0,0 +1,330 @@
|
||||
"""聚合阶段 2～6 的真实案例、外部服务和恢复证据，生成统一蓝图退出报告。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from django.db import DatabaseError
|
||||
|
||||
from apps.tickets.evaluation import build_stage_2_acceptance_report
|
||||
from apps.tickets.support_evaluation import build_stage_3_acceptance_report
|
||||
|
||||
SENSITIVE_KEY_PARTS = ("password", "secret", "token", "credential")
|
||||
|
||||
EXTERNAL_GATE_SPECS: dict[str, dict[str, Any]] = {
|
||||
"stage3_retrieval": {
|
||||
"title": "阶段 3：PostgreSQL 混合检索与重排",
|
||||
"required_true": ["thresholds_met"],
|
||||
"required_nonempty": [
|
||||
"embedding_deployment",
|
||||
"rerank_deployment",
|
||||
"metrics",
|
||||
"hnsw",
|
||||
],
|
||||
"required_positive": ["sample_count"],
|
||||
"required_values": {"environment": "postgresql"},
|
||||
},
|
||||
"stage4_distillation": {
|
||||
"title": "阶段 4：真实模型知识蒸馏闭环",
|
||||
"required_true": [
|
||||
"independent_review",
|
||||
"anonymization_review_passed",
|
||||
"published",
|
||||
"retrieved_after_publish",
|
||||
],
|
||||
"required_positive": ["sample_count"],
|
||||
"required_sets": {
|
||||
"services": {"postgresql", "embedding", "rerank", "text_model"},
|
||||
},
|
||||
},
|
||||
"stage4_documents": {
|
||||
"title": "阶段 4：真实文档安全导入闭环",
|
||||
"required_true": ["failure_recovery_verified"],
|
||||
"required_sets": {
|
||||
"formats": {"PDF", "TXT", "LOG", "DOCX", "XLSX", "PPTX"},
|
||||
"services": {"minio", "clamav", "postgresql", "embedding"},
|
||||
},
|
||||
},
|
||||
"stage6_channel": {
|
||||
"title": "阶段 6：真实企业渠道",
|
||||
"required_true": [
|
||||
"signature_verified",
|
||||
"inbound_verified",
|
||||
"outbound_verified",
|
||||
"receipt_verified",
|
||||
"idempotency_verified",
|
||||
],
|
||||
"required_nonempty": ["provider"],
|
||||
},
|
||||
"stage6_identity": {
|
||||
"title": "阶段 6：真实企业身份与目录",
|
||||
"required_true": [
|
||||
"oidc_login_verified",
|
||||
"group_claim_verified",
|
||||
"failure_fallback_verified",
|
||||
"ldap_directory_sync_verified",
|
||||
],
|
||||
"required_nonempty": ["oidc_provider", "ldap_directory"],
|
||||
},
|
||||
"stage6_production_ops": {
|
||||
"title": "阶段 6：生产告警、调度与异地备份",
|
||||
"required_true": [
|
||||
"alerts_verified",
|
||||
"backup_schedule_installed",
|
||||
"offsite_encrypted_copy_verified",
|
||||
],
|
||||
"required_nonempty": ["alert_receiver", "backup_schedule", "offsite_storage"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def build_blueprint_readiness_report(
|
||||
*,
|
||||
include_synthetic: bool = False,
|
||||
external_evidence: dict[str, Any] | None = None,
|
||||
recovery_evidence: dict[str, Any] | None = None,
|
||||
stage2_report: dict[str, Any] | None = None,
|
||||
stage3_report: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""生成九门槛退出报告；只有全部真实证据通过时才返回 ``exit_ready``。"""
|
||||
|
||||
stage2_report = (
|
||||
stage2_report
|
||||
if stage2_report is not None
|
||||
else _safe_evaluation_report(
|
||||
build_stage_2_acceptance_report,
|
||||
"阶段 2",
|
||||
include_synthetic,
|
||||
)
|
||||
)
|
||||
stage3_report = (
|
||||
stage3_report
|
||||
if stage3_report is not None
|
||||
else _safe_evaluation_report(
|
||||
build_stage_3_acceptance_report,
|
||||
"阶段 3",
|
||||
include_synthetic,
|
||||
)
|
||||
)
|
||||
gates = [
|
||||
_evaluation_gate(
|
||||
"stage2_intake",
|
||||
"阶段 2：脱敏真实案例智能建单",
|
||||
stage2_report,
|
||||
),
|
||||
_evaluation_gate(
|
||||
"stage3_support",
|
||||
"阶段 3：脱敏真实案例支持闭环",
|
||||
stage3_report,
|
||||
),
|
||||
]
|
||||
|
||||
manifest_reasons = _manifest_reasons(external_evidence)
|
||||
evidence_gates = (
|
||||
external_evidence.get("gates", {})
|
||||
if isinstance(external_evidence, dict)
|
||||
and isinstance(external_evidence.get("gates"), dict)
|
||||
else {}
|
||||
)
|
||||
for code, spec in EXTERNAL_GATE_SPECS.items():
|
||||
record = evidence_gates.get(code)
|
||||
reasons = (
|
||||
[*manifest_reasons]
|
||||
if manifest_reasons
|
||||
else _external_gate_reasons(code, record, spec)
|
||||
)
|
||||
gates.append(
|
||||
{
|
||||
"code": code,
|
||||
"title": spec["title"],
|
||||
"status": "passed" if not reasons else "not_ready",
|
||||
"source": "external_evidence_manifest",
|
||||
"reasons": reasons,
|
||||
"evidence": record if isinstance(record, dict) else {},
|
||||
}
|
||||
)
|
||||
|
||||
recovery_reasons = _recovery_reasons(recovery_evidence)
|
||||
gates.append(
|
||||
{
|
||||
"code": "stage6_recovery",
|
||||
"title": "阶段 6：隔离一致性恢复",
|
||||
"status": "passed" if not recovery_reasons else "not_ready",
|
||||
"source": "recovery_evidence",
|
||||
"reasons": recovery_reasons,
|
||||
"evidence": recovery_evidence if isinstance(recovery_evidence, dict) else {},
|
||||
}
|
||||
)
|
||||
|
||||
passed_count = sum(gate["status"] == "passed" for gate in gates)
|
||||
exit_ready = passed_count == len(gates)
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"generated_at": datetime.now(tz=UTC).isoformat(),
|
||||
"status": "passed" if exit_ready else "not_ready",
|
||||
"exit_ready": exit_ready,
|
||||
"summary": {
|
||||
"passed_gate_count": passed_count,
|
||||
"total_gate_count": len(gates),
|
||||
"remaining_gate_count": len(gates) - passed_count,
|
||||
},
|
||||
"gates": gates,
|
||||
}
|
||||
|
||||
|
||||
def _safe_evaluation_report(builder, stage: str, include_synthetic: bool) -> dict[str, Any]:
|
||||
"""将目标数据库未迁移或不可用转换为明确的未就绪结果。"""
|
||||
|
||||
try:
|
||||
return builder(include_synthetic=include_synthetic)
|
||||
except DatabaseError:
|
||||
return {
|
||||
"status": "not_ready",
|
||||
"exit_ready": False,
|
||||
"metrics": {},
|
||||
"reasons": [
|
||||
{
|
||||
"code": "database_schema_unavailable",
|
||||
"message": f"{stage} 验收数据不可读取，请先在目标环境执行最新数据库迁移",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _evaluation_gate(code: str, title: str, report: dict[str, Any]) -> dict[str, Any]:
|
||||
reasons = [
|
||||
str(reason.get("message") or reason.get("code") or "验收条件未满足")
|
||||
for reason in report.get("reasons", [])
|
||||
if isinstance(reason, dict)
|
||||
]
|
||||
if not report.get("exit_ready") and not reasons:
|
||||
reasons.append(f"{title}尚无足够真实案例证据")
|
||||
return {
|
||||
"code": code,
|
||||
"title": title,
|
||||
"status": "passed" if report.get("exit_ready") else "not_ready",
|
||||
"source": "database_evaluation",
|
||||
"reasons": reasons,
|
||||
"evidence": report,
|
||||
}
|
||||
|
||||
|
||||
def _manifest_reasons(payload: dict[str, Any] | None) -> list[str]:
|
||||
if payload is None:
|
||||
return ["未提供外部验收证据清单"]
|
||||
if not isinstance(payload, dict):
|
||||
return ["外部验收证据必须是 JSON 对象"]
|
||||
if payload.get("schema_version") != 1:
|
||||
return ["外部验收证据 schema_version 必须为 1"]
|
||||
if _find_sensitive_keys(payload):
|
||||
return ["外部验收证据不得包含密码、密钥、令牌或凭据字段"]
|
||||
if not isinstance(payload.get("gates"), dict):
|
||||
return ["外部验收证据缺少 gates 对象"]
|
||||
return []
|
||||
|
||||
|
||||
def _external_gate_reasons(code: str, record: Any, spec: dict[str, Any]) -> list[str]:
|
||||
if not isinstance(record, dict):
|
||||
return ["缺少该退出条件的外部验收记录"]
|
||||
reasons = []
|
||||
if record.get("status") != "passed":
|
||||
reasons.append("验收状态尚未标记为 passed")
|
||||
if not _valid_timestamp(record.get("verified_at")):
|
||||
reasons.append("verified_at 必须是带时区的 ISO 8601 时间")
|
||||
if not str(record.get("reviewer") or "").strip():
|
||||
reasons.append("缺少验收人 reviewer")
|
||||
artifacts = record.get("artifact_refs")
|
||||
if not isinstance(artifacts, list) or not artifacts or not all(
|
||||
isinstance(item, str) and item.strip() for item in artifacts
|
||||
):
|
||||
reasons.append("artifact_refs 必须包含至少一个非空证据引用")
|
||||
for field in spec.get("required_true", []):
|
||||
if record.get(field) is not True:
|
||||
reasons.append(f"{field} 必须为 true")
|
||||
for field in spec.get("required_nonempty", []):
|
||||
if record.get(field) in (None, "", [], {}):
|
||||
reasons.append(f"{field} 不能为空")
|
||||
for field in spec.get("required_positive", []):
|
||||
value = record.get(field)
|
||||
if not isinstance(value, (int, float)) or isinstance(value, bool) or value <= 0:
|
||||
reasons.append(f"{field} 必须大于 0")
|
||||
for field, expected in spec.get("required_values", {}).items():
|
||||
if record.get(field) != expected:
|
||||
reasons.append(f"{field} 必须为 {expected}")
|
||||
for field, expected_items in spec.get("required_sets", {}).items():
|
||||
value = record.get(field)
|
||||
actual = set(value) if isinstance(value, list) else set()
|
||||
missing = sorted(expected_items - actual)
|
||||
if missing:
|
||||
reasons.append(f"{field} 缺少: {', '.join(missing)}")
|
||||
if record.get("status") == "passed" and code == "stage3_retrieval":
|
||||
reasons.extend(_retrieval_metric_reasons(record))
|
||||
return reasons
|
||||
|
||||
|
||||
def _retrieval_metric_reasons(record: dict[str, Any]) -> list[str]:
|
||||
metrics = record.get("metrics")
|
||||
hnsw = record.get("hnsw")
|
||||
reasons = []
|
||||
if not isinstance(metrics, dict):
|
||||
return ["metrics 必须记录 recall_at_5 和 p95_latency_ms"]
|
||||
recall = metrics.get("recall_at_5")
|
||||
if not isinstance(recall, (int, float)) or isinstance(recall, bool) or not 0 <= recall <= 1:
|
||||
reasons.append("metrics.recall_at_5 必须位于 0 到 1")
|
||||
latency = metrics.get("p95_latency_ms")
|
||||
if not isinstance(latency, (int, float)) or isinstance(latency, bool) or latency <= 0:
|
||||
reasons.append("metrics.p95_latency_ms 必须大于 0")
|
||||
if not isinstance(hnsw, dict) or any(
|
||||
not isinstance(hnsw.get(key), int) or hnsw[key] <= 0
|
||||
for key in ("m", "ef_construction", "ef_search")
|
||||
):
|
||||
reasons.append("hnsw 必须记录正整数 m、ef_construction 和 ef_search")
|
||||
return reasons
|
||||
|
||||
|
||||
def _recovery_reasons(payload: dict[str, Any] | None) -> list[str]:
|
||||
if not isinstance(payload, dict):
|
||||
return ["缺少机器可读的隔离恢复演练证据"]
|
||||
reasons = []
|
||||
expected = {
|
||||
"status": "passed",
|
||||
"database_smoke": "ticket_and_attachment_restored",
|
||||
"object_storage_smoke": "marker_restored",
|
||||
"django_check": "passed",
|
||||
}
|
||||
for field, value in expected.items():
|
||||
if payload.get(field) != value:
|
||||
reasons.append(f"{field} 必须为 {value}")
|
||||
if not str(payload.get("backup_id") or "").strip():
|
||||
reasons.append("缺少 backup_id")
|
||||
duration = payload.get("duration_seconds")
|
||||
if not isinstance(duration, (int, float)) or isinstance(duration, bool) or duration <= 0:
|
||||
reasons.append("duration_seconds 必须大于 0")
|
||||
return reasons
|
||||
|
||||
|
||||
def _valid_timestamp(value: Any) -> bool:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
return False
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return False
|
||||
return parsed.tzinfo is not None
|
||||
|
||||
|
||||
def _find_sensitive_keys(value: Any, path: str = "") -> list[str]:
|
||||
"""递归定位可能承载密码、密钥、令牌或凭据的字段路径。"""
|
||||
|
||||
found = []
|
||||
if isinstance(value, dict):
|
||||
for key, item in value.items():
|
||||
next_path = f"{path}.{key}" if path else str(key)
|
||||
if any(part in str(key).lower() for part in SENSITIVE_KEY_PARTS):
|
||||
found.append(next_path)
|
||||
found.extend(_find_sensitive_keys(item, next_path))
|
||||
elif isinstance(value, list):
|
||||
for index, item in enumerate(value):
|
||||
found.extend(_find_sensitive_keys(item, f"{path}[{index}]"))
|
||||
return found
|
||||
1
backend/apps/system/management/__init__.py
Normal file
1
backend/apps/system/management/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
|
||||
1
backend/apps/system/management/commands/__init__.py
Normal file
1
backend/apps/system/management/commands/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from apps.system.blueprint_readiness import build_blueprint_readiness_report
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "生成覆盖阶段 2～6 退出条件的统一蓝图就绪报告"
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument("--evidence", help="外部验收证据 JSON；不得包含任何凭据")
|
||||
parser.add_argument(
|
||||
"--recovery-evidence",
|
||||
help="隔离恢复演练 JSON；默认读取仓库内最新已登记记录",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--include-synthetic",
|
||||
action="store_true",
|
||||
help="包含合成案例用于诊断；合成案例不会使阶段 2/3 通过",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--format",
|
||||
choices=["text", "json"],
|
||||
default="text",
|
||||
dest="output_format",
|
||||
)
|
||||
parser.add_argument("--output", help="可选 JSON 报告输出路径")
|
||||
parser.add_argument(
|
||||
"--fail-on-not-ready",
|
||||
action="store_true",
|
||||
help="任一蓝图退出条件未满足时返回非零退出码",
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
workspace_root = Path(settings.BASE_DIR)
|
||||
recovery_path = Path(
|
||||
options.get("recovery_evidence")
|
||||
or workspace_root / "docs/operations/recovery-drill-2026-07-27.json"
|
||||
)
|
||||
external = _load_json(options.get("evidence"), label="外部验收证据")
|
||||
recovery = _load_json(recovery_path, label="隔离恢复证据")
|
||||
report = build_blueprint_readiness_report(
|
||||
include_synthetic=options["include_synthetic"],
|
||||
external_evidence=external,
|
||||
recovery_evidence=recovery,
|
||||
)
|
||||
report_json = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
|
||||
output_path = options.get("output")
|
||||
if output_path:
|
||||
path = Path(output_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(f"{report_json}\n", encoding="utf-8")
|
||||
|
||||
if options["output_format"] == "json":
|
||||
self.stdout.write(report_json)
|
||||
else:
|
||||
summary = report["summary"]
|
||||
self.stdout.write(
|
||||
f"蓝图验收状态：{report['status']}；"
|
||||
f"已通过 {summary['passed_gate_count']}/{summary['total_gate_count']}；"
|
||||
f"剩余 {summary['remaining_gate_count']}"
|
||||
)
|
||||
for gate in report["gates"]:
|
||||
marker = "通过" if gate["status"] == "passed" else "待验收"
|
||||
self.stdout.write(f"[{marker}] {gate['title']}")
|
||||
for reason in gate["reasons"]:
|
||||
self.stdout.write(self.style.WARNING(f" - {reason}"))
|
||||
if output_path:
|
||||
self.stdout.write(f"JSON 报告：{Path(output_path).resolve()}")
|
||||
|
||||
if options["fail_on_not_ready"] and not report["exit_ready"]:
|
||||
raise CommandError("产品蓝图尚未满足全部退出条件")
|
||||
|
||||
|
||||
def _load_json(path_value, *, label: str):
|
||||
if not path_value:
|
||||
return None
|
||||
path = Path(path_value)
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as error:
|
||||
raise CommandError(f"{label}无法读取：{error}") from error
|
||||
if not isinstance(payload, dict):
|
||||
raise CommandError(f"{label}必须是 JSON 对象")
|
||||
return payload
|
||||
176
backend/apps/system/tests/test_blueprint_readiness.py
Normal file
176
backend/apps/system/tests/test_blueprint_readiness.py
Normal file
@@ -0,0 +1,176 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from django.core.management import call_command
|
||||
from django.core.management.base import CommandError
|
||||
from django.db import DatabaseError
|
||||
|
||||
from apps.system import blueprint_readiness
|
||||
from apps.system.blueprint_readiness import (
|
||||
EXTERNAL_GATE_SPECS,
|
||||
build_blueprint_readiness_report,
|
||||
)
|
||||
|
||||
|
||||
def passed_evaluation_report():
|
||||
return {"status": "passed", "exit_ready": True, "reasons": [], "metrics": {}}
|
||||
|
||||
|
||||
def complete_external_evidence():
|
||||
common = {
|
||||
"status": "passed",
|
||||
"verified_at": "2026-07-27T09:30:00Z",
|
||||
"reviewer": "acceptance-reviewer",
|
||||
"artifact_refs": ["evidence/run-001.json"],
|
||||
}
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"gates": {
|
||||
"stage3_retrieval": {
|
||||
**common,
|
||||
"environment": "postgresql",
|
||||
"embedding_deployment": "embedding-prod",
|
||||
"rerank_deployment": "rerank-prod",
|
||||
"sample_count": 50,
|
||||
"thresholds_met": True,
|
||||
"metrics": {"recall_at_5": 0.92, "p95_latency_ms": 180},
|
||||
"hnsw": {"m": 16, "ef_construction": 64, "ef_search": 40},
|
||||
},
|
||||
"stage4_distillation": {
|
||||
**common,
|
||||
"sample_count": 20,
|
||||
"independent_review": True,
|
||||
"anonymization_review_passed": True,
|
||||
"published": True,
|
||||
"retrieved_after_publish": True,
|
||||
"services": ["postgresql", "embedding", "rerank", "text_model"],
|
||||
},
|
||||
"stage4_documents": {
|
||||
**common,
|
||||
"failure_recovery_verified": True,
|
||||
"formats": ["PDF", "TXT", "LOG", "DOCX", "XLSX", "PPTX"],
|
||||
"services": ["minio", "clamav", "postgresql", "embedding"],
|
||||
},
|
||||
"stage6_channel": {
|
||||
**common,
|
||||
"provider": "enterprise-chat",
|
||||
"signature_verified": True,
|
||||
"inbound_verified": True,
|
||||
"outbound_verified": True,
|
||||
"receipt_verified": True,
|
||||
"idempotency_verified": True,
|
||||
},
|
||||
"stage6_identity": {
|
||||
**common,
|
||||
"oidc_provider": "enterprise-idp",
|
||||
"ldap_directory": "enterprise-directory",
|
||||
"oidc_login_verified": True,
|
||||
"group_claim_verified": True,
|
||||
"failure_fallback_verified": True,
|
||||
"ldap_directory_sync_verified": True,
|
||||
},
|
||||
"stage6_production_ops": {
|
||||
**common,
|
||||
"alert_receiver": "on-call",
|
||||
"backup_schedule": "daily",
|
||||
"offsite_storage": "encrypted-vault",
|
||||
"alerts_verified": True,
|
||||
"backup_schedule_installed": True,
|
||||
"offsite_encrypted_copy_verified": True,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def recovery_evidence():
|
||||
return {
|
||||
"status": "passed",
|
||||
"backup_id": "20260727T092235Z",
|
||||
"duration_seconds": 106.57,
|
||||
"database_smoke": "ticket_and_attachment_restored",
|
||||
"object_storage_smoke": "marker_restored",
|
||||
"django_check": "passed",
|
||||
}
|
||||
|
||||
|
||||
def test_complete_blueprint_evidence_passes_all_gates():
|
||||
report = build_blueprint_readiness_report(
|
||||
stage2_report=passed_evaluation_report(),
|
||||
stage3_report=passed_evaluation_report(),
|
||||
external_evidence=complete_external_evidence(),
|
||||
recovery_evidence=recovery_evidence(),
|
||||
)
|
||||
|
||||
assert report["exit_ready"] is True
|
||||
assert report["status"] == "passed"
|
||||
assert report["summary"] == {
|
||||
"passed_gate_count": 9,
|
||||
"total_gate_count": 9,
|
||||
"remaining_gate_count": 0,
|
||||
}
|
||||
assert {gate["code"] for gate in report["gates"]} == {
|
||||
"stage2_intake",
|
||||
"stage3_support",
|
||||
*EXTERNAL_GATE_SPECS,
|
||||
"stage6_recovery",
|
||||
}
|
||||
|
||||
|
||||
def test_external_evidence_rejects_sensitive_keys_and_incomplete_contract():
|
||||
evidence = complete_external_evidence()
|
||||
evidence["gates"]["stage6_identity"]["client_secret"] = "must-not-be-recorded"
|
||||
report = build_blueprint_readiness_report(
|
||||
stage2_report=passed_evaluation_report(),
|
||||
stage3_report=passed_evaluation_report(),
|
||||
external_evidence=evidence,
|
||||
recovery_evidence=recovery_evidence(),
|
||||
)
|
||||
|
||||
external_gates = [
|
||||
gate for gate in report["gates"] if gate["source"] == "external_evidence_manifest"
|
||||
]
|
||||
assert all(gate["status"] == "not_ready" for gate in external_gates)
|
||||
assert all("不得包含密码" in gate["reasons"][0] for gate in external_gates)
|
||||
|
||||
|
||||
def test_database_schema_failure_becomes_not_ready_gate(monkeypatch):
|
||||
def unavailable_report(*, include_synthetic):
|
||||
raise DatabaseError("schema unavailable")
|
||||
|
||||
monkeypatch.setattr(
|
||||
blueprint_readiness,
|
||||
"build_stage_2_acceptance_report",
|
||||
unavailable_report,
|
||||
)
|
||||
report = build_blueprint_readiness_report(
|
||||
stage3_report=passed_evaluation_report(),
|
||||
external_evidence=complete_external_evidence(),
|
||||
recovery_evidence=recovery_evidence(),
|
||||
)
|
||||
|
||||
stage2_gate = next(gate for gate in report["gates"] if gate["code"] == "stage2_intake")
|
||||
assert stage2_gate["status"] == "not_ready"
|
||||
assert "数据库迁移" in stage2_gate["reasons"][0]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_blueprint_command_writes_not_ready_report_and_can_fail_ci(tmp_path):
|
||||
output = tmp_path / "blueprint-readiness.json"
|
||||
missing_recovery = tmp_path / "missing-recovery.json"
|
||||
|
||||
call_command(
|
||||
"report_blueprint_readiness",
|
||||
output=str(output),
|
||||
)
|
||||
report = json.loads(output.read_text(encoding="utf-8"))
|
||||
assert report["status"] == "not_ready"
|
||||
assert report["exit_ready"] is False
|
||||
recovery_gate = next(gate for gate in report["gates"] if gate["code"] == "stage6_recovery")
|
||||
assert recovery_gate["status"] == "passed"
|
||||
|
||||
with pytest.raises(CommandError, match="尚未满足"):
|
||||
call_command(
|
||||
"report_blueprint_readiness",
|
||||
recovery_evidence=str(missing_recovery),
|
||||
fail_on_not_ready=True,
|
||||
)
|
||||
@@ -1,5 +1,51 @@
|
||||
import pytest
|
||||
from django.test import override_settings
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
|
||||
def test_health_endpoint_returns_ok(client):
|
||||
response = client.get("/api/system/health")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"status": "ok"}
|
||||
|
||||
|
||||
def test_liveness_and_readiness_endpoints(client):
|
||||
live_response = client.get("/api/system/health/live")
|
||||
ready_response = client.get("/api/system/health/ready")
|
||||
|
||||
assert live_response.status_code == 200
|
||||
assert live_response.json() == {"status": "alive"}
|
||||
assert ready_response.status_code == 200
|
||||
assert ready_response.json() == {
|
||||
"status": "ready",
|
||||
"checks": {"database": True, "cache": True},
|
||||
}
|
||||
|
||||
|
||||
def test_metrics_endpoint_exposes_low_cardinality_metrics(admin_client):
|
||||
response = admin_client.get("/api/system/metrics")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response["Content-Type"].startswith("text/plain")
|
||||
content = response.content.decode()
|
||||
assert "ai_ticket_channel_messages_total" in content
|
||||
assert "ai_ticket_agent_runs_total" in content
|
||||
assert "ai_ticket_tool_calls_total" in content
|
||||
assert "ai_ticket_work_items_active" in content
|
||||
assert "ai_ticket_channel_messages_failed_recent" in content
|
||||
assert "ai_ticket_channel_messages_stuck" in content
|
||||
assert "ai_ticket_agent_runs_failed_recent" in content
|
||||
assert "ai_ticket_tool_calls_failed_recent" in content
|
||||
|
||||
|
||||
@override_settings(METRICS_TOKEN="prometheus-secret")
|
||||
def test_metrics_endpoint_accepts_bearer_token_for_prometheus(client):
|
||||
response = client.get(
|
||||
"/api/system/metrics",
|
||||
HTTP_AUTHORIZATION="Bearer prometheus-secret",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "ai_ticket_channel_messages_total" in response.content.decode()
|
||||
|
||||
@@ -6,6 +6,7 @@ requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"Django>=5.0,<5.3",
|
||||
"django-ninja>=1.3,<2.0",
|
||||
"gunicorn>=23.0,<24.0",
|
||||
"psycopg[binary]>=3.2,<4.0",
|
||||
"celery>=5.4,<6.0",
|
||||
"redis>=5.0,<6.0",
|
||||
@@ -17,6 +18,13 @@ dependencies = [
|
||||
"jsonschema>=4.23,<5.0",
|
||||
"minio>=7.2,<8.0",
|
||||
"Pillow>=11.0,<12.0",
|
||||
"pypdf>=5.0,<7.0",
|
||||
"python-docx>=1.1,<2.0",
|
||||
"openpyxl>=3.1,<4.0",
|
||||
"python-pptx>=1.0,<2.0",
|
||||
"pgvector>=0.4,<0.5",
|
||||
"Authlib>=1.6,<2.0",
|
||||
"requests>=2.32,<3.0",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
@@ -43,6 +51,7 @@ testpaths = ["apps", "tests"]
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
target-version = "py312"
|
||||
extend-exclude = ["**/migrations/*.py"]
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "UP", "B"]
|
||||
|
||||
Reference in New Issue
Block a user