ops: add production observability and recovery

.env.example

@@ -3,6 +3,10 @@ DJANGO_DEBUG=true
 DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1
 CORS_ALLOWED_ORIGINS=http://localhost:5173
 CSRF_TRUSTED_ORIGINS=http://localhost:5173,http://127.0.0.1:5173
+# OIDC 完成后重定向到的前端根地址；生产环境必须使用 HTTPS。
+FRONTEND_APP_URL=http://localhost:5173
+# 对外可访问的后端根地址，例如 https://tickets-api.example.com；留空时从当前请求推导。
+OIDC_CALLBACK_BASE_URL=
 DJANGO_SUPERUSER_USERNAME=admin
 DJANGO_SUPERUSER_PASSWORD=admin12345
 DJANGO_SUPERUSER_EMAIL=admin@example.com
@@ -15,9 +19,40 @@ POSTGRES_PORT=5432
 
 CELERY_BROKER_URL=redis://redis:6379/0
 CELERY_RESULT_BACKEND=redis://redis:6379/1
+# 必须是稳定的 Fernet 密钥；生成后请妥善保存，后续更换会导致已有模型凭据无法解密。
+# 生成命令：python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"
 AI_CREDENTIAL_ENCRYPTION_KEY=
+# 内部只读工单 MCP 的 Bearer Token；为空时 /internal-mcp 返回 503。
+INTERNAL_MCP_TOKEN=
+METRICS_TOKEN=
 AI_RUN_RETENTION_DAYS=90
+AI_SLOW_RUN_THRESHOLD_SECONDS=20
+AI_AGENT_MODEL_TIMEOUT_SECONDS=20
+KNOWLEDGE_EMBEDDING_DEPLOYMENT_ID=0
+KNOWLEDGE_EMBEDDING_DIMENSIONS=1536
+KNOWLEDGE_EMBEDDING_ALLOW_HASH_FALLBACK=false
+KNOWLEDGE_RETRIEVAL_TEXT_WEIGHT=0.45
+KNOWLEDGE_RETRIEVAL_VECTOR_WEIGHT=0.35
+KNOWLEDGE_RETRIEVAL_SCOPE_WEIGHT=0.20
+KNOWLEDGE_RERANK_DEPLOYMENT_ID=0
+KNOWLEDGE_RERANK_CANDIDATE_LIMIT=30
+KNOWLEDGE_RERANK_MODEL_WEIGHT=0.60
+TICKET_AUTO_RESOLVE_ENABLED=false
+TICKET_AUTO_RESOLVE_MIN_SCORE=0.85
+WORK_ITEM_DEFAULT_SLA_HOURS=24
+WORK_ITEM_INTAKE_FAILURE_SLA_HOURS=4
+WORK_ITEM_SUPPORT_FAILURE_SLA_HOURS=4
+WORK_ITEM_CUSTOMER_REPLY_SLA_HOURS=8
+WORK_ITEM_TICKET_CLOSE_SLA_HOURS=24
 TICKET_DRAFT_RETENTION_HOURS=24
+TICKET_MATERIAL_MAX_FILE_BYTES=20971520
+TICKET_MATERIAL_MAX_FILES=10
+TICKET_MATERIAL_MAX_TOTAL_BYTES=104857600
+TICKET_MATERIAL_MAX_PARSED_CHARS=200000
+TICKET_MATERIAL_SCANNER_BACKEND=clamav
+CLAMAV_HOST=clamav
+CLAMAV_PORT=3310
+CLAMAV_TIMEOUT_SECONDS=30
 
 MINIO_ROOT_USER=minio
 MINIO_ROOT_PASSWORD=minio123

.gitignore

@@ -40,3 +40,4 @@ skills-lock.json
 frontend/skills-lock.json
 
 infra/data/
+backups/

backend/apps/system/api.py

@@ -1,4 +1,19 @@
-from ninja import Router
+import hmac
+from datetime import timedelta
+
+from django.conf import settings
+from django.core.cache import cache
+from django.db import connection
+from django.db.models import Count
+from django.http import HttpResponse
+from django.utils import timezone
+from ninja import Router, Status
+
+from apps.accounts.api import require_permission
+from apps.agents.models import AgentRun
+from apps.channels.models import ChannelMessage
+from apps.tools.models import ToolCall
+from apps.workitems.models import WorkItem
 
 router = Router(tags=["system"])
 
@@ -6,3 +21,114 @@ router = Router(tags=["system"])
 @router.get("/health")
 def health(request):
     return {"status": "ok"}
+
+
+@router.get("/health/live", auth=None)
+def liveness(request):
+    return {"status": "alive"}
+
+
+@router.get("/health/ready", response={200: dict, 503: dict}, auth=None)
+def readiness(request):
+    checks = {}
+    try:
+        with connection.cursor() as cursor:
+            cursor.execute("SELECT 1")
+            checks["database"] = cursor.fetchone()[0] == 1
+    except Exception:
+        checks["database"] = False
+    try:
+        cache_key = "system-readiness"
+        cache.set(cache_key, "ready", timeout=5)
+        checks["cache"] = cache.get(cache_key) == "ready"
+        cache.delete(cache_key)
+    except Exception:
+        checks["cache"] = False
+    ready = all(checks.values())
+    return Status(
+        200 if ready else 503,
+        {
+            "status": "ready" if ready else "not_ready",
+            "checks": checks,
+        },
+    )
+
+
+def require_metrics_access(request) -> None:
+    expected = settings.METRICS_TOKEN
+    provided = request.headers.get("X-Metrics-Token", "")
+    authorization = request.headers.get("Authorization", "")
+    if not provided and authorization.startswith("Bearer "):
+        provided = authorization.removeprefix("Bearer ").strip()
+    if expected and hmac.compare_digest(expected, provided):
+        return
+    require_permission(request, "system.audit.view")
+
+
+def metric_lines(name: str, queryset) -> list[str]:
+    return [
+        f'{name}{{status="{row["status"]}"}} {row["total"]}'
+        for row in queryset.values("status").annotate(total=Count("id")).order_by("status")
+    ]
+
+
+@router.get("/metrics")
+def metrics(request):
+    require_metrics_access(request)
+    now = timezone.now()
+    recent_cutoff = now - timedelta(minutes=15)
+    stale_cutoff = now - timedelta(minutes=10)
+    lines = [
+        "# HELP ai_ticket_channel_messages_total Channel messages by delivery status.",
+        "# TYPE ai_ticket_channel_messages_total gauge",
+        *metric_lines("ai_ticket_channel_messages_total", ChannelMessage.objects.all()),
+        "# HELP ai_ticket_agent_runs_total Agent runs by status.",
+        "# TYPE ai_ticket_agent_runs_total gauge",
+        *metric_lines("ai_ticket_agent_runs_total", AgentRun.objects.all()),
+        "# HELP ai_ticket_tool_calls_total Tool calls by status.",
+        "# TYPE ai_ticket_tool_calls_total gauge",
+        *metric_lines("ai_ticket_tool_calls_total", ToolCall.objects.all()),
+        "# HELP ai_ticket_work_items_active Active work items.",
+        "# TYPE ai_ticket_work_items_active gauge",
+        f'ai_ticket_work_items_active {WorkItem.objects.filter(status="active").count()}',
+        (
+            "# HELP ai_ticket_channel_messages_failed_recent "
+            "Failed channel messages updated in the last 15 minutes."
+        ),
+        "# TYPE ai_ticket_channel_messages_failed_recent gauge",
+        (
+            "ai_ticket_channel_messages_failed_recent "
+            f"{ChannelMessage.objects.filter(
+                status='failed',
+                updated_at__gte=recent_cutoff,
+            ).count()}"
+        ),
+        "# HELP ai_ticket_channel_messages_stuck Channel messages queued for more than 10 minutes.",
+        "# TYPE ai_ticket_channel_messages_stuck gauge",
+        (
+            "ai_ticket_channel_messages_stuck "
+            f'{ChannelMessage.objects.filter(status="queued", created_at__lt=stale_cutoff).count()}'
+        ),
+        "# HELP ai_ticket_agent_runs_failed_recent Agent runs failed in the last 15 minutes.",
+        "# TYPE ai_ticket_agent_runs_failed_recent gauge",
+        (
+            "ai_ticket_agent_runs_failed_recent "
+            f"{AgentRun.objects.filter(
+                status='failed',
+                finished_at__gte=recent_cutoff,
+            ).count()}"
+        ),
+        "# HELP ai_ticket_tool_calls_failed_recent Tool calls failed in the last 15 minutes.",
+        "# TYPE ai_ticket_tool_calls_failed_recent gauge",
+        (
+            "ai_ticket_tool_calls_failed_recent "
+            f"{ToolCall.objects.filter(
+                status='failed',
+                finished_at__gte=recent_cutoff,
+            ).count()}"
+        ),
+    ]
+    return HttpResponse(
+        "\n".join(lines) + "\n",
+        content_type="text/plain; version=0.0.4; charset=utf-8",
+    )

backend/apps/system/blueprint_readiness.py

@@ -0,0 +1,330 @@
+"""聚合阶段 2～6 的真实案例、外部服务和恢复证据，生成统一蓝图退出报告。"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Any
+
+from django.db import DatabaseError
+
+from apps.tickets.evaluation import build_stage_2_acceptance_report
+from apps.tickets.support_evaluation import build_stage_3_acceptance_report
+
+SENSITIVE_KEY_PARTS = ("password", "secret", "token", "credential")
+
+EXTERNAL_GATE_SPECS: dict[str, dict[str, Any]] = {
+    "stage3_retrieval": {
+        "title": "阶段 3：PostgreSQL 混合检索与重排",
+        "required_true": ["thresholds_met"],
+        "required_nonempty": [
+            "embedding_deployment",
+            "rerank_deployment",
+            "metrics",
+            "hnsw",
+        ],
+        "required_positive": ["sample_count"],
+        "required_values": {"environment": "postgresql"},
+    },
+    "stage4_distillation": {
+        "title": "阶段 4：真实模型知识蒸馏闭环",
+        "required_true": [
+            "independent_review",
+            "anonymization_review_passed",
+            "published",
+            "retrieved_after_publish",
+        ],
+        "required_positive": ["sample_count"],
+        "required_sets": {
+            "services": {"postgresql", "embedding", "rerank", "text_model"},
+        },
+    },
+    "stage4_documents": {
+        "title": "阶段 4：真实文档安全导入闭环",
+        "required_true": ["failure_recovery_verified"],
+        "required_sets": {
+            "formats": {"PDF", "TXT", "LOG", "DOCX", "XLSX", "PPTX"},
+            "services": {"minio", "clamav", "postgresql", "embedding"},
+        },
+    },
+    "stage6_channel": {
+        "title": "阶段 6：真实企业渠道",
+        "required_true": [
+            "signature_verified",
+            "inbound_verified",
+            "outbound_verified",
+            "receipt_verified",
+            "idempotency_verified",
+        ],
+        "required_nonempty": ["provider"],
+    },
+    "stage6_identity": {
+        "title": "阶段 6：真实企业身份与目录",
+        "required_true": [
+            "oidc_login_verified",
+            "group_claim_verified",
+            "failure_fallback_verified",
+            "ldap_directory_sync_verified",
+        ],
+        "required_nonempty": ["oidc_provider", "ldap_directory"],
+    },
+    "stage6_production_ops": {
+        "title": "阶段 6：生产告警、调度与异地备份",
+        "required_true": [
+            "alerts_verified",
+            "backup_schedule_installed",
+            "offsite_encrypted_copy_verified",
+        ],
+        "required_nonempty": ["alert_receiver", "backup_schedule", "offsite_storage"],
+    },
+}
+
+
+def build_blueprint_readiness_report(
+    *,
+    include_synthetic: bool = False,
+    external_evidence: dict[str, Any] | None = None,
+    recovery_evidence: dict[str, Any] | None = None,
+    stage2_report: dict[str, Any] | None = None,
+    stage3_report: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """生成九门槛退出报告；只有全部真实证据通过时才返回 ``exit_ready``。"""
+
+    stage2_report = (
+        stage2_report
+        if stage2_report is not None
+        else _safe_evaluation_report(
+            build_stage_2_acceptance_report,
+            "阶段 2",
+            include_synthetic,
+        )
+    )
+    stage3_report = (
+        stage3_report
+        if stage3_report is not None
+        else _safe_evaluation_report(
+            build_stage_3_acceptance_report,
+            "阶段 3",
+            include_synthetic,
+        )
+    )
+    gates = [
+        _evaluation_gate(
+            "stage2_intake",
+            "阶段 2：脱敏真实案例智能建单",
+            stage2_report,
+        ),
+        _evaluation_gate(
+            "stage3_support",
+            "阶段 3：脱敏真实案例支持闭环",
+            stage3_report,
+        ),
+    ]
+
+    manifest_reasons = _manifest_reasons(external_evidence)
+    evidence_gates = (
+        external_evidence.get("gates", {})
+        if isinstance(external_evidence, dict)
+        and isinstance(external_evidence.get("gates"), dict)
+        else {}
+    )
+    for code, spec in EXTERNAL_GATE_SPECS.items():
+        record = evidence_gates.get(code)
+        reasons = (
+            [*manifest_reasons]
+            if manifest_reasons
+            else _external_gate_reasons(code, record, spec)
+        )
+        gates.append(
+            {
+                "code": code,
+                "title": spec["title"],
+                "status": "passed" if not reasons else "not_ready",
+                "source": "external_evidence_manifest",
+                "reasons": reasons,
+                "evidence": record if isinstance(record, dict) else {},
+            }
+        )
+
+    recovery_reasons = _recovery_reasons(recovery_evidence)
+    gates.append(
+        {
+            "code": "stage6_recovery",
+            "title": "阶段 6：隔离一致性恢复",
+            "status": "passed" if not recovery_reasons else "not_ready",
+            "source": "recovery_evidence",
+            "reasons": recovery_reasons,
+            "evidence": recovery_evidence if isinstance(recovery_evidence, dict) else {},
+        }
+    )
+
+    passed_count = sum(gate["status"] == "passed" for gate in gates)
+    exit_ready = passed_count == len(gates)
+    return {
+        "schema_version": 1,
+        "generated_at": datetime.now(tz=UTC).isoformat(),
+        "status": "passed" if exit_ready else "not_ready",
+        "exit_ready": exit_ready,
+        "summary": {
+            "passed_gate_count": passed_count,
+            "total_gate_count": len(gates),
+            "remaining_gate_count": len(gates) - passed_count,
+        },
+        "gates": gates,
+    }
+
+
+def _safe_evaluation_report(builder, stage: str, include_synthetic: bool) -> dict[str, Any]:
+    """将目标数据库未迁移或不可用转换为明确的未就绪结果。"""
+
+    try:
+        return builder(include_synthetic=include_synthetic)
+    except DatabaseError:
+        return {
+            "status": "not_ready",
+            "exit_ready": False,
+            "metrics": {},
+            "reasons": [
+                {
+                    "code": "database_schema_unavailable",
+                    "message": f"{stage} 验收数据不可读取，请先在目标环境执行最新数据库迁移",
+                }
+            ],
+        }
+
+
+def _evaluation_gate(code: str, title: str, report: dict[str, Any]) -> dict[str, Any]:
+    reasons = [
+        str(reason.get("message") or reason.get("code") or "验收条件未满足")
+        for reason in report.get("reasons", [])
+        if isinstance(reason, dict)
+    ]
+    if not report.get("exit_ready") and not reasons:
+        reasons.append(f"{title}尚无足够真实案例证据")
+    return {
+        "code": code,
+        "title": title,
+        "status": "passed" if report.get("exit_ready") else "not_ready",
+        "source": "database_evaluation",
+        "reasons": reasons,
+        "evidence": report,
+    }
+
+
+def _manifest_reasons(payload: dict[str, Any] | None) -> list[str]:
+    if payload is None:
+        return ["未提供外部验收证据清单"]
+    if not isinstance(payload, dict):
+        return ["外部验收证据必须是 JSON 对象"]
+    if payload.get("schema_version") != 1:
+        return ["外部验收证据 schema_version 必须为 1"]
+    if _find_sensitive_keys(payload):
+        return ["外部验收证据不得包含密码、密钥、令牌或凭据字段"]
+    if not isinstance(payload.get("gates"), dict):
+        return ["外部验收证据缺少 gates 对象"]
+    return []
+
+
+def _external_gate_reasons(code: str, record: Any, spec: dict[str, Any]) -> list[str]:
+    if not isinstance(record, dict):
+        return ["缺少该退出条件的外部验收记录"]
+    reasons = []
+    if record.get("status") != "passed":
+        reasons.append("验收状态尚未标记为 passed")
+    if not _valid_timestamp(record.get("verified_at")):
+        reasons.append("verified_at 必须是带时区的 ISO 8601 时间")
+    if not str(record.get("reviewer") or "").strip():
+        reasons.append("缺少验收人 reviewer")
+    artifacts = record.get("artifact_refs")
+    if not isinstance(artifacts, list) or not artifacts or not all(
+        isinstance(item, str) and item.strip() for item in artifacts
+    ):
+        reasons.append("artifact_refs 必须包含至少一个非空证据引用")
+    for field in spec.get("required_true", []):
+        if record.get(field) is not True:
+            reasons.append(f"{field} 必须为 true")
+    for field in spec.get("required_nonempty", []):
+        if record.get(field) in (None, "", [], {}):
+            reasons.append(f"{field} 不能为空")
+    for field in spec.get("required_positive", []):
+        value = record.get(field)
+        if not isinstance(value, (int, float)) or isinstance(value, bool) or value <= 0:
+            reasons.append(f"{field} 必须大于 0")
+    for field, expected in spec.get("required_values", {}).items():
+        if record.get(field) != expected:
+            reasons.append(f"{field} 必须为 {expected}")
+    for field, expected_items in spec.get("required_sets", {}).items():
+        value = record.get(field)
+        actual = set(value) if isinstance(value, list) else set()
+        missing = sorted(expected_items - actual)
+        if missing:
+            reasons.append(f"{field} 缺少: {', '.join(missing)}")
+    if record.get("status") == "passed" and code == "stage3_retrieval":
+        reasons.extend(_retrieval_metric_reasons(record))
+    return reasons
+
+
+def _retrieval_metric_reasons(record: dict[str, Any]) -> list[str]:
+    metrics = record.get("metrics")
+    hnsw = record.get("hnsw")
+    reasons = []
+    if not isinstance(metrics, dict):
+        return ["metrics 必须记录 recall_at_5 和 p95_latency_ms"]
+    recall = metrics.get("recall_at_5")
+    if not isinstance(recall, (int, float)) or isinstance(recall, bool) or not 0 <= recall <= 1:
+        reasons.append("metrics.recall_at_5 必须位于 0 到 1")
+    latency = metrics.get("p95_latency_ms")
+    if not isinstance(latency, (int, float)) or isinstance(latency, bool) or latency <= 0:
+        reasons.append("metrics.p95_latency_ms 必须大于 0")
+    if not isinstance(hnsw, dict) or any(
+        not isinstance(hnsw.get(key), int) or hnsw[key] <= 0
+        for key in ("m", "ef_construction", "ef_search")
+    ):
+        reasons.append("hnsw 必须记录正整数 m、ef_construction 和 ef_search")
+    return reasons
+
+
+def _recovery_reasons(payload: dict[str, Any] | None) -> list[str]:
+    if not isinstance(payload, dict):
+        return ["缺少机器可读的隔离恢复演练证据"]
+    reasons = []
+    expected = {
+        "status": "passed",
+        "database_smoke": "ticket_and_attachment_restored",
+        "object_storage_smoke": "marker_restored",
+        "django_check": "passed",
+    }
+    for field, value in expected.items():
+        if payload.get(field) != value:
+            reasons.append(f"{field} 必须为 {value}")
+    if not str(payload.get("backup_id") or "").strip():
+        reasons.append("缺少 backup_id")
+    duration = payload.get("duration_seconds")
+    if not isinstance(duration, (int, float)) or isinstance(duration, bool) or duration <= 0:
+        reasons.append("duration_seconds 必须大于 0")
+    return reasons
+
+
+def _valid_timestamp(value: Any) -> bool:
+    if not isinstance(value, str) or not value.strip():
+        return False
+    try:
+        parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except ValueError:
+        return False
+    return parsed.tzinfo is not None
+
+
+def _find_sensitive_keys(value: Any, path: str = "") -> list[str]:
+    """递归定位可能承载密码、密钥、令牌或凭据的字段路径。"""
+
+    found = []
+    if isinstance(value, dict):
+        for key, item in value.items():
+            next_path = f"{path}.{key}" if path else str(key)
+            if any(part in str(key).lower() for part in SENSITIVE_KEY_PARTS):
+                found.append(next_path)
+            found.extend(_find_sensitive_keys(item, next_path))
+    elif isinstance(value, list):
+        for index, item in enumerate(value):
+            found.extend(_find_sensitive_keys(item, f"{path}[{index}]"))
+    return found

backend/apps/system/management/__init__.py

@@ -0,0 +1 @@
+

backend/apps/system/management/commands/__init__.py

@@ -0,0 +1 @@
+

backend/apps/system/management/commands/report_blueprint_readiness.py

@@ -0,0 +1,91 @@
+import json
+from pathlib import Path
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+
+from apps.system.blueprint_readiness import build_blueprint_readiness_report
+
+
+class Command(BaseCommand):
+    help = "生成覆盖阶段 2～6 退出条件的统一蓝图就绪报告"
+
+    def add_arguments(self, parser):
+        parser.add_argument("--evidence", help="外部验收证据 JSON；不得包含任何凭据")
+        parser.add_argument(
+            "--recovery-evidence",
+            help="隔离恢复演练 JSON；默认读取仓库内最新已登记记录",
+        )
+        parser.add_argument(
+            "--include-synthetic",
+            action="store_true",
+            help="包含合成案例用于诊断；合成案例不会使阶段 2/3 通过",
+        )
+        parser.add_argument(
+            "--format",
+            choices=["text", "json"],
+            default="text",
+            dest="output_format",
+        )
+        parser.add_argument("--output", help="可选 JSON 报告输出路径")
+        parser.add_argument(
+            "--fail-on-not-ready",
+            action="store_true",
+            help="任一蓝图退出条件未满足时返回非零退出码",
+        )
+
+    def handle(self, *args, **options):
+        workspace_root = Path(settings.BASE_DIR)
+        recovery_path = Path(
+            options.get("recovery_evidence")
+            or workspace_root / "docs/operations/recovery-drill-2026-07-27.json"
+        )
+        external = _load_json(options.get("evidence"), label="外部验收证据")
+        recovery = _load_json(recovery_path, label="隔离恢复证据")
+        report = build_blueprint_readiness_report(
+            include_synthetic=options["include_synthetic"],
+            external_evidence=external,
+            recovery_evidence=recovery,
+        )
+        report_json = json.dumps(report, ensure_ascii=False, indent=2)
+
+        output_path = options.get("output")
+        if output_path:
+            path = Path(output_path)
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_text(f"{report_json}\n", encoding="utf-8")
+
+        if options["output_format"] == "json":
+            self.stdout.write(report_json)
+        else:
+            summary = report["summary"]
+            self.stdout.write(
+                f"蓝图验收状态：{report['status']}；"
+                f"已通过 {summary['passed_gate_count']}/{summary['total_gate_count']}；"
+                f"剩余 {summary['remaining_gate_count']}"
+            )
+            for gate in report["gates"]:
+                marker = "通过" if gate["status"] == "passed" else "待验收"
+                self.stdout.write(f"[{marker}] {gate['title']}")
+                for reason in gate["reasons"]:
+                    self.stdout.write(self.style.WARNING(f"  - {reason}"))
+            if output_path:
+                self.stdout.write(f"JSON 报告：{Path(output_path).resolve()}")
+
+        if options["fail_on_not_ready"] and not report["exit_ready"]:
+            raise CommandError("产品蓝图尚未满足全部退出条件")
+
+
+def _load_json(path_value, *, label: str):
+    if not path_value:
+        return None
+    path = Path(path_value)
+    if not path.exists():
+        return None
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as error:
+        raise CommandError(f"{label}无法读取：{error}") from error
+    if not isinstance(payload, dict):
+        raise CommandError(f"{label}必须是 JSON 对象")
+    return payload

backend/apps/system/tests/test_blueprint_readiness.py

@@ -0,0 +1,176 @@
+import json
+
+import pytest
+from django.core.management import call_command
+from django.core.management.base import CommandError
+from django.db import DatabaseError
+
+from apps.system import blueprint_readiness
+from apps.system.blueprint_readiness import (
+    EXTERNAL_GATE_SPECS,
+    build_blueprint_readiness_report,
+)
+
+
+def passed_evaluation_report():
+    return {"status": "passed", "exit_ready": True, "reasons": [], "metrics": {}}
+
+
+def complete_external_evidence():
+    common = {
+        "status": "passed",
+        "verified_at": "2026-07-27T09:30:00Z",
+        "reviewer": "acceptance-reviewer",
+        "artifact_refs": ["evidence/run-001.json"],
+    }
+    return {
+        "schema_version": 1,
+        "gates": {
+            "stage3_retrieval": {
+                **common,
+                "environment": "postgresql",
+                "embedding_deployment": "embedding-prod",
+                "rerank_deployment": "rerank-prod",
+                "sample_count": 50,
+                "thresholds_met": True,
+                "metrics": {"recall_at_5": 0.92, "p95_latency_ms": 180},
+                "hnsw": {"m": 16, "ef_construction": 64, "ef_search": 40},
+            },
+            "stage4_distillation": {
+                **common,
+                "sample_count": 20,
+                "independent_review": True,
+                "anonymization_review_passed": True,
+                "published": True,
+                "retrieved_after_publish": True,
+                "services": ["postgresql", "embedding", "rerank", "text_model"],
+            },
+            "stage4_documents": {
+                **common,
+                "failure_recovery_verified": True,
+                "formats": ["PDF", "TXT", "LOG", "DOCX", "XLSX", "PPTX"],
+                "services": ["minio", "clamav", "postgresql", "embedding"],
+            },
+            "stage6_channel": {
+                **common,
+                "provider": "enterprise-chat",
+                "signature_verified": True,
+                "inbound_verified": True,
+                "outbound_verified": True,
+                "receipt_verified": True,
+                "idempotency_verified": True,
+            },
+            "stage6_identity": {
+                **common,
+                "oidc_provider": "enterprise-idp",
+                "ldap_directory": "enterprise-directory",
+                "oidc_login_verified": True,
+                "group_claim_verified": True,
+                "failure_fallback_verified": True,
+                "ldap_directory_sync_verified": True,
+            },
+            "stage6_production_ops": {
+                **common,
+                "alert_receiver": "on-call",
+                "backup_schedule": "daily",
+                "offsite_storage": "encrypted-vault",
+                "alerts_verified": True,
+                "backup_schedule_installed": True,
+                "offsite_encrypted_copy_verified": True,
+            },
+        },
+    }
+
+
+def recovery_evidence():
+    return {
+        "status": "passed",
+        "backup_id": "20260727T092235Z",
+        "duration_seconds": 106.57,
+        "database_smoke": "ticket_and_attachment_restored",
+        "object_storage_smoke": "marker_restored",
+        "django_check": "passed",
+    }
+
+
+def test_complete_blueprint_evidence_passes_all_gates():
+    report = build_blueprint_readiness_report(
+        stage2_report=passed_evaluation_report(),
+        stage3_report=passed_evaluation_report(),
+        external_evidence=complete_external_evidence(),
+        recovery_evidence=recovery_evidence(),
+    )
+
+    assert report["exit_ready"] is True
+    assert report["status"] == "passed"
+    assert report["summary"] == {
+        "passed_gate_count": 9,
+        "total_gate_count": 9,
+        "remaining_gate_count": 0,
+    }
+    assert {gate["code"] for gate in report["gates"]} == {
+        "stage2_intake",
+        "stage3_support",
+        *EXTERNAL_GATE_SPECS,
+        "stage6_recovery",
+    }
+
+
+def test_external_evidence_rejects_sensitive_keys_and_incomplete_contract():
+    evidence = complete_external_evidence()
+    evidence["gates"]["stage6_identity"]["client_secret"] = "must-not-be-recorded"
+    report = build_blueprint_readiness_report(
+        stage2_report=passed_evaluation_report(),
+        stage3_report=passed_evaluation_report(),
+        external_evidence=evidence,
+        recovery_evidence=recovery_evidence(),
+    )
+
+    external_gates = [
+        gate for gate in report["gates"] if gate["source"] == "external_evidence_manifest"
+    ]
+    assert all(gate["status"] == "not_ready" for gate in external_gates)
+    assert all("不得包含密码" in gate["reasons"][0] for gate in external_gates)
+
+
+def test_database_schema_failure_becomes_not_ready_gate(monkeypatch):
+    def unavailable_report(*, include_synthetic):
+        raise DatabaseError("schema unavailable")
+
+    monkeypatch.setattr(
+        blueprint_readiness,
+        "build_stage_2_acceptance_report",
+        unavailable_report,
+    )
+    report = build_blueprint_readiness_report(
+        stage3_report=passed_evaluation_report(),
+        external_evidence=complete_external_evidence(),
+        recovery_evidence=recovery_evidence(),
+    )
+
+    stage2_gate = next(gate for gate in report["gates"] if gate["code"] == "stage2_intake")
+    assert stage2_gate["status"] == "not_ready"
+    assert "数据库迁移" in stage2_gate["reasons"][0]
+
+
+@pytest.mark.django_db
+def test_blueprint_command_writes_not_ready_report_and_can_fail_ci(tmp_path):
+    output = tmp_path / "blueprint-readiness.json"
+    missing_recovery = tmp_path / "missing-recovery.json"
+
+    call_command(
+        "report_blueprint_readiness",
+        output=str(output),
+    )
+    report = json.loads(output.read_text(encoding="utf-8"))
+    assert report["status"] == "not_ready"
+    assert report["exit_ready"] is False
+    recovery_gate = next(gate for gate in report["gates"] if gate["code"] == "stage6_recovery")
+    assert recovery_gate["status"] == "passed"
+
+    with pytest.raises(CommandError, match="尚未满足"):
+        call_command(
+            "report_blueprint_readiness",
+            recovery_evidence=str(missing_recovery),
+            fail_on_not_ready=True,
+        )

backend/apps/system/tests/test_health_api.py

@@ -1,5 +1,51 @@
+import pytest
+from django.test import override_settings
+
+pytestmark = pytest.mark.django_db
+
+
 def test_health_endpoint_returns_ok(client):
     response = client.get("/api/system/health")
 
     assert response.status_code == 200
     assert response.json() == {"status": "ok"}
+
+
+def test_liveness_and_readiness_endpoints(client):
+    live_response = client.get("/api/system/health/live")
+    ready_response = client.get("/api/system/health/ready")
+
+    assert live_response.status_code == 200
+    assert live_response.json() == {"status": "alive"}
+    assert ready_response.status_code == 200
+    assert ready_response.json() == {
+        "status": "ready",
+        "checks": {"database": True, "cache": True},
+    }
+
+
+def test_metrics_endpoint_exposes_low_cardinality_metrics(admin_client):
+    response = admin_client.get("/api/system/metrics")
+
+    assert response.status_code == 200
+    assert response["Content-Type"].startswith("text/plain")
+    content = response.content.decode()
+    assert "ai_ticket_channel_messages_total" in content
+    assert "ai_ticket_agent_runs_total" in content
+    assert "ai_ticket_tool_calls_total" in content
+    assert "ai_ticket_work_items_active" in content
+    assert "ai_ticket_channel_messages_failed_recent" in content
+    assert "ai_ticket_channel_messages_stuck" in content
+    assert "ai_ticket_agent_runs_failed_recent" in content
+    assert "ai_ticket_tool_calls_failed_recent" in content
+
+
+@override_settings(METRICS_TOKEN="prometheus-secret")
+def test_metrics_endpoint_accepts_bearer_token_for_prometheus(client):
+    response = client.get(
+        "/api/system/metrics",
+        HTTP_AUTHORIZATION="Bearer prometheus-secret",
+    )
+
+    assert response.status_code == 200
+    assert "ai_ticket_channel_messages_total" in response.content.decode()

backend/pyproject.toml

@@ -6,6 +6,7 @@ requires-python = ">=3.12"
 dependencies = [
   "Django>=5.0,<5.3",
   "django-ninja>=1.3,<2.0",
+  "gunicorn>=23.0,<24.0",
   "psycopg[binary]>=3.2,<4.0",
   "celery>=5.4,<6.0",
   "redis>=5.0,<6.0",
@@ -17,6 +18,13 @@ dependencies = [
   "jsonschema>=4.23,<5.0",
   "minio>=7.2,<8.0",
   "Pillow>=11.0,<12.0",
+  "pypdf>=5.0,<7.0",
+  "python-docx>=1.1,<2.0",
+  "openpyxl>=3.1,<4.0",
+  "python-pptx>=1.0,<2.0",
+  "pgvector>=0.4,<0.5",
+  "Authlib>=1.6,<2.0",
+  "requests>=2.32,<3.0",
 ]
 
 [build-system]
@@ -43,6 +51,7 @@ testpaths = ["apps", "tests"]
 [tool.ruff]
 line-length = 100
 target-version = "py312"
+extend-exclude = ["**/migrations/*.py"]
 
 [tool.ruff.lint]
 select = ["E", "F", "I", "UP", "B"]

frontend/package.json

@@ -2,6 +2,7 @@
   "name": "ai-ticket-platform-frontend",
   "version": "0.1.0",
   "private": true,
+  "packageManager": "pnpm@11.9.0",
   "type": "module",
   "scripts": {
     "dev": "vite",

frontend/pnpm-workspace.yaml

@@ -0,0 +1,2 @@
+allowBuilds:
+  esbuild: true

infra/backup-cron.example

@@ -0,0 +1,3 @@
+# 每日 02:17 创建备份、校验并清理超过 30 天的标准备份目录。
+# 请把 /opt/ai-ticket-platform 替换为实际绝对路径，并由专用系统账号安装。
+17 2 * * * cd /opt/ai-ticket-platform && BACKUP_RETENTION_DAYS=30 sh infra/scripts/backup.sh && BACKUP_RETENTION_DAYS=30 sh infra/scripts/prune-backups.sh >> /var/log/ai-ticket-backup.log 2>&1

infra/docker-compose.local.yml

@@ -32,3 +32,10 @@ services:
       - ./data/minio:/data
     restart: unless-stopped
 
+  clamav:
+    image: clamav/clamav:stable
+    ports:
+      - "3310:3310"
+    volumes:
+      - ./data/clamav:/var/lib/clamav
+    restart: unless-stopped

infra/docker-compose.prod.yml

@@ -0,0 +1,184 @@
+services:
+  postgres:
+    image: pgvector/pgvector:pg16
+    environment:
+      POSTGRES_DB: ${POSTGRES_DB:-ai_ticket_platform}
+      POSTGRES_USER: ${POSTGRES_USER:-ai_ticket}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+
+  redis:
+    image: redis:7-alpine
+    command: ["redis-server", "--appendonly", "yes"]
+    volumes:
+      - redis-data:/data
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+
+  minio:
+    image: minio/minio:latest
+    command: server /data --console-address ":9001"
+    environment:
+      MINIO_ROOT_USER: ${MINIO_ROOT_USER:?MINIO_ROOT_USER is required}
+      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?MINIO_ROOT_PASSWORD is required}
+    volumes:
+      - minio-data:/data
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+
+  clamav:
+    image: clamav/clamav:stable
+    volumes:
+      - clamav-data:/var/lib/clamav
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD-SHELL", "echo PING | nc 127.0.0.1 3310 | grep PONG"]
+      interval: 30s
+      timeout: 10s
+      retries: 10
+      start_period: 60s
+
+  backend:
+    build:
+      context: ..
+      dockerfile: infra/backend.Dockerfile
+    entrypoint: ["/entrypoint.sh"]
+    command:
+      [
+        "gunicorn",
+        "config.wsgi:application",
+        "--bind",
+        "0.0.0.0:8000",
+        "--workers",
+        "3",
+        "--timeout",
+        "300",
+        "--access-logfile",
+        "-"
+      ]
+    env_file: ../.env
+    restart: unless-stopped
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      minio:
+        condition: service_healthy
+      clamav:
+        condition: service_healthy
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "python",
+          "-c",
+          "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/api/system/health/ready', timeout=5)"
+        ]
+      interval: 15s
+      timeout: 8s
+      retries: 10
+
+  worker:
+    build:
+      context: ..
+      dockerfile: infra/backend.Dockerfile
+    command: ["celery", "-A", "config.celery", "worker", "-l", "info"]
+    env_file: ../.env
+    restart: unless-stopped
+    depends_on:
+      backend:
+        condition: service_healthy
+
+  beat:
+    build:
+      context: ..
+      dockerfile: infra/backend.Dockerfile
+    command: ["celery", "-A", "config.celery", "beat", "-l", "info"]
+    env_file: ../.env
+    restart: unless-stopped
+    depends_on:
+      backend:
+        condition: service_healthy
+
+  frontend:
+    build:
+      context: ..
+      dockerfile: infra/frontend-prod.Dockerfile
+    ports:
+      - "${HTTP_PORT:-8080}:80"
+    restart: unless-stopped
+    depends_on:
+      backend:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1/"]
+      interval: 15s
+      timeout: 5s
+      retries: 10
+
+  prometheus:
+    image: prom/prometheus:v3.5.0
+    profiles: ["monitoring"]
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.path=/prometheus
+      - --storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-30d}
+    volumes:
+      - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./monitoring/alerts.yml:/etc/prometheus/alerts.yml:ro
+      - prometheus-data:/prometheus
+    secrets:
+      - metrics_token
+    ports:
+      - "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"
+    restart: unless-stopped
+    depends_on:
+      backend:
+        condition: service_healthy
+      alertmanager:
+        condition: service_started
+
+  alertmanager:
+    image: prom/alertmanager:v0.28.1
+    profiles: ["monitoring"]
+    command:
+      - --config.file=/etc/alertmanager/alertmanager.yml
+      - --storage.path=/alertmanager
+    volumes:
+      - ./monitoring/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
+      - alertmanager-data:/alertmanager
+    secrets:
+      - alert_webhook_url
+    ports:
+      - "127.0.0.1:${ALERTMANAGER_PORT:-9093}:9093"
+    restart: unless-stopped
+
+volumes:
+  postgres-data:
+  redis-data:
+  minio-data:
+  clamav-data:
+  prometheus-data:
+  alertmanager-data:
+
+secrets:
+  metrics_token:
+    file: ${METRICS_TOKEN_FILE:-./secrets/metrics_token.example}
+  alert_webhook_url:
+    file: ${ALERT_WEBHOOK_URL_FILE:-./secrets/alert_webhook_url.example}

infra/docker-compose.yml

@@ -27,6 +27,13 @@ services:
     volumes:
       - minio-data:/data
 
+  clamav:
+    image: clamav/clamav:stable
+    ports:
+      - "3310:3310"
+    volumes:
+      - clamav-data:/var/lib/clamav
+
   backend:
     build:
       context: ..
@@ -43,6 +50,8 @@ services:
     depends_on:
       - postgres
       - redis
+      - minio
+      - clamav
     volumes:
       - ../backend:/app
 
@@ -59,6 +68,7 @@ services:
     depends_on:
       - backend
       - redis
+      - clamav
     volumes:
       - ../backend:/app
 
@@ -96,4 +106,5 @@ services:
 volumes:
   postgres-data:
   minio-data:
+  clamav-data:
   frontend-node-modules:

infra/frontend-prod.Dockerfile

@@ -0,0 +1,12 @@
+FROM node:22-slim AS build
+
+WORKDIR /app
+RUN corepack enable && corepack prepare pnpm@11.9.0 --activate
+COPY frontend/package.json frontend/pnpm-lock.yaml frontend/pnpm-workspace.yaml ./
+RUN pnpm install --frozen-lockfile
+COPY frontend ./
+RUN pnpm build
+
+FROM nginx:1.27-alpine
+COPY infra/nginx.conf /etc/nginx/conf.d/default.conf
+COPY --from=build /app/dist /usr/share/nginx/html

infra/monitoring/alertmanager.yml

@@ -0,0 +1,14 @@
+route:
+  receiver: operations-webhook
+  group_by:
+    - alertname
+    - severity
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 4h
+
+receivers:
+  - name: operations-webhook
+    webhook_configs:
+      - url_file: /run/secrets/alert_webhook_url
+        send_resolved: true

infra/monitoring/alerts.yml

@@ -0,0 +1,56 @@
+groups:
+  - name: ai-ticket-platform
+    rules:
+      - alert: AiTicketBackendDown
+        expr: up{job="ai-ticket-backend"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: AI 工单平台后端不可用
+          description: Prometheus 连续 2 分钟无法抓取后端指标。
+
+      - alert: AiTicketChannelDeliveryFailures
+        expr: ai_ticket_channel_messages_failed_recent > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: 渠道消息投递持续失败
+          description: 最近 15 分钟存在失败的渠道消息，请检查渠道凭证、地址和重试队列。
+
+      - alert: AiTicketChannelQueueStuck
+        expr: ai_ticket_channel_messages_stuck > 0
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: 渠道消息队列出现积压
+          description: 存在排队超过 10 分钟的出站消息，请检查 Celery Worker 和渠道网络。
+
+      - alert: AiTicketAgentRunFailures
+        expr: ai_ticket_agent_runs_failed_recent > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: Agent 运行持续失败
+          description: 最近 15 分钟存在失败的 Agent 运行，请检查模型、工具和运行日志。
+
+      - alert: AiTicketToolCallFailures
+        expr: ai_ticket_tool_calls_failed_recent > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: Tool/MCP 调用持续失败
+          description: 最近 15 分钟存在失败的工具调用，请检查授权、MCP 健康和审计记录。
+
+      - alert: AiTicketWorkItemBacklog
+        expr: ai_ticket_work_items_active > 200
+        for: 30m
+        labels:
+          severity: warning
+        annotations:
+          summary: 活动任务积压
+          description: 活动任务连续 30 分钟超过 200 个，请检查人工处理容量和自动任务异常。

infra/monitoring/prometheus.yml

@@ -0,0 +1,21 @@
+global:
+  scrape_interval: 30s
+  evaluation_interval: 30s
+
+rule_files:
+  - /etc/prometheus/alerts.yml
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+            - alertmanager:9093
+
+scrape_configs:
+  - job_name: ai-ticket-backend
+    metrics_path: /api/system/metrics
+    authorization:
+      credentials_file: /run/secrets/metrics_token
+    static_configs:
+      - targets:
+          - backend:8000

infra/nginx.conf

@@ -0,0 +1,22 @@
+server {
+    listen 80;
+    server_name _;
+
+    root /usr/share/nginx/html;
+    index index.html;
+
+    location /api/ {
+        proxy_pass http://backend:8000;
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_buffering off;
+        proxy_read_timeout 300s;
+    }
+
+    location / {
+        try_files $uri $uri/ /index.html;
+    }
+}

infra/scripts/backup.ps1

@@ -0,0 +1,142 @@
+[CmdletBinding()]
+param(
+    [string]$ComposeFile = "",
+    [string]$BackupRoot = "",
+    [string]$PostgresDatabase = $env:POSTGRES_DB,
+    [string]$PostgresUser = $env:POSTGRES_USER,
+    [string]$BackupHelperImage = "alpine:3.21",
+    [switch]$SkipApplicationServices
+)
+
+$ErrorActionPreference = "Stop"
+$scriptDirectory = Split-Path -Parent $MyInvocation.MyCommand.Path
+$infraDirectory = [IO.Path]::GetFullPath((Join-Path $scriptDirectory ".."))
+$projectDirectory = [IO.Path]::GetFullPath((Join-Path $infraDirectory ".."))
+
+if (-not $ComposeFile) {
+    $ComposeFile = Join-Path $infraDirectory "docker-compose.prod.yml"
+}
+$ComposeFile = (Resolve-Path -LiteralPath $ComposeFile).Path
+if (-not $BackupRoot) {
+    $BackupRoot = Join-Path $projectDirectory "backups"
+}
+$BackupRoot = [IO.Path]::GetFullPath($BackupRoot)
+$projectPrefix = $projectDirectory.TrimEnd("\", "/") + [IO.Path]::DirectorySeparatorChar
+if (-not $BackupRoot.StartsWith($projectPrefix, [StringComparison]::OrdinalIgnoreCase)) {
+    throw "拒绝写入项目目录之外的备份路径：$BackupRoot"
+}
+
+if (-not $PostgresDatabase) { $PostgresDatabase = "ai_ticket_platform" }
+if (-not $PostgresUser) { $PostgresUser = "ai_ticket" }
+
+$stamp = (Get-Date).ToUniversalTime().ToString("yyyyMMddTHHmmssZ")
+$target = [IO.Path]::GetFullPath((Join-Path $BackupRoot $stamp))
+if (-not $target.StartsWith($projectPrefix, [StringComparison]::OrdinalIgnoreCase)) {
+    throw "拒绝写入项目目录之外的备份路径：$target"
+}
+New-Item -ItemType Directory -Path $target -Force | Out-Null
+
+$postgresTemp = "/tmp/ai-ticket-postgres-$PID.dump"
+
+function Invoke-Compose {
+    param([Parameter(Mandatory = $true)][string[]]$Arguments)
+    & docker compose -f $ComposeFile @Arguments
+    if ($LASTEXITCODE -ne 0) {
+        throw "docker compose 执行失败：$($Arguments -join ' ')"
+    }
+}
+
+$runningServices = & docker compose -f $ComposeFile ps --services --filter status=running
+if ($LASTEXITCODE -ne 0) {
+    throw "无法读取 Compose 服务状态"
+}
+$minioWasRunning = $runningServices -contains "minio"
+$minioContainer = & docker compose -f $ComposeFile ps -q minio
+if ($LASTEXITCODE -ne 0 -or -not "$minioContainer".Trim()) {
+    throw "未找到 MinIO 容器"
+}
+$minioContainer = "$minioContainer".Trim()
+
+$runningApplicationServices = @()
+if (-not $SkipApplicationServices) {
+    $runningApplicationServices = @(
+        "backend", "worker", "beat" |
+            Where-Object { $runningServices -contains $_ }
+    )
+    if ($runningApplicationServices.Count -gt 0) {
+        Invoke-Compose (@("stop") + $runningApplicationServices)
+    }
+}
+if ($minioWasRunning) {
+    Invoke-Compose @("stop", "minio")
+}
+
+try {
+    Invoke-Compose @(
+        "exec", "-T", "postgres",
+        "pg_dump", "-U", $PostgresUser, "-d", $PostgresDatabase,
+        "--format=custom", "--file=$postgresTemp"
+    )
+    Invoke-Compose @("cp", "postgres:$postgresTemp", (Join-Path $target "postgres.dump"))
+
+    $contents = & docker compose -f $ComposeFile exec -T postgres `
+        pg_restore --list $postgresTemp
+    if ($LASTEXITCODE -ne 0) {
+        throw "PostgreSQL 备份结构校验失败"
+    }
+    [IO.File]::WriteAllLines(
+        (Join-Path $target "postgres.contents"),
+        [string[]]$contents,
+        [Text.UTF8Encoding]::new($false)
+    )
+
+    & docker run --rm `
+        --volumes-from $minioContainer `
+        -v "${target}:/backup" `
+        $BackupHelperImage `
+        tar -C /data -czf /backup/minio-data.tar.gz .
+    if ($LASTEXITCODE -ne 0) {
+        throw "MinIO 数据归档失败"
+    }
+
+    $checksumLines = @("postgres.dump", "postgres.contents", "minio-data.tar.gz") | ForEach-Object {
+        $hash = (Get-FileHash -Algorithm SHA256 -LiteralPath (Join-Path $target $_)).Hash.ToLowerInvariant()
+        "$hash  $_"
+    }
+    [IO.File]::WriteAllLines(
+        (Join-Path $target "SHA256SUMS"),
+        [string[]]$checksumLines,
+        [Text.UTF8Encoding]::new($false)
+    )
+    $manifestLines = @(
+        "created_at=$stamp",
+        "postgres_database=$PostgresDatabase",
+        "postgres_user=$PostgresUser",
+        "compose_file=$ComposeFile",
+        "compose_project=$($env:COMPOSE_PROJECT_NAME)"
+    )
+    [IO.File]::WriteAllLines(
+        (Join-Path $target "manifest.txt"),
+        [string[]]$manifestLines,
+        [Text.UTF8Encoding]::new($false)
+    )
+}
+finally {
+    & docker compose -f $ComposeFile exec -T postgres rm -f $postgresTemp 2>$null
+    $minioReady = $true
+    if ($minioWasRunning) {
+        & docker compose -f $ComposeFile up -d --wait minio
+        if ($LASTEXITCODE -ne 0) {
+            $minioReady = $false
+            Write-Warning "备份结束后 MinIO 未恢复健康，应用服务保持停止"
+        }
+    }
+    if ($minioReady -and $runningApplicationServices.Count -gt 0) {
+        & docker compose -f $ComposeFile start @runningApplicationServices
+        if ($LASTEXITCODE -ne 0) {
+            Write-Warning "备份结束后无法恢复原应用服务：$($runningApplicationServices -join ', ')"
+        }
+    }
+}
+
+Write-Output "备份已创建并完成结构校验：$target"

infra/scripts/backup.sh

@@ -0,0 +1,86 @@
+#!/bin/sh
+set -eu
+
+SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+INFRA_DIR=$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd)
+PROJECT_DIR=$(CDPATH= cd -- "$INFRA_DIR/.." && pwd)
+COMPOSE_FILE=${COMPOSE_FILE:-"$INFRA_DIR/docker-compose.prod.yml"}
+BACKUP_ROOT=${BACKUP_ROOT:-"$PROJECT_DIR/backups"}
+STAMP=$(date -u +%Y%m%dT%H%M%SZ)
+TARGET="$BACKUP_ROOT/$STAMP"
+
+mkdir -p "$TARGET"
+TARGET=$(CDPATH= cd -- "$TARGET" && pwd)
+case "$TARGET" in
+  "$PROJECT_DIR"/*) ;;
+  *) echo "拒绝写入项目目录之外的备份路径：$TARGET" >&2; exit 2 ;;
+esac
+
+POSTGRES_DB=${POSTGRES_DB:-ai_ticket_platform}
+POSTGRES_USER=${POSTGRES_USER:-ai_ticket}
+BACKUP_HELPER_IMAGE=${BACKUP_HELPER_IMAGE:-alpine:3.21}
+RUNNING_APPLICATION_SERVICES=""
+MINIO_WAS_RUNNING=false
+
+cleanup() {
+  MINIO_READY=true
+  if [ "$MINIO_WAS_RUNNING" = "true" ]; then
+    if ! docker compose -f "$COMPOSE_FILE" up -d --wait minio >/dev/null; then
+      echo "备份结束后 MinIO 未恢复健康，应用服务保持停止" >&2
+      MINIO_READY=false
+    fi
+  fi
+  if [ "$MINIO_READY" = "true" ] && [ -n "$RUNNING_APPLICATION_SERVICES" ]; then
+    docker compose -f "$COMPOSE_FILE" start $RUNNING_APPLICATION_SERVICES >/dev/null || true
+  fi
+}
+trap cleanup EXIT INT TERM
+
+RUNNING_SERVICES=$(docker compose -f "$COMPOSE_FILE" ps --services --filter status=running)
+if printf '%s\n' "$RUNNING_SERVICES" | grep -qx "minio"; then
+  MINIO_WAS_RUNNING=true
+fi
+if [ "${BACKUP_SKIP_APPLICATION_SERVICES:-false}" != "true" ]; then
+  for service in backend worker beat; do
+    if printf '%s\n' "$RUNNING_SERVICES" | grep -qx "$service"; then
+      RUNNING_APPLICATION_SERVICES="$RUNNING_APPLICATION_SERVICES $service"
+    fi
+  done
+  if [ -n "$RUNNING_APPLICATION_SERVICES" ]; then
+    docker compose -f "$COMPOSE_FILE" stop $RUNNING_APPLICATION_SERVICES
+  fi
+fi
+if [ "$MINIO_WAS_RUNNING" = "true" ]; then
+  docker compose -f "$COMPOSE_FILE" stop minio
+fi
+
+docker compose -f "$COMPOSE_FILE" exec -T postgres \
+  pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom > "$TARGET/postgres.dump"
+docker compose -f "$COMPOSE_FILE" exec -T postgres \
+  pg_restore --list < "$TARGET/postgres.dump" > "$TARGET/postgres.contents"
+MINIO_CONTAINER=$(docker compose -f "$COMPOSE_FILE" ps -q minio)
+if [ -z "$MINIO_CONTAINER" ]; then
+  echo "未找到 MinIO 容器" >&2
+  exit 3
+fi
+docker run --rm \
+  --volumes-from "$MINIO_CONTAINER" \
+  -v "$TARGET:/backup" \
+  "$BACKUP_HELPER_IMAGE" \
+  tar -C /data -czf /backup/minio-data.tar.gz .
+
+(
+  cd "$TARGET"
+  sha256sum postgres.dump postgres.contents minio-data.tar.gz > SHA256SUMS
+)
+
+cat > "$TARGET/manifest.txt" <<EOF
+created_at=$STAMP
+postgres_database=$POSTGRES_DB
+postgres_user=$POSTGRES_USER
+compose_file=$COMPOSE_FILE
+EOF
+
+cleanup
+trap - EXIT INT TERM
+echo "备份已创建并完成结构校验：$TARGET"

infra/scripts/drill-restore.ps1

@@ -0,0 +1,194 @@
+[CmdletBinding()]
+param(
+    [string]$ComposeFile = "",
+    [switch]$SkipBuild
+)
+
+$ErrorActionPreference = "Stop"
+$scriptDirectory = Split-Path -Parent $MyInvocation.MyCommand.Path
+$infraDirectory = [IO.Path]::GetFullPath((Join-Path $scriptDirectory ".."))
+$projectDirectory = [IO.Path]::GetFullPath((Join-Path $infraDirectory ".."))
+if (-not $ComposeFile) {
+    $ComposeFile = Join-Path $infraDirectory "docker-compose.prod.yml"
+}
+$ComposeFile = (Resolve-Path -LiteralPath $ComposeFile).Path
+
+$projectName = "ai-ticket-restore-drill-$PID"
+$databasePassword = "restore-drill-postgres-$PID"
+$minioUser = "restore-drill-minio"
+$minioPassword = "restore-drill-minio-$PID"
+$backupRoot = Join-Path $projectDirectory "backups\$projectName"
+$stopwatch = [Diagnostics.Stopwatch]::StartNew()
+
+$savedEnvironment = @{
+    COMPOSE_PROJECT_NAME = $env:COMPOSE_PROJECT_NAME
+    POSTGRES_DB = $env:POSTGRES_DB
+    POSTGRES_USER = $env:POSTGRES_USER
+    POSTGRES_PASSWORD = $env:POSTGRES_PASSWORD
+    MINIO_ROOT_USER = $env:MINIO_ROOT_USER
+    MINIO_ROOT_PASSWORD = $env:MINIO_ROOT_PASSWORD
+}
+
+$env:COMPOSE_PROJECT_NAME = $projectName
+$env:POSTGRES_DB = "ai_ticket_platform"
+$env:POSTGRES_USER = "ai_ticket"
+$env:POSTGRES_PASSWORD = $databasePassword
+$env:MINIO_ROOT_USER = $minioUser
+$env:MINIO_ROOT_PASSWORD = $minioPassword
+
+function Invoke-Docker {
+    param([Parameter(Mandatory = $true)][string[]]$Arguments)
+    & docker @Arguments
+    if ($LASTEXITCODE -ne 0) {
+        throw "Docker 执行失败：docker $($Arguments -join ' ')"
+    }
+}
+
+function Invoke-DjangoCode {
+    param(
+        [Parameter(Mandatory = $true)][string[]]$RunPrefix,
+        [Parameter(Mandatory = $true)][string]$Code,
+        [Parameter(Mandatory = $true)][string]$FailureMessage
+    )
+    $bytes = [Text.Encoding]::UTF8.GetBytes($Code)
+    $runner = "exec(bytes([" + ($bytes -join ",") + "]))"
+    & docker @RunPrefix python manage.py shell -c $runner
+    if ($LASTEXITCODE -ne 0) {
+        throw $FailureMessage
+    }
+}
+
+$backendEnvironment = @(
+    "-e", "POSTGRES_HOST=postgres",
+    "-e", "POSTGRES_DB=ai_ticket_platform",
+    "-e", "POSTGRES_USER=ai_ticket",
+    "-e", "POSTGRES_PASSWORD=$databasePassword",
+    "-e", "MINIO_ENDPOINT=minio:9000",
+    "-e", "MINIO_ROOT_USER=$minioUser",
+    "-e", "MINIO_ROOT_PASSWORD=$minioPassword",
+    "-e", "CELERY_BROKER_URL=redis://redis:6379/0",
+    "-e", "CELERY_RESULT_BACKEND=redis://redis:6379/1"
+)
+
+$seedCode = @'
+from django.contrib.auth import get_user_model
+from apps.tickets.models import Ticket, TicketAttachment
+
+user, _ = get_user_model().objects.get_or_create(
+    username="restore_drill",
+    defaults={"email": "restore-drill@example.invalid"},
+)
+ticket, _ = Ticket.objects.update_or_create(
+    ticket_no="RESTORE-DRILL-001",
+    defaults={
+        "title": "隔离恢复演练工单",
+        "description": "restore-drill-marker",
+        "requester": user,
+        "created_by": user,
+    },
+)
+TicketAttachment.objects.update_or_create(
+    ticket=ticket,
+    storage_path="restore-drill/marker.txt",
+    defaults={
+        "file_name": "marker.txt",
+        "mime_type": "text/plain",
+        "file_size": 20,
+        "created_by": user,
+    },
+)
+print(f"seeded_ticket={ticket.ticket_no}")
+'@
+
+$deleteCode = @'
+from apps.tickets.models import Ticket
+Ticket.objects.filter(ticket_no="RESTORE-DRILL-001").delete()
+print("seed_deleted=true")
+'@
+
+$verifyCode = @'
+from apps.tickets.models import Ticket
+ticket = Ticket.objects.get(ticket_no="RESTORE-DRILL-001")
+attachment = ticket.attachments.get(storage_path="restore-drill/marker.txt")
+assert ticket.description == "restore-drill-marker"
+assert attachment.file_name == "marker.txt"
+print(f"restored_ticket={ticket.ticket_no}")
+print(f"restored_attachment={attachment.storage_path}")
+'@
+
+try {
+    Invoke-Docker @("compose", "-f", $ComposeFile, "up", "-d", "--wait", "postgres", "minio")
+    if (-not $SkipBuild) {
+        Invoke-Docker @("compose", "-f", $ComposeFile, "build", "backend")
+    }
+
+    $runPrefix = @(
+        "compose", "-f", $ComposeFile,
+        "run", "--rm", "-T", "--no-deps"
+    ) + $backendEnvironment + @("backend")
+
+    Invoke-Docker ($runPrefix + @("python", "manage.py", "migrate", "--noinput"))
+    Invoke-DjangoCode -RunPrefix $runPrefix -Code $seedCode -FailureMessage "演练数据创建失败"
+
+    Invoke-Docker @(
+        "compose", "-f", $ComposeFile, "exec", "-T", "minio",
+        "sh", "-c",
+        'mc alias set drill http://127.0.0.1:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null && mc mb --ignore-existing drill/ai-ticket-platform >/dev/null && printf restore-drill-marker | mc pipe drill/ai-ticket-platform/restore-drill/marker.txt >/dev/null'
+    )
+
+    & (Join-Path $scriptDirectory "backup.ps1") `
+        -ComposeFile $ComposeFile `
+        -BackupRoot $backupRoot `
+        -PostgresDatabase "ai_ticket_platform" `
+        -PostgresUser "ai_ticket" `
+        -SkipApplicationServices
+    $backupDirectory = Get-ChildItem -LiteralPath $backupRoot -Directory |
+        Sort-Object Name -Descending |
+        Select-Object -First 1
+    if (-not $backupDirectory) { throw "未找到演练备份目录" }
+
+    Invoke-DjangoCode -RunPrefix $runPrefix -Code $deleteCode -FailureMessage "演练数据破坏步骤失败"
+    Invoke-Docker @(
+        "compose", "-f", $ComposeFile, "exec", "-T", "minio",
+        "mc", "rm", "--force", "drill/ai-ticket-platform/restore-drill/marker.txt"
+    )
+
+    & (Join-Path $scriptDirectory "restore.ps1") `
+        -BackupDirectory $backupDirectory.FullName `
+        -ComposeFile $ComposeFile `
+        -PostgresDatabase "ai_ticket_platform" `
+        -PostgresUser "ai_ticket" `
+        -Confirm "restore-ai-ticket-platform" `
+        -SkipApplicationServices
+
+    Invoke-DjangoCode -RunPrefix $runPrefix -Code $verifyCode -FailureMessage "数据库业务冒烟失败"
+    $objectMarker = & docker compose -f $ComposeFile exec -T minio `
+        mc cat drill/ai-ticket-platform/restore-drill/marker.txt
+    if ($LASTEXITCODE -ne 0 -or "$objectMarker".Trim() -ne "restore-drill-marker") {
+        throw "对象存储业务冒烟失败"
+    }
+    Invoke-Docker ($runPrefix + @("python", "manage.py", "check"))
+
+    $stopwatch.Stop()
+    [pscustomobject]@{
+        status = "passed"
+        compose_project = $projectName
+        backup_id = $backupDirectory.Name
+        duration_seconds = [Math]::Round($stopwatch.Elapsed.TotalSeconds, 2)
+        database_smoke = "ticket_and_attachment_restored"
+        object_storage_smoke = "marker_restored"
+        django_check = "passed"
+        backup_directory = $backupDirectory.FullName
+    } | ConvertTo-Json
+}
+finally {
+    & docker compose -f $ComposeFile down -v --remove-orphans --rmi local
+    foreach ($entry in $savedEnvironment.GetEnumerator()) {
+        if ($null -eq $entry.Value) {
+            Remove-Item "Env:$($entry.Key)" -ErrorAction SilentlyContinue
+        }
+        else {
+            Set-Item "Env:$($entry.Key)" $entry.Value
+        }
+    }
+}

infra/scripts/prune-backups.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+set -eu
+
+SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+INFRA_DIR=$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd)
+PROJECT_DIR=$(CDPATH= cd -- "$INFRA_DIR/.." && pwd)
+BACKUP_ROOT=${BACKUP_ROOT:-"$PROJECT_DIR/backups"}
+RETENTION_DAYS=${BACKUP_RETENTION_DAYS:-30}
+
+case "$RETENTION_DAYS" in
+  *[!0-9]*|"") echo "BACKUP_RETENTION_DAYS 必须是正整数" >&2; exit 2 ;;
+esac
+if [ "$RETENTION_DAYS" -lt 1 ]; then
+  echo "BACKUP_RETENTION_DAYS 必须至少为 1" >&2
+  exit 2
+fi
+
+mkdir -p "$BACKUP_ROOT"
+BACKUP_ROOT=$(CDPATH= cd -- "$BACKUP_ROOT" && pwd)
+case "$BACKUP_ROOT" in
+  "$PROJECT_DIR/backups"|"$PROJECT_DIR/backups/"*) ;;
+  *) echo "拒绝清理项目 backups 目录之外的路径：$BACKUP_ROOT" >&2; exit 2 ;;
+esac
+
+find "$BACKUP_ROOT" -mindepth 1 -maxdepth 1 -type d -mtime "+$RETENTION_DAYS" -print |
+while IFS= read -r candidate; do
+  name=$(basename "$candidate")
+  if ! printf '%s\n' "$name" | grep -Eq '^20[0-9]{6}T[0-9]{6}Z$'; then
+    echo "跳过非标准备份目录：$candidate"
+    continue
+  fi
+  if [ ! -f "$candidate/manifest.txt" ] || [ ! -f "$candidate/SHA256SUMS" ]; then
+    echo "跳过缺少清单的目录：$candidate"
+    continue
+  fi
+  rm -rf -- "$candidate"
+  echo "已按保留策略删除：$candidate"
+done

infra/scripts/restore.ps1

@@ -0,0 +1,84 @@
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory = $true)]
+    [string]$BackupDirectory,
+    [string]$ComposeFile = "",
+    [string]$PostgresDatabase = $env:POSTGRES_DB,
+    [string]$PostgresUser = $env:POSTGRES_USER,
+    [string]$Confirm = $env:RESTORE_CONFIRM,
+    [string]$BackupHelperImage = "alpine:3.21",
+    [switch]$SkipApplicationServices
+)
+
+$ErrorActionPreference = "Stop"
+if ($Confirm -ne "restore-ai-ticket-platform") {
+    throw "恢复会替换当前 PostgreSQL 与 MinIO 数据；请传入 -Confirm restore-ai-ticket-platform。"
+}
+
+$scriptDirectory = Split-Path -Parent $MyInvocation.MyCommand.Path
+$infraDirectory = [IO.Path]::GetFullPath((Join-Path $scriptDirectory ".."))
+if (-not $ComposeFile) {
+    $ComposeFile = Join-Path $infraDirectory "docker-compose.prod.yml"
+}
+$ComposeFile = (Resolve-Path -LiteralPath $ComposeFile).Path
+$target = (Resolve-Path -LiteralPath $BackupDirectory).Path
+if (-not $PostgresDatabase) { $PostgresDatabase = "ai_ticket_platform" }
+if (-not $PostgresUser) { $PostgresUser = "ai_ticket" }
+
+& (Join-Path $scriptDirectory "verify-backup.ps1") -BackupDirectory $target
+
+$postgresTemp = "/tmp/ai-ticket-restore-$PID.dump"
+$minioContainer = & docker compose -f $ComposeFile ps -q minio
+if ($LASTEXITCODE -ne 0 -or -not "$minioContainer".Trim()) {
+    throw "未找到 MinIO 容器"
+}
+$minioContainer = "$minioContainer".Trim()
+
+function Invoke-Compose {
+    param([Parameter(Mandatory = $true)][string[]]$Arguments)
+    & docker compose -f $ComposeFile @Arguments
+    if ($LASTEXITCODE -ne 0) {
+        throw "docker compose 执行失败：$($Arguments -join ' ')"
+    }
+}
+
+if (-not $SkipApplicationServices) {
+    Invoke-Compose @("stop", "backend", "worker", "beat")
+}
+Invoke-Compose @("stop", "minio")
+
+try {
+    Invoke-Compose @("cp", (Join-Path $target "postgres.dump"), "postgres:$postgresTemp")
+    Invoke-Compose @(
+        "exec", "-T", "postgres",
+        "dropdb", "-U", $PostgresUser, "--if-exists", "--force", $PostgresDatabase
+    )
+    Invoke-Compose @(
+        "exec", "-T", "postgres",
+        "createdb", "-U", $PostgresUser, $PostgresDatabase
+    )
+    Invoke-Compose @(
+        "exec", "-T", "postgres",
+        "pg_restore", "-U", $PostgresUser, "-d", $PostgresDatabase,
+        "--no-owner", "--exit-on-error", $postgresTemp
+    )
+
+    & docker run --rm `
+        --volumes-from $minioContainer `
+        -v "${target}:/backup:ro" `
+        $BackupHelperImage `
+        sh -c "find /data -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + && tar -C /data -xzf /backup/minio-data.tar.gz"
+    if ($LASTEXITCODE -ne 0) {
+        throw "MinIO 数据恢复失败"
+    }
+    Invoke-Compose @("up", "-d", "--wait", "minio")
+}
+finally {
+    & docker compose -f $ComposeFile exec -T postgres rm -f $postgresTemp 2>$null
+}
+
+if (-not $SkipApplicationServices) {
+    Invoke-Compose @("start", "backend", "worker", "beat")
+}
+
+Write-Output "恢复完成。请立即检查 /api/system/health/ready 并执行核心业务冒烟测试。"

infra/scripts/restore.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+set -eu
+
+if [ "$#" -ne 1 ] || [ "${RESTORE_CONFIRM:-}" != "restore-ai-ticket-platform" ]; then
+  echo "用法：RESTORE_CONFIRM=restore-ai-ticket-platform $0 <backup-directory>" >&2
+  echo "恢复会替换当前 PostgreSQL 数据库和 MinIO 数据。" >&2
+  exit 2
+fi
+
+SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+INFRA_DIR=$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd)
+COMPOSE_FILE=${COMPOSE_FILE:-"$INFRA_DIR/docker-compose.prod.yml"}
+TARGET=$(CDPATH= cd -- "$1" && pwd)
+
+"$SCRIPT_DIR/verify-backup.sh" "$TARGET"
+
+POSTGRES_DB=${POSTGRES_DB:-ai_ticket_platform}
+POSTGRES_USER=${POSTGRES_USER:-ai_ticket}
+BACKUP_HELPER_IMAGE=${BACKUP_HELPER_IMAGE:-alpine:3.21}
+
+if [ "${RESTORE_SKIP_APPLICATION_SERVICES:-false}" != "true" ]; then
+  docker compose -f "$COMPOSE_FILE" stop backend worker beat
+fi
+MINIO_CONTAINER=$(docker compose -f "$COMPOSE_FILE" ps -q minio)
+if [ -z "$MINIO_CONTAINER" ]; then
+  echo "未找到 MinIO 容器" >&2
+  exit 3
+fi
+docker compose -f "$COMPOSE_FILE" stop minio
+docker compose -f "$COMPOSE_FILE" exec -T postgres \
+  dropdb -U "$POSTGRES_USER" --force "$POSTGRES_DB"
+docker compose -f "$COMPOSE_FILE" exec -T postgres \
+  createdb -U "$POSTGRES_USER" "$POSTGRES_DB"
+docker compose -f "$COMPOSE_FILE" exec -T postgres \
+  pg_restore -U "$POSTGRES_USER" -d "$POSTGRES_DB" --no-owner --exit-on-error \
+  < "$TARGET/postgres.dump"
+docker run --rm \
+  --volumes-from "$MINIO_CONTAINER" \
+  -v "$TARGET:/backup:ro" \
+  "$BACKUP_HELPER_IMAGE" \
+  sh -c 'find /data -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + && tar -C /data -xzf /backup/minio-data.tar.gz'
+docker compose -f "$COMPOSE_FILE" up -d --wait minio
+if [ "${RESTORE_SKIP_APPLICATION_SERVICES:-false}" != "true" ]; then
+  docker compose -f "$COMPOSE_FILE" start backend worker beat
+fi
+
+echo "恢复完成。请立即检查 /api/system/health/ready 并执行核心业务冒烟测试。"

infra/scripts/verify-backup.ps1

@@ -0,0 +1,38 @@
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory = $true)]
+    [string]$BackupDirectory
+)
+
+$ErrorActionPreference = "Stop"
+$target = (Resolve-Path -LiteralPath $BackupDirectory).Path
+$requiredFiles = @(
+    "SHA256SUMS",
+    "postgres.dump",
+    "postgres.contents",
+    "minio-data.tar.gz",
+    "manifest.txt"
+)
+foreach ($file in $requiredFiles) {
+    if (-not (Test-Path -LiteralPath (Join-Path $target $file) -PathType Leaf)) {
+        throw "备份缺少文件：$file"
+    }
+}
+
+foreach ($line in Get-Content -LiteralPath (Join-Path $target "SHA256SUMS")) {
+    if ($line -notmatch "^([0-9a-fA-F]{64})\s{2}(.+)$") {
+        throw "SHA256SUMS 格式无效：$line"
+    }
+    $expected = $Matches[1].ToLowerInvariant()
+    $file = $Matches[2]
+    $path = Join-Path $target $file
+    if (-not (Test-Path -LiteralPath $path -PathType Leaf)) {
+        throw "校验文件不存在：$file"
+    }
+    $actual = (Get-FileHash -Algorithm SHA256 -LiteralPath $path).Hash.ToLowerInvariant()
+    if ($actual -ne $expected) {
+        throw "备份校验失败：$file"
+    }
+}
+
+Write-Output "备份校验通过：$target"

infra/scripts/verify-backup.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+set -eu
+
+if [ "$#" -ne 1 ]; then
+  echo "用法：$0 <backup-directory>" >&2
+  exit 2
+fi
+
+TARGET=$(CDPATH= cd -- "$1" && pwd)
+for file in SHA256SUMS postgres.dump postgres.contents minio-data.tar.gz manifest.txt; do
+  if [ ! -f "$TARGET/$file" ]; then
+    echo "备份缺少文件：$file" >&2
+    exit 3
+  fi
+done
+
+(
+  cd "$TARGET"
+  sha256sum -c SHA256SUMS
+)
+
+echo "备份校验通过：$TARGET"

infra/secrets/alert_webhook_url.example

@@ -0,0 +1 @@
+http://host.docker.internal:9099/alerts

infra/secrets/metrics_token.example

@@ -0,0 +1 @@
+replace-with-the-same-METRICS_TOKEN-used-by-backend

start-backend.ps1

@@ -0,0 +1,95 @@
+[CmdletBinding()]
+param(
+    [switch]$Restart
+)
+
+$ErrorActionPreference = "Stop"
+$ProjectRoot = $PSScriptRoot
+$BackendDir = Join-Path $ProjectRoot "backend"
+$EnvFile = Join-Path $ProjectRoot ".env"
+
+function Get-MatchingProcesses {
+    param([string]$Pattern)
+
+    return @(Get-CimInstance Win32_Process | Where-Object {
+        $_.CommandLine -and $_.CommandLine -match $Pattern
+    })
+}
+
+function Stop-MatchingProcesses {
+    param([string]$Name, [string]$Pattern)
+
+    $processes = Get-MatchingProcesses -Pattern $Pattern
+    if (-not $processes) {
+        return
+    }
+    $hasStaleEnvironment = @($processes | Where-Object {
+        $_.CreationDate -lt $script:EnvLastWriteTime
+    }).Count -gt 0
+    if (-not $Restart -and -not $hasStaleEnvironment) {
+        Write-Host "[skip] $Name 已运行" -ForegroundColor Yellow
+        return
+    }
+    foreach ($process in $processes) {
+        Stop-Process -Id $process.ProcessId -Force -ErrorAction SilentlyContinue
+    }
+    $reason = if ($Restart) { "显式重启" } else { ".env 晚于进程启动时间" }
+    Write-Host "[restart] 已停止旧 $Name 进程（$reason）" -ForegroundColor Yellow
+}
+
+function Start-PythonProcess {
+    param(
+        [string]$Name,
+        [string]$Pattern,
+        [string]$Command
+    )
+
+    if (Get-MatchingProcesses -Pattern $Pattern) {
+        return
+    }
+    $escapedDirectory = $BackendDir.Replace("'", "''")
+    $windowCommand = "Set-Location -LiteralPath '$escapedDirectory'; `$Host.UI.RawUI.WindowTitle = '$Name'; $Command"
+    Start-Process powershell.exe -ArgumentList @("-NoExit", "-Command", $windowCommand)
+    Write-Host "[start] $Name" -ForegroundColor Green
+}
+
+if (-not (Test-Path -LiteralPath $EnvFile)) {
+    throw "未找到 $EnvFile，请先复制 .env.example 并完成配置。"
+}
+if (-not (Get-Command python -ErrorAction SilentlyContinue)) {
+    throw "未找到 python，请先安装 Python 3.12 并加入 PATH。"
+}
+$script:EnvLastWriteTime = (Get-Item -LiteralPath $EnvFile).LastWriteTime
+
+Push-Location $ProjectRoot
+try {
+    python -c "from cryptography.fernet import Fernet; from dotenv import dotenv_values; key=dotenv_values('.env').get('AI_CREDENTIAL_ENCRYPTION_KEY',''); Fernet(key.encode()); print('[ok] AI credential encryption key is valid')"
+    if ($LASTEXITCODE -ne 0) {
+        throw ".env 中的 AI_CREDENTIAL_ENCRYPTION_KEY 缺失或不是有效 Fernet 密钥。"
+    }
+
+    Stop-MatchingProcesses -Name "Django API" -Pattern "manage\.py runserver 0\.0\.0\.0:8000"
+    Stop-MatchingProcesses -Name "Celery Worker" -Pattern "celery.*-A config\.celery worker"
+    Stop-MatchingProcesses -Name "Celery Beat" -Pattern "celery.*-A config\.celery beat"
+
+    Start-PythonProcess -Name "AI Ticket - Django API" `
+        -Pattern "manage\.py runserver 0\.0\.0\.0:8000" `
+        -Command "python manage.py runserver 0.0.0.0:8000"
+    Start-PythonProcess -Name "AI Ticket - Celery Worker" `
+        -Pattern "celery.*-A config\.celery worker" `
+        -Command "python -m celery -A config.celery worker -l info -P solo"
+    Start-PythonProcess -Name "AI Ticket - Celery Beat" `
+        -Pattern "celery.*-A config\.celery beat" `
+        -Command "python -m celery -A config.celery beat -l info"
+
+    Write-Host ""
+    Write-Host "Python 后端程序已启动：" -ForegroundColor Cyan
+    Write-Host "  Django API     http://localhost:8000/api/docs"
+    Write-Host "  Celery Worker  已启动"
+    Write-Host "  Celery Beat    已启动"
+    Write-Host ""
+    Write-Host "修改 .env 后可执行：.\start-backend.ps1 -Restart" -ForegroundColor Yellow
+}
+finally {
+    Pop-Location
+}