ops: add production observability and recovery

2026-07-27 19:06:34 +08:00
parent e69368a2f3
commit b02d18d3b9
33 changed files with 8832 additions and 1 deletions
--- a/infra/backup-cron.example
+++ b/infra/backup-cron.example
@@ -0,0 +1,3 @@
+# 每日 02:17 创建备份、校验并清理超过 30 天的标准备份目录。
+# 请把 /opt/ai-ticket-platform 替换为实际绝对路径，并由专用系统账号安装。
+17 2 * * * cd /opt/ai-ticket-platform && BACKUP_RETENTION_DAYS=30 sh infra/scripts/backup.sh && BACKUP_RETENTION_DAYS=30 sh infra/scripts/prune-backups.sh >> /var/log/ai-ticket-backup.log 2>&1
--- a/infra/docker-compose.local.yml
+++ b/infra/docker-compose.local.yml
@@ -32,3 +32,10 @@ services:
      - ./data/minio:/data
    restart: unless-stopped

+  clamav:
+    image: clamav/clamav:stable
+    ports:
+      - "3310:3310"
+    volumes:
+      - ./data/clamav:/var/lib/clamav
+    restart: unless-stopped
--- a/infra/docker-compose.prod.yml
+++ b/infra/docker-compose.prod.yml
@@ -0,0 +1,184 @@
+services:
+  postgres:
+    image: pgvector/pgvector:pg16
+    environment:
+      POSTGRES_DB: ${POSTGRES_DB:-ai_ticket_platform}
+      POSTGRES_USER: ${POSTGRES_USER:-ai_ticket}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+
+  redis:
+    image: redis:7-alpine
+    command: ["redis-server", "--appendonly", "yes"]
+    volumes:
+      - redis-data:/data
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+
+  minio:
+    image: minio/minio:latest
+    command: server /data --console-address ":9001"
+    environment:
+      MINIO_ROOT_USER: ${MINIO_ROOT_USER:?MINIO_ROOT_USER is required}
+      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?MINIO_ROOT_PASSWORD is required}
+    volumes:
+      - minio-data:/data
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+
+  clamav:
+    image: clamav/clamav:stable
+    volumes:
+      - clamav-data:/var/lib/clamav
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD-SHELL", "echo PING | nc 127.0.0.1 3310 | grep PONG"]
+      interval: 30s
+      timeout: 10s
+      retries: 10
+      start_period: 60s
+
+  backend:
+    build:
+      context: ..
+      dockerfile: infra/backend.Dockerfile
+    entrypoint: ["/entrypoint.sh"]
+    command:
+      [
+        "gunicorn",
+        "config.wsgi:application",
+        "--bind",
+        "0.0.0.0:8000",
+        "--workers",
+        "3",
+        "--timeout",
+        "300",
+        "--access-logfile",
+        "-"
+      ]
+    env_file: ../.env
+    restart: unless-stopped
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      minio:
+        condition: service_healthy
+      clamav:
+        condition: service_healthy
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "python",
+          "-c",
+          "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/api/system/health/ready', timeout=5)"
+        ]
+      interval: 15s
+      timeout: 8s
+      retries: 10
+
+  worker:
+    build:
+      context: ..
+      dockerfile: infra/backend.Dockerfile
+    command: ["celery", "-A", "config.celery", "worker", "-l", "info"]
+    env_file: ../.env
+    restart: unless-stopped
+    depends_on:
+      backend:
+        condition: service_healthy
+
+  beat:
+    build:
+      context: ..
+      dockerfile: infra/backend.Dockerfile
+    command: ["celery", "-A", "config.celery", "beat", "-l", "info"]
+    env_file: ../.env
+    restart: unless-stopped
+    depends_on:
+      backend:
+        condition: service_healthy
+
+  frontend:
+    build:
+      context: ..
+      dockerfile: infra/frontend-prod.Dockerfile
+    ports:
+      - "${HTTP_PORT:-8080}:80"
+    restart: unless-stopped
+    depends_on:
+      backend:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1/"]
+      interval: 15s
+      timeout: 5s
+      retries: 10
+
+  prometheus:
+    image: prom/prometheus:v3.5.0
+    profiles: ["monitoring"]
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.path=/prometheus
+      - --storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-30d}
+    volumes:
+      - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./monitoring/alerts.yml:/etc/prometheus/alerts.yml:ro
+      - prometheus-data:/prometheus
+    secrets:
+      - metrics_token
+    ports:
+      - "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"
+    restart: unless-stopped
+    depends_on:
+      backend:
+        condition: service_healthy
+      alertmanager:
+        condition: service_started
+
+  alertmanager:
+    image: prom/alertmanager:v0.28.1
+    profiles: ["monitoring"]
+    command:
+      - --config.file=/etc/alertmanager/alertmanager.yml
+      - --storage.path=/alertmanager
+    volumes:
+      - ./monitoring/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
+      - alertmanager-data:/alertmanager
+    secrets:
+      - alert_webhook_url
+    ports:
+      - "127.0.0.1:${ALERTMANAGER_PORT:-9093}:9093"
+    restart: unless-stopped
+
+volumes:
+  postgres-data:
+  redis-data:
+  minio-data:
+  clamav-data:
+  prometheus-data:
+  alertmanager-data:
+
+secrets:
+  metrics_token:
+    file: ${METRICS_TOKEN_FILE:-./secrets/metrics_token.example}
+  alert_webhook_url:
+    file: ${ALERT_WEBHOOK_URL_FILE:-./secrets/alert_webhook_url.example}
--- a/infra/docker-compose.yml
+++ b/infra/docker-compose.yml
@@ -27,6 +27,13 @@ services:
    volumes:
      - minio-data:/data

+  clamav:
+    image: clamav/clamav:stable
+    ports:
+      - "3310:3310"
+    volumes:
+      - clamav-data:/var/lib/clamav
+
  backend:
    build:
      context: ..
@@ -43,6 +50,8 @@ services:
    depends_on:
      - postgres
      - redis
+      - minio
+      - clamav
    volumes:
      - ../backend:/app

@@ -59,6 +68,7 @@ services:
    depends_on:
      - backend
      - redis
+      - clamav
    volumes:
      - ../backend:/app

@@ -96,4 +106,5 @@ services:
 volumes:
  postgres-data:
  minio-data:
+  clamav-data:
  frontend-node-modules:
--- a/infra/frontend-prod.Dockerfile
+++ b/infra/frontend-prod.Dockerfile
@@ -0,0 +1,12 @@
+FROM node:22-slim AS build
+
+WORKDIR /app
+RUN corepack enable && corepack prepare pnpm@11.9.0 --activate
+COPY frontend/package.json frontend/pnpm-lock.yaml frontend/pnpm-workspace.yaml ./
+RUN pnpm install --frozen-lockfile
+COPY frontend ./
+RUN pnpm build
+
+FROM nginx:1.27-alpine
+COPY infra/nginx.conf /etc/nginx/conf.d/default.conf
+COPY --from=build /app/dist /usr/share/nginx/html
--- a/infra/monitoring/alertmanager.yml
+++ b/infra/monitoring/alertmanager.yml
@@ -0,0 +1,14 @@
+route:
+  receiver: operations-webhook
+  group_by:
+    - alertname
+    - severity
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 4h
+
+receivers:
+  - name: operations-webhook
+    webhook_configs:
+      - url_file: /run/secrets/alert_webhook_url
+        send_resolved: true
--- a/infra/monitoring/alerts.yml
+++ b/infra/monitoring/alerts.yml
@@ -0,0 +1,56 @@
+groups:
+  - name: ai-ticket-platform
+    rules:
+      - alert: AiTicketBackendDown
+        expr: up{job="ai-ticket-backend"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: AI 工单平台后端不可用
+          description: Prometheus 连续 2 分钟无法抓取后端指标。
+
+      - alert: AiTicketChannelDeliveryFailures
+        expr: ai_ticket_channel_messages_failed_recent > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: 渠道消息投递持续失败
+          description: 最近 15 分钟存在失败的渠道消息，请检查渠道凭证、地址和重试队列。
+
+      - alert: AiTicketChannelQueueStuck
+        expr: ai_ticket_channel_messages_stuck > 0
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: 渠道消息队列出现积压
+          description: 存在排队超过 10 分钟的出站消息，请检查 Celery Worker 和渠道网络。
+
+      - alert: AiTicketAgentRunFailures
+        expr: ai_ticket_agent_runs_failed_recent > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: Agent 运行持续失败
+          description: 最近 15 分钟存在失败的 Agent 运行，请检查模型、工具和运行日志。
+
+      - alert: AiTicketToolCallFailures
+        expr: ai_ticket_tool_calls_failed_recent > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: Tool/MCP 调用持续失败
+          description: 最近 15 分钟存在失败的工具调用，请检查授权、MCP 健康和审计记录。
+
+      - alert: AiTicketWorkItemBacklog
+        expr: ai_ticket_work_items_active > 200
+        for: 30m
+        labels:
+          severity: warning
+        annotations:
+          summary: 活动任务积压
+          description: 活动任务连续 30 分钟超过 200 个，请检查人工处理容量和自动任务异常。
--- a/infra/monitoring/prometheus.yml
+++ b/infra/monitoring/prometheus.yml
@@ -0,0 +1,21 @@
+global:
+  scrape_interval: 30s
+  evaluation_interval: 30s
+
+rule_files:
+  - /etc/prometheus/alerts.yml
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+            - alertmanager:9093
+
+scrape_configs:
+  - job_name: ai-ticket-backend
+    metrics_path: /api/system/metrics
+    authorization:
+      credentials_file: /run/secrets/metrics_token
+    static_configs:
+      - targets:
+          - backend:8000
--- a/infra/nginx.conf
+++ b/infra/nginx.conf
@@ -0,0 +1,22 @@
+server {
+    listen 80;
+    server_name _;
+
+    root /usr/share/nginx/html;
+    index index.html;
+
+    location /api/ {
+        proxy_pass http://backend:8000;
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_buffering off;
+        proxy_read_timeout 300s;
+    }
+
+    location / {
+        try_files $uri $uri/ /index.html;
+    }
+}
--- a/infra/scripts/backup.ps1
+++ b/infra/scripts/backup.ps1
@@ -0,0 +1,142 @@
+[CmdletBinding()]
+param(
+    [string]$ComposeFile = "",
+    [string]$BackupRoot = "",
+    [string]$PostgresDatabase = $env:POSTGRES_DB,
+    [string]$PostgresUser = $env:POSTGRES_USER,
+    [string]$BackupHelperImage = "alpine:3.21",
+    [switch]$SkipApplicationServices
+)
+
+$ErrorActionPreference = "Stop"
+$scriptDirectory = Split-Path -Parent $MyInvocation.MyCommand.Path
+$infraDirectory = [IO.Path]::GetFullPath((Join-Path $scriptDirectory ".."))
+$projectDirectory = [IO.Path]::GetFullPath((Join-Path $infraDirectory ".."))
+
+if (-not $ComposeFile) {
+    $ComposeFile = Join-Path $infraDirectory "docker-compose.prod.yml"
+}
+$ComposeFile = (Resolve-Path -LiteralPath $ComposeFile).Path
+if (-not $BackupRoot) {
+    $BackupRoot = Join-Path $projectDirectory "backups"
+}
+$BackupRoot = [IO.Path]::GetFullPath($BackupRoot)
+$projectPrefix = $projectDirectory.TrimEnd("\", "/") + [IO.Path]::DirectorySeparatorChar
+if (-not $BackupRoot.StartsWith($projectPrefix, [StringComparison]::OrdinalIgnoreCase)) {
+    throw "拒绝写入项目目录之外的备份路径：$BackupRoot"
+}
+
+if (-not $PostgresDatabase) { $PostgresDatabase = "ai_ticket_platform" }
+if (-not $PostgresUser) { $PostgresUser = "ai_ticket" }
+
+$stamp = (Get-Date).ToUniversalTime().ToString("yyyyMMddTHHmmssZ")
+$target = [IO.Path]::GetFullPath((Join-Path $BackupRoot $stamp))
+if (-not $target.StartsWith($projectPrefix, [StringComparison]::OrdinalIgnoreCase)) {
+    throw "拒绝写入项目目录之外的备份路径：$target"
+}
+New-Item -ItemType Directory -Path $target -Force | Out-Null
+
+$postgresTemp = "/tmp/ai-ticket-postgres-$PID.dump"
+
+function Invoke-Compose {
+    param([Parameter(Mandatory = $true)][string[]]$Arguments)
+    & docker compose -f $ComposeFile @Arguments
+    if ($LASTEXITCODE -ne 0) {
+        throw "docker compose 执行失败：$($Arguments -join ' ')"
+    }
+}
+
+$runningServices = & docker compose -f $ComposeFile ps --services --filter status=running
+if ($LASTEXITCODE -ne 0) {
+    throw "无法读取 Compose 服务状态"
+}
+$minioWasRunning = $runningServices -contains "minio"
+$minioContainer = & docker compose -f $ComposeFile ps -q minio
+if ($LASTEXITCODE -ne 0 -or -not "$minioContainer".Trim()) {
+    throw "未找到 MinIO 容器"
+}
+$minioContainer = "$minioContainer".Trim()
+
+$runningApplicationServices = @()
+if (-not $SkipApplicationServices) {
+    $runningApplicationServices = @(
+        "backend", "worker", "beat" |
+            Where-Object { $runningServices -contains $_ }
+    )
+    if ($runningApplicationServices.Count -gt 0) {
+        Invoke-Compose (@("stop") + $runningApplicationServices)
+    }
+}
+if ($minioWasRunning) {
+    Invoke-Compose @("stop", "minio")
+}
+
+try {
+    Invoke-Compose @(
+        "exec", "-T", "postgres",
+        "pg_dump", "-U", $PostgresUser, "-d", $PostgresDatabase,
+        "--format=custom", "--file=$postgresTemp"
+    )
+    Invoke-Compose @("cp", "postgres:$postgresTemp", (Join-Path $target "postgres.dump"))
+
+    $contents = & docker compose -f $ComposeFile exec -T postgres `
+        pg_restore --list $postgresTemp
+    if ($LASTEXITCODE -ne 0) {
+        throw "PostgreSQL 备份结构校验失败"
+    }
+    [IO.File]::WriteAllLines(
+        (Join-Path $target "postgres.contents"),
+        [string[]]$contents,
+        [Text.UTF8Encoding]::new($false)
+    )
+
+    & docker run --rm `
+        --volumes-from $minioContainer `
+        -v "${target}:/backup" `
+        $BackupHelperImage `
+        tar -C /data -czf /backup/minio-data.tar.gz .
+    if ($LASTEXITCODE -ne 0) {
+        throw "MinIO 数据归档失败"
+    }
+
+    $checksumLines = @("postgres.dump", "postgres.contents", "minio-data.tar.gz") | ForEach-Object {
+        $hash = (Get-FileHash -Algorithm SHA256 -LiteralPath (Join-Path $target $_)).Hash.ToLowerInvariant()
+        "$hash  $_"
+    }
+    [IO.File]::WriteAllLines(
+        (Join-Path $target "SHA256SUMS"),
+        [string[]]$checksumLines,
+        [Text.UTF8Encoding]::new($false)
+    )
+    $manifestLines = @(
+        "created_at=$stamp",
+        "postgres_database=$PostgresDatabase",
+        "postgres_user=$PostgresUser",
+        "compose_file=$ComposeFile",
+        "compose_project=$($env:COMPOSE_PROJECT_NAME)"
+    )
+    [IO.File]::WriteAllLines(
+        (Join-Path $target "manifest.txt"),
+        [string[]]$manifestLines,
+        [Text.UTF8Encoding]::new($false)
+    )
+}
+finally {
+    & docker compose -f $ComposeFile exec -T postgres rm -f $postgresTemp 2>$null
+    $minioReady = $true
+    if ($minioWasRunning) {
+        & docker compose -f $ComposeFile up -d --wait minio
+        if ($LASTEXITCODE -ne 0) {
+            $minioReady = $false
+            Write-Warning "备份结束后 MinIO 未恢复健康，应用服务保持停止"
+        }
+    }
+    if ($minioReady -and $runningApplicationServices.Count -gt 0) {
+        & docker compose -f $ComposeFile start @runningApplicationServices
+        if ($LASTEXITCODE -ne 0) {
+            Write-Warning "备份结束后无法恢复原应用服务：$($runningApplicationServices -join ', ')"
+        }
+    }
+}
+
+Write-Output "备份已创建并完成结构校验：$target"
--- a/infra/scripts/backup.sh
+++ b/infra/scripts/backup.sh
@@ -0,0 +1,86 @@
+#!/bin/sh
+set -eu
+
+SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+INFRA_DIR=$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd)
+PROJECT_DIR=$(CDPATH= cd -- "$INFRA_DIR/.." && pwd)
+COMPOSE_FILE=${COMPOSE_FILE:-"$INFRA_DIR/docker-compose.prod.yml"}
+BACKUP_ROOT=${BACKUP_ROOT:-"$PROJECT_DIR/backups"}
+STAMP=$(date -u +%Y%m%dT%H%M%SZ)
+TARGET="$BACKUP_ROOT/$STAMP"
+
+mkdir -p "$TARGET"
+TARGET=$(CDPATH= cd -- "$TARGET" && pwd)
+case "$TARGET" in
+  "$PROJECT_DIR"/*) ;;
+  *) echo "拒绝写入项目目录之外的备份路径：$TARGET" >&2; exit 2 ;;
+esac
+
+POSTGRES_DB=${POSTGRES_DB:-ai_ticket_platform}
+POSTGRES_USER=${POSTGRES_USER:-ai_ticket}
+BACKUP_HELPER_IMAGE=${BACKUP_HELPER_IMAGE:-alpine:3.21}
+RUNNING_APPLICATION_SERVICES=""
+MINIO_WAS_RUNNING=false
+
+cleanup() {
+  MINIO_READY=true
+  if [ "$MINIO_WAS_RUNNING" = "true" ]; then
+    if ! docker compose -f "$COMPOSE_FILE" up -d --wait minio >/dev/null; then
+      echo "备份结束后 MinIO 未恢复健康，应用服务保持停止" >&2
+      MINIO_READY=false
+    fi
+  fi
+  if [ "$MINIO_READY" = "true" ] && [ -n "$RUNNING_APPLICATION_SERVICES" ]; then
+    docker compose -f "$COMPOSE_FILE" start $RUNNING_APPLICATION_SERVICES >/dev/null || true
+  fi
+}
+trap cleanup EXIT INT TERM
+
+RUNNING_SERVICES=$(docker compose -f "$COMPOSE_FILE" ps --services --filter status=running)
+if printf '%s\n' "$RUNNING_SERVICES" | grep -qx "minio"; then
+  MINIO_WAS_RUNNING=true
+fi
+if [ "${BACKUP_SKIP_APPLICATION_SERVICES:-false}" != "true" ]; then
+  for service in backend worker beat; do
+    if printf '%s\n' "$RUNNING_SERVICES" | grep -qx "$service"; then
+      RUNNING_APPLICATION_SERVICES="$RUNNING_APPLICATION_SERVICES $service"
+    fi
+  done
+  if [ -n "$RUNNING_APPLICATION_SERVICES" ]; then
+    docker compose -f "$COMPOSE_FILE" stop $RUNNING_APPLICATION_SERVICES
+  fi
+fi
+if [ "$MINIO_WAS_RUNNING" = "true" ]; then
+  docker compose -f "$COMPOSE_FILE" stop minio
+fi
+
+docker compose -f "$COMPOSE_FILE" exec -T postgres \
+  pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom > "$TARGET/postgres.dump"
+docker compose -f "$COMPOSE_FILE" exec -T postgres \
+  pg_restore --list < "$TARGET/postgres.dump" > "$TARGET/postgres.contents"
+MINIO_CONTAINER=$(docker compose -f "$COMPOSE_FILE" ps -q minio)
+if [ -z "$MINIO_CONTAINER" ]; then
+  echo "未找到 MinIO 容器" >&2
+  exit 3
+fi
+docker run --rm \
+  --volumes-from "$MINIO_CONTAINER" \
+  -v "$TARGET:/backup" \
+  "$BACKUP_HELPER_IMAGE" \
+  tar -C /data -czf /backup/minio-data.tar.gz .
+
+(
+  cd "$TARGET"
+  sha256sum postgres.dump postgres.contents minio-data.tar.gz > SHA256SUMS
+)
+
+cat > "$TARGET/manifest.txt" <<EOF
+created_at=$STAMP
+postgres_database=$POSTGRES_DB
+postgres_user=$POSTGRES_USER
+compose_file=$COMPOSE_FILE
+EOF
+
+cleanup
+trap - EXIT INT TERM
+echo "备份已创建并完成结构校验：$TARGET"
--- a/infra/scripts/drill-restore.ps1
+++ b/infra/scripts/drill-restore.ps1
@@ -0,0 +1,194 @@
+[CmdletBinding()]
+param(
+    [string]$ComposeFile = "",
+    [switch]$SkipBuild
+)
+
+$ErrorActionPreference = "Stop"
+$scriptDirectory = Split-Path -Parent $MyInvocation.MyCommand.Path
+$infraDirectory = [IO.Path]::GetFullPath((Join-Path $scriptDirectory ".."))
+$projectDirectory = [IO.Path]::GetFullPath((Join-Path $infraDirectory ".."))
+if (-not $ComposeFile) {
+    $ComposeFile = Join-Path $infraDirectory "docker-compose.prod.yml"
+}
+$ComposeFile = (Resolve-Path -LiteralPath $ComposeFile).Path
+
+$projectName = "ai-ticket-restore-drill-$PID"
+$databasePassword = "restore-drill-postgres-$PID"
+$minioUser = "restore-drill-minio"
+$minioPassword = "restore-drill-minio-$PID"
+$backupRoot = Join-Path $projectDirectory "backups\$projectName"
+$stopwatch = [Diagnostics.Stopwatch]::StartNew()
+
+$savedEnvironment = @{
+    COMPOSE_PROJECT_NAME = $env:COMPOSE_PROJECT_NAME
+    POSTGRES_DB = $env:POSTGRES_DB
+    POSTGRES_USER = $env:POSTGRES_USER
+    POSTGRES_PASSWORD = $env:POSTGRES_PASSWORD
+    MINIO_ROOT_USER = $env:MINIO_ROOT_USER
+    MINIO_ROOT_PASSWORD = $env:MINIO_ROOT_PASSWORD
+}
+
+$env:COMPOSE_PROJECT_NAME = $projectName
+$env:POSTGRES_DB = "ai_ticket_platform"
+$env:POSTGRES_USER = "ai_ticket"
+$env:POSTGRES_PASSWORD = $databasePassword
+$env:MINIO_ROOT_USER = $minioUser
+$env:MINIO_ROOT_PASSWORD = $minioPassword
+
+function Invoke-Docker {
+    param([Parameter(Mandatory = $true)][string[]]$Arguments)
+    & docker @Arguments
+    if ($LASTEXITCODE -ne 0) {
+        throw "Docker 执行失败：docker $($Arguments -join ' ')"
+    }
+}
+
+function Invoke-DjangoCode {
+    param(
+        [Parameter(Mandatory = $true)][string[]]$RunPrefix,
+        [Parameter(Mandatory = $true)][string]$Code,
+        [Parameter(Mandatory = $true)][string]$FailureMessage
+    )
+    $bytes = [Text.Encoding]::UTF8.GetBytes($Code)
+    $runner = "exec(bytes([" + ($bytes -join ",") + "]))"
+    & docker @RunPrefix python manage.py shell -c $runner
+    if ($LASTEXITCODE -ne 0) {
+        throw $FailureMessage
+    }
+}
+
+$backendEnvironment = @(
+    "-e", "POSTGRES_HOST=postgres",
+    "-e", "POSTGRES_DB=ai_ticket_platform",
+    "-e", "POSTGRES_USER=ai_ticket",
+    "-e", "POSTGRES_PASSWORD=$databasePassword",
+    "-e", "MINIO_ENDPOINT=minio:9000",
+    "-e", "MINIO_ROOT_USER=$minioUser",
+    "-e", "MINIO_ROOT_PASSWORD=$minioPassword",
+    "-e", "CELERY_BROKER_URL=redis://redis:6379/0",
+    "-e", "CELERY_RESULT_BACKEND=redis://redis:6379/1"
+)
+
+$seedCode = @'
+from django.contrib.auth import get_user_model
+from apps.tickets.models import Ticket, TicketAttachment
+
+user, _ = get_user_model().objects.get_or_create(
+    username="restore_drill",
+    defaults={"email": "restore-drill@example.invalid"},
+)
+ticket, _ = Ticket.objects.update_or_create(
+    ticket_no="RESTORE-DRILL-001",
+    defaults={
+        "title": "隔离恢复演练工单",
+        "description": "restore-drill-marker",
+        "requester": user,
+        "created_by": user,
+    },
+)
+TicketAttachment.objects.update_or_create(
+    ticket=ticket,
+    storage_path="restore-drill/marker.txt",
+    defaults={
+        "file_name": "marker.txt",
+        "mime_type": "text/plain",
+        "file_size": 20,
+        "created_by": user,
+    },
+)
+print(f"seeded_ticket={ticket.ticket_no}")
+'@
+
+$deleteCode = @'
+from apps.tickets.models import Ticket
+Ticket.objects.filter(ticket_no="RESTORE-DRILL-001").delete()
+print("seed_deleted=true")
+'@
+
+$verifyCode = @'
+from apps.tickets.models import Ticket
+ticket = Ticket.objects.get(ticket_no="RESTORE-DRILL-001")
+attachment = ticket.attachments.get(storage_path="restore-drill/marker.txt")
+assert ticket.description == "restore-drill-marker"
+assert attachment.file_name == "marker.txt"
+print(f"restored_ticket={ticket.ticket_no}")
+print(f"restored_attachment={attachment.storage_path}")
+'@
+
+try {
+    Invoke-Docker @("compose", "-f", $ComposeFile, "up", "-d", "--wait", "postgres", "minio")
+    if (-not $SkipBuild) {
+        Invoke-Docker @("compose", "-f", $ComposeFile, "build", "backend")
+    }
+
+    $runPrefix = @(
+        "compose", "-f", $ComposeFile,
+        "run", "--rm", "-T", "--no-deps"
+    ) + $backendEnvironment + @("backend")
+
+    Invoke-Docker ($runPrefix + @("python", "manage.py", "migrate", "--noinput"))
+    Invoke-DjangoCode -RunPrefix $runPrefix -Code $seedCode -FailureMessage "演练数据创建失败"
+
+    Invoke-Docker @(
+        "compose", "-f", $ComposeFile, "exec", "-T", "minio",
+        "sh", "-c",
+        'mc alias set drill http://127.0.0.1:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null && mc mb --ignore-existing drill/ai-ticket-platform >/dev/null && printf restore-drill-marker | mc pipe drill/ai-ticket-platform/restore-drill/marker.txt >/dev/null'
+    )
+
+    & (Join-Path $scriptDirectory "backup.ps1") `
+        -ComposeFile $ComposeFile `
+        -BackupRoot $backupRoot `
+        -PostgresDatabase "ai_ticket_platform" `
+        -PostgresUser "ai_ticket" `
+        -SkipApplicationServices
+    $backupDirectory = Get-ChildItem -LiteralPath $backupRoot -Directory |
+        Sort-Object Name -Descending |
+        Select-Object -First 1
+    if (-not $backupDirectory) { throw "未找到演练备份目录" }
+
+    Invoke-DjangoCode -RunPrefix $runPrefix -Code $deleteCode -FailureMessage "演练数据破坏步骤失败"
+    Invoke-Docker @(
+        "compose", "-f", $ComposeFile, "exec", "-T", "minio",
+        "mc", "rm", "--force", "drill/ai-ticket-platform/restore-drill/marker.txt"
+    )
+
+    & (Join-Path $scriptDirectory "restore.ps1") `
+        -BackupDirectory $backupDirectory.FullName `
+        -ComposeFile $ComposeFile `
+        -PostgresDatabase "ai_ticket_platform" `
+        -PostgresUser "ai_ticket" `
+        -Confirm "restore-ai-ticket-platform" `
+        -SkipApplicationServices
+
+    Invoke-DjangoCode -RunPrefix $runPrefix -Code $verifyCode -FailureMessage "数据库业务冒烟失败"
+    $objectMarker = & docker compose -f $ComposeFile exec -T minio `
+        mc cat drill/ai-ticket-platform/restore-drill/marker.txt
+    if ($LASTEXITCODE -ne 0 -or "$objectMarker".Trim() -ne "restore-drill-marker") {
+        throw "对象存储业务冒烟失败"
+    }
+    Invoke-Docker ($runPrefix + @("python", "manage.py", "check"))
+
+    $stopwatch.Stop()
+    [pscustomobject]@{
+        status = "passed"
+        compose_project = $projectName
+        backup_id = $backupDirectory.Name
+        duration_seconds = [Math]::Round($stopwatch.Elapsed.TotalSeconds, 2)
+        database_smoke = "ticket_and_attachment_restored"
+        object_storage_smoke = "marker_restored"
+        django_check = "passed"
+        backup_directory = $backupDirectory.FullName
+    } | ConvertTo-Json
+}
+finally {
+    & docker compose -f $ComposeFile down -v --remove-orphans --rmi local
+    foreach ($entry in $savedEnvironment.GetEnumerator()) {
+        if ($null -eq $entry.Value) {
+            Remove-Item "Env:$($entry.Key)" -ErrorAction SilentlyContinue
+        }
+        else {
+            Set-Item "Env:$($entry.Key)" $entry.Value
+        }
+    }
+}
--- a/infra/scripts/prune-backups.sh
+++ b/infra/scripts/prune-backups.sh
@@ -0,0 +1,38 @@
+#!/bin/sh
+set -eu
+
+SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+INFRA_DIR=$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd)
+PROJECT_DIR=$(CDPATH= cd -- "$INFRA_DIR/.." && pwd)
+BACKUP_ROOT=${BACKUP_ROOT:-"$PROJECT_DIR/backups"}
+RETENTION_DAYS=${BACKUP_RETENTION_DAYS:-30}
+
+case "$RETENTION_DAYS" in
+  *[!0-9]*|"") echo "BACKUP_RETENTION_DAYS 必须是正整数" >&2; exit 2 ;;
+esac
+if [ "$RETENTION_DAYS" -lt 1 ]; then
+  echo "BACKUP_RETENTION_DAYS 必须至少为 1" >&2
+  exit 2
+fi
+
+mkdir -p "$BACKUP_ROOT"
+BACKUP_ROOT=$(CDPATH= cd -- "$BACKUP_ROOT" && pwd)
+case "$BACKUP_ROOT" in
+  "$PROJECT_DIR/backups"|"$PROJECT_DIR/backups/"*) ;;
+  *) echo "拒绝清理项目 backups 目录之外的路径：$BACKUP_ROOT" >&2; exit 2 ;;
+esac
+
+find "$BACKUP_ROOT" -mindepth 1 -maxdepth 1 -type d -mtime "+$RETENTION_DAYS" -print |
+while IFS= read -r candidate; do
+  name=$(basename "$candidate")
+  if ! printf '%s\n' "$name" | grep -Eq '^20[0-9]{6}T[0-9]{6}Z$'; then
+    echo "跳过非标准备份目录：$candidate"
+    continue
+  fi
+  if [ ! -f "$candidate/manifest.txt" ] || [ ! -f "$candidate/SHA256SUMS" ]; then
+    echo "跳过缺少清单的目录：$candidate"
+    continue
+  fi
+  rm -rf -- "$candidate"
+  echo "已按保留策略删除：$candidate"
+done
--- a/infra/scripts/restore.ps1
+++ b/infra/scripts/restore.ps1
@@ -0,0 +1,84 @@
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory = $true)]
+    [string]$BackupDirectory,
+    [string]$ComposeFile = "",
+    [string]$PostgresDatabase = $env:POSTGRES_DB,
+    [string]$PostgresUser = $env:POSTGRES_USER,
+    [string]$Confirm = $env:RESTORE_CONFIRM,
+    [string]$BackupHelperImage = "alpine:3.21",
+    [switch]$SkipApplicationServices
+)
+
+$ErrorActionPreference = "Stop"
+if ($Confirm -ne "restore-ai-ticket-platform") {
+    throw "恢复会替换当前 PostgreSQL 与 MinIO 数据；请传入 -Confirm restore-ai-ticket-platform。"
+}
+
+$scriptDirectory = Split-Path -Parent $MyInvocation.MyCommand.Path
+$infraDirectory = [IO.Path]::GetFullPath((Join-Path $scriptDirectory ".."))
+if (-not $ComposeFile) {
+    $ComposeFile = Join-Path $infraDirectory "docker-compose.prod.yml"
+}
+$ComposeFile = (Resolve-Path -LiteralPath $ComposeFile).Path
+$target = (Resolve-Path -LiteralPath $BackupDirectory).Path
+if (-not $PostgresDatabase) { $PostgresDatabase = "ai_ticket_platform" }
+if (-not $PostgresUser) { $PostgresUser = "ai_ticket" }
+
+& (Join-Path $scriptDirectory "verify-backup.ps1") -BackupDirectory $target
+
+$postgresTemp = "/tmp/ai-ticket-restore-$PID.dump"
+$minioContainer = & docker compose -f $ComposeFile ps -q minio
+if ($LASTEXITCODE -ne 0 -or -not "$minioContainer".Trim()) {
+    throw "未找到 MinIO 容器"
+}
+$minioContainer = "$minioContainer".Trim()
+
+function Invoke-Compose {
+    param([Parameter(Mandatory = $true)][string[]]$Arguments)
+    & docker compose -f $ComposeFile @Arguments
+    if ($LASTEXITCODE -ne 0) {
+        throw "docker compose 执行失败：$($Arguments -join ' ')"
+    }
+}
+
+if (-not $SkipApplicationServices) {
+    Invoke-Compose @("stop", "backend", "worker", "beat")
+}
+Invoke-Compose @("stop", "minio")
+
+try {
+    Invoke-Compose @("cp", (Join-Path $target "postgres.dump"), "postgres:$postgresTemp")
+    Invoke-Compose @(
+        "exec", "-T", "postgres",
+        "dropdb", "-U", $PostgresUser, "--if-exists", "--force", $PostgresDatabase
+    )
+    Invoke-Compose @(
+        "exec", "-T", "postgres",
+        "createdb", "-U", $PostgresUser, $PostgresDatabase
+    )
+    Invoke-Compose @(
+        "exec", "-T", "postgres",
+        "pg_restore", "-U", $PostgresUser, "-d", $PostgresDatabase,
+        "--no-owner", "--exit-on-error", $postgresTemp
+    )
+
+    & docker run --rm `
+        --volumes-from $minioContainer `
+        -v "${target}:/backup:ro" `
+        $BackupHelperImage `
+        sh -c "find /data -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + && tar -C /data -xzf /backup/minio-data.tar.gz"
+    if ($LASTEXITCODE -ne 0) {
+        throw "MinIO 数据恢复失败"
+    }
+    Invoke-Compose @("up", "-d", "--wait", "minio")
+}
+finally {
+    & docker compose -f $ComposeFile exec -T postgres rm -f $postgresTemp 2>$null
+}
+
+if (-not $SkipApplicationServices) {
+    Invoke-Compose @("start", "backend", "worker", "beat")
+}
+
+Write-Output "恢复完成。请立即检查 /api/system/health/ready 并执行核心业务冒烟测试。"
--- a/infra/scripts/restore.sh
+++ b/infra/scripts/restore.sh
@@ -0,0 +1,47 @@
+#!/bin/sh
+set -eu
+
+if [ "$#" -ne 1 ] || [ "${RESTORE_CONFIRM:-}" != "restore-ai-ticket-platform" ]; then
+  echo "用法：RESTORE_CONFIRM=restore-ai-ticket-platform $0 <backup-directory>" >&2
+  echo "恢复会替换当前 PostgreSQL 数据库和 MinIO 数据。" >&2
+  exit 2
+fi
+
+SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+INFRA_DIR=$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd)
+COMPOSE_FILE=${COMPOSE_FILE:-"$INFRA_DIR/docker-compose.prod.yml"}
+TARGET=$(CDPATH= cd -- "$1" && pwd)
+
+"$SCRIPT_DIR/verify-backup.sh" "$TARGET"
+
+POSTGRES_DB=${POSTGRES_DB:-ai_ticket_platform}
+POSTGRES_USER=${POSTGRES_USER:-ai_ticket}
+BACKUP_HELPER_IMAGE=${BACKUP_HELPER_IMAGE:-alpine:3.21}
+
+if [ "${RESTORE_SKIP_APPLICATION_SERVICES:-false}" != "true" ]; then
+  docker compose -f "$COMPOSE_FILE" stop backend worker beat
+fi
+MINIO_CONTAINER=$(docker compose -f "$COMPOSE_FILE" ps -q minio)
+if [ -z "$MINIO_CONTAINER" ]; then
+  echo "未找到 MinIO 容器" >&2
+  exit 3
+fi
+docker compose -f "$COMPOSE_FILE" stop minio
+docker compose -f "$COMPOSE_FILE" exec -T postgres \
+  dropdb -U "$POSTGRES_USER" --force "$POSTGRES_DB"
+docker compose -f "$COMPOSE_FILE" exec -T postgres \
+  createdb -U "$POSTGRES_USER" "$POSTGRES_DB"
+docker compose -f "$COMPOSE_FILE" exec -T postgres \
+  pg_restore -U "$POSTGRES_USER" -d "$POSTGRES_DB" --no-owner --exit-on-error \
+  < "$TARGET/postgres.dump"
+docker run --rm \
+  --volumes-from "$MINIO_CONTAINER" \
+  -v "$TARGET:/backup:ro" \
+  "$BACKUP_HELPER_IMAGE" \
+  sh -c 'find /data -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + && tar -C /data -xzf /backup/minio-data.tar.gz'
+docker compose -f "$COMPOSE_FILE" up -d --wait minio
+if [ "${RESTORE_SKIP_APPLICATION_SERVICES:-false}" != "true" ]; then
+  docker compose -f "$COMPOSE_FILE" start backend worker beat
+fi
+
+echo "恢复完成。请立即检查 /api/system/health/ready 并执行核心业务冒烟测试。"
--- a/infra/scripts/verify-backup.ps1
+++ b/infra/scripts/verify-backup.ps1
@@ -0,0 +1,38 @@
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory = $true)]
+    [string]$BackupDirectory
+)
+
+$ErrorActionPreference = "Stop"
+$target = (Resolve-Path -LiteralPath $BackupDirectory).Path
+$requiredFiles = @(
+    "SHA256SUMS",
+    "postgres.dump",
+    "postgres.contents",
+    "minio-data.tar.gz",
+    "manifest.txt"
+)
+foreach ($file in $requiredFiles) {
+    if (-not (Test-Path -LiteralPath (Join-Path $target $file) -PathType Leaf)) {
+        throw "备份缺少文件：$file"
+    }
+}
+
+foreach ($line in Get-Content -LiteralPath (Join-Path $target "SHA256SUMS")) {
+    if ($line -notmatch "^([0-9a-fA-F]{64})\s{2}(.+)$") {
+        throw "SHA256SUMS 格式无效：$line"
+    }
+    $expected = $Matches[1].ToLowerInvariant()
+    $file = $Matches[2]
+    $path = Join-Path $target $file
+    if (-not (Test-Path -LiteralPath $path -PathType Leaf)) {
+        throw "校验文件不存在：$file"
+    }
+    $actual = (Get-FileHash -Algorithm SHA256 -LiteralPath $path).Hash.ToLowerInvariant()
+    if ($actual -ne $expected) {
+        throw "备份校验失败：$file"
+    }
+}
+
+Write-Output "备份校验通过：$target"
--- a/infra/scripts/verify-backup.sh
+++ b/infra/scripts/verify-backup.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+set -eu
+
+if [ "$#" -ne 1 ]; then
+  echo "用法：$0 <backup-directory>" >&2
+  exit 2
+fi
+
+TARGET=$(CDPATH= cd -- "$1" && pwd)
+for file in SHA256SUMS postgres.dump postgres.contents minio-data.tar.gz manifest.txt; do
+  if [ ! -f "$TARGET/$file" ]; then
+    echo "备份缺少文件：$file" >&2
+    exit 3
+  fi
+done
+
+(
+  cd "$TARGET"
+  sha256sum -c SHA256SUMS
+)
+
+echo "备份校验通过：$TARGET"
--- a/infra/secrets/alert_webhook_url.example
+++ b/infra/secrets/alert_webhook_url.example
@@ -0,0 +1 @@
+http://host.docker.internal:9099/alerts
--- a/infra/secrets/metrics_token.example
+++ b/infra/secrets/metrics_token.example
@@ -0,0 +1 @@
+replace-with-the-same-METRICS_TOKEN-used-by-backend
				`@@ -0,0 +1 @@`
				`replace-with-the-same-METRICS_TOKEN-used-by-backend`