ops: add production observability and recovery

2026-07-27 19:06:34 +08:00
parent e69368a2f3
commit b02d18d3b9
33 changed files with 8832 additions and 1 deletions
--- a/infra/scripts/backup.ps1
+++ b/infra/scripts/backup.ps1
@@ -0,0 +1,142 @@
+[CmdletBinding()]
+param(
+    [string]$ComposeFile = "",
+    [string]$BackupRoot = "",
+    [string]$PostgresDatabase = $env:POSTGRES_DB,
+    [string]$PostgresUser = $env:POSTGRES_USER,
+    [string]$BackupHelperImage = "alpine:3.21",
+    [switch]$SkipApplicationServices
+)
+
+$ErrorActionPreference = "Stop"
+$scriptDirectory = Split-Path -Parent $MyInvocation.MyCommand.Path
+$infraDirectory = [IO.Path]::GetFullPath((Join-Path $scriptDirectory ".."))
+$projectDirectory = [IO.Path]::GetFullPath((Join-Path $infraDirectory ".."))
+
+if (-not $ComposeFile) {
+    $ComposeFile = Join-Path $infraDirectory "docker-compose.prod.yml"
+}
+$ComposeFile = (Resolve-Path -LiteralPath $ComposeFile).Path
+if (-not $BackupRoot) {
+    $BackupRoot = Join-Path $projectDirectory "backups"
+}
+$BackupRoot = [IO.Path]::GetFullPath($BackupRoot)
+$projectPrefix = $projectDirectory.TrimEnd("\", "/") + [IO.Path]::DirectorySeparatorChar
+if (-not $BackupRoot.StartsWith($projectPrefix, [StringComparison]::OrdinalIgnoreCase)) {
+    throw "拒绝写入项目目录之外的备份路径：$BackupRoot"
+}
+
+if (-not $PostgresDatabase) { $PostgresDatabase = "ai_ticket_platform" }
+if (-not $PostgresUser) { $PostgresUser = "ai_ticket" }
+
+$stamp = (Get-Date).ToUniversalTime().ToString("yyyyMMddTHHmmssZ")
+$target = [IO.Path]::GetFullPath((Join-Path $BackupRoot $stamp))
+if (-not $target.StartsWith($projectPrefix, [StringComparison]::OrdinalIgnoreCase)) {
+    throw "拒绝写入项目目录之外的备份路径：$target"
+}
+New-Item -ItemType Directory -Path $target -Force | Out-Null
+
+$postgresTemp = "/tmp/ai-ticket-postgres-$PID.dump"
+
+function Invoke-Compose {
+    param([Parameter(Mandatory = $true)][string[]]$Arguments)
+    & docker compose -f $ComposeFile @Arguments
+    if ($LASTEXITCODE -ne 0) {
+        throw "docker compose 执行失败：$($Arguments -join ' ')"
+    }
+}
+
+$runningServices = & docker compose -f $ComposeFile ps --services --filter status=running
+if ($LASTEXITCODE -ne 0) {
+    throw "无法读取 Compose 服务状态"
+}
+$minioWasRunning = $runningServices -contains "minio"
+$minioContainer = & docker compose -f $ComposeFile ps -q minio
+if ($LASTEXITCODE -ne 0 -or -not "$minioContainer".Trim()) {
+    throw "未找到 MinIO 容器"
+}
+$minioContainer = "$minioContainer".Trim()
+
+$runningApplicationServices = @()
+if (-not $SkipApplicationServices) {
+    $runningApplicationServices = @(
+        "backend", "worker", "beat" |
+            Where-Object { $runningServices -contains $_ }
+    )
+    if ($runningApplicationServices.Count -gt 0) {
+        Invoke-Compose (@("stop") + $runningApplicationServices)
+    }
+}
+if ($minioWasRunning) {
+    Invoke-Compose @("stop", "minio")
+}
+
+try {
+    Invoke-Compose @(
+        "exec", "-T", "postgres",
+        "pg_dump", "-U", $PostgresUser, "-d", $PostgresDatabase,
+        "--format=custom", "--file=$postgresTemp"
+    )
+    Invoke-Compose @("cp", "postgres:$postgresTemp", (Join-Path $target "postgres.dump"))
+
+    $contents = & docker compose -f $ComposeFile exec -T postgres `
+        pg_restore --list $postgresTemp
+    if ($LASTEXITCODE -ne 0) {
+        throw "PostgreSQL 备份结构校验失败"
+    }
+    [IO.File]::WriteAllLines(
+        (Join-Path $target "postgres.contents"),
+        [string[]]$contents,
+        [Text.UTF8Encoding]::new($false)
+    )
+
+    & docker run --rm `
+        --volumes-from $minioContainer `
+        -v "${target}:/backup" `
+        $BackupHelperImage `
+        tar -C /data -czf /backup/minio-data.tar.gz .
+    if ($LASTEXITCODE -ne 0) {
+        throw "MinIO 数据归档失败"
+    }
+
+    $checksumLines = @("postgres.dump", "postgres.contents", "minio-data.tar.gz") | ForEach-Object {
+        $hash = (Get-FileHash -Algorithm SHA256 -LiteralPath (Join-Path $target $_)).Hash.ToLowerInvariant()
+        "$hash  $_"
+    }
+    [IO.File]::WriteAllLines(
+        (Join-Path $target "SHA256SUMS"),
+        [string[]]$checksumLines,
+        [Text.UTF8Encoding]::new($false)
+    )
+    $manifestLines = @(
+        "created_at=$stamp",
+        "postgres_database=$PostgresDatabase",
+        "postgres_user=$PostgresUser",
+        "compose_file=$ComposeFile",
+        "compose_project=$($env:COMPOSE_PROJECT_NAME)"
+    )
+    [IO.File]::WriteAllLines(
+        (Join-Path $target "manifest.txt"),
+        [string[]]$manifestLines,
+        [Text.UTF8Encoding]::new($false)
+    )
+}
+finally {
+    & docker compose -f $ComposeFile exec -T postgres rm -f $postgresTemp 2>$null
+    $minioReady = $true
+    if ($minioWasRunning) {
+        & docker compose -f $ComposeFile up -d --wait minio
+        if ($LASTEXITCODE -ne 0) {
+            $minioReady = $false
+            Write-Warning "备份结束后 MinIO 未恢复健康，应用服务保持停止"
+        }
+    }
+    if ($minioReady -and $runningApplicationServices.Count -gt 0) {
+        & docker compose -f $ComposeFile start @runningApplicationServices
+        if ($LASTEXITCODE -ne 0) {
+            Write-Warning "备份结束后无法恢复原应用服务：$($runningApplicationServices -join ', ')"
+        }
+    }
+}
+
+Write-Output "备份已创建并完成结构校验：$target"
--- a/infra/scripts/backup.sh
+++ b/infra/scripts/backup.sh
@@ -0,0 +1,86 @@
+#!/bin/sh
+set -eu
+
+SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+INFRA_DIR=$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd)
+PROJECT_DIR=$(CDPATH= cd -- "$INFRA_DIR/.." && pwd)
+COMPOSE_FILE=${COMPOSE_FILE:-"$INFRA_DIR/docker-compose.prod.yml"}
+BACKUP_ROOT=${BACKUP_ROOT:-"$PROJECT_DIR/backups"}
+STAMP=$(date -u +%Y%m%dT%H%M%SZ)
+TARGET="$BACKUP_ROOT/$STAMP"
+
+mkdir -p "$TARGET"
+TARGET=$(CDPATH= cd -- "$TARGET" && pwd)
+case "$TARGET" in
+  "$PROJECT_DIR"/*) ;;
+  *) echo "拒绝写入项目目录之外的备份路径：$TARGET" >&2; exit 2 ;;
+esac
+
+POSTGRES_DB=${POSTGRES_DB:-ai_ticket_platform}
+POSTGRES_USER=${POSTGRES_USER:-ai_ticket}
+BACKUP_HELPER_IMAGE=${BACKUP_HELPER_IMAGE:-alpine:3.21}
+RUNNING_APPLICATION_SERVICES=""
+MINIO_WAS_RUNNING=false
+
+cleanup() {
+  MINIO_READY=true
+  if [ "$MINIO_WAS_RUNNING" = "true" ]; then
+    if ! docker compose -f "$COMPOSE_FILE" up -d --wait minio >/dev/null; then
+      echo "备份结束后 MinIO 未恢复健康，应用服务保持停止" >&2
+      MINIO_READY=false
+    fi
+  fi
+  if [ "$MINIO_READY" = "true" ] && [ -n "$RUNNING_APPLICATION_SERVICES" ]; then
+    docker compose -f "$COMPOSE_FILE" start $RUNNING_APPLICATION_SERVICES >/dev/null || true
+  fi
+}
+trap cleanup EXIT INT TERM
+
+RUNNING_SERVICES=$(docker compose -f "$COMPOSE_FILE" ps --services --filter status=running)
+if printf '%s\n' "$RUNNING_SERVICES" | grep -qx "minio"; then
+  MINIO_WAS_RUNNING=true
+fi
+if [ "${BACKUP_SKIP_APPLICATION_SERVICES:-false}" != "true" ]; then
+  for service in backend worker beat; do
+    if printf '%s\n' "$RUNNING_SERVICES" | grep -qx "$service"; then
+      RUNNING_APPLICATION_SERVICES="$RUNNING_APPLICATION_SERVICES $service"
+    fi
+  done
+  if [ -n "$RUNNING_APPLICATION_SERVICES" ]; then
+    docker compose -f "$COMPOSE_FILE" stop $RUNNING_APPLICATION_SERVICES
+  fi
+fi
+if [ "$MINIO_WAS_RUNNING" = "true" ]; then
+  docker compose -f "$COMPOSE_FILE" stop minio
+fi
+
+docker compose -f "$COMPOSE_FILE" exec -T postgres \
+  pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom > "$TARGET/postgres.dump"
+docker compose -f "$COMPOSE_FILE" exec -T postgres \
+  pg_restore --list < "$TARGET/postgres.dump" > "$TARGET/postgres.contents"
+MINIO_CONTAINER=$(docker compose -f "$COMPOSE_FILE" ps -q minio)
+if [ -z "$MINIO_CONTAINER" ]; then
+  echo "未找到 MinIO 容器" >&2
+  exit 3
+fi
+docker run --rm \
+  --volumes-from "$MINIO_CONTAINER" \
+  -v "$TARGET:/backup" \
+  "$BACKUP_HELPER_IMAGE" \
+  tar -C /data -czf /backup/minio-data.tar.gz .
+
+(
+  cd "$TARGET"
+  sha256sum postgres.dump postgres.contents minio-data.tar.gz > SHA256SUMS
+)
+
+cat > "$TARGET/manifest.txt" <<EOF
+created_at=$STAMP
+postgres_database=$POSTGRES_DB
+postgres_user=$POSTGRES_USER
+compose_file=$COMPOSE_FILE
+EOF
+
+cleanup
+trap - EXIT INT TERM
+echo "备份已创建并完成结构校验：$TARGET"
--- a/infra/scripts/drill-restore.ps1
+++ b/infra/scripts/drill-restore.ps1
@@ -0,0 +1,194 @@
+[CmdletBinding()]
+param(
+    [string]$ComposeFile = "",
+    [switch]$SkipBuild
+)
+
+$ErrorActionPreference = "Stop"
+$scriptDirectory = Split-Path -Parent $MyInvocation.MyCommand.Path
+$infraDirectory = [IO.Path]::GetFullPath((Join-Path $scriptDirectory ".."))
+$projectDirectory = [IO.Path]::GetFullPath((Join-Path $infraDirectory ".."))
+if (-not $ComposeFile) {
+    $ComposeFile = Join-Path $infraDirectory "docker-compose.prod.yml"
+}
+$ComposeFile = (Resolve-Path -LiteralPath $ComposeFile).Path
+
+$projectName = "ai-ticket-restore-drill-$PID"
+$databasePassword = "restore-drill-postgres-$PID"
+$minioUser = "restore-drill-minio"
+$minioPassword = "restore-drill-minio-$PID"
+$backupRoot = Join-Path $projectDirectory "backups\$projectName"
+$stopwatch = [Diagnostics.Stopwatch]::StartNew()
+
+$savedEnvironment = @{
+    COMPOSE_PROJECT_NAME = $env:COMPOSE_PROJECT_NAME
+    POSTGRES_DB = $env:POSTGRES_DB
+    POSTGRES_USER = $env:POSTGRES_USER
+    POSTGRES_PASSWORD = $env:POSTGRES_PASSWORD
+    MINIO_ROOT_USER = $env:MINIO_ROOT_USER
+    MINIO_ROOT_PASSWORD = $env:MINIO_ROOT_PASSWORD
+}
+
+$env:COMPOSE_PROJECT_NAME = $projectName
+$env:POSTGRES_DB = "ai_ticket_platform"
+$env:POSTGRES_USER = "ai_ticket"
+$env:POSTGRES_PASSWORD = $databasePassword
+$env:MINIO_ROOT_USER = $minioUser
+$env:MINIO_ROOT_PASSWORD = $minioPassword
+
+function Invoke-Docker {
+    param([Parameter(Mandatory = $true)][string[]]$Arguments)
+    & docker @Arguments
+    if ($LASTEXITCODE -ne 0) {
+        throw "Docker 执行失败：docker $($Arguments -join ' ')"
+    }
+}
+
+function Invoke-DjangoCode {
+    param(
+        [Parameter(Mandatory = $true)][string[]]$RunPrefix,
+        [Parameter(Mandatory = $true)][string]$Code,
+        [Parameter(Mandatory = $true)][string]$FailureMessage
+    )
+    $bytes = [Text.Encoding]::UTF8.GetBytes($Code)
+    $runner = "exec(bytes([" + ($bytes -join ",") + "]))"
+    & docker @RunPrefix python manage.py shell -c $runner
+    if ($LASTEXITCODE -ne 0) {
+        throw $FailureMessage
+    }
+}
+
+$backendEnvironment = @(
+    "-e", "POSTGRES_HOST=postgres",
+    "-e", "POSTGRES_DB=ai_ticket_platform",
+    "-e", "POSTGRES_USER=ai_ticket",
+    "-e", "POSTGRES_PASSWORD=$databasePassword",
+    "-e", "MINIO_ENDPOINT=minio:9000",
+    "-e", "MINIO_ROOT_USER=$minioUser",
+    "-e", "MINIO_ROOT_PASSWORD=$minioPassword",
+    "-e", "CELERY_BROKER_URL=redis://redis:6379/0",
+    "-e", "CELERY_RESULT_BACKEND=redis://redis:6379/1"
+)
+
+$seedCode = @'
+from django.contrib.auth import get_user_model
+from apps.tickets.models import Ticket, TicketAttachment
+
+user, _ = get_user_model().objects.get_or_create(
+    username="restore_drill",
+    defaults={"email": "restore-drill@example.invalid"},
+)
+ticket, _ = Ticket.objects.update_or_create(
+    ticket_no="RESTORE-DRILL-001",
+    defaults={
+        "title": "隔离恢复演练工单",
+        "description": "restore-drill-marker",
+        "requester": user,
+        "created_by": user,
+    },
+)
+TicketAttachment.objects.update_or_create(
+    ticket=ticket,
+    storage_path="restore-drill/marker.txt",
+    defaults={
+        "file_name": "marker.txt",
+        "mime_type": "text/plain",
+        "file_size": 20,
+        "created_by": user,
+    },
+)
+print(f"seeded_ticket={ticket.ticket_no}")
+'@
+
+$deleteCode = @'
+from apps.tickets.models import Ticket
+Ticket.objects.filter(ticket_no="RESTORE-DRILL-001").delete()
+print("seed_deleted=true")
+'@
+
+$verifyCode = @'
+from apps.tickets.models import Ticket
+ticket = Ticket.objects.get(ticket_no="RESTORE-DRILL-001")
+attachment = ticket.attachments.get(storage_path="restore-drill/marker.txt")
+assert ticket.description == "restore-drill-marker"
+assert attachment.file_name == "marker.txt"
+print(f"restored_ticket={ticket.ticket_no}")
+print(f"restored_attachment={attachment.storage_path}")
+'@
+
+try {
+    Invoke-Docker @("compose", "-f", $ComposeFile, "up", "-d", "--wait", "postgres", "minio")
+    if (-not $SkipBuild) {
+        Invoke-Docker @("compose", "-f", $ComposeFile, "build", "backend")
+    }
+
+    $runPrefix = @(
+        "compose", "-f", $ComposeFile,
+        "run", "--rm", "-T", "--no-deps"
+    ) + $backendEnvironment + @("backend")
+
+    Invoke-Docker ($runPrefix + @("python", "manage.py", "migrate", "--noinput"))
+    Invoke-DjangoCode -RunPrefix $runPrefix -Code $seedCode -FailureMessage "演练数据创建失败"
+
+    Invoke-Docker @(
+        "compose", "-f", $ComposeFile, "exec", "-T", "minio",
+        "sh", "-c",
+        'mc alias set drill http://127.0.0.1:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null && mc mb --ignore-existing drill/ai-ticket-platform >/dev/null && printf restore-drill-marker | mc pipe drill/ai-ticket-platform/restore-drill/marker.txt >/dev/null'
+    )
+
+    & (Join-Path $scriptDirectory "backup.ps1") `
+        -ComposeFile $ComposeFile `
+        -BackupRoot $backupRoot `
+        -PostgresDatabase "ai_ticket_platform" `
+        -PostgresUser "ai_ticket" `
+        -SkipApplicationServices
+    $backupDirectory = Get-ChildItem -LiteralPath $backupRoot -Directory |
+        Sort-Object Name -Descending |
+        Select-Object -First 1
+    if (-not $backupDirectory) { throw "未找到演练备份目录" }
+
+    Invoke-DjangoCode -RunPrefix $runPrefix -Code $deleteCode -FailureMessage "演练数据破坏步骤失败"
+    Invoke-Docker @(
+        "compose", "-f", $ComposeFile, "exec", "-T", "minio",
+        "mc", "rm", "--force", "drill/ai-ticket-platform/restore-drill/marker.txt"
+    )
+
+    & (Join-Path $scriptDirectory "restore.ps1") `
+        -BackupDirectory $backupDirectory.FullName `
+        -ComposeFile $ComposeFile `
+        -PostgresDatabase "ai_ticket_platform" `
+        -PostgresUser "ai_ticket" `
+        -Confirm "restore-ai-ticket-platform" `
+        -SkipApplicationServices
+
+    Invoke-DjangoCode -RunPrefix $runPrefix -Code $verifyCode -FailureMessage "数据库业务冒烟失败"
+    $objectMarker = & docker compose -f $ComposeFile exec -T minio `
+        mc cat drill/ai-ticket-platform/restore-drill/marker.txt
+    if ($LASTEXITCODE -ne 0 -or "$objectMarker".Trim() -ne "restore-drill-marker") {
+        throw "对象存储业务冒烟失败"
+    }
+    Invoke-Docker ($runPrefix + @("python", "manage.py", "check"))
+
+    $stopwatch.Stop()
+    [pscustomobject]@{
+        status = "passed"
+        compose_project = $projectName
+        backup_id = $backupDirectory.Name
+        duration_seconds = [Math]::Round($stopwatch.Elapsed.TotalSeconds, 2)
+        database_smoke = "ticket_and_attachment_restored"
+        object_storage_smoke = "marker_restored"
+        django_check = "passed"
+        backup_directory = $backupDirectory.FullName
+    } | ConvertTo-Json
+}
+finally {
+    & docker compose -f $ComposeFile down -v --remove-orphans --rmi local
+    foreach ($entry in $savedEnvironment.GetEnumerator()) {
+        if ($null -eq $entry.Value) {
+            Remove-Item "Env:$($entry.Key)" -ErrorAction SilentlyContinue
+        }
+        else {
+            Set-Item "Env:$($entry.Key)" $entry.Value
+        }
+    }
+}
--- a/infra/scripts/prune-backups.sh
+++ b/infra/scripts/prune-backups.sh
@@ -0,0 +1,38 @@
+#!/bin/sh
+set -eu
+
+SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+INFRA_DIR=$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd)
+PROJECT_DIR=$(CDPATH= cd -- "$INFRA_DIR/.." && pwd)
+BACKUP_ROOT=${BACKUP_ROOT:-"$PROJECT_DIR/backups"}
+RETENTION_DAYS=${BACKUP_RETENTION_DAYS:-30}
+
+case "$RETENTION_DAYS" in
+  *[!0-9]*|"") echo "BACKUP_RETENTION_DAYS 必须是正整数" >&2; exit 2 ;;
+esac
+if [ "$RETENTION_DAYS" -lt 1 ]; then
+  echo "BACKUP_RETENTION_DAYS 必须至少为 1" >&2
+  exit 2
+fi
+
+mkdir -p "$BACKUP_ROOT"
+BACKUP_ROOT=$(CDPATH= cd -- "$BACKUP_ROOT" && pwd)
+case "$BACKUP_ROOT" in
+  "$PROJECT_DIR/backups"|"$PROJECT_DIR/backups/"*) ;;
+  *) echo "拒绝清理项目 backups 目录之外的路径：$BACKUP_ROOT" >&2; exit 2 ;;
+esac
+
+find "$BACKUP_ROOT" -mindepth 1 -maxdepth 1 -type d -mtime "+$RETENTION_DAYS" -print |
+while IFS= read -r candidate; do
+  name=$(basename "$candidate")
+  if ! printf '%s\n' "$name" | grep -Eq '^20[0-9]{6}T[0-9]{6}Z$'; then
+    echo "跳过非标准备份目录：$candidate"
+    continue
+  fi
+  if [ ! -f "$candidate/manifest.txt" ] || [ ! -f "$candidate/SHA256SUMS" ]; then
+    echo "跳过缺少清单的目录：$candidate"
+    continue
+  fi
+  rm -rf -- "$candidate"
+  echo "已按保留策略删除：$candidate"
+done
--- a/infra/scripts/restore.ps1
+++ b/infra/scripts/restore.ps1
@@ -0,0 +1,84 @@
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory = $true)]
+    [string]$BackupDirectory,
+    [string]$ComposeFile = "",
+    [string]$PostgresDatabase = $env:POSTGRES_DB,
+    [string]$PostgresUser = $env:POSTGRES_USER,
+    [string]$Confirm = $env:RESTORE_CONFIRM,
+    [string]$BackupHelperImage = "alpine:3.21",
+    [switch]$SkipApplicationServices
+)
+
+$ErrorActionPreference = "Stop"
+if ($Confirm -ne "restore-ai-ticket-platform") {
+    throw "恢复会替换当前 PostgreSQL 与 MinIO 数据；请传入 -Confirm restore-ai-ticket-platform。"
+}
+
+$scriptDirectory = Split-Path -Parent $MyInvocation.MyCommand.Path
+$infraDirectory = [IO.Path]::GetFullPath((Join-Path $scriptDirectory ".."))
+if (-not $ComposeFile) {
+    $ComposeFile = Join-Path $infraDirectory "docker-compose.prod.yml"
+}
+$ComposeFile = (Resolve-Path -LiteralPath $ComposeFile).Path
+$target = (Resolve-Path -LiteralPath $BackupDirectory).Path
+if (-not $PostgresDatabase) { $PostgresDatabase = "ai_ticket_platform" }
+if (-not $PostgresUser) { $PostgresUser = "ai_ticket" }
+
+& (Join-Path $scriptDirectory "verify-backup.ps1") -BackupDirectory $target
+
+$postgresTemp = "/tmp/ai-ticket-restore-$PID.dump"
+$minioContainer = & docker compose -f $ComposeFile ps -q minio
+if ($LASTEXITCODE -ne 0 -or -not "$minioContainer".Trim()) {
+    throw "未找到 MinIO 容器"
+}
+$minioContainer = "$minioContainer".Trim()
+
+function Invoke-Compose {
+    param([Parameter(Mandatory = $true)][string[]]$Arguments)
+    & docker compose -f $ComposeFile @Arguments
+    if ($LASTEXITCODE -ne 0) {
+        throw "docker compose 执行失败：$($Arguments -join ' ')"
+    }
+}
+
+if (-not $SkipApplicationServices) {
+    Invoke-Compose @("stop", "backend", "worker", "beat")
+}
+Invoke-Compose @("stop", "minio")
+
+try {
+    Invoke-Compose @("cp", (Join-Path $target "postgres.dump"), "postgres:$postgresTemp")
+    Invoke-Compose @(
+        "exec", "-T", "postgres",
+        "dropdb", "-U", $PostgresUser, "--if-exists", "--force", $PostgresDatabase
+    )
+    Invoke-Compose @(
+        "exec", "-T", "postgres",
+        "createdb", "-U", $PostgresUser, $PostgresDatabase
+    )
+    Invoke-Compose @(
+        "exec", "-T", "postgres",
+        "pg_restore", "-U", $PostgresUser, "-d", $PostgresDatabase,
+        "--no-owner", "--exit-on-error", $postgresTemp
+    )
+
+    & docker run --rm `
+        --volumes-from $minioContainer `
+        -v "${target}:/backup:ro" `
+        $BackupHelperImage `
+        sh -c "find /data -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + && tar -C /data -xzf /backup/minio-data.tar.gz"
+    if ($LASTEXITCODE -ne 0) {
+        throw "MinIO 数据恢复失败"
+    }
+    Invoke-Compose @("up", "-d", "--wait", "minio")
+}
+finally {
+    & docker compose -f $ComposeFile exec -T postgres rm -f $postgresTemp 2>$null
+}
+
+if (-not $SkipApplicationServices) {
+    Invoke-Compose @("start", "backend", "worker", "beat")
+}
+
+Write-Output "恢复完成。请立即检查 /api/system/health/ready 并执行核心业务冒烟测试。"
--- a/infra/scripts/restore.sh
+++ b/infra/scripts/restore.sh
@@ -0,0 +1,47 @@
+#!/bin/sh
+set -eu
+
+if [ "$#" -ne 1 ] || [ "${RESTORE_CONFIRM:-}" != "restore-ai-ticket-platform" ]; then
+  echo "用法：RESTORE_CONFIRM=restore-ai-ticket-platform $0 <backup-directory>" >&2
+  echo "恢复会替换当前 PostgreSQL 数据库和 MinIO 数据。" >&2
+  exit 2
+fi
+
+SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+INFRA_DIR=$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd)
+COMPOSE_FILE=${COMPOSE_FILE:-"$INFRA_DIR/docker-compose.prod.yml"}
+TARGET=$(CDPATH= cd -- "$1" && pwd)
+
+"$SCRIPT_DIR/verify-backup.sh" "$TARGET"
+
+POSTGRES_DB=${POSTGRES_DB:-ai_ticket_platform}
+POSTGRES_USER=${POSTGRES_USER:-ai_ticket}
+BACKUP_HELPER_IMAGE=${BACKUP_HELPER_IMAGE:-alpine:3.21}
+
+if [ "${RESTORE_SKIP_APPLICATION_SERVICES:-false}" != "true" ]; then
+  docker compose -f "$COMPOSE_FILE" stop backend worker beat
+fi
+MINIO_CONTAINER=$(docker compose -f "$COMPOSE_FILE" ps -q minio)
+if [ -z "$MINIO_CONTAINER" ]; then
+  echo "未找到 MinIO 容器" >&2
+  exit 3
+fi
+docker compose -f "$COMPOSE_FILE" stop minio
+docker compose -f "$COMPOSE_FILE" exec -T postgres \
+  dropdb -U "$POSTGRES_USER" --force "$POSTGRES_DB"
+docker compose -f "$COMPOSE_FILE" exec -T postgres \
+  createdb -U "$POSTGRES_USER" "$POSTGRES_DB"
+docker compose -f "$COMPOSE_FILE" exec -T postgres \
+  pg_restore -U "$POSTGRES_USER" -d "$POSTGRES_DB" --no-owner --exit-on-error \
+  < "$TARGET/postgres.dump"
+docker run --rm \
+  --volumes-from "$MINIO_CONTAINER" \
+  -v "$TARGET:/backup:ro" \
+  "$BACKUP_HELPER_IMAGE" \
+  sh -c 'find /data -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + && tar -C /data -xzf /backup/minio-data.tar.gz'
+docker compose -f "$COMPOSE_FILE" up -d --wait minio
+if [ "${RESTORE_SKIP_APPLICATION_SERVICES:-false}" != "true" ]; then
+  docker compose -f "$COMPOSE_FILE" start backend worker beat
+fi
+
+echo "恢复完成。请立即检查 /api/system/health/ready 并执行核心业务冒烟测试。"
--- a/infra/scripts/verify-backup.ps1
+++ b/infra/scripts/verify-backup.ps1
@@ -0,0 +1,38 @@
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory = $true)]
+    [string]$BackupDirectory
+)
+
+$ErrorActionPreference = "Stop"
+$target = (Resolve-Path -LiteralPath $BackupDirectory).Path
+$requiredFiles = @(
+    "SHA256SUMS",
+    "postgres.dump",
+    "postgres.contents",
+    "minio-data.tar.gz",
+    "manifest.txt"
+)
+foreach ($file in $requiredFiles) {
+    if (-not (Test-Path -LiteralPath (Join-Path $target $file) -PathType Leaf)) {
+        throw "备份缺少文件：$file"
+    }
+}
+
+foreach ($line in Get-Content -LiteralPath (Join-Path $target "SHA256SUMS")) {
+    if ($line -notmatch "^([0-9a-fA-F]{64})\s{2}(.+)$") {
+        throw "SHA256SUMS 格式无效：$line"
+    }
+    $expected = $Matches[1].ToLowerInvariant()
+    $file = $Matches[2]
+    $path = Join-Path $target $file
+    if (-not (Test-Path -LiteralPath $path -PathType Leaf)) {
+        throw "校验文件不存在：$file"
+    }
+    $actual = (Get-FileHash -Algorithm SHA256 -LiteralPath $path).Hash.ToLowerInvariant()
+    if ($actual -ne $expected) {
+        throw "备份校验失败：$file"
+    }
+}
+
+Write-Output "备份校验通过：$target"
--- a/infra/scripts/verify-backup.sh
+++ b/infra/scripts/verify-backup.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+set -eu
+
+if [ "$#" -ne 1 ]; then
+  echo "用法：$0 <backup-directory>" >&2
+  exit 2
+fi
+
+TARGET=$(CDPATH= cd -- "$1" && pwd)
+for file in SHA256SUMS postgres.dump postgres.contents minio-data.tar.gz manifest.txt; do
+  if [ ! -f "$TARGET/$file" ]; then
+    echo "备份缺少文件：$file" >&2
+    exit 3
+  fi
+done
+
+(
+  cd "$TARGET"
+  sha256sum -c SHA256SUMS
+)
+
+echo "备份校验通过：$TARGET"