ops: add production observability and recovery

2026-07-27 19:06:34 +08:00
parent e69368a2f3
commit b02d18d3b9
33 changed files with 8832 additions and 1 deletions
--- a/infra/scripts/restore.ps1
+++ b/infra/scripts/restore.ps1
@@ -0,0 +1,84 @@
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory = $true)]
+    [string]$BackupDirectory,
+    [string]$ComposeFile = "",
+    [string]$PostgresDatabase = $env:POSTGRES_DB,
+    [string]$PostgresUser = $env:POSTGRES_USER,
+    [string]$Confirm = $env:RESTORE_CONFIRM,
+    [string]$BackupHelperImage = "alpine:3.21",
+    [switch]$SkipApplicationServices
+)
+
+$ErrorActionPreference = "Stop"
+if ($Confirm -ne "restore-ai-ticket-platform") {
+    throw "恢复会替换当前 PostgreSQL 与 MinIO 数据；请传入 -Confirm restore-ai-ticket-platform。"
+}
+
+$scriptDirectory = Split-Path -Parent $MyInvocation.MyCommand.Path
+$infraDirectory = [IO.Path]::GetFullPath((Join-Path $scriptDirectory ".."))
+if (-not $ComposeFile) {
+    $ComposeFile = Join-Path $infraDirectory "docker-compose.prod.yml"
+}
+$ComposeFile = (Resolve-Path -LiteralPath $ComposeFile).Path
+$target = (Resolve-Path -LiteralPath $BackupDirectory).Path
+if (-not $PostgresDatabase) { $PostgresDatabase = "ai_ticket_platform" }
+if (-not $PostgresUser) { $PostgresUser = "ai_ticket" }
+
+& (Join-Path $scriptDirectory "verify-backup.ps1") -BackupDirectory $target
+
+$postgresTemp = "/tmp/ai-ticket-restore-$PID.dump"
+$minioContainer = & docker compose -f $ComposeFile ps -q minio
+if ($LASTEXITCODE -ne 0 -or -not "$minioContainer".Trim()) {
+    throw "未找到 MinIO 容器"
+}
+$minioContainer = "$minioContainer".Trim()
+
+function Invoke-Compose {
+    param([Parameter(Mandatory = $true)][string[]]$Arguments)
+    & docker compose -f $ComposeFile @Arguments
+    if ($LASTEXITCODE -ne 0) {
+        throw "docker compose 执行失败：$($Arguments -join ' ')"
+    }
+}
+
+if (-not $SkipApplicationServices) {
+    Invoke-Compose @("stop", "backend", "worker", "beat")
+}
+Invoke-Compose @("stop", "minio")
+
+try {
+    Invoke-Compose @("cp", (Join-Path $target "postgres.dump"), "postgres:$postgresTemp")
+    Invoke-Compose @(
+        "exec", "-T", "postgres",
+        "dropdb", "-U", $PostgresUser, "--if-exists", "--force", $PostgresDatabase
+    )
+    Invoke-Compose @(
+        "exec", "-T", "postgres",
+        "createdb", "-U", $PostgresUser, $PostgresDatabase
+    )
+    Invoke-Compose @(
+        "exec", "-T", "postgres",
+        "pg_restore", "-U", $PostgresUser, "-d", $PostgresDatabase,
+        "--no-owner", "--exit-on-error", $postgresTemp
+    )
+
+    & docker run --rm `
+        --volumes-from $minioContainer `
+        -v "${target}:/backup:ro" `
+        $BackupHelperImage `
+        sh -c "find /data -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + && tar -C /data -xzf /backup/minio-data.tar.gz"
+    if ($LASTEXITCODE -ne 0) {
+        throw "MinIO 数据恢复失败"
+    }
+    Invoke-Compose @("up", "-d", "--wait", "minio")
+}
+finally {
+    & docker compose -f $ComposeFile exec -T postgres rm -f $postgresTemp 2>$null
+}
+
+if (-not $SkipApplicationServices) {
+    Invoke-Compose @("start", "backend", "worker", "beat")
+}
+
+Write-Output "恢复完成。请立即检查 /api/system/health/ready 并执行核心业务冒烟测试。"