ops: add production observability and recovery

2026-07-27 19:06:34 +08:00
parent e69368a2f3
commit b02d18d3b9
33 changed files with 8832 additions and 1 deletions
--- a/infra/scripts/backup.ps1
+++ b/infra/scripts/backup.ps1
@@ -0,0 +1,142 @@
+[CmdletBinding()]
+param(
+    [string]$ComposeFile = "",
+    [string]$BackupRoot = "",
+    [string]$PostgresDatabase = $env:POSTGRES_DB,
+    [string]$PostgresUser = $env:POSTGRES_USER,
+    [string]$BackupHelperImage = "alpine:3.21",
+    [switch]$SkipApplicationServices
+)
+
+$ErrorActionPreference = "Stop"
+$scriptDirectory = Split-Path -Parent $MyInvocation.MyCommand.Path
+$infraDirectory = [IO.Path]::GetFullPath((Join-Path $scriptDirectory ".."))
+$projectDirectory = [IO.Path]::GetFullPath((Join-Path $infraDirectory ".."))
+
+if (-not $ComposeFile) {
+    $ComposeFile = Join-Path $infraDirectory "docker-compose.prod.yml"
+}
+$ComposeFile = (Resolve-Path -LiteralPath $ComposeFile).Path
+if (-not $BackupRoot) {
+    $BackupRoot = Join-Path $projectDirectory "backups"
+}
+$BackupRoot = [IO.Path]::GetFullPath($BackupRoot)
+$projectPrefix = $projectDirectory.TrimEnd("\", "/") + [IO.Path]::DirectorySeparatorChar
+if (-not $BackupRoot.StartsWith($projectPrefix, [StringComparison]::OrdinalIgnoreCase)) {
+    throw "拒绝写入项目目录之外的备份路径：$BackupRoot"
+}
+
+if (-not $PostgresDatabase) { $PostgresDatabase = "ai_ticket_platform" }
+if (-not $PostgresUser) { $PostgresUser = "ai_ticket" }
+
+$stamp = (Get-Date).ToUniversalTime().ToString("yyyyMMddTHHmmssZ")
+$target = [IO.Path]::GetFullPath((Join-Path $BackupRoot $stamp))
+if (-not $target.StartsWith($projectPrefix, [StringComparison]::OrdinalIgnoreCase)) {
+    throw "拒绝写入项目目录之外的备份路径：$target"
+}
+New-Item -ItemType Directory -Path $target -Force | Out-Null
+
+$postgresTemp = "/tmp/ai-ticket-postgres-$PID.dump"
+
+function Invoke-Compose {
+    param([Parameter(Mandatory = $true)][string[]]$Arguments)
+    & docker compose -f $ComposeFile @Arguments
+    if ($LASTEXITCODE -ne 0) {
+        throw "docker compose 执行失败：$($Arguments -join ' ')"
+    }
+}
+
+$runningServices = & docker compose -f $ComposeFile ps --services --filter status=running
+if ($LASTEXITCODE -ne 0) {
+    throw "无法读取 Compose 服务状态"
+}
+$minioWasRunning = $runningServices -contains "minio"
+$minioContainer = & docker compose -f $ComposeFile ps -q minio
+if ($LASTEXITCODE -ne 0 -or -not "$minioContainer".Trim()) {
+    throw "未找到 MinIO 容器"
+}
+$minioContainer = "$minioContainer".Trim()
+
+$runningApplicationServices = @()
+if (-not $SkipApplicationServices) {
+    $runningApplicationServices = @(
+        "backend", "worker", "beat" |
+            Where-Object { $runningServices -contains $_ }
+    )
+    if ($runningApplicationServices.Count -gt 0) {
+        Invoke-Compose (@("stop") + $runningApplicationServices)
+    }
+}
+if ($minioWasRunning) {
+    Invoke-Compose @("stop", "minio")
+}
+
+try {
+    Invoke-Compose @(
+        "exec", "-T", "postgres",
+        "pg_dump", "-U", $PostgresUser, "-d", $PostgresDatabase,
+        "--format=custom", "--file=$postgresTemp"
+    )
+    Invoke-Compose @("cp", "postgres:$postgresTemp", (Join-Path $target "postgres.dump"))
+
+    $contents = & docker compose -f $ComposeFile exec -T postgres `
+        pg_restore --list $postgresTemp
+    if ($LASTEXITCODE -ne 0) {
+        throw "PostgreSQL 备份结构校验失败"
+    }
+    [IO.File]::WriteAllLines(
+        (Join-Path $target "postgres.contents"),
+        [string[]]$contents,
+        [Text.UTF8Encoding]::new($false)
+    )
+
+    & docker run --rm `
+        --volumes-from $minioContainer `
+        -v "${target}:/backup" `
+        $BackupHelperImage `
+        tar -C /data -czf /backup/minio-data.tar.gz .
+    if ($LASTEXITCODE -ne 0) {
+        throw "MinIO 数据归档失败"
+    }
+
+    $checksumLines = @("postgres.dump", "postgres.contents", "minio-data.tar.gz") | ForEach-Object {
+        $hash = (Get-FileHash -Algorithm SHA256 -LiteralPath (Join-Path $target $_)).Hash.ToLowerInvariant()
+        "$hash  $_"
+    }
+    [IO.File]::WriteAllLines(
+        (Join-Path $target "SHA256SUMS"),
+        [string[]]$checksumLines,
+        [Text.UTF8Encoding]::new($false)
+    )
+    $manifestLines = @(
+        "created_at=$stamp",
+        "postgres_database=$PostgresDatabase",
+        "postgres_user=$PostgresUser",
+        "compose_file=$ComposeFile",
+        "compose_project=$($env:COMPOSE_PROJECT_NAME)"
+    )
+    [IO.File]::WriteAllLines(
+        (Join-Path $target "manifest.txt"),
+        [string[]]$manifestLines,
+        [Text.UTF8Encoding]::new($false)
+    )
+}
+finally {
+    & docker compose -f $ComposeFile exec -T postgres rm -f $postgresTemp 2>$null
+    $minioReady = $true
+    if ($minioWasRunning) {
+        & docker compose -f $ComposeFile up -d --wait minio
+        if ($LASTEXITCODE -ne 0) {
+            $minioReady = $false
+            Write-Warning "备份结束后 MinIO 未恢复健康，应用服务保持停止"
+        }
+    }
+    if ($minioReady -and $runningApplicationServices.Count -gt 0) {
+        & docker compose -f $ComposeFile start @runningApplicationServices
+        if ($LASTEXITCODE -ne 0) {
+            Write-Warning "备份结束后无法恢复原应用服务：$($runningApplicationServices -join ', ')"
+        }
+    }
+}
+
+Write-Output "备份已创建并完成结构校验：$target"