ops: add production observability and recovery
This commit is contained in:
hailin
194
infra/scripts/drill-restore.ps1
Normal file
194
infra/scripts/drill-restore.ps1
Normal file
@@ -0,0 +1,194 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$ComposeFile = "",
|
||||
[switch]$SkipBuild
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$scriptDirectory = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$infraDirectory = [IO.Path]::GetFullPath((Join-Path $scriptDirectory ".."))
|
||||
$projectDirectory = [IO.Path]::GetFullPath((Join-Path $infraDirectory ".."))
|
||||
if (-not $ComposeFile) {
|
||||
$ComposeFile = Join-Path $infraDirectory "docker-compose.prod.yml"
|
||||
}
|
||||
$ComposeFile = (Resolve-Path -LiteralPath $ComposeFile).Path
|
||||
|
||||
$projectName = "ai-ticket-restore-drill-$PID"
|
||||
$databasePassword = "restore-drill-postgres-$PID"
|
||||
$minioUser = "restore-drill-minio"
|
||||
$minioPassword = "restore-drill-minio-$PID"
|
||||
$backupRoot = Join-Path $projectDirectory "backups\$projectName"
|
||||
$stopwatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
|
||||
$savedEnvironment = @{
|
||||
COMPOSE_PROJECT_NAME = $env:COMPOSE_PROJECT_NAME
|
||||
POSTGRES_DB = $env:POSTGRES_DB
|
||||
POSTGRES_USER = $env:POSTGRES_USER
|
||||
POSTGRES_PASSWORD = $env:POSTGRES_PASSWORD
|
||||
MINIO_ROOT_USER = $env:MINIO_ROOT_USER
|
||||
MINIO_ROOT_PASSWORD = $env:MINIO_ROOT_PASSWORD
|
||||
}
|
||||
|
||||
$env:COMPOSE_PROJECT_NAME = $projectName
|
||||
$env:POSTGRES_DB = "ai_ticket_platform"
|
||||
$env:POSTGRES_USER = "ai_ticket"
|
||||
$env:POSTGRES_PASSWORD = $databasePassword
|
||||
$env:MINIO_ROOT_USER = $minioUser
|
||||
$env:MINIO_ROOT_PASSWORD = $minioPassword
|
||||
|
||||
function Invoke-Docker {
|
||||
param([Parameter(Mandatory = $true)][string[]]$Arguments)
|
||||
& docker @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Docker 执行失败：docker $($Arguments -join ' ')"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-DjangoCode {
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string[]]$RunPrefix,
|
||||
[Parameter(Mandatory = $true)][string]$Code,
|
||||
[Parameter(Mandatory = $true)][string]$FailureMessage
|
||||
)
|
||||
$bytes = [Text.Encoding]::UTF8.GetBytes($Code)
|
||||
$runner = "exec(bytes([" + ($bytes -join ",") + "]))"
|
||||
& docker @RunPrefix python manage.py shell -c $runner
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw $FailureMessage
|
||||
}
|
||||
}
|
||||
|
||||
$backendEnvironment = @(
|
||||
"-e", "POSTGRES_HOST=postgres",
|
||||
"-e", "POSTGRES_DB=ai_ticket_platform",
|
||||
"-e", "POSTGRES_USER=ai_ticket",
|
||||
"-e", "POSTGRES_PASSWORD=$databasePassword",
|
||||
"-e", "MINIO_ENDPOINT=minio:9000",
|
||||
"-e", "MINIO_ROOT_USER=$minioUser",
|
||||
"-e", "MINIO_ROOT_PASSWORD=$minioPassword",
|
||||
"-e", "CELERY_BROKER_URL=redis://redis:6379/0",
|
||||
"-e", "CELERY_RESULT_BACKEND=redis://redis:6379/1"
|
||||
)
|
||||
|
||||
$seedCode = @'
|
||||
from django.contrib.auth import get_user_model
|
||||
from apps.tickets.models import Ticket, TicketAttachment
|
||||
|
||||
user, _ = get_user_model().objects.get_or_create(
|
||||
username="restore_drill",
|
||||
defaults={"email": "restore-drill@example.invalid"},
|
||||
)
|
||||
ticket, _ = Ticket.objects.update_or_create(
|
||||
ticket_no="RESTORE-DRILL-001",
|
||||
defaults={
|
||||
"title": "隔离恢复演练工单",
|
||||
"description": "restore-drill-marker",
|
||||
"requester": user,
|
||||
"created_by": user,
|
||||
},
|
||||
)
|
||||
TicketAttachment.objects.update_or_create(
|
||||
ticket=ticket,
|
||||
storage_path="restore-drill/marker.txt",
|
||||
defaults={
|
||||
"file_name": "marker.txt",
|
||||
"mime_type": "text/plain",
|
||||
"file_size": 20,
|
||||
"created_by": user,
|
||||
},
|
||||
)
|
||||
print(f"seeded_ticket={ticket.ticket_no}")
|
||||
'@
|
||||
|
||||
$deleteCode = @'
|
||||
from apps.tickets.models import Ticket
|
||||
Ticket.objects.filter(ticket_no="RESTORE-DRILL-001").delete()
|
||||
print("seed_deleted=true")
|
||||
'@
|
||||
|
||||
$verifyCode = @'
|
||||
from apps.tickets.models import Ticket
|
||||
ticket = Ticket.objects.get(ticket_no="RESTORE-DRILL-001")
|
||||
attachment = ticket.attachments.get(storage_path="restore-drill/marker.txt")
|
||||
assert ticket.description == "restore-drill-marker"
|
||||
assert attachment.file_name == "marker.txt"
|
||||
print(f"restored_ticket={ticket.ticket_no}")
|
||||
print(f"restored_attachment={attachment.storage_path}")
|
||||
'@
|
||||
|
||||
try {
|
||||
Invoke-Docker @("compose", "-f", $ComposeFile, "up", "-d", "--wait", "postgres", "minio")
|
||||
if (-not $SkipBuild) {
|
||||
Invoke-Docker @("compose", "-f", $ComposeFile, "build", "backend")
|
||||
}
|
||||
|
||||
$runPrefix = @(
|
||||
"compose", "-f", $ComposeFile,
|
||||
"run", "--rm", "-T", "--no-deps"
|
||||
) + $backendEnvironment + @("backend")
|
||||
|
||||
Invoke-Docker ($runPrefix + @("python", "manage.py", "migrate", "--noinput"))
|
||||
Invoke-DjangoCode -RunPrefix $runPrefix -Code $seedCode -FailureMessage "演练数据创建失败"
|
||||
|
||||
Invoke-Docker @(
|
||||
"compose", "-f", $ComposeFile, "exec", "-T", "minio",
|
||||
"sh", "-c",
|
||||
'mc alias set drill http://127.0.0.1:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null && mc mb --ignore-existing drill/ai-ticket-platform >/dev/null && printf restore-drill-marker | mc pipe drill/ai-ticket-platform/restore-drill/marker.txt >/dev/null'
|
||||
)
|
||||
|
||||
& (Join-Path $scriptDirectory "backup.ps1") `
|
||||
-ComposeFile $ComposeFile `
|
||||
-BackupRoot $backupRoot `
|
||||
-PostgresDatabase "ai_ticket_platform" `
|
||||
-PostgresUser "ai_ticket" `
|
||||
-SkipApplicationServices
|
||||
$backupDirectory = Get-ChildItem -LiteralPath $backupRoot -Directory |
|
||||
Sort-Object Name -Descending |
|
||||
Select-Object -First 1
|
||||
if (-not $backupDirectory) { throw "未找到演练备份目录" }
|
||||
|
||||
Invoke-DjangoCode -RunPrefix $runPrefix -Code $deleteCode -FailureMessage "演练数据破坏步骤失败"
|
||||
Invoke-Docker @(
|
||||
"compose", "-f", $ComposeFile, "exec", "-T", "minio",
|
||||
"mc", "rm", "--force", "drill/ai-ticket-platform/restore-drill/marker.txt"
|
||||
)
|
||||
|
||||
& (Join-Path $scriptDirectory "restore.ps1") `
|
||||
-BackupDirectory $backupDirectory.FullName `
|
||||
-ComposeFile $ComposeFile `
|
||||
-PostgresDatabase "ai_ticket_platform" `
|
||||
-PostgresUser "ai_ticket" `
|
||||
-Confirm "restore-ai-ticket-platform" `
|
||||
-SkipApplicationServices
|
||||
|
||||
Invoke-DjangoCode -RunPrefix $runPrefix -Code $verifyCode -FailureMessage "数据库业务冒烟失败"
|
||||
$objectMarker = & docker compose -f $ComposeFile exec -T minio `
|
||||
mc cat drill/ai-ticket-platform/restore-drill/marker.txt
|
||||
if ($LASTEXITCODE -ne 0 -or "$objectMarker".Trim() -ne "restore-drill-marker") {
|
||||
throw "对象存储业务冒烟失败"
|
||||
}
|
||||
Invoke-Docker ($runPrefix + @("python", "manage.py", "check"))
|
||||
|
||||
$stopwatch.Stop()
|
||||
[pscustomobject]@{
|
||||
status = "passed"
|
||||
compose_project = $projectName
|
||||
backup_id = $backupDirectory.Name
|
||||
duration_seconds = [Math]::Round($stopwatch.Elapsed.TotalSeconds, 2)
|
||||
database_smoke = "ticket_and_attachment_restored"
|
||||
object_storage_smoke = "marker_restored"
|
||||
django_check = "passed"
|
||||
backup_directory = $backupDirectory.FullName
|
||||
} | ConvertTo-Json
|
||||
}
|
||||
finally {
|
||||
& docker compose -f $ComposeFile down -v --remove-orphans --rmi local
|
||||
foreach ($entry in $savedEnvironment.GetEnumerator()) {
|
||||
if ($null -eq $entry.Value) {
|
||||
Remove-Item "Env:$($entry.Key)" -ErrorAction SilentlyContinue
|
||||
}
|
||||
else {
|
||||
Set-Item "Env:$($entry.Key)" $entry.Value
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user