ops: add production observability and recovery

This commit is contained in:
hailin
2026-07-27 19:06:34 +08:00
parent e69368a2f3
commit b02d18d3b9
33 changed files with 8832 additions and 1 deletions

47
infra/scripts/restore.sh Normal file
View File

@@ -0,0 +1,47 @@
#!/bin/sh
set -eu
if [ "$#" -ne 1 ] || [ "${RESTORE_CONFIRM:-}" != "restore-ai-ticket-platform" ]; then
echo "用法:RESTORE_CONFIRM=restore-ai-ticket-platform $0 <backup-directory>" >&2
echo "恢复会替换当前 PostgreSQL 数据库和 MinIO 数据。" >&2
exit 2
fi
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
INFRA_DIR=$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd)
COMPOSE_FILE=${COMPOSE_FILE:-"$INFRA_DIR/docker-compose.prod.yml"}
TARGET=$(CDPATH= cd -- "$1" && pwd)
"$SCRIPT_DIR/verify-backup.sh" "$TARGET"
POSTGRES_DB=${POSTGRES_DB:-ai_ticket_platform}
POSTGRES_USER=${POSTGRES_USER:-ai_ticket}
BACKUP_HELPER_IMAGE=${BACKUP_HELPER_IMAGE:-alpine:3.21}
if [ "${RESTORE_SKIP_APPLICATION_SERVICES:-false}" != "true" ]; then
docker compose -f "$COMPOSE_FILE" stop backend worker beat
fi
MINIO_CONTAINER=$(docker compose -f "$COMPOSE_FILE" ps -q minio)
if [ -z "$MINIO_CONTAINER" ]; then
echo "未找到 MinIO 容器" >&2
exit 3
fi
docker compose -f "$COMPOSE_FILE" stop minio
docker compose -f "$COMPOSE_FILE" exec -T postgres \
dropdb -U "$POSTGRES_USER" --force "$POSTGRES_DB"
docker compose -f "$COMPOSE_FILE" exec -T postgres \
createdb -U "$POSTGRES_USER" "$POSTGRES_DB"
docker compose -f "$COMPOSE_FILE" exec -T postgres \
pg_restore -U "$POSTGRES_USER" -d "$POSTGRES_DB" --no-owner --exit-on-error \
< "$TARGET/postgres.dump"
docker run --rm \
--volumes-from "$MINIO_CONTAINER" \
-v "$TARGET:/backup:ro" \
"$BACKUP_HELPER_IMAGE" \
sh -c 'find /data -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + && tar -C /data -xzf /backup/minio-data.tar.gz'
docker compose -f "$COMPOSE_FILE" up -d --wait minio
if [ "${RESTORE_SKIP_APPLICATION_SERVICES:-false}" != "true" ]; then
docker compose -f "$COMPOSE_FILE" start backend worker beat
fi
echo "恢复完成。请立即检查 /api/system/health/ready 并执行核心业务冒烟测试。"