ops: add production observability and recovery

infra/scripts/backup.sh

@@ -0,0 +1,86 @@
+#!/bin/sh
+set -eu
+
+SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+INFRA_DIR=$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd)
+PROJECT_DIR=$(CDPATH= cd -- "$INFRA_DIR/.." && pwd)
+COMPOSE_FILE=${COMPOSE_FILE:-"$INFRA_DIR/docker-compose.prod.yml"}
+BACKUP_ROOT=${BACKUP_ROOT:-"$PROJECT_DIR/backups"}
+STAMP=$(date -u +%Y%m%dT%H%M%SZ)
+TARGET="$BACKUP_ROOT/$STAMP"
+
+mkdir -p "$TARGET"
+TARGET=$(CDPATH= cd -- "$TARGET" && pwd)
+case "$TARGET" in
+  "$PROJECT_DIR"/*) ;;
+  *) echo "拒绝写入项目目录之外的备份路径：$TARGET" >&2; exit 2 ;;
+esac
+
+POSTGRES_DB=${POSTGRES_DB:-ai_ticket_platform}
+POSTGRES_USER=${POSTGRES_USER:-ai_ticket}
+BACKUP_HELPER_IMAGE=${BACKUP_HELPER_IMAGE:-alpine:3.21}
+RUNNING_APPLICATION_SERVICES=""
+MINIO_WAS_RUNNING=false
+
+cleanup() {
+  MINIO_READY=true
+  if [ "$MINIO_WAS_RUNNING" = "true" ]; then
+    if ! docker compose -f "$COMPOSE_FILE" up -d --wait minio >/dev/null; then
+      echo "备份结束后 MinIO 未恢复健康，应用服务保持停止" >&2
+      MINIO_READY=false
+    fi
+  fi
+  if [ "$MINIO_READY" = "true" ] && [ -n "$RUNNING_APPLICATION_SERVICES" ]; then
+    docker compose -f "$COMPOSE_FILE" start $RUNNING_APPLICATION_SERVICES >/dev/null || true
+  fi
+}
+trap cleanup EXIT INT TERM
+
+RUNNING_SERVICES=$(docker compose -f "$COMPOSE_FILE" ps --services --filter status=running)
+if printf '%s\n' "$RUNNING_SERVICES" | grep -qx "minio"; then
+  MINIO_WAS_RUNNING=true
+fi
+if [ "${BACKUP_SKIP_APPLICATION_SERVICES:-false}" != "true" ]; then
+  for service in backend worker beat; do
+    if printf '%s\n' "$RUNNING_SERVICES" | grep -qx "$service"; then
+      RUNNING_APPLICATION_SERVICES="$RUNNING_APPLICATION_SERVICES $service"
+    fi
+  done
+  if [ -n "$RUNNING_APPLICATION_SERVICES" ]; then
+    docker compose -f "$COMPOSE_FILE" stop $RUNNING_APPLICATION_SERVICES
+  fi
+fi
+if [ "$MINIO_WAS_RUNNING" = "true" ]; then
+  docker compose -f "$COMPOSE_FILE" stop minio
+fi
+
+docker compose -f "$COMPOSE_FILE" exec -T postgres \
+  pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom > "$TARGET/postgres.dump"
+docker compose -f "$COMPOSE_FILE" exec -T postgres \
+  pg_restore --list < "$TARGET/postgres.dump" > "$TARGET/postgres.contents"
+MINIO_CONTAINER=$(docker compose -f "$COMPOSE_FILE" ps -q minio)
+if [ -z "$MINIO_CONTAINER" ]; then
+  echo "未找到 MinIO 容器" >&2
+  exit 3
+fi
+docker run --rm \
+  --volumes-from "$MINIO_CONTAINER" \
+  -v "$TARGET:/backup" \
+  "$BACKUP_HELPER_IMAGE" \
+  tar -C /data -czf /backup/minio-data.tar.gz .
+
+(
+  cd "$TARGET"
+  sha256sum postgres.dump postgres.contents minio-data.tar.gz > SHA256SUMS
+)
+
+cat > "$TARGET/manifest.txt" <<EOF
+created_at=$STAMP
+postgres_database=$POSTGRES_DB
+postgres_user=$POSTGRES_USER
+compose_file=$COMPOSE_FILE
+EOF
+
+cleanup
+trap - EXIT INT TERM
+echo "备份已创建并完成结构校验：$TARGET"