ops: add production observability and recovery

infra/monitoring/alertmanager.yml

@@ -0,0 +1,14 @@
+route:
+  receiver: operations-webhook
+  group_by:
+    - alertname
+    - severity
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 4h
+
+receivers:
+  - name: operations-webhook
+    webhook_configs:
+      - url_file: /run/secrets/alert_webhook_url
+        send_resolved: true

infra/monitoring/alerts.yml

@@ -0,0 +1,56 @@
+groups:
+  - name: ai-ticket-platform
+    rules:
+      - alert: AiTicketBackendDown
+        expr: up{job="ai-ticket-backend"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: AI 工单平台后端不可用
+          description: Prometheus 连续 2 分钟无法抓取后端指标。
+
+      - alert: AiTicketChannelDeliveryFailures
+        expr: ai_ticket_channel_messages_failed_recent > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: 渠道消息投递持续失败
+          description: 最近 15 分钟存在失败的渠道消息，请检查渠道凭证、地址和重试队列。
+
+      - alert: AiTicketChannelQueueStuck
+        expr: ai_ticket_channel_messages_stuck > 0
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: 渠道消息队列出现积压
+          description: 存在排队超过 10 分钟的出站消息，请检查 Celery Worker 和渠道网络。
+
+      - alert: AiTicketAgentRunFailures
+        expr: ai_ticket_agent_runs_failed_recent > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: Agent 运行持续失败
+          description: 最近 15 分钟存在失败的 Agent 运行，请检查模型、工具和运行日志。
+
+      - alert: AiTicketToolCallFailures
+        expr: ai_ticket_tool_calls_failed_recent > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: Tool/MCP 调用持续失败
+          description: 最近 15 分钟存在失败的工具调用，请检查授权、MCP 健康和审计记录。
+
+      - alert: AiTicketWorkItemBacklog
+        expr: ai_ticket_work_items_active > 200
+        for: 30m
+        labels:
+          severity: warning
+        annotations:
+          summary: 活动任务积压
+          description: 活动任务连续 30 分钟超过 200 个，请检查人工处理容量和自动任务异常。

infra/monitoring/prometheus.yml

@@ -0,0 +1,21 @@
+global:
+  scrape_interval: 30s
+  evaluation_interval: 30s
+
+rule_files:
+  - /etc/prometheus/alerts.yml
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+            - alertmanager:9093
+
+scrape_configs:
+  - job_name: ai-ticket-backend
+    metrics_path: /api/system/metrics
+    authorization:
+      credentials_file: /run/secrets/metrics_token
+    static_configs:
+      - targets:
+          - backend:8000