ops: add production observability and recovery

backend/apps/system/api.py

@@ -1,4 +1,19 @@
-from ninja import Router
+import hmac
+from datetime import timedelta
+
+from django.conf import settings
+from django.core.cache import cache
+from django.db import connection
+from django.db.models import Count
+from django.http import HttpResponse
+from django.utils import timezone
+from ninja import Router, Status
+
+from apps.accounts.api import require_permission
+from apps.agents.models import AgentRun
+from apps.channels.models import ChannelMessage
+from apps.tools.models import ToolCall
+from apps.workitems.models import WorkItem
 
 router = Router(tags=["system"])
 
@@ -6,3 +21,114 @@ router = Router(tags=["system"])
 @router.get("/health")
 def health(request):
     return {"status": "ok"}
+
+
+@router.get("/health/live", auth=None)
+def liveness(request):
+    return {"status": "alive"}
+
+
+@router.get("/health/ready", response={200: dict, 503: dict}, auth=None)
+def readiness(request):
+    checks = {}
+    try:
+        with connection.cursor() as cursor:
+            cursor.execute("SELECT 1")
+            checks["database"] = cursor.fetchone()[0] == 1
+    except Exception:
+        checks["database"] = False
+    try:
+        cache_key = "system-readiness"
+        cache.set(cache_key, "ready", timeout=5)
+        checks["cache"] = cache.get(cache_key) == "ready"
+        cache.delete(cache_key)
+    except Exception:
+        checks["cache"] = False
+    ready = all(checks.values())
+    return Status(
+        200 if ready else 503,
+        {
+            "status": "ready" if ready else "not_ready",
+            "checks": checks,
+        },
+    )
+
+
+def require_metrics_access(request) -> None:
+    expected = settings.METRICS_TOKEN
+    provided = request.headers.get("X-Metrics-Token", "")
+    authorization = request.headers.get("Authorization", "")
+    if not provided and authorization.startswith("Bearer "):
+        provided = authorization.removeprefix("Bearer ").strip()
+    if expected and hmac.compare_digest(expected, provided):
+        return
+    require_permission(request, "system.audit.view")
+
+
+def metric_lines(name: str, queryset) -> list[str]:
+    return [
+        f'{name}{{status="{row["status"]}"}} {row["total"]}'
+        for row in queryset.values("status").annotate(total=Count("id")).order_by("status")
+    ]
+
+
+@router.get("/metrics")
+def metrics(request):
+    require_metrics_access(request)
+    now = timezone.now()
+    recent_cutoff = now - timedelta(minutes=15)
+    stale_cutoff = now - timedelta(minutes=10)
+    lines = [
+        "# HELP ai_ticket_channel_messages_total Channel messages by delivery status.",
+        "# TYPE ai_ticket_channel_messages_total gauge",
+        *metric_lines("ai_ticket_channel_messages_total", ChannelMessage.objects.all()),
+        "# HELP ai_ticket_agent_runs_total Agent runs by status.",
+        "# TYPE ai_ticket_agent_runs_total gauge",
+        *metric_lines("ai_ticket_agent_runs_total", AgentRun.objects.all()),
+        "# HELP ai_ticket_tool_calls_total Tool calls by status.",
+        "# TYPE ai_ticket_tool_calls_total gauge",
+        *metric_lines("ai_ticket_tool_calls_total", ToolCall.objects.all()),
+        "# HELP ai_ticket_work_items_active Active work items.",
+        "# TYPE ai_ticket_work_items_active gauge",
+        f'ai_ticket_work_items_active {WorkItem.objects.filter(status="active").count()}',
+        (
+            "# HELP ai_ticket_channel_messages_failed_recent "
+            "Failed channel messages updated in the last 15 minutes."
+        ),
+        "# TYPE ai_ticket_channel_messages_failed_recent gauge",
+        (
+            "ai_ticket_channel_messages_failed_recent "
+            f"{ChannelMessage.objects.filter(
+                status='failed',
+                updated_at__gte=recent_cutoff,
+            ).count()}"
+        ),
+        "# HELP ai_ticket_channel_messages_stuck Channel messages queued for more than 10 minutes.",
+        "# TYPE ai_ticket_channel_messages_stuck gauge",
+        (
+            "ai_ticket_channel_messages_stuck "
+            f'{ChannelMessage.objects.filter(status="queued", created_at__lt=stale_cutoff).count()}'
+        ),
+        "# HELP ai_ticket_agent_runs_failed_recent Agent runs failed in the last 15 minutes.",
+        "# TYPE ai_ticket_agent_runs_failed_recent gauge",
+        (
+            "ai_ticket_agent_runs_failed_recent "
+            f"{AgentRun.objects.filter(
+                status='failed',
+                finished_at__gte=recent_cutoff,
+            ).count()}"
+        ),
+        "# HELP ai_ticket_tool_calls_failed_recent Tool calls failed in the last 15 minutes.",
+        "# TYPE ai_ticket_tool_calls_failed_recent gauge",
+        (
+            "ai_ticket_tool_calls_failed_recent "
+            f"{ToolCall.objects.filter(
+                status='failed',
+                finished_at__gte=recent_cutoff,
+            ).count()}"
+        ),
+    ]
+    return HttpResponse(
+        "\n".join(lines) + "\n",
+        content_type="text/plain; version=0.0.4; charset=utf-8",
+    )