import hmac from datetime import timedelta from django.conf import settings from django.core.cache import cache from django.db import connection from django.db.models import Count from django.http import HttpResponse from django.utils import timezone from ninja import Router, Status from apps.accounts.api import require_permission from apps.agents.models import AgentRun from apps.channels.models import ChannelMessage from apps.tools.models import ToolCall from apps.workitems.models import WorkItem router = Router(tags=["system"]) @router.get("/health") def health(request): return {"status": "ok"} @router.get("/health/live", auth=None) def liveness(request): return {"status": "alive"} @router.get("/health/ready", response={200: dict, 503: dict}, auth=None) def readiness(request): checks = {} try: with connection.cursor() as cursor: cursor.execute("SELECT 1") checks["database"] = cursor.fetchone()[0] == 1 except Exception: checks["database"] = False try: cache_key = "system-readiness" cache.set(cache_key, "ready", timeout=5) checks["cache"] = cache.get(cache_key) == "ready" cache.delete(cache_key) except Exception: checks["cache"] = False ready = all(checks.values()) return Status( 200 if ready else 503, { "status": "ready" if ready else "not_ready", "checks": checks, }, ) def require_metrics_access(request) -> None: expected = settings.METRICS_TOKEN provided = request.headers.get("X-Metrics-Token", "") authorization = request.headers.get("Authorization", "") if not provided and authorization.startswith("Bearer "): provided = authorization.removeprefix("Bearer ").strip() if expected and hmac.compare_digest(expected, provided): return require_permission(request, "system.audit.view") def metric_lines(name: str, queryset) -> list[str]: return [ f'{name}{{status="{row["status"]}"}} {row["total"]}' for row in queryset.values("status").annotate(total=Count("id")).order_by("status") ] @router.get("/metrics") def metrics(request): require_metrics_access(request) now = timezone.now() recent_cutoff = now - timedelta(minutes=15) stale_cutoff = now - timedelta(minutes=10) lines = [ "# HELP ai_ticket_channel_messages_total Channel messages by delivery status.", "# TYPE ai_ticket_channel_messages_total gauge", *metric_lines("ai_ticket_channel_messages_total", ChannelMessage.objects.all()), "# HELP ai_ticket_agent_runs_total Agent runs by status.", "# TYPE ai_ticket_agent_runs_total gauge", *metric_lines("ai_ticket_agent_runs_total", AgentRun.objects.all()), "# HELP ai_ticket_tool_calls_total Tool calls by status.", "# TYPE ai_ticket_tool_calls_total gauge", *metric_lines("ai_ticket_tool_calls_total", ToolCall.objects.all()), "# HELP ai_ticket_work_items_active Active work items.", "# TYPE ai_ticket_work_items_active gauge", f'ai_ticket_work_items_active {WorkItem.objects.filter(status="active").count()}', ( "# HELP ai_ticket_channel_messages_failed_recent " "Failed channel messages updated in the last 15 minutes." ), "# TYPE ai_ticket_channel_messages_failed_recent gauge", ( "ai_ticket_channel_messages_failed_recent " f"{ChannelMessage.objects.filter( status='failed', updated_at__gte=recent_cutoff, ).count()}" ), "# HELP ai_ticket_channel_messages_stuck Channel messages queued for more than 10 minutes.", "# TYPE ai_ticket_channel_messages_stuck gauge", ( "ai_ticket_channel_messages_stuck " f'{ChannelMessage.objects.filter(status="queued", created_at__lt=stale_cutoff).count()}' ), "# HELP ai_ticket_agent_runs_failed_recent Agent runs failed in the last 15 minutes.", "# TYPE ai_ticket_agent_runs_failed_recent gauge", ( "ai_ticket_agent_runs_failed_recent " f"{AgentRun.objects.filter( status='failed', finished_at__gte=recent_cutoff, ).count()}" ), "# HELP ai_ticket_tool_calls_failed_recent Tool calls failed in the last 15 minutes.", "# TYPE ai_ticket_tool_calls_failed_recent gauge", ( "ai_ticket_tool_calls_failed_recent " f"{ToolCall.objects.filter( status='failed', finished_at__gte=recent_cutoff, ).count()}" ), ] return HttpResponse( "\n".join(lines) + "\n", content_type="text/plain; version=0.0.4; charset=utf-8", )