135 lines
4.7 KiB
Python
135 lines
4.7 KiB
Python
import hmac
|
|
from datetime import timedelta
|
|
|
|
from django.conf import settings
|
|
from django.core.cache import cache
|
|
from django.db import connection
|
|
from django.db.models import Count
|
|
from django.http import HttpResponse
|
|
from django.utils import timezone
|
|
from ninja import Router, Status
|
|
|
|
from apps.accounts.api import require_permission
|
|
from apps.agents.models import AgentRun
|
|
from apps.channels.models import ChannelMessage
|
|
from apps.tools.models import ToolCall
|
|
from apps.workitems.models import WorkItem
|
|
|
|
router = Router(tags=["system"])
|
|
|
|
|
|
@router.get("/health")
|
|
def health(request):
|
|
return {"status": "ok"}
|
|
|
|
|
|
@router.get("/health/live", auth=None)
|
|
def liveness(request):
|
|
return {"status": "alive"}
|
|
|
|
|
|
@router.get("/health/ready", response={200: dict, 503: dict}, auth=None)
|
|
def readiness(request):
|
|
checks = {}
|
|
try:
|
|
with connection.cursor() as cursor:
|
|
cursor.execute("SELECT 1")
|
|
checks["database"] = cursor.fetchone()[0] == 1
|
|
except Exception:
|
|
checks["database"] = False
|
|
try:
|
|
cache_key = "system-readiness"
|
|
cache.set(cache_key, "ready", timeout=5)
|
|
checks["cache"] = cache.get(cache_key) == "ready"
|
|
cache.delete(cache_key)
|
|
except Exception:
|
|
checks["cache"] = False
|
|
ready = all(checks.values())
|
|
return Status(
|
|
200 if ready else 503,
|
|
{
|
|
"status": "ready" if ready else "not_ready",
|
|
"checks": checks,
|
|
},
|
|
)
|
|
|
|
|
|
def require_metrics_access(request) -> None:
|
|
expected = settings.METRICS_TOKEN
|
|
provided = request.headers.get("X-Metrics-Token", "")
|
|
authorization = request.headers.get("Authorization", "")
|
|
if not provided and authorization.startswith("Bearer "):
|
|
provided = authorization.removeprefix("Bearer ").strip()
|
|
if expected and hmac.compare_digest(expected, provided):
|
|
return
|
|
require_permission(request, "system.audit.view")
|
|
|
|
|
|
def metric_lines(name: str, queryset) -> list[str]:
|
|
return [
|
|
f'{name}{{status="{row["status"]}"}} {row["total"]}'
|
|
for row in queryset.values("status").annotate(total=Count("id")).order_by("status")
|
|
]
|
|
|
|
|
|
@router.get("/metrics")
|
|
def metrics(request):
|
|
require_metrics_access(request)
|
|
now = timezone.now()
|
|
recent_cutoff = now - timedelta(minutes=15)
|
|
stale_cutoff = now - timedelta(minutes=10)
|
|
lines = [
|
|
"# HELP ai_ticket_channel_messages_total Channel messages by delivery status.",
|
|
"# TYPE ai_ticket_channel_messages_total gauge",
|
|
*metric_lines("ai_ticket_channel_messages_total", ChannelMessage.objects.all()),
|
|
"# HELP ai_ticket_agent_runs_total Agent runs by status.",
|
|
"# TYPE ai_ticket_agent_runs_total gauge",
|
|
*metric_lines("ai_ticket_agent_runs_total", AgentRun.objects.all()),
|
|
"# HELP ai_ticket_tool_calls_total Tool calls by status.",
|
|
"# TYPE ai_ticket_tool_calls_total gauge",
|
|
*metric_lines("ai_ticket_tool_calls_total", ToolCall.objects.all()),
|
|
"# HELP ai_ticket_work_items_active Active work items.",
|
|
"# TYPE ai_ticket_work_items_active gauge",
|
|
f'ai_ticket_work_items_active {WorkItem.objects.filter(status="active").count()}',
|
|
(
|
|
"# HELP ai_ticket_channel_messages_failed_recent "
|
|
"Failed channel messages updated in the last 15 minutes."
|
|
),
|
|
"# TYPE ai_ticket_channel_messages_failed_recent gauge",
|
|
(
|
|
"ai_ticket_channel_messages_failed_recent "
|
|
f"{ChannelMessage.objects.filter(
|
|
status='failed',
|
|
updated_at__gte=recent_cutoff,
|
|
).count()}"
|
|
),
|
|
"# HELP ai_ticket_channel_messages_stuck Channel messages queued for more than 10 minutes.",
|
|
"# TYPE ai_ticket_channel_messages_stuck gauge",
|
|
(
|
|
"ai_ticket_channel_messages_stuck "
|
|
f'{ChannelMessage.objects.filter(status="queued", created_at__lt=stale_cutoff).count()}'
|
|
),
|
|
"# HELP ai_ticket_agent_runs_failed_recent Agent runs failed in the last 15 minutes.",
|
|
"# TYPE ai_ticket_agent_runs_failed_recent gauge",
|
|
(
|
|
"ai_ticket_agent_runs_failed_recent "
|
|
f"{AgentRun.objects.filter(
|
|
status='failed',
|
|
finished_at__gte=recent_cutoff,
|
|
).count()}"
|
|
),
|
|
"# HELP ai_ticket_tool_calls_failed_recent Tool calls failed in the last 15 minutes.",
|
|
"# TYPE ai_ticket_tool_calls_failed_recent gauge",
|
|
(
|
|
"ai_ticket_tool_calls_failed_recent "
|
|
f"{ToolCall.objects.filter(
|
|
status='failed',
|
|
finished_at__gte=recent_cutoff,
|
|
).count()}"
|
|
),
|
|
]
|
|
return HttpResponse(
|
|
"\n".join(lines) + "\n",
|
|
content_type="text/plain; version=0.0.4; charset=utf-8",
|
|
)
|