ops: add production observability and recovery

backend/apps/system/tests/test_health_api.py

@@ -1,5 +1,51 @@
+import pytest
+from django.test import override_settings
+
+pytestmark = pytest.mark.django_db
+
+
 def test_health_endpoint_returns_ok(client):
     response = client.get("/api/system/health")
 
     assert response.status_code == 200
     assert response.json() == {"status": "ok"}
+
+
+def test_liveness_and_readiness_endpoints(client):
+    live_response = client.get("/api/system/health/live")
+    ready_response = client.get("/api/system/health/ready")
+
+    assert live_response.status_code == 200
+    assert live_response.json() == {"status": "alive"}
+    assert ready_response.status_code == 200
+    assert ready_response.json() == {
+        "status": "ready",
+        "checks": {"database": True, "cache": True},
+    }
+
+
+def test_metrics_endpoint_exposes_low_cardinality_metrics(admin_client):
+    response = admin_client.get("/api/system/metrics")
+
+    assert response.status_code == 200
+    assert response["Content-Type"].startswith("text/plain")
+    content = response.content.decode()
+    assert "ai_ticket_channel_messages_total" in content
+    assert "ai_ticket_agent_runs_total" in content
+    assert "ai_ticket_tool_calls_total" in content
+    assert "ai_ticket_work_items_active" in content
+    assert "ai_ticket_channel_messages_failed_recent" in content
+    assert "ai_ticket_channel_messages_stuck" in content
+    assert "ai_ticket_agent_runs_failed_recent" in content
+    assert "ai_ticket_tool_calls_failed_recent" in content
+
+
+@override_settings(METRICS_TOKEN="prometheus-secret")
+def test_metrics_endpoint_accepts_bearer_token_for_prometheus(client):
+    response = client.get(
+        "/api/system/metrics",
+        HTTP_AUTHORIZATION="Bearer prometheus-secret",
+    )
+
+    assert response.status_code == 200
+    assert "ai_ticket_channel_messages_total" in response.content.decode()