173 lines
7.2 KiB
Python
173 lines
7.2 KiB
Python
from django.conf import settings
|
|
from django.db.models import Count, Q, Sum
|
|
|
|
|
|
def build_structured_output_metrics(
|
|
*,
|
|
deployment_id: int | None = None,
|
|
skill_version_id: int | None = None,
|
|
) -> dict:
|
|
"""按模型或 Skill 汇总结构调用，底层尝试记录仍保留完整审计关联。"""
|
|
|
|
from apps.agents.models import AgentRunAttempt, AgentRunStep
|
|
|
|
attempt_filters = {"attempt_type__in": {"initial", "structure_repair"}}
|
|
step_filters = {"error_code": "structured_output_invalid"}
|
|
if deployment_id is not None:
|
|
attempt_filters["model_invocation__deployment_id"] = deployment_id
|
|
step_filters["model_deployment_id"] = deployment_id
|
|
if skill_version_id is not None:
|
|
attempt_filters["step__skill_version_id"] = skill_version_id
|
|
step_filters["skill_version_id"] = skill_version_id
|
|
|
|
attempts = AgentRunAttempt.objects.filter(**attempt_filters)
|
|
totals = attempts.aggregate(
|
|
initial_count=Count("id", filter=Q(attempt_type="initial")),
|
|
first_pass_count=Count(
|
|
"id",
|
|
filter=Q(attempt_type="initial", status="succeeded", validation_error=""),
|
|
),
|
|
repair_count=Count("id", filter=Q(attempt_type="structure_repair")),
|
|
input_tokens=Sum("input_tokens"),
|
|
output_tokens=Sum("output_tokens"),
|
|
duration_ms=Sum("duration_ms"),
|
|
)
|
|
initial_count = totals["initial_count"] or 0
|
|
first_pass_count = totals["first_pass_count"] or 0
|
|
repair_count = totals["repair_count"] or 0
|
|
structural_failure_count = AgentRunStep.objects.filter(**step_filters).count()
|
|
repair_rate = repair_count / initial_count if initial_count else 0
|
|
return {
|
|
"initial_count": initial_count,
|
|
"first_pass_count": first_pass_count,
|
|
"first_pass_rate": first_pass_count / initial_count if initial_count else 0,
|
|
"repair_count": repair_count,
|
|
"repair_rate": repair_rate,
|
|
"structural_failure_count": structural_failure_count,
|
|
"structural_failure_rate": (
|
|
structural_failure_count / initial_count if initial_count else 0
|
|
),
|
|
"invocation_count": initial_count + repair_count,
|
|
"input_tokens": totals["input_tokens"] or 0,
|
|
"output_tokens": totals["output_tokens"] or 0,
|
|
"duration_ms": totals["duration_ms"] or 0,
|
|
"repair_alert": (
|
|
initial_count >= 5
|
|
and repair_rate > settings.AI_STRUCTURED_OUTPUT_REPAIR_ALERT_THRESHOLD
|
|
),
|
|
"repair_alert_threshold": settings.AI_STRUCTURED_OUTPUT_REPAIR_ALERT_THRESHOLD,
|
|
}
|
|
|
|
|
|
def build_deployment_structured_output_metrics(
|
|
deployment_ids: list[int],
|
|
) -> dict[int, dict]:
|
|
"""批量汇总模型结构化输出指标，避免模型列表逐行查询。"""
|
|
|
|
if not deployment_ids:
|
|
return {}
|
|
from apps.agents.models import AgentRunAttempt, AgentRunStep
|
|
from apps.modelhub.models import ModelInvocation
|
|
|
|
rows = (
|
|
AgentRunAttempt.objects.filter(
|
|
model_invocation__deployment_id__in=deployment_ids,
|
|
attempt_type__in={"initial", "structure_repair"},
|
|
)
|
|
.values("model_invocation__deployment_id")
|
|
.annotate(
|
|
initial_count=Count("id", filter=Q(attempt_type="initial")),
|
|
first_pass_count=Count(
|
|
"id",
|
|
filter=Q(
|
|
attempt_type="initial",
|
|
status="succeeded",
|
|
validation_error="",
|
|
),
|
|
),
|
|
repair_count=Count("id", filter=Q(attempt_type="structure_repair")),
|
|
input_tokens=Sum("input_tokens"),
|
|
output_tokens=Sum("output_tokens"),
|
|
duration_ms=Sum("duration_ms"),
|
|
)
|
|
)
|
|
failures = dict(
|
|
AgentRunStep.objects.filter(
|
|
model_deployment_id__in=deployment_ids,
|
|
error_code="structured_output_invalid",
|
|
)
|
|
.values("model_deployment_id")
|
|
.annotate(total=Count("id"))
|
|
.values_list("model_deployment_id", "total")
|
|
)
|
|
invocation_rows = {
|
|
row["deployment_id"]: row
|
|
for row in ModelInvocation.objects.filter(deployment_id__in=deployment_ids)
|
|
.values("deployment_id")
|
|
.annotate(
|
|
model_invocation_count=Count("id"),
|
|
model_success_count=Count("id", filter=Q(status="succeeded")),
|
|
model_failure_count=Count("id", filter=Q(status="failed")),
|
|
model_input_tokens=Sum("input_tokens"),
|
|
model_output_tokens=Sum("output_tokens"),
|
|
model_duration_ms=Sum("duration_ms"),
|
|
)
|
|
}
|
|
result: dict[int, dict] = {}
|
|
for row in rows:
|
|
deployment_id = row["model_invocation__deployment_id"]
|
|
initial_count = row["initial_count"] or 0
|
|
repair_count = row["repair_count"] or 0
|
|
first_pass_count = row["first_pass_count"] or 0
|
|
failure_count = failures.get(deployment_id, 0)
|
|
repair_rate = repair_count / initial_count if initial_count else 0
|
|
result[deployment_id] = {
|
|
"initial_count": initial_count,
|
|
"first_pass_count": first_pass_count,
|
|
"first_pass_rate": first_pass_count / initial_count if initial_count else 0,
|
|
"repair_count": repair_count,
|
|
"repair_rate": repair_rate,
|
|
"structural_failure_count": failure_count,
|
|
"structural_failure_rate": failure_count / initial_count if initial_count else 0,
|
|
"invocation_count": initial_count + repair_count,
|
|
"input_tokens": row["input_tokens"] or 0,
|
|
"output_tokens": row["output_tokens"] or 0,
|
|
"duration_ms": row["duration_ms"] or 0,
|
|
"repair_alert": (
|
|
initial_count >= 5
|
|
and repair_rate > settings.AI_STRUCTURED_OUTPUT_REPAIR_ALERT_THRESHOLD
|
|
),
|
|
"repair_alert_threshold": settings.AI_STRUCTURED_OUTPUT_REPAIR_ALERT_THRESHOLD,
|
|
}
|
|
empty = {
|
|
"initial_count": 0,
|
|
"first_pass_count": 0,
|
|
"first_pass_rate": 0,
|
|
"repair_count": 0,
|
|
"repair_rate": 0,
|
|
"structural_failure_count": 0,
|
|
"structural_failure_rate": 0,
|
|
"invocation_count": 0,
|
|
"input_tokens": 0,
|
|
"output_tokens": 0,
|
|
"duration_ms": 0,
|
|
"repair_alert": False,
|
|
"repair_alert_threshold": settings.AI_STRUCTURED_OUTPUT_REPAIR_ALERT_THRESHOLD,
|
|
}
|
|
for deployment_id in deployment_ids:
|
|
result.setdefault(deployment_id, empty)
|
|
invocation_row = invocation_rows.get(deployment_id, {})
|
|
invocation_count = invocation_row.get("model_invocation_count") or 0
|
|
success_count = invocation_row.get("model_success_count") or 0
|
|
failure_count = invocation_row.get("model_failure_count") or 0
|
|
result[deployment_id] = {
|
|
**result[deployment_id],
|
|
"model_invocation_count": invocation_count,
|
|
"model_success_rate": success_count / invocation_count if invocation_count else 0,
|
|
"model_failure_rate": failure_count / invocation_count if invocation_count else 0,
|
|
"model_input_tokens": invocation_row.get("model_input_tokens") or 0,
|
|
"model_output_tokens": invocation_row.get("model_output_tokens") or 0,
|
|
"model_duration_ms": invocation_row.get("model_duration_ms") or 0,
|
|
}
|
|
return result
|