feat: init commint

backend/apps/agents/tests/test_runtime.py

@@ -6,6 +6,7 @@ from django.utils import timezone
 
 from apps.agents.models import AgentRun
 from apps.agents.services import build_config_snapshot, execute_agent_run
+from apps.agents.structured_metrics import build_structured_output_metrics
 from apps.agents.tasks import cleanup_expired_agent_runs
 from apps.agents.tasks import execute_agent_run as execute_agent_run_task
 from apps.audit.models import OperationAuditLog
@@ -47,6 +48,19 @@ def test_ordered_runtime_records_steps_usage_and_final_output(db, monkeypatch):
     assert run.input_tokens == 22
     assert run.output_tokens == 11
     assert list(run.steps.values_list("code", flat=True)) == ["structure", "reply"]
+    assert run.steps.count() == 2
+    assert sum(step.attempts.count() for step in run.steps.all()) == 2
+    first_invocation = run.steps.first().attempts.get().model_invocation
+    assert first_invocation.model_snapshot["structured_output"]["mode"] == "native_json_schema"
+    assert first_invocation.model_snapshot["structured_output"]["schema_hash"]
+    assert first_invocation.input_snapshot["output_schema"]
+    metrics = build_structured_output_metrics(
+        deployment_id=version.model_deployment_id,
+    )
+    assert metrics["initial_count"] == 2
+    assert metrics["first_pass_rate"] == 1
+    assert metrics["repair_count"] == 0
+    assert metrics["invocation_count"] == 2
     assert run.progress_stage == "completed"
     assert run.progress_message == "草稿生成完成"
 
@@ -75,9 +89,14 @@ def test_invalid_structure_is_repaired_once(db, monkeypatch):
         "initial",
         "structure_repair",
     ]
+    metrics = build_structured_output_metrics(skill_version_id=step.skill_version_id)
+    assert metrics["initial_count"] == 1
+    assert metrics["first_pass_rate"] == 0
+    assert metrics["repair_count"] == 1
+    assert metrics["repair_rate"] == 1
 
 
-def test_flattened_fact_output_is_normalized_without_model_repair(db, monkeypatch):
+def test_flattened_fact_output_never_passes_as_final_result(db, monkeypatch):
     _, version = create_agent(two_steps=False)
     skill_version = version.steps.get().skill_version
     fact_object = {
@@ -114,16 +133,13 @@ def test_flattened_fact_output_is_normalized_without_model_repair(db, monkeypatc
 
     run.refresh_from_db()
     step = run.steps.get()
-    assert run.status == "succeeded"
-    assert step.attempts.count() == 1
-    assert run.final_output == {
-        "facts": {"title": "登录失败", "description": ""},
-        "evidence": {"title": "无法登录", "description": ""},
-        "missing_fields": ["description"],
-    }
+    assert run.status == "failed"
+    assert run.error_code == "structured_output_invalid"
+    assert step.attempts.count() == 2
+    assert run.final_output is None
 
 
-def test_nested_fact_output_with_missing_string_is_completed_without_repair(db, monkeypatch):
+def test_missing_nested_fact_field_never_passes_as_final_result(db, monkeypatch):
     _, version = create_agent(two_steps=False)
     skill_version = version.steps.get().skill_version
     fact_object = {
@@ -160,14 +176,13 @@ def test_nested_fact_output_with_missing_string_is_completed_without_repair(db,
     execute_agent_run(run.id)
 
     run.refresh_from_db()
-    assert run.status == "succeeded"
-    assert run.steps.get().attempts.count() == 1
-    assert run.final_output["facts"]["problem_performance"] == ""
-    assert run.final_output["evidence"]["problem_performance"] == ""
-    assert run.final_output["missing_fields"] == ["problem_performance"]
+    assert run.status == "failed"
+    assert run.error_code == "structured_output_invalid"
+    assert run.steps.get().attempts.count() == 2
+    assert run.final_output is None
 
 
-def test_flattened_draft_output_accepts_follow_up_questions_without_repair(db, monkeypatch):
+def test_old_flattened_draft_fields_never_pass_as_final_result(db, monkeypatch):
     _, version = create_agent(two_steps=False)
     skill_version = version.steps.get().skill_version
     skill_version.output_schema = {
@@ -203,9 +218,70 @@ def test_flattened_draft_output_accepts_follow_up_questions_without_repair(db, m
     execute_agent_run(run.id)
 
     run.refresh_from_db()
-    assert run.status == "succeeded"
-    assert run.steps.get().attempts.count() == 1
-    assert run.final_output["questions"] == ["请补充现象"]
+    assert run.status == "failed"
+    assert run.error_code == "structured_output_invalid"
+    assert run.steps.get().attempts.count() == 2
+    assert run.final_output is None
+
+
+def test_wrong_evidence_properties_never_pass_as_final_result(db, monkeypatch):
+    _, version = create_agent(two_steps=False)
+    skill_version = version.steps.get().skill_version
+    evidence_schema = {
+        "type": "object",
+        "properties": {"title": {"type": "string"}},
+        "required": ["title"],
+        "additionalProperties": False,
+    }
+    skill_version.output_schema = {
+        "type": "object",
+        "properties": {
+            "facts": evidence_schema,
+            "evidence": evidence_schema,
+            "missing_fields": {"type": "array", "items": {"type": "string"}},
+        },
+        "required": ["facts", "evidence", "missing_fields"],
+        "additionalProperties": False,
+    }
+    skill_version.save(update_fields=["output_schema"])
+    monkeypatch.setattr(
+        "apps.modelhub.services._request_completion",
+        lambda *args, **kwargs: CompletionResult(
+            '{"facts":{"title":"登录失败"},'
+            '"evidence":{"source":"旧证据字段"},"missing_fields":[]}',
+            10,
+            5,
+        ),
+    )
+    run = create_run(version)
+
+    execute_agent_run(run.id)
+
+    run.refresh_from_db()
+    assert run.status == "failed"
+    assert run.error_code == "structured_output_invalid"
+    assert run.steps.get().attempts.count() == 2
+    assert run.final_output is None
+
+
+def test_unverified_model_fails_before_any_model_call(db, monkeypatch):
+    _, version = create_agent(two_steps=False)
+    version.model_deployment.structured_output_test_status = "untested"
+    version.model_deployment.save(update_fields=["structured_output_test_status"])
+    monkeypatch.setattr(
+        "apps.modelhub.services._request_completion",
+        lambda *args, **kwargs: pytest.fail("未验证模型不应被调用"),
+    )
+    run = create_run(version)
+
+    execute_agent_run(run.id)
+
+    run.refresh_from_db()
+    step = run.steps.get()
+    assert run.status == "failed"
+    assert run.error_code == "structured_output_not_verified"
+    assert step.error_code == "structured_output_not_verified"
+    assert step.attempts.count() == 0
 
 
 def test_second_invalid_structure_fails_and_stops(db, monkeypatch):
@@ -226,8 +302,13 @@ def test_second_invalid_structure_fails_and_stops(db, monkeypatch):
 
     run.refresh_from_db()
     assert run.status == "failed"
+    assert run.error_code == "structured_output_invalid"
+    assert run.final_output is None
     assert run.steps.count() == 1
-    assert run.steps.get().status == "failed"
+    step = run.steps.get()
+    assert step.status == "failed"
+    assert step.attempts.count() == 2
+    assert step.attempts.get(attempt_type="structure_repair").validation_error
 
 
 def test_cancellation_before_first_step_stops_run(db):
@@ -350,10 +431,12 @@ def test_step_model_override_controls_deployment_and_image_egress(db, monkeypatc
     assert run.config_snapshot["steps"][0]["model"] == {
         "deployment_id": vision.id,
         "inherited": False,
-        "provider_name": vision.provider.name,
-        "model_name": vision.model_name,
-        "capability": "vision_chat",
-    }
+            "provider_name": vision.provider.name,
+            "model_name": vision.model_name,
+            "capability": "vision_chat",
+            "structured_output_mode": "native_json_schema",
+            "structured_output_test_status": "passed",
+        }
     assert run.config_snapshot["steps"][1]["model"]["inherited"] is True
     assert run.config_snapshot["steps"][1]["model"]["capability"] == "text_chat"