156 lines
4.9 KiB
Python
156 lines
4.9 KiB
Python
import pytest
|
|
|
|
from apps.modelhub.structured_output import (
|
|
StructuredOutputError,
|
|
build_structured_request,
|
|
ensure_compatible,
|
|
schema_hash,
|
|
)
|
|
|
|
OUTPUT_SCHEMA = {
|
|
"type": "object",
|
|
"properties": {"answer": {"type": "string"}},
|
|
"required": ["answer"],
|
|
"additionalProperties": False,
|
|
}
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("mode", "expected_response_type", "schema_in_system_message"),
|
|
[
|
|
("native_json_schema", "json_schema", False),
|
|
("json_object", "json_object", True),
|
|
("prompt_schema", None, True),
|
|
],
|
|
)
|
|
def test_structured_modes_build_stable_requests(
|
|
mode,
|
|
expected_response_type,
|
|
schema_in_system_message,
|
|
):
|
|
messages = [{"role": "user", "content": "业务输入"}]
|
|
|
|
request_messages, response_format, metadata = build_structured_request(
|
|
messages,
|
|
mode=mode,
|
|
output_schema=OUTPUT_SCHEMA,
|
|
schema_name="draft.result/v1",
|
|
)
|
|
|
|
assert messages == [{"role": "user", "content": "业务输入"}]
|
|
assert metadata == {
|
|
"mode": mode,
|
|
"schema_name": "draft_result_v1",
|
|
"schema_hash": schema_hash(OUTPUT_SCHEMA),
|
|
"response_format_included": expected_response_type is not None,
|
|
"schema_in_system_message": schema_in_system_message,
|
|
}
|
|
if expected_response_type is None:
|
|
assert response_format is None
|
|
else:
|
|
assert response_format["type"] == expected_response_type
|
|
if mode == "native_json_schema":
|
|
assert response_format["json_schema"] == {
|
|
"name": "draft_result_v1",
|
|
"strict": True,
|
|
"schema": OUTPUT_SCHEMA,
|
|
}
|
|
assert request_messages == messages
|
|
else:
|
|
assert request_messages[0]["role"] == "system"
|
|
assert "<output_schema>" in request_messages[0]["content"]
|
|
assert '"additionalProperties":false' in request_messages[0]["content"]
|
|
assert request_messages[1:] == messages
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("mode", "enforcement", "compatible"),
|
|
[
|
|
("native_json_schema", "native_json_schema", True),
|
|
("native_json_schema", "json_object", True),
|
|
("native_json_schema", "validated_json", True),
|
|
("json_object", "native_json_schema", False),
|
|
("json_object", "json_object", True),
|
|
("json_object", "validated_json", True),
|
|
("prompt_schema", "native_json_schema", False),
|
|
("prompt_schema", "json_object", False),
|
|
("prompt_schema", "validated_json", True),
|
|
],
|
|
)
|
|
def test_all_model_and_skill_compatibility_combinations(mode, enforcement, compatible):
|
|
if compatible:
|
|
ensure_compatible(mode=mode, test_status="passed", enforcement=enforcement)
|
|
return
|
|
with pytest.raises(StructuredOutputError) as caught:
|
|
ensure_compatible(mode=mode, test_status="passed", enforcement=enforcement)
|
|
assert caught.value.code == "structured_output_incompatible"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("mode", "status", "expected_code"),
|
|
[
|
|
("unconfigured", "untested", "structured_output_not_configured"),
|
|
("native_json_schema", "untested", "structured_output_not_verified"),
|
|
("native_json_schema", "failed", "structured_output_not_verified"),
|
|
],
|
|
)
|
|
def test_unconfigured_or_unverified_models_are_rejected(mode, status, expected_code):
|
|
with pytest.raises(StructuredOutputError) as caught:
|
|
ensure_compatible(
|
|
mode=mode,
|
|
test_status=status,
|
|
enforcement="validated_json",
|
|
)
|
|
assert caught.value.code == expected_code
|
|
|
|
|
|
def test_native_mode_rejects_unsupported_schema_keyword_before_provider_call():
|
|
schema = {
|
|
"type": "object",
|
|
"properties": {
|
|
"missing_fields": {
|
|
"type": "array",
|
|
"items": {"type": "string"},
|
|
"uniqueItems": True,
|
|
}
|
|
},
|
|
"required": ["missing_fields"],
|
|
"additionalProperties": False,
|
|
}
|
|
|
|
with pytest.raises(StructuredOutputError) as caught:
|
|
build_structured_request(
|
|
[{"role": "user", "content": "业务输入"}],
|
|
mode="native_json_schema",
|
|
output_schema=schema,
|
|
schema_name="facts",
|
|
)
|
|
|
|
assert caught.value.code == "structured_output_schema_unsupported"
|
|
assert "$.properties.missing_fields.uniqueItems" in str(caught.value)
|
|
|
|
|
|
def test_prompt_modes_keep_full_business_schema():
|
|
schema = {
|
|
"type": "object",
|
|
"properties": {
|
|
"values": {
|
|
"type": "array",
|
|
"items": {"type": "string"},
|
|
"uniqueItems": True,
|
|
}
|
|
},
|
|
"required": ["values"],
|
|
"additionalProperties": False,
|
|
}
|
|
|
|
messages, response_format, _ = build_structured_request(
|
|
[{"role": "user", "content": "业务输入"}],
|
|
mode="prompt_schema",
|
|
output_schema=schema,
|
|
schema_name="values",
|
|
)
|
|
|
|
assert response_format is None
|
|
assert '"uniqueItems":true' in messages[0]["content"]
|