feat: init commint

This commit is contained in:
2026-09-18 15:26:19 +08:00
parent 3b61a767a0
commit 9f47dbb5cc
95 changed files with 7712 additions and 1137 deletions

View File

@@ -1,28 +1,113 @@
from django.db import IntegrityError
from datetime import date, timedelta
from math import ceil
from django.conf import settings
from django.db.models import Count, Q
from django.utils import timezone
from ninja import Router
from ninja.errors import HttpError
from apps.accounts.api import require_permission
from apps.accounts.api import require_login, require_permission
from apps.audit.models import OperationAuditLog
from apps.modelhub.crypto import CredentialEncryptionError, encrypt_secret, mask_secret
from apps.modelhub.crypto import mask_secret
from apps.modelhub.governance import (
ModelGovernanceError,
create_deployment_config,
create_provider_config,
deployment_impact,
provider_impact,
remove_deployment,
remove_provider,
restore_deployment,
restore_provider,
update_deployment_config,
update_provider_config,
)
from apps.modelhub.models import ModelDeployment, ModelInvocation, ModelProvider
from apps.modelhub.schemas import (
DeletionImpactOut,
LifecycleResultOut,
ModelDeploymentIn,
ModelDeploymentOptionOut,
ModelDeploymentOut,
ModelDeploymentPageOut,
ModelDeploymentUpdateIn,
ModelInvocationDiagnosticsOut,
ModelInvocationOut,
ModelInvocationPageOut,
ModelProviderIn,
ModelProviderOut,
ModelProviderPageOut,
ModelProviderUpdateIn,
ModelTestIn,
)
from apps.modelhub.services import create_model_test
from apps.modelhub.services import create_model_test, create_structured_output_test
from apps.modelhub.structured_output import (
CHAT_CAPABILITIES,
StructuredOutputError,
validate_mode_for_capability,
)
from apps.modelhub.tasks import run_model_test
router = Router(tags=["modelhub"])
def governance_http_error(error: ModelGovernanceError) -> HttpError:
return HttpError(error.status, f"{error.code}: {error}")
def structured_http_error(error: StructuredOutputError) -> HttpError:
return HttpError(400, f"{error.code}: {error}")
def require_superuser(request):
user = require_login(request)
if not user.is_superuser:
raise HttpError(403, "superuser_required: 仅超级管理员可执行生命周期操作")
return user
def require_any_permission(request, *codes: str):
user = require_login(request)
if user.is_superuser or any(user.has_perm(f"system.{code}") for code in codes):
return user
raise HttpError(403, "Permission denied")
def _user_label(user) -> str | None:
if not user:
return None
return user.get_full_name() or user.username
def _provider_status(provider: ModelProvider) -> str:
if provider.archived_at:
return "archived"
if not provider.enabled:
return "disabled"
if not provider.api_key_encrypted:
return "credential_missing"
return "ready"
def _deployment_status(deployment: ModelDeployment) -> str:
if deployment.archived_at:
return "archived"
if deployment.provider.archived_at:
return "provider_archived"
if not deployment.enabled:
return "disabled"
provider_status = _provider_status(deployment.provider)
if provider_status != "ready":
return f"provider_{provider_status}"
if (
deployment.capability in CHAT_CAPABILITIES
and deployment.structured_output_test_status != "passed"
):
return "unverified"
return "ready"
def serialize_provider(provider: ModelProvider) -> dict:
return {
"id": provider.id,
@@ -33,19 +118,58 @@ def serialize_provider(provider: ModelProvider) -> dict:
"credential_set": bool(provider.api_key_encrypted),
"credential_masked": mask_secret(provider.api_key_encrypted),
"image_egress_approved": provider.image_egress_approved,
"image_egress_approved_by": (
provider.image_egress_approved_by.get_full_name()
or provider.image_egress_approved_by.username
if provider.image_egress_approved_by_id
else None
),
"image_egress_approved_by": _user_label(provider.image_egress_approved_by),
"image_egress_approved_at": provider.image_egress_approved_at,
"credential_updated_at": provider.credential_updated_at,
"credential_updated_by": _user_label(provider.credential_updated_by),
"archived_at": provider.archived_at,
"archived_by": _user_label(provider.archived_by),
"deployment_count": (
provider.deployment_count
if hasattr(provider, "deployment_count")
else provider.deployments.count()
),
"active_deployment_count": (
provider.active_deployment_count
if hasattr(provider, "active_deployment_count")
else provider.deployments.filter(archived_at__isnull=True).count()
),
"effective_status": _provider_status(provider),
"created_at": provider.created_at,
"updated_at": provider.updated_at,
}
def serialize_deployment(deployment: ModelDeployment) -> dict:
def _reference_summary(deployment: ModelDeployment) -> dict[str, int]:
return {
"agent_version_count": (
deployment.agent_version_count
if hasattr(deployment, "agent_version_count")
else deployment.agent_versions.count()
),
"step_binding_count": (
deployment.step_binding_count
if hasattr(deployment, "step_binding_count")
else deployment.agent_step_bindings.count()
),
"run_step_count": (
deployment.run_step_count
if hasattr(deployment, "run_step_count")
else deployment.agent_run_steps.count()
),
"invocation_count": (
deployment.invocation_count
if hasattr(deployment, "invocation_count")
else deployment.invocations.count()
),
}
def serialize_deployment(deployment: ModelDeployment, metrics: dict | None = None) -> dict:
if metrics is None:
from apps.agents.structured_metrics import build_structured_output_metrics
metrics = build_structured_output_metrics(deployment_id=deployment.id)
return {
"id": deployment.id,
"provider_id": deployment.provider_id,
@@ -55,7 +179,74 @@ def serialize_deployment(deployment: ModelDeployment) -> dict:
"capability": deployment.capability,
"default_temperature": deployment.default_temperature,
"max_output_tokens": deployment.max_output_tokens,
"structured_output_mode": deployment.structured_output_mode,
"structured_output_test_status": deployment.structured_output_test_status,
"structured_output_verified_at": deployment.structured_output_verified_at,
"structured_output_test_message": deployment.structured_output_test_message,
"structured_output_metrics": metrics,
"enabled": deployment.enabled,
"effective_status": _deployment_status(deployment),
"provider_effective_status": _provider_status(deployment.provider),
"reference_summary": _reference_summary(deployment),
"archived_at": deployment.archived_at,
"archived_by": _user_label(deployment.archived_by),
"archived_via_provider": deployment.archived_via_provider,
}
def _diagnostics_expires_at(invocation: ModelInvocation):
return invocation.created_at + timedelta(
days=settings.MODEL_INVOCATION_DIAGNOSTIC_RETENTION_DAYS
)
def serialize_invocation(
invocation: ModelInvocation,
*,
include_diagnostics: bool = False,
) -> dict:
snapshot = invocation.model_snapshot or {}
deployment = invocation.deployment
expires_at = _diagnostics_expires_at(invocation)
available = (
invocation.diagnostics_scrubbed_at is None and expires_at > timezone.now()
)
summary = {
key: snapshot.get(key)
for key in (
"provider_id",
"provider_name",
"deployment_id",
"model_name",
"capability",
"structured_output",
)
if key in snapshot
}
return {
"public_id": invocation.public_id,
"deployment_id": invocation.deployment_id,
"deployment_display_name": deployment.display_name if deployment else None,
"provider_name": (
deployment.provider.name if deployment else snapshot.get("provider_name")
),
"invocation_type": invocation.invocation_type,
"status": invocation.status,
"model_snapshot": snapshot if include_diagnostics else {},
"model_snapshot_summary": summary,
"output_text": invocation.output_text if include_diagnostics and available else "",
"input_tokens": invocation.input_tokens,
"output_tokens": invocation.output_tokens,
"duration_ms": invocation.duration_ms,
"retry_count": invocation.retry_count,
"error_code": invocation.error_code,
"error_message": invocation.error_message[:500],
"created_at": invocation.created_at,
"started_at": invocation.started_at,
"finished_at": invocation.finished_at,
"diagnostics_scrubbed_at": invocation.diagnostics_scrubbed_at,
"diagnostics_expires_at": expires_at,
"diagnostics_available": available,
}
@@ -71,37 +262,58 @@ def audit(user, action: str, target_type: str, target_id: str, snapshot: dict) -
)
@router.get("/providers", response=list[ModelProviderOut])
def list_providers(request):
def _page(queryset, page: int, page_size: int):
page = max(page, 1)
page_size = min(max(page_size, 1), 100)
total = queryset.count()
start = (page - 1) * page_size
return list(queryset[start : start + page_size]), {
"page": page,
"page_size": page_size,
"total": total,
"total_pages": ceil(total / page_size) if total else 0,
}
@router.get("/providers", response=ModelProviderPageOut)
def list_providers(
request,
q: str = "",
enabled: str = "all",
archived: str = "active",
page: int = 1,
page_size: int = 20,
):
require_permission(request, "model.manage")
return [
serialize_provider(provider)
for provider in ModelProvider.objects.select_related("image_egress_approved_by")
]
queryset = ModelProvider.objects.select_related(
"image_egress_approved_by", "credential_updated_by", "archived_by"
).annotate(
deployment_count=Count("deployments", distinct=True),
active_deployment_count=Count(
"deployments",
filter=Q(deployments__archived_at__isnull=True),
distinct=True,
),
)
if q:
queryset = queryset.filter(Q(name__icontains=q) | Q(base_url__icontains=q))
if enabled in {"true", "false"}:
queryset = queryset.filter(enabled=enabled == "true")
if archived == "active":
queryset = queryset.filter(archived_at__isnull=True)
elif archived == "archived":
queryset = queryset.filter(archived_at__isnull=False)
rows, pagination = _page(queryset, page, page_size)
return {"items": [serialize_provider(item) for item in rows], **pagination}
@router.post("/providers", response=ModelProviderOut)
def create_provider(request, payload: ModelProviderIn):
require_permission(request, "model.manage")
if not 1 <= payload.default_timeout_seconds <= 300:
raise HttpError(400, "Timeout must be between 1 and 300 seconds")
try:
encrypted = encrypt_secret(payload.api_key)
provider = ModelProvider.objects.create(
name=payload.name,
base_url=payload.base_url.rstrip("/"),
api_key_encrypted=encrypted,
enabled=payload.enabled,
default_timeout_seconds=payload.default_timeout_seconds,
image_egress_approved=payload.image_egress_approved,
image_egress_approved_by=request.user if payload.image_egress_approved else None,
image_egress_approved_at=timezone.now() if payload.image_egress_approved else None,
created_by=request.user,
)
except CredentialEncryptionError as error:
raise HttpError(400, str(error)) from error
except IntegrityError as error:
raise HttpError(400, "Provider name already exists") from error
provider = create_provider_config(payload.dict(), request.user)
except ModelGovernanceError as error:
raise governance_http_error(error) from error
audit(request.user, "create", "model_provider", str(provider.id), {"name": provider.name})
return serialize_provider(provider)
@@ -109,58 +321,168 @@ def create_provider(request, payload: ModelProviderIn):
@router.patch("/providers/{provider_id}", response=ModelProviderOut)
def update_provider(request, provider_id: int, payload: ModelProviderUpdateIn):
require_permission(request, "model.manage")
provider = ModelProvider.objects.get(id=provider_id)
values = payload.dict(exclude_unset=True)
api_key = values.pop("api_key", None)
image_egress_approved = values.pop("image_egress_approved", None)
if api_key:
try:
provider.api_key_encrypted = encrypt_secret(api_key)
except CredentialEncryptionError as error:
raise HttpError(400, str(error)) from error
if "default_timeout_seconds" in values and not 1 <= values["default_timeout_seconds"] <= 300:
raise HttpError(400, "Timeout must be between 1 and 300 seconds")
for field, value in values.items():
setattr(provider, field, value.rstrip("/") if field == "base_url" else value)
if image_egress_approved is not None:
provider.image_egress_approved = image_egress_approved
provider.image_egress_approved_by = request.user if image_egress_approved else None
provider.image_egress_approved_at = timezone.now() if image_egress_approved else None
provider.save()
audit(
request.user,
"update",
"model_provider",
str(provider.id),
{
"fields": sorted(
[*values, *(["image_egress_approved"] if image_egress_approved is not None else [])]
),
"credential_replaced": bool(api_key),
},
)
provider = ModelProvider.objects.select_related(
"image_egress_approved_by", "credential_updated_by", "archived_by"
).get(id=provider_id)
try:
provider, snapshot = update_provider_config(
provider, payload.dict(exclude_unset=True), request.user
)
except ModelGovernanceError as error:
raise governance_http_error(error) from error
audit(request.user, "update", "model_provider", str(provider.id), snapshot)
return serialize_provider(provider)
@router.get("/deployments", response=list[ModelDeploymentOut])
def list_deployments(request, provider_id: int | None = None):
@router.get("/providers/{provider_id}/deletion-impact", response=DeletionImpactOut)
def get_provider_deletion_impact(request, provider_id: int):
require_superuser(request)
return provider_impact(ModelProvider.objects.get(id=provider_id))
@router.delete("/providers/{provider_id}", response=LifecycleResultOut)
def delete_provider(request, provider_id: int):
user = require_superuser(request)
try:
result, impact = remove_provider(provider_id, user)
except ModelGovernanceError as error:
raise governance_http_error(error) from error
audit(user, result, "model_provider", str(provider_id), {"impact": impact})
return {"result": result, "impact": impact}
@router.post("/providers/{provider_id}/restore", response=ModelProviderOut)
def restore_provider_api(request, provider_id: int):
user = require_superuser(request)
try:
provider = restore_provider(provider_id)
except ModelGovernanceError as error:
raise governance_http_error(error) from error
audit(user, "restore", "model_provider", str(provider_id), {"enabled": False})
return serialize_provider(provider)
@router.get("/deployments", response=ModelDeploymentPageOut)
def list_deployments(
request,
provider_id: int | None = None,
capability: str = "",
effective_status: str = "",
structured_test_status: str = "",
archived: str = "active",
q: str = "",
page: int = 1,
page_size: int = 20,
):
require_permission(request, "model.manage")
queryset = ModelDeployment.objects.select_related("provider")
queryset = ModelDeployment.objects.select_related(
"provider", "archived_by"
).annotate(
agent_version_count=Count("agent_versions", distinct=True),
step_binding_count=Count("agent_step_bindings", distinct=True),
run_step_count=Count("agent_run_steps", distinct=True),
invocation_count=Count("invocations", distinct=True),
)
if provider_id:
queryset = queryset.filter(provider_id=provider_id)
return [serialize_deployment(item) for item in queryset]
if capability:
queryset = queryset.filter(capability=capability)
if structured_test_status:
queryset = queryset.filter(
structured_output_test_status=structured_test_status
)
if q:
queryset = queryset.filter(
Q(display_name__icontains=q)
| Q(model_name__icontains=q)
| Q(provider__name__icontains=q)
)
if archived == "active":
queryset = queryset.filter(archived_at__isnull=True)
elif archived == "archived":
queryset = queryset.filter(archived_at__isnull=False)
if effective_status:
if effective_status == "archived":
queryset = queryset.filter(archived_at__isnull=False)
elif effective_status == "provider_archived":
queryset = queryset.filter(
archived_at__isnull=True,
provider__archived_at__isnull=False,
)
elif effective_status == "disabled":
queryset = queryset.filter(archived_at__isnull=True, enabled=False)
elif effective_status == "provider_disabled":
queryset = queryset.filter(
archived_at__isnull=True,
enabled=True,
provider__archived_at__isnull=True,
provider__enabled=False,
)
elif effective_status == "unverified":
queryset = queryset.filter(
archived_at__isnull=True,
provider__archived_at__isnull=True,
enabled=True,
provider__enabled=True,
capability__in=CHAT_CAPABILITIES,
).exclude(structured_output_test_status="passed")
elif effective_status == "ready":
queryset = queryset.filter(
archived_at__isnull=True,
provider__archived_at__isnull=True,
enabled=True,
provider__enabled=True,
).exclude(provider__api_key_encrypted="").filter(
Q(capability__in={"embedding", "rerank"})
| Q(structured_output_test_status="passed")
)
rows, pagination = _page(queryset, page, page_size)
from apps.agents.structured_metrics import build_deployment_structured_output_metrics
metrics = build_deployment_structured_output_metrics([item.id for item in rows])
return {
"items": [serialize_deployment(item, metrics[item.id]) for item in rows],
**pagination,
}
@router.get("/deployment-options", response=list[ModelDeploymentOptionOut])
def list_deployment_options(request):
require_any_permission(request, "agent.manage", "model.manage")
queryset = ModelDeployment.objects.select_related("provider").filter(
archived_at__isnull=True,
provider__archived_at__isnull=True,
enabled=True,
provider__enabled=True,
).exclude(provider__api_key_encrypted="")
items = [item for item in queryset if _deployment_status(item) == "ready"]
return [
{
"id": item.id,
"provider_id": item.provider_id,
"provider_name": item.provider.name,
"display_name": item.display_name,
"model_name": item.model_name,
"capability": item.capability,
"default_temperature": item.default_temperature,
"max_output_tokens": item.max_output_tokens,
"structured_output_mode": item.structured_output_mode,
"structured_output_test_status": item.structured_output_test_status,
"structured_output_verified_at": item.structured_output_verified_at,
"structured_output_test_message": item.structured_output_test_message,
"enabled": item.enabled,
}
for item in items
]
@router.post("/deployments", response=ModelDeploymentOut)
def create_deployment(request, payload: ModelDeploymentIn):
require_permission(request, "model.manage")
if not 0 <= payload.default_temperature <= 2:
raise HttpError(400, "Temperature must be between 0 and 2")
if not 1 <= payload.max_output_tokens <= 32768:
raise HttpError(400, "Max output tokens are out of range")
if payload.capability not in {"text_chat", "vision_chat", "embedding", "rerank"}:
raise HttpError(400, "Unknown model capability")
deployment = ModelDeployment.objects.create(**payload.dict())
try:
deployment = create_deployment_config(payload.dict())
except ModelGovernanceError as error:
raise governance_http_error(error) from error
deployment = ModelDeployment.objects.select_related("provider").get(id=deployment.id)
audit(
request.user,
@@ -175,34 +497,33 @@ def create_deployment(request, payload: ModelDeploymentIn):
@router.patch("/deployments/{deployment_id}", response=ModelDeploymentOut)
def update_deployment(request, deployment_id: int, payload: ModelDeploymentUpdateIn):
require_permission(request, "model.manage")
deployment = ModelDeployment.objects.select_related("provider").get(id=deployment_id)
values = payload.dict(exclude_unset=True)
if "capability" in values and values["capability"] not in {
"text_chat",
"vision_chat",
"embedding",
"rerank",
}:
raise HttpError(400, "Unknown model capability")
for field, value in values.items():
setattr(deployment, field, value)
deployment.save()
audit(
request.user,
"update",
"model_deployment",
str(deployment.id),
{"fields": sorted(values)},
deployment = ModelDeployment.objects.select_related("provider", "archived_by").get(
id=deployment_id
)
try:
deployment, snapshot = update_deployment_config(
deployment, payload.dict(exclude_unset=True)
)
except ModelGovernanceError as error:
raise governance_http_error(error) from error
audit(request.user, "update", "model_deployment", str(deployment.id), snapshot)
return serialize_deployment(deployment)
def _ensure_testable(deployment: ModelDeployment):
if deployment.archived_at or deployment.provider.archived_at:
raise HttpError(
400, "model_config_archived: 归档模型不允许编辑、测试或启用"
)
@router.post("/deployments/{deployment_id}/test", response=ModelInvocationOut)
def test_deployment(request, deployment_id: int, payload: ModelTestIn):
require_permission(request, "model.manage")
deployment = ModelDeployment.objects.select_related("provider").get(id=deployment_id)
if deployment.capability == "embedding":
raise HttpError(400, "Embedding 模型请通过知识检索验证")
_ensure_testable(deployment)
if deployment.capability not in CHAT_CAPABILITIES:
raise HttpError(400, "Embedding 或重排模型请通过对应能力入口验证")
invocation = create_model_test(deployment, payload.prompt, request.user)
run_model_test.delay(invocation.id)
invocation.refresh_from_db()
@@ -213,10 +534,152 @@ def test_deployment(request, deployment_id: int, payload: ModelTestIn):
str(deployment.id),
{"invocation_id": str(invocation.public_id)},
)
return invocation
return serialize_invocation(invocation, include_diagnostics=True)
@router.post(
"/deployments/{deployment_id}/structured-output-test",
response=ModelInvocationOut,
)
def test_structured_output(request, deployment_id: int):
require_permission(request, "model.manage")
deployment = ModelDeployment.objects.select_related("provider").get(id=deployment_id)
_ensure_testable(deployment)
if deployment.capability not in CHAT_CAPABILITIES:
raise HttpError(400, "该模型能力不适用结构化输出测试")
try:
validate_mode_for_capability(
deployment.capability, deployment.structured_output_mode
)
except StructuredOutputError as error:
raise structured_http_error(error) from error
deployment.structured_output_test_status = "untested"
deployment.structured_output_verified_at = None
deployment.structured_output_test_message = "结构化输出测试执行中"
deployment.save(
update_fields=[
"structured_output_test_status",
"structured_output_verified_at",
"structured_output_test_message",
"updated_at",
]
)
invocation = create_structured_output_test(deployment, request.user)
run_model_test.delay(invocation.id)
invocation.refresh_from_db()
audit(
request.user,
"structured_output_test",
"model_deployment",
str(deployment.id),
{
"invocation_id": str(invocation.public_id),
"mode": deployment.structured_output_mode,
},
)
return serialize_invocation(invocation, include_diagnostics=True)
@router.get("/deployments/{deployment_id}/deletion-impact", response=DeletionImpactOut)
def get_deployment_deletion_impact(request, deployment_id: int):
require_superuser(request)
return deployment_impact(ModelDeployment.objects.get(id=deployment_id))
@router.delete("/deployments/{deployment_id}", response=LifecycleResultOut)
def delete_deployment(request, deployment_id: int):
user = require_superuser(request)
try:
result, impact = remove_deployment(deployment_id, user)
except ModelGovernanceError as error:
raise governance_http_error(error) from error
audit(user, result, "model_deployment", str(deployment_id), {"impact": impact})
return {"result": result, "impact": impact}
@router.post("/deployments/{deployment_id}/restore", response=ModelDeploymentOut)
def restore_deployment_api(request, deployment_id: int):
user = require_superuser(request)
try:
deployment = restore_deployment(deployment_id)
except ModelGovernanceError as error:
raise governance_http_error(error) from error
audit(user, "restore", "model_deployment", str(deployment_id), {"enabled": False})
return serialize_deployment(deployment)
@router.get("/invocations", response=ModelInvocationPageOut)
def list_invocations(
request,
deployment_id: int | None = None,
provider_id: int | None = None,
invocation_type: str = "",
status: str = "",
date_from: date | None = None,
date_to: date | None = None,
error_code: str = "",
page: int = 1,
page_size: int = 20,
):
require_permission(request, "model.manage")
queryset = ModelInvocation.objects.select_related("deployment__provider")
if deployment_id:
queryset = queryset.filter(deployment_id=deployment_id)
if provider_id:
queryset = queryset.filter(deployment__provider_id=provider_id)
if invocation_type:
queryset = queryset.filter(invocation_type=invocation_type)
if status:
queryset = queryset.filter(status=status)
if date_from:
queryset = queryset.filter(created_at__date__gte=date_from)
if date_to:
queryset = queryset.filter(created_at__date__lte=date_to)
if error_code:
queryset = queryset.filter(error_code=error_code)
rows, pagination = _page(queryset, page, page_size)
return {"items": [serialize_invocation(item) for item in rows], **pagination}
@router.get("/invocations/{public_id}", response=ModelInvocationOut)
def get_invocation(request, public_id: str):
require_permission(request, "model.manage")
invocation = ModelInvocation.objects.select_related("deployment__provider").get(
public_id=public_id
)
return serialize_invocation(invocation)
@router.get(
"/invocations/{public_id}/diagnostics",
response=ModelInvocationDiagnosticsOut,
)
def get_invocation_diagnostics(request, public_id: str):
require_permission(request, "model.invocation.view")
invocation = ModelInvocation.objects.get(public_id=public_id)
expires_at = _diagnostics_expires_at(invocation)
available = (
invocation.diagnostics_scrubbed_at is None and expires_at > timezone.now()
)
if not available:
raise HttpError(
410, "diagnostics_expired: 模型调用诊断内容已超过保留期"
)
return {
"public_id": invocation.public_id,
"input_snapshot": invocation.input_snapshot,
"output_text": invocation.output_text,
"diagnostics_scrubbed_at": invocation.diagnostics_scrubbed_at,
"diagnostics_expires_at": expires_at,
"diagnostics_available": True,
}
@router.get("/model-tests/{public_id}", response=ModelInvocationOut)
def get_model_test(request, public_id: str):
require_permission(request, "model.manage")
return ModelInvocation.objects.get(public_id=public_id, invocation_type="model_test")
invocation = ModelInvocation.objects.select_related("deployment__provider").get(
public_id=public_id,
invocation_type__in={"model_test", "structured_output_test"},
)
return serialize_invocation(invocation, include_diagnostics=True)

View File

@@ -0,0 +1,439 @@
from __future__ import annotations
from typing import Any
from django.core.exceptions import ValidationError
from django.core.validators import URLValidator
from django.db import IntegrityError, transaction
from django.utils import timezone
from apps.modelhub.crypto import CredentialEncryptionError, encrypt_secret
from apps.modelhub.models import ModelDeployment, ModelProvider
from apps.modelhub.structured_output import (
StructuredOutputError,
validate_mode_for_capability,
)
CHAT_CAPABILITIES = {"text_chat", "vision_chat"}
CAPABILITIES = CHAT_CAPABILITIES | {"embedding", "rerank"}
class ModelGovernanceError(Exception):
def __init__(self, code: str, message: str, *, status: int = 400):
super().__init__(message)
self.code = code
self.status = status
def _clean_name(value: str, field_label: str) -> str:
cleaned = value.strip()
if not cleaned:
raise ModelGovernanceError("invalid_model_config", f"{field_label}不能为空")
return cleaned
def validate_provider_values(values: dict[str, Any], provider: ModelProvider | None = None) -> dict:
cleaned = dict(values)
if "name" in cleaned:
cleaned["name"] = _clean_name(cleaned["name"], "渠道名称")
if "base_url" in cleaned:
base_url = cleaned["base_url"].strip().rstrip("/")
try:
URLValidator(schemes=["http", "https"])(base_url)
except ValidationError as error:
raise ModelGovernanceError(
"invalid_provider_url", "渠道地址必须是有效的 HTTP(S) URL"
) from error
cleaned["base_url"] = base_url
timeout = cleaned.get(
"default_timeout_seconds",
provider.default_timeout_seconds if provider else 60,
)
if not 1 <= timeout <= 300:
raise ModelGovernanceError(
"invalid_provider_timeout", "请求超时必须在 1 到 300 秒之间"
)
if cleaned.get("clear_api_key") and cleaned.get("api_key"):
raise ModelGovernanceError(
"credential_action_conflict", "不能同时替换和清除渠道凭证"
)
return cleaned
def validate_deployment_values(
values: dict[str, Any],
deployment: ModelDeployment | None = None,
) -> dict:
cleaned = dict(values)
for field, label in (("display_name", "显示名称"), ("model_name", "实际模型名")):
if field in cleaned:
cleaned[field] = _clean_name(cleaned[field], label)
capability = cleaned.get(
"capability", deployment.capability if deployment else "text_chat"
)
if capability not in CAPABILITIES:
raise ModelGovernanceError("invalid_model_capability", "未知的模型能力")
if capability not in CHAT_CAPABILITIES:
cleaned["structured_output_mode"] = "none"
temperature = cleaned.get(
"default_temperature",
deployment.default_temperature if deployment else 0.2,
)
if not 0 <= temperature <= 2:
raise ModelGovernanceError(
"invalid_model_temperature", "Temperature 必须在 0 到 2 之间"
)
max_tokens = cleaned.get(
"max_output_tokens", deployment.max_output_tokens if deployment else 2048
)
if not 1 <= max_tokens <= 32768:
raise ModelGovernanceError(
"invalid_model_max_tokens", "最大输出 Token 必须在 1 到 32768 之间"
)
mode = cleaned.get(
"structured_output_mode",
deployment.structured_output_mode if deployment else "unconfigured",
)
try:
validate_mode_for_capability(capability, mode)
except StructuredOutputError as error:
raise ModelGovernanceError(error.code, str(error)) from error
return cleaned
def create_provider_config(values: dict[str, Any], user) -> ModelProvider:
cleaned = validate_provider_values(values)
api_key = cleaned.pop("api_key").strip()
if not api_key:
raise ModelGovernanceError(
"credential_required", "创建渠道时必须录入有效凭证"
)
cleaned.pop("clear_api_key", None)
approved = cleaned.get("image_egress_approved", False)
now = timezone.now()
try:
encrypted = encrypt_secret(api_key)
return ModelProvider.objects.create(
**cleaned,
api_key_encrypted=encrypted,
credential_updated_at=now,
credential_updated_by=user,
image_egress_approved_by=user if approved else None,
image_egress_approved_at=now if approved else None,
created_by=user,
)
except CredentialEncryptionError as error:
raise ModelGovernanceError("credential_encryption_failed", str(error)) from error
except IntegrityError as error:
raise ModelGovernanceError(
"provider_name_conflict", "活跃渠道名称已存在", status=409
) from error
def update_provider_config(
provider: ModelProvider,
values: dict[str, Any],
user,
) -> tuple[ModelProvider, dict[str, Any]]:
if provider.archived_at:
raise ModelGovernanceError(
"model_config_archived", "归档渠道不允许编辑、测试或启用"
)
cleaned = validate_provider_values(values, provider)
api_key = cleaned.pop("api_key", None)
clear_api_key = cleaned.pop("clear_api_key", False)
approved = cleaned.pop("image_egress_approved", None)
if (
cleaned.get("enabled") is True
and not clear_api_key
and not api_key
and not provider.api_key_encrypted
):
raise ModelGovernanceError(
"credential_required", "启用渠道前必须录入有效凭证"
)
base_url_changed = (
"base_url" in cleaned and cleaned["base_url"] != provider.base_url.rstrip("/")
)
credential_action = "unchanged"
now = timezone.now()
try:
if api_key:
provider.api_key_encrypted = encrypt_secret(api_key)
provider.credential_updated_at = now
provider.credential_updated_by = user
credential_action = "replaced"
elif clear_api_key:
provider.api_key_encrypted = ""
provider.credential_updated_at = now
provider.credential_updated_by = user
provider.enabled = False
credential_action = "cleared"
for field, value in cleaned.items():
setattr(provider, field, value)
if clear_api_key:
provider.enabled = False
if approved is not None:
provider.image_egress_approved = approved
provider.image_egress_approved_by = user if approved else None
provider.image_egress_approved_at = now if approved else None
provider.save()
except CredentialEncryptionError as error:
raise ModelGovernanceError("credential_encryption_failed", str(error)) from error
except IntegrityError as error:
raise ModelGovernanceError(
"provider_name_conflict", "活跃渠道名称已存在", status=409
) from error
if base_url_changed:
provider.deployments.filter(
archived_at__isnull=True,
capability__in=CHAT_CAPABILITIES,
).update(
structured_output_test_status="untested",
structured_output_verified_at=None,
structured_output_test_message="渠道地址已变更,请重新验证结构化输出",
)
return provider, {
"fields": sorted([*cleaned, *(["image_egress_approved"] if approved is not None else [])]),
"credential_action": credential_action,
}
def create_deployment_config(values: dict[str, Any]) -> ModelDeployment:
cleaned = validate_deployment_values(values)
provider = ModelProvider.objects.get(id=cleaned.pop("provider_id"))
if provider.archived_at:
raise ModelGovernanceError(
"model_config_archived", "不能在已归档渠道下创建模型"
)
try:
return ModelDeployment.objects.create(provider=provider, **cleaned)
except IntegrityError as error:
raise ModelGovernanceError(
"deployment_name_conflict",
"同一活跃渠道下实际模型名已存在",
status=409,
) from error
def update_deployment_config(
deployment: ModelDeployment,
values: dict[str, Any],
) -> tuple[ModelDeployment, dict[str, Any]]:
if deployment.archived_at or deployment.provider.archived_at:
raise ModelGovernanceError(
"model_config_archived", "归档模型不允许编辑、测试或启用"
)
cleaned = validate_deployment_values(values, deployment)
reset_test = any(
field in cleaned and cleaned[field] != getattr(deployment, field)
for field in {"model_name", "structured_output_mode", "capability"}
)
for field, value in cleaned.items():
setattr(deployment, field, value)
if reset_test:
deployment.structured_output_test_status = "untested"
deployment.structured_output_verified_at = None
deployment.structured_output_test_message = (
"模型配置已变更,请重新验证结构化输出"
if deployment.capability in CHAT_CAPABILITIES
else ""
)
try:
deployment.save()
except IntegrityError as error:
raise ModelGovernanceError(
"deployment_name_conflict",
"同一活跃渠道下实际模型名已存在",
status=409,
) from error
return deployment, {"fields": sorted(cleaned), "structured_test_reset": reset_test}
def deployment_impact(deployment: ModelDeployment) -> dict[str, int | str]:
counts = {
"agent_version_count": deployment.agent_versions.count(),
"step_binding_count": deployment.agent_step_bindings.count(),
"run_step_count": deployment.agent_run_steps.count(),
"invocation_count": deployment.invocations.count(),
}
return {
"suggested_action": "archive" if any(counts.values()) else "delete",
"deployment_count": 1,
**counts,
}
def provider_impact(provider: ModelProvider) -> dict[str, int | str]:
from django.db.models import Count, Sum
deployments = provider.deployments.annotate(
agent_version_refs=Count("agent_versions", distinct=True),
step_binding_refs=Count("agent_step_bindings", distinct=True),
run_step_refs=Count("agent_run_steps", distinct=True),
invocation_refs=Count("invocations", distinct=True),
)
totals = deployments.aggregate(
agent_version_count=Sum("agent_version_refs"),
step_binding_count=Sum("step_binding_refs"),
run_step_count=Sum("run_step_refs"),
invocation_count=Sum("invocation_refs"),
)
counts = {key: value or 0 for key, value in totals.items()}
return {
"suggested_action": "archive" if any(counts.values()) else "delete",
"deployment_count": deployments.count(),
**counts,
}
@transaction.atomic
def remove_deployment(deployment_id: int, user) -> tuple[str, dict]:
deployment = (
ModelDeployment.objects.select_for_update()
.select_related("provider")
.get(id=deployment_id)
)
impact = deployment_impact(deployment)
if deployment.archived_at:
return "archived", impact
if impact["suggested_action"] == "delete":
deployment.delete()
return "deleted", impact
deployment.enabled = False
deployment.archived_at = timezone.now()
deployment.archived_by = user
deployment.archived_via_provider = False
deployment.save(
update_fields=[
"enabled",
"archived_at",
"archived_by",
"archived_via_provider",
"updated_at",
]
)
return "archived", impact
@transaction.atomic
def remove_provider(provider_id: int, user) -> tuple[str, dict]:
provider = ModelProvider.objects.select_for_update().get(id=provider_id)
list(provider.deployments.select_for_update())
impact = provider_impact(provider)
if provider.archived_at:
return "archived", impact
if impact["suggested_action"] == "delete":
provider.deployments.all().delete()
provider.delete()
return "deleted", impact
now = timezone.now()
provider.deployments.filter(archived_at__isnull=True).update(
enabled=False,
archived_at=now,
archived_by=user,
archived_via_provider=True,
)
provider.enabled = False
provider.archived_at = now
provider.archived_by = user
provider.api_key_encrypted = ""
provider.credential_updated_at = now
provider.credential_updated_by = user
provider.image_egress_approved = False
provider.image_egress_approved_by = None
provider.image_egress_approved_at = None
provider.save()
return "archived", impact
@transaction.atomic
def restore_deployment(deployment_id: int) -> ModelDeployment:
deployment = (
ModelDeployment.objects.select_for_update()
.select_related("provider")
.get(id=deployment_id)
)
if deployment.archived_at is None:
deployment.enabled = False
deployment.save(update_fields=["enabled", "updated_at"])
return deployment
if deployment.provider.archived_at:
raise ModelGovernanceError(
"provider_archived", "请先恢复模型所属渠道", status=409
)
if ModelDeployment.objects.filter(
provider_id=deployment.provider_id,
model_name=deployment.model_name,
archived_at__isnull=True,
).exclude(id=deployment.id).exists():
raise ModelGovernanceError(
"lifecycle_name_conflict", "恢复失败:活跃模型已复用该实际模型名", status=409
)
deployment.archived_at = None
deployment.archived_by = None
deployment.archived_via_provider = False
deployment.enabled = False
deployment.structured_output_test_status = "untested"
deployment.structured_output_verified_at = None
deployment.structured_output_test_message = (
"模型已恢复,请重新验证"
if deployment.capability in CHAT_CAPABILITIES
else ""
)
if deployment.capability not in CHAT_CAPABILITIES:
deployment.structured_output_mode = "none"
deployment.save()
return deployment
@transaction.atomic
def restore_provider(provider_id: int) -> ModelProvider:
provider = ModelProvider.objects.select_for_update().get(id=provider_id)
children = list(provider.deployments.select_for_update())
if ModelProvider.objects.filter(
name=provider.name, archived_at__isnull=True
).exclude(id=provider.id).exists():
raise ModelGovernanceError(
"lifecycle_name_conflict", "恢复失败:活跃渠道已复用该名称", status=409
)
for deployment in children:
if not deployment.archived_via_provider:
continue
if ModelDeployment.objects.filter(
provider_id=provider.id,
model_name=deployment.model_name,
archived_at__isnull=True,
).exclude(id=deployment.id).exists():
raise ModelGovernanceError(
"lifecycle_name_conflict",
f"恢复失败:模型名 {deployment.model_name} 已被复用",
status=409,
)
provider.archived_at = None
provider.archived_by = None
provider.enabled = False
provider.save(update_fields=["archived_at", "archived_by", "enabled", "updated_at"])
restored_ids = list(
provider.deployments.filter(archived_via_provider=True).values_list(
"id",
flat=True,
)
)
restored_deployments = provider.deployments.filter(id__in=restored_ids)
restored_deployments.update(
archived_at=None,
archived_by=None,
archived_via_provider=False,
enabled=False,
structured_output_test_status="untested",
structured_output_verified_at=None,
)
restored_deployments.filter(capability__in=CHAT_CAPABILITIES).update(
structured_output_test_message="渠道已恢复,请重新录入凭证并验证",
)
restored_deployments.exclude(capability__in=CHAT_CAPABILITIES).update(
structured_output_mode="none",
structured_output_test_message="",
)
return provider

View File

@@ -0,0 +1,56 @@
from django.db import migrations, models
def configure_non_chat_models(apps, schema_editor):
ModelDeployment = apps.get_model("modelhub", "ModelDeployment")
ModelDeployment.objects.filter(capability__in={"embedding", "rerank"}).update(
structured_output_mode="none",
)
class Migration(migrations.Migration):
dependencies = [
("modelhub", "0002_modelprovider_image_egress_approved_and_more"),
]
operations = [
migrations.AddField(
model_name="modeldeployment",
name="structured_output_mode",
field=models.CharField(
choices=[
("unconfigured", "未配置"),
("native_json_schema", "原生 JSON Schema"),
("json_object", "JSON Object"),
("prompt_schema", "Prompt Schema"),
("none", "不支持"),
],
default="unconfigured",
max_length=32,
),
),
migrations.AddField(
model_name="modeldeployment",
name="structured_output_test_status",
field=models.CharField(
choices=[
("untested", "未验证"),
("passed", "已通过"),
("failed", "验证失败"),
],
default="untested",
max_length=16,
),
),
migrations.AddField(
model_name="modeldeployment",
name="structured_output_test_message",
field=models.TextField(blank=True),
),
migrations.AddField(
model_name="modeldeployment",
name="structured_output_verified_at",
field=models.DateTimeField(blank=True, null=True),
),
migrations.RunPython(configure_non_chat_models, migrations.RunPython.noop),
]

View File

@@ -0,0 +1,103 @@
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
("modelhub", "0003_structured_output_governance"),
]
operations = [
migrations.AlterField(
model_name="modelprovider",
name="name",
field=models.CharField(max_length=120),
),
migrations.AddField(
model_name="modelprovider",
name="archived_at",
field=models.DateTimeField(blank=True, db_index=True, null=True),
),
migrations.AddField(
model_name="modelprovider",
name="archived_by",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="archived_model_providers",
to=settings.AUTH_USER_MODEL,
),
),
migrations.AddField(
model_name="modelprovider",
name="credential_updated_at",
field=models.DateTimeField(blank=True, null=True),
),
migrations.AddField(
model_name="modelprovider",
name="credential_updated_by",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="updated_model_provider_credentials",
to=settings.AUTH_USER_MODEL,
),
),
migrations.AddField(
model_name="modeldeployment",
name="archived_at",
field=models.DateTimeField(blank=True, db_index=True, null=True),
),
migrations.AddField(
model_name="modeldeployment",
name="archived_by",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="archived_model_deployments",
to=settings.AUTH_USER_MODEL,
),
),
migrations.AddField(
model_name="modeldeployment",
name="archived_via_provider",
field=models.BooleanField(default=False),
),
migrations.AddField(
model_name="modelinvocation",
name="diagnostics_scrubbed_at",
field=models.DateTimeField(blank=True, null=True),
),
migrations.RemoveConstraint(
model_name="modeldeployment",
name="unique_provider_model_name",
),
migrations.AddConstraint(
model_name="modelprovider",
constraint=models.UniqueConstraint(
condition=models.Q(("archived_at__isnull", True)),
fields=("name",),
name="unique_active_model_provider_name",
),
),
migrations.AddConstraint(
model_name="modeldeployment",
constraint=models.UniqueConstraint(
condition=models.Q(("archived_at__isnull", True)),
fields=("provider", "model_name"),
name="unique_active_provider_model_name",
),
),
migrations.AddIndex(
model_name="modelinvocation",
index=models.Index(
fields=["deployment", "created_at"],
name="modelhub_mo_deploym_844a9d_idx",
),
),
]

View File

@@ -0,0 +1,24 @@
from django.db import migrations
def normalize_non_chat_structured_output(apps, schema_editor):
ModelDeployment = apps.get_model("modelhub", "ModelDeployment")
ModelDeployment.objects.filter(capability__in={"embedding", "rerank"}).update(
structured_output_mode="none",
structured_output_test_status="untested",
structured_output_verified_at=None,
structured_output_test_message="",
)
class Migration(migrations.Migration):
dependencies = [
("modelhub", "0004_modelhub_lifecycle_governance"),
]
operations = [
migrations.RunPython(
normalize_non_chat_structured_output,
migrations.RunPython.noop,
),
]

View File

@@ -2,10 +2,11 @@ import uuid
from django.conf import settings
from django.db import models
from django.db.models import Q
class ModelProvider(models.Model):
name = models.CharField(max_length=120, unique=True)
name = models.CharField(max_length=120)
base_url = models.URLField(max_length=500)
api_key_encrypted = models.TextField(blank=True)
enabled = models.BooleanField(default=True)
@@ -19,6 +20,22 @@ class ModelProvider(models.Model):
related_name="approved_image_model_providers",
)
image_egress_approved_at = models.DateTimeField(null=True, blank=True)
credential_updated_at = models.DateTimeField(null=True, blank=True)
credential_updated_by = models.ForeignKey(
settings.AUTH_USER_MODEL,
null=True,
blank=True,
on_delete=models.SET_NULL,
related_name="updated_model_provider_credentials",
)
archived_at = models.DateTimeField(null=True, blank=True, db_index=True)
archived_by = models.ForeignKey(
settings.AUTH_USER_MODEL,
null=True,
blank=True,
on_delete=models.SET_NULL,
related_name="archived_model_providers",
)
created_by = models.ForeignKey(
settings.AUTH_USER_MODEL,
null=True,
@@ -31,9 +48,28 @@ class ModelProvider(models.Model):
class Meta:
ordering = ["name"]
constraints = [
models.UniqueConstraint(
fields=["name"],
condition=Q(archived_at__isnull=True),
name="unique_active_model_provider_name",
)
]
class ModelDeployment(models.Model):
class StructuredOutputMode(models.TextChoices):
UNCONFIGURED = "unconfigured", "未配置"
NATIVE_JSON_SCHEMA = "native_json_schema", "原生 JSON Schema"
JSON_OBJECT = "json_object", "JSON Object"
PROMPT_SCHEMA = "prompt_schema", "Prompt Schema"
NONE = "none", "不支持"
class StructuredOutputTestStatus(models.TextChoices):
UNTESTED = "untested", "未验证"
PASSED = "passed", "已通过"
FAILED = "failed", "验证失败"
provider = models.ForeignKey(
ModelProvider,
on_delete=models.PROTECT,
@@ -44,7 +80,28 @@ class ModelDeployment(models.Model):
capability = models.CharField(max_length=40, default="text_chat")
default_temperature = models.FloatField(default=0.2)
max_output_tokens = models.PositiveIntegerField(default=2048)
structured_output_mode = models.CharField(
max_length=32,
choices=StructuredOutputMode.choices,
default=StructuredOutputMode.UNCONFIGURED,
)
structured_output_test_status = models.CharField(
max_length=16,
choices=StructuredOutputTestStatus.choices,
default=StructuredOutputTestStatus.UNTESTED,
)
structured_output_verified_at = models.DateTimeField(null=True, blank=True)
structured_output_test_message = models.TextField(blank=True)
enabled = models.BooleanField(default=True)
archived_at = models.DateTimeField(null=True, blank=True, db_index=True)
archived_by = models.ForeignKey(
settings.AUTH_USER_MODEL,
null=True,
blank=True,
on_delete=models.SET_NULL,
related_name="archived_model_deployments",
)
archived_via_provider = models.BooleanField(default=False)
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
@@ -53,7 +110,8 @@ class ModelDeployment(models.Model):
constraints = [
models.UniqueConstraint(
fields=["provider", "model_name"],
name="unique_provider_model_name",
condition=Q(archived_at__isnull=True),
name="unique_active_provider_model_name",
)
]
@@ -78,6 +136,7 @@ class ModelInvocation(models.Model):
retry_count = models.PositiveIntegerField(default=0)
error_code = models.CharField(max_length=80, blank=True)
error_message = models.TextField(blank=True)
diagnostics_scrubbed_at = models.DateTimeField(null=True, blank=True)
created_by = models.ForeignKey(
settings.AUTH_USER_MODEL,
null=True,
@@ -94,4 +153,5 @@ class ModelInvocation(models.Model):
indexes = [
models.Index(fields=["status", "created_at"]),
models.Index(fields=["invocation_type", "created_at"]),
models.Index(fields=["deployment", "created_at"]),
]

View File

@@ -18,6 +18,7 @@ class ModelProviderUpdateIn(Schema):
name: str | None = None
base_url: str | None = None
api_key: str | None = None
clear_api_key: bool = False
enabled: bool | None = None
default_timeout_seconds: int | None = None
image_egress_approved: bool | None = None
@@ -34,6 +35,13 @@ class ModelProviderOut(Schema):
image_egress_approved: bool
image_egress_approved_by: str | None
image_egress_approved_at: datetime | None
credential_updated_at: datetime | None
credential_updated_by: str | None
archived_at: datetime | None
archived_by: str | None
deployment_count: int = 0
active_deployment_count: int = 0
effective_status: str
created_at: datetime
updated_at: datetime
@@ -45,6 +53,7 @@ class ModelDeploymentIn(Schema):
capability: str = "text_chat"
default_temperature: float = 0.2
max_output_tokens: int = 2048
structured_output_mode: str = "unconfigured"
enabled: bool = True
@@ -54,6 +63,7 @@ class ModelDeploymentUpdateIn(Schema):
capability: str | None = None
default_temperature: float | None = None
max_output_tokens: int | None = None
structured_output_mode: str | None = None
enabled: bool | None = None
@@ -66,6 +76,49 @@ class ModelDeploymentOut(Schema):
capability: str
default_temperature: float
max_output_tokens: int
structured_output_mode: str
structured_output_test_status: str
structured_output_verified_at: datetime | None
structured_output_test_message: str
structured_output_metrics: dict[str, Any]
enabled: bool
effective_status: str
provider_effective_status: str
reference_summary: dict[str, int]
archived_at: datetime | None
archived_by: str | None
archived_via_provider: bool
class ModelProviderPageOut(Schema):
items: list[ModelProviderOut]
page: int
page_size: int
total: int
total_pages: int
class ModelDeploymentPageOut(Schema):
items: list[ModelDeploymentOut]
page: int
page_size: int
total: int
total_pages: int
class ModelDeploymentOptionOut(Schema):
id: int
provider_id: int
provider_name: str
display_name: str
model_name: str
capability: str
default_temperature: float
max_output_tokens: int
structured_output_mode: str
structured_output_test_status: str
structured_output_verified_at: datetime | None
structured_output_test_message: str
enabled: bool
@@ -76,6 +129,7 @@ class ModelTestIn(Schema):
class ModelInvocationOut(Schema):
public_id: UUID
deployment_id: int | None
invocation_type: str
status: str
model_snapshot: dict[str, Any]
output_text: str
@@ -88,3 +142,40 @@ class ModelInvocationOut(Schema):
created_at: datetime
started_at: datetime | None
finished_at: datetime | None
diagnostics_scrubbed_at: datetime | None = None
diagnostics_expires_at: datetime | None = None
diagnostics_available: bool = True
deployment_display_name: str | None = None
provider_name: str | None = None
model_snapshot_summary: dict[str, Any] = {}
class ModelInvocationPageOut(Schema):
items: list[ModelInvocationOut]
page: int
page_size: int
total: int
total_pages: int
class ModelInvocationDiagnosticsOut(Schema):
public_id: UUID
input_snapshot: dict[str, Any]
output_text: str
diagnostics_scrubbed_at: datetime | None
diagnostics_expires_at: datetime
diagnostics_available: bool
class DeletionImpactOut(Schema):
suggested_action: str
deployment_count: int
agent_version_count: int
step_binding_count: int
run_step_count: int
invocation_count: int
class LifecycleResultOut(Schema):
result: str
impact: DeletionImpactOut

View File

@@ -21,6 +21,12 @@ from openai import (
from apps.modelhub.crypto import CredentialEncryptionError, decrypt_secret
from apps.modelhub.models import ModelDeployment, ModelInvocation
from apps.modelhub.structured_output import (
STRUCTURED_OUTPUT_TEST_PROMPT,
STRUCTURED_OUTPUT_TEST_SCHEMA,
StructuredOutputError,
build_structured_request,
)
TRANSIENT_ERRORS = (APIConnectionError, APITimeoutError, RateLimitError)
@@ -44,6 +50,7 @@ def _request_completion(
*,
temperature: float,
max_output_tokens: int,
response_format: dict[str, Any] | None = None,
timeout_seconds: float | None = None,
) -> CompletionResult:
provider = deployment.provider
@@ -57,11 +64,16 @@ def _request_completion(
# Retry here so the platform can expose it instead of waiting on opaque SDK retries.
max_retries=0,
)
request_options: dict[str, Any] = {
"model": deployment.model_name,
"messages": messages,
"temperature": temperature,
"max_tokens": max_output_tokens,
}
if response_format is not None:
request_options["response_format"] = response_format
response = client.chat.completions.create(
model=deployment.model_name,
messages=messages,
temperature=temperature,
max_tokens=max_output_tokens,
**request_options,
)
content = response.choices[0].message.content or ""
usage = response.usage
@@ -96,13 +108,21 @@ def execute_invocation(
*,
temperature: float | None = None,
max_output_tokens: int | None = None,
output_schema: dict[str, Any] | None = None,
schema_name: str | None = None,
on_retry: Callable[[int, str], None] | None = None,
total_timeout_seconds: float | None = None,
) -> ModelInvocation:
"""在总时间预算内调用模型,分类错误、限制重试并记录脱敏终态。"""
deployment = invocation.deployment
if deployment is None or not deployment.enabled or not deployment.provider.enabled:
if (
deployment is None
or deployment.archived_at is not None
or deployment.provider.archived_at is not None
or not deployment.enabled
or not deployment.provider.enabled
):
invocation.status = "failed"
invocation.error_code = "deployment_unavailable"
invocation.error_message = "模型配置或渠道未启用"
@@ -110,6 +130,27 @@ def execute_invocation(
invocation.save(update_fields=["status", "error_code", "error_message", "finished_at"])
return invocation
request_messages = messages
response_format = None
structured_metadata = None
if output_schema is not None:
try:
request_messages, response_format, structured_metadata = build_structured_request(
messages,
mode=deployment.structured_output_mode,
output_schema=output_schema,
schema_name=schema_name or f"invocation_{invocation.public_id.hex}",
)
except StructuredOutputError as error:
invocation.status = "failed"
invocation.error_code = error.code
invocation.error_message = str(error)
invocation.finished_at = timezone.now()
invocation.save(
update_fields=["status", "error_code", "error_message", "finished_at"]
)
return invocation
invocation.status = "running"
invocation.started_at = timezone.now()
invocation.model_snapshot = {
@@ -121,8 +162,18 @@ def execute_invocation(
"capability": deployment.capability,
"temperature": temperature if temperature is not None else deployment.default_temperature,
"max_output_tokens": max_output_tokens or deployment.max_output_tokens,
"structured_output": structured_metadata,
}
invocation.save(update_fields=["status", "started_at", "model_snapshot"])
if structured_metadata is not None:
invocation.input_snapshot = {
**invocation.input_snapshot,
"structured_output": structured_metadata,
"output_schema": output_schema,
"response_format": response_format,
}
invocation.save(
update_fields=["status", "started_at", "model_snapshot", "input_snapshot"]
)
started = monotonic()
result: CompletionResult | None = None
last_error: Exception | None = None
@@ -136,17 +187,21 @@ def execute_invocation(
try:
result = _request_completion(
deployment,
messages,
request_messages,
temperature=temperature
if temperature is not None
else deployment.default_temperature,
max_output_tokens=max_output_tokens or deployment.max_output_tokens,
response_format=response_format,
timeout_seconds=remaining,
)
invocation.retry_count = attempt
break
except Exception as error: # provider libraries expose several subclasses
code, message, transient = _error_details(error)
if response_format is not None and code == "bad_request":
code = "structured_output_unsupported"
message = "模型渠道拒绝当前结构化输出配置"
last_error = error
invocation.retry_count = attempt
if not transient or attempt == 1:
@@ -199,3 +254,21 @@ def create_model_test(
input_snapshot={"messages": [{"role": "user", "content": prompt}]},
created_by=user,
)
def create_structured_output_test(
deployment: ModelDeployment,
user=None,
) -> ModelInvocation:
"""创建单次、不可修复的结构化输出能力测试。"""
return ModelInvocation.objects.create(
deployment=deployment,
invocation_type="structured_output_test",
input_snapshot={
"messages": [{"role": "user", "content": STRUCTURED_OUTPUT_TEST_PROMPT}],
"output_schema": STRUCTURED_OUTPUT_TEST_SCHEMA,
"schema_name": f"deployment_{deployment.id}_structured_output_test",
},
created_by=user,
)

View File

@@ -0,0 +1,216 @@
import hashlib
import json
import re
from copy import deepcopy
from typing import Any
CHAT_CAPABILITIES = {"text_chat", "vision_chat"}
STRUCTURED_OUTPUT_MODES = {
"native_json_schema",
"json_object",
"prompt_schema",
}
MODE_STRENGTH = {
"prompt_schema": 1,
"json_object": 2,
"native_json_schema": 3,
}
ENFORCEMENT_STRENGTH = {
"validated_json": 1,
"json_object": 2,
"native_json_schema": 3,
}
NATIVE_JSON_SCHEMA_UNSUPPORTED_KEYWORDS = {
"allOf",
"dependentRequired",
"dependentSchemas",
"else",
"if",
"not",
"then",
"uniqueItems",
}
STRUCTURED_OUTPUT_TEST_SCHEMA = {
"type": "object",
"properties": {
"status": {"type": "string", "enum": ["ok"]},
"items": {
"type": "array",
"items": {
"type": "object",
"properties": {
"code": {"type": "string", "enum": ["A1"]},
"score": {"type": "integer"},
},
"required": ["code", "score"],
"additionalProperties": False,
},
},
},
"required": ["status", "items"],
"additionalProperties": False,
}
STRUCTURED_OUTPUT_TEST_EXPECTED = {
"status": "ok",
"items": [{"code": "A1", "score": 1}],
}
STRUCTURED_OUTPUT_TEST_PROMPT = (
"返回结构化能力测试结果:status 必须为 ok,items 只包含一项,"
"该项 code 为 A1、score 为 1。不要输出解释。"
)
class StructuredOutputError(ValueError):
def __init__(self, code: str, message: str):
super().__init__(message)
self.code = code
def validate_mode_for_capability(capability: str, mode: str) -> None:
if capability in CHAT_CAPABILITIES:
if mode not in STRUCTURED_OUTPUT_MODES:
raise StructuredOutputError(
"structured_output_not_configured",
"文本或视觉模型必须配置结构化输出方式",
)
# Embedding 与重排不产生业务 JSON,结构化输出不属于它们的运行能力边界。
# 持久化层会将这两类模型统一归一为 none,此处不以该字段阻断保存或使用。
def ensure_compatible(
*,
mode: str,
test_status: str,
enforcement: str,
) -> None:
if mode not in MODE_STRENGTH:
raise StructuredOutputError(
"structured_output_not_configured",
"模型尚未配置结构化输出方式",
)
if test_status != "passed":
raise StructuredOutputError(
"structured_output_not_verified",
"模型结构化输出能力尚未验证通过",
)
if MODE_STRENGTH[mode] < ENFORCEMENT_STRENGTH.get(enforcement, 1):
raise StructuredOutputError(
"structured_output_incompatible",
f"模型输出模式 {mode} 不满足 Skill 的 {enforcement} 要求",
)
def schema_hash(schema: dict[str, Any]) -> str:
canonical = json.dumps(
schema,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def normalize_schema_name(value: str) -> str:
normalized = re.sub(r"[^A-Za-z0-9_-]+", "_", value).strip("_")
return (normalized or "structured_output")[:64]
def validate_native_schema_subset(
schema: dict[str, Any],
*,
path: str = "$",
) -> None:
"""校验平台原生 Structured Outputs 采用的可移植 JSON Schema 子集。"""
for keyword in sorted(NATIVE_JSON_SCHEMA_UNSUPPORTED_KEYWORDS):
if keyword in schema:
raise StructuredOutputError(
"structured_output_schema_unsupported",
f"原生结构化输出不支持 Schema 关键字 {keyword}: {path}.{keyword}",
)
for container in ("properties", "patternProperties", "$defs", "definitions"):
children = schema.get(container)
if isinstance(children, dict):
for name, child in children.items():
if isinstance(child, dict):
validate_native_schema_subset(
child,
path=f"{path}.{container}.{name}",
)
for keyword in ("additionalProperties", "contains", "items", "propertyNames"):
child = schema.get(keyword)
if isinstance(child, dict):
validate_native_schema_subset(child, path=f"{path}.{keyword}")
for keyword in ("anyOf", "oneOf", "prefixItems"):
children = schema.get(keyword)
if isinstance(children, list):
for index, child in enumerate(children):
if isinstance(child, dict):
validate_native_schema_subset(
child,
path=f"{path}.{keyword}[{index}]",
)
def build_structured_request(
messages: list[dict[str, Any]],
*,
mode: str,
output_schema: dict[str, Any],
schema_name: str,
) -> tuple[list[dict[str, Any]], dict[str, Any] | None, dict[str, Any]]:
request_messages = deepcopy(messages)
normalized_name = normalize_schema_name(schema_name)
digest = schema_hash(output_schema)
response_format: dict[str, Any] | None = None
schema_in_system_message = False
if mode == "native_json_schema":
validate_native_schema_subset(output_schema)
response_format = {
"type": "json_schema",
"json_schema": {
"name": normalized_name,
"strict": True,
"schema": deepcopy(output_schema),
},
}
elif mode in {"json_object", "prompt_schema"}:
if mode == "json_object":
response_format = {"type": "json_object"}
compact_schema = json.dumps(
output_schema,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
)
request_messages.insert(
0,
{
"role": "system",
"content": (
"你必须只输出一个符合以下 JSON Schema 的 JSON 对象,不得增加解释、"
"Markdown 或 Schema 外字段。\n"
f"<output_schema>{compact_schema}</output_schema>"
),
},
)
schema_in_system_message = True
else:
raise StructuredOutputError(
"structured_output_not_configured",
f"不支持的结构化输出模式:{mode}",
)
metadata = {
"mode": mode,
"schema_name": normalized_name,
"schema_hash": digest,
"response_format_included": response_format is not None,
"schema_in_system_message": schema_in_system_message,
}
return request_messages, response_format, metadata

View File

@@ -1,7 +1,18 @@
import json
from datetime import timedelta
from celery import shared_task
from django.conf import settings
from django.db import transaction
from django.utils import timezone
from jsonschema import Draft202012Validator, ValidationError
from apps.modelhub.models import ModelInvocation
from apps.modelhub.services import execute_invocation
from apps.modelhub.structured_output import (
STRUCTURED_OUTPUT_TEST_EXPECTED,
STRUCTURED_OUTPUT_TEST_SCHEMA,
)
@shared_task(name="apps.modelhub.tasks.run_model_test")
@@ -10,4 +21,121 @@ def run_model_test(invocation_id: int) -> None:
id=invocation_id
)
messages = invocation.input_snapshot.get("messages", [])
execute_invocation(invocation, messages)
output_schema = invocation.input_snapshot.get("output_schema")
execute_invocation(
invocation,
messages,
output_schema=output_schema,
schema_name=invocation.input_snapshot.get("schema_name"),
)
if invocation.invocation_type != "structured_output_test":
return
deployment = invocation.deployment
if deployment is None:
return
message = invocation.error_message
passed = invocation.status == "succeeded"
if passed:
try:
output = json.loads(invocation.output_text)
Draft202012Validator(STRUCTURED_OUTPUT_TEST_SCHEMA).validate(output)
if output != STRUCTURED_OUTPUT_TEST_EXPECTED:
raise ValidationError("测试值与预期不一致")
except (json.JSONDecodeError, ValidationError) as error:
passed = False
invocation.status = "failed"
invocation.error_code = "structured_output_invalid"
invocation.error_message = f"结构化输出测试未通过:{error}"
invocation.save(update_fields=["status", "error_code", "error_message"])
message = invocation.error_message
deployment.structured_output_test_status = "passed" if passed else "failed"
deployment.structured_output_verified_at = timezone.now() if passed else None
deployment.structured_output_test_message = (
"结构化输出测试已通过" if passed else (message or "结构化输出测试失败")
)
deployment.save(
update_fields=[
"structured_output_test_status",
"structured_output_verified_at",
"structured_output_test_message",
"updated_at",
]
)
from apps.audit.models import OperationAuditLog
OperationAuditLog.objects.create(
user=invocation.created_by,
module="modelhub",
action="structured_output_test_result",
target_type="model_deployment",
target_id=str(deployment.id),
request_snapshot={
"invocation_id": str(invocation.public_id),
"status": deployment.structured_output_test_status,
"error_code": invocation.error_code,
},
result="success" if passed else "failed",
)
def _retained_input_metadata(snapshot: dict) -> dict:
structured = snapshot.get("structured_output")
retained: dict = {}
if isinstance(structured, dict):
retained["structured_output"] = {
key: structured.get(key)
for key in (
"mode",
"schema_name",
"schema_hash",
"response_format_sent",
)
if key in structured
}
if snapshot.get("schema_name"):
retained["schema_name"] = snapshot["schema_name"]
retained["diagnostics_scrubbed"] = True
return retained
@shared_task(name="apps.modelhub.tasks.scrub_expired_model_invocation_diagnostics")
def scrub_expired_model_invocation_diagnostics() -> int:
"""擦除过期诊断正文,保留调用元数据、Schema 证据、Token 与终态。"""
cutoff = timezone.now() - timedelta(
days=settings.MODEL_INVOCATION_DIAGNOSTIC_RETENTION_DAYS
)
from apps.agents.models import AgentRunAttempt
scrubbed_count = 0
while True:
invocation_ids = list(
ModelInvocation.objects.filter(
created_at__lt=cutoff,
diagnostics_scrubbed_at__isnull=True,
).values_list("id", flat=True)[:1000]
)
if not invocation_ids:
break
now = timezone.now()
with transaction.atomic():
invocations = list(
ModelInvocation.objects.select_for_update().filter(id__in=invocation_ids)
)
for invocation in invocations:
invocation.input_snapshot = _retained_input_metadata(
invocation.input_snapshot or {}
)
invocation.output_text = ""
invocation.diagnostics_scrubbed_at = now
ModelInvocation.objects.bulk_update(
invocations,
["input_snapshot", "output_text", "diagnostics_scrubbed_at"],
)
AgentRunAttempt.objects.filter(
model_invocation_id__in=invocation_ids
).update(raw_output="")
scrubbed_count += len(invocations)
return scrubbed_count

View File

@@ -1,9 +1,14 @@
from datetime import timedelta
from django.contrib.auth import get_user_model
from django.contrib.auth.models import Permission
from django.core.management import call_command
from django.utils import timezone
from apps.modelhub.crypto import decrypt_secret
from apps.modelhub.models import ModelProvider
from apps.modelhub.models import ModelDeployment, ModelInvocation, ModelProvider
from apps.modelhub.services import CompletionResult
from apps.modelhub.tasks import scrub_expired_model_invocation_diagnostics
def login_admin(client):
@@ -65,8 +70,9 @@ def test_model_test_runs_through_recorded_invocation(client, db, monkeypatch):
"/api/modelhub/deployments",
data={
"provider_id": provider_id,
"display_name": "支持模型",
"model_name": "support-model",
"display_name": "支持模型",
"model_name": "support-model",
"structured_output_mode": "prompt_schema",
},
content_type="application/json",
)
@@ -96,15 +102,27 @@ def test_embedding_deployment_can_be_configured_but_uses_knowledge_search_for_te
"/api/modelhub/deployments",
data={
"provider_id": provider_id,
"display_name": "知识向量模型",
"model_name": "embedding-model",
"capability": "embedding",
"display_name": "知识向量模型",
"model_name": "embedding-model",
"capability": "embedding",
},
content_type="application/json",
)
assert deployment.status_code == 200
assert deployment.json()["capability"] == "embedding"
assert deployment.json()["structured_output_mode"] == "none"
stored_deployment = ModelDeployment.objects.get(id=deployment.json()["id"])
stored_deployment.structured_output_mode = "unconfigured"
stored_deployment.save(update_fields=["structured_output_mode"])
updated = client.patch(
f"/api/modelhub/deployments/{stored_deployment.id}",
data={"display_name": "知识向量模型(更新)"},
content_type="application/json",
)
assert updated.status_code == 200
assert updated.json()["structured_output_mode"] == "none"
response = client.post(
f"/api/modelhub/deployments/{deployment.json()['id']}/test",
@@ -112,7 +130,7 @@ def test_embedding_deployment_can_be_configured_but_uses_knowledge_search_for_te
content_type="application/json",
)
assert response.status_code == 400
assert "知识检索" in response.json()["detail"]
assert "对应能力入口" in response.json()["detail"]
rerank = client.post(
"/api/modelhub/deployments",
@@ -121,11 +139,116 @@ def test_embedding_deployment_can_be_configured_but_uses_knowledge_search_for_te
"display_name": "知识重排模型",
"model_name": "rerank-model",
"capability": "rerank",
"structured_output_mode": "native_json_schema",
},
content_type="application/json",
)
assert rerank.status_code == 200
assert rerank.json()["capability"] == "rerank"
assert rerank.json()["structured_output_mode"] == "none"
def test_text_deployment_requires_explicit_structured_output_mode(client, db):
login_admin(client)
provider_id = create_provider(client).json()["id"]
response = client.post(
"/api/modelhub/deployments",
data={
"provider_id": provider_id,
"display_name": "未配置模型",
"model_name": "unconfigured-model",
},
content_type="application/json",
)
assert response.status_code == 400
assert response.json()["detail"].startswith("structured_output_not_configured:")
assert "结构化输出" in response.json()["detail"]
def test_structured_output_test_passes_and_configuration_change_resets_it(
client,
db,
monkeypatch,
):
login_admin(client)
provider_id = create_provider(client).json()["id"]
deployment_response = client.post(
"/api/modelhub/deployments",
data={
"provider_id": provider_id,
"display_name": "原生结构化模型",
"model_name": "native-model",
"structured_output_mode": "native_json_schema",
},
content_type="application/json",
)
deployment_id = deployment_response.json()["id"]
monkeypatch.setattr(
"apps.modelhub.services._request_completion",
lambda *args, **kwargs: CompletionResult(
'{"status":"ok","items":[{"code":"A1","score":1}]}',
20,
10,
),
)
tested = client.post(
f"/api/modelhub/deployments/{deployment_id}/structured-output-test",
content_type="application/json",
)
assert tested.status_code == 200
assert tested.json()["invocation_type"] == "structured_output_test"
assert tested.json()["status"] == "succeeded"
deployment = ModelDeployment.objects.get(id=deployment_id)
assert deployment.structured_output_test_status == "passed"
assert deployment.structured_output_verified_at is not None
updated = client.patch(
f"/api/modelhub/deployments/{deployment_id}",
data={"model_name": "native-model-v2"},
content_type="application/json",
)
assert updated.status_code == 200
assert updated.json()["structured_output_test_status"] == "untested"
assert updated.json()["structured_output_verified_at"] is None
deployment.structured_output_test_status = "passed"
deployment.save(update_fields=["structured_output_test_status"])
provider_updated = client.patch(
f"/api/modelhub/providers/{provider_id}",
data={"base_url": "https://models-v2.example.com/v1"},
content_type="application/json",
)
assert provider_updated.status_code == 200
deployment.refresh_from_db()
assert deployment.structured_output_test_status == "untested"
def test_non_chat_model_cannot_start_structured_output_test(client, db):
login_admin(client)
provider_id = create_provider(client).json()["id"]
deployment = client.post(
"/api/modelhub/deployments",
data={
"provider_id": provider_id,
"display_name": "知识重排模型",
"model_name": "rerank-structured-test",
"capability": "rerank",
"structured_output_mode": "none",
},
content_type="application/json",
)
response = client.post(
f"/api/modelhub/deployments/{deployment.json()['id']}/structured-output-test",
content_type="application/json",
)
assert response.status_code == 400
assert "不适用" in response.json()["detail"]
def test_non_ai_admin_cannot_manage_modelhub(client, db):
@@ -135,3 +258,190 @@ def test_non_ai_admin_cannot_manage_modelhub(client, db):
response = client.get("/api/modelhub/providers")
assert response.status_code == 403
def test_unused_provider_and_deployment_are_physically_deleted(client, db):
login_admin(client)
provider_id = create_provider(client).json()["id"]
deployment = client.post(
"/api/modelhub/deployments",
data={
"provider_id": provider_id,
"display_name": "待删除模型",
"model_name": "unused-model",
"structured_output_mode": "prompt_schema",
},
content_type="application/json",
)
impact = client.get(
f"/api/modelhub/deployments/{deployment.json()['id']}/deletion-impact"
)
deleted = client.delete(
f"/api/modelhub/deployments/{deployment.json()['id']}"
)
assert impact.status_code == 200
assert impact.json()["suggested_action"] == "delete"
assert deleted.json()["result"] == "deleted"
assert not ModelDeployment.objects.filter(id=deployment.json()["id"]).exists()
provider_deleted = client.delete(f"/api/modelhub/providers/{provider_id}")
assert provider_deleted.json()["result"] == "deleted"
assert not ModelProvider.objects.filter(id=provider_id).exists()
def test_provider_with_history_is_atomically_archived_and_restored_disabled(client, db):
admin = login_admin(client)
provider_id = create_provider(client).json()["id"]
deployment_id = client.post(
"/api/modelhub/deployments",
data={
"provider_id": provider_id,
"display_name": "历史模型",
"model_name": "history-model",
"structured_output_mode": "prompt_schema",
},
content_type="application/json",
).json()["id"]
ModelInvocation.objects.create(
deployment_id=deployment_id,
status="succeeded",
output_text="sensitive output",
created_by=admin,
)
response = client.delete(f"/api/modelhub/providers/{provider_id}")
assert response.status_code == 200
assert response.json()["result"] == "archived"
provider = ModelProvider.objects.get(id=provider_id)
deployment = ModelDeployment.objects.get(id=deployment_id)
assert provider.archived_at is not None
assert provider.api_key_encrypted == ""
assert provider.image_egress_approved is False
assert deployment.archived_at is not None
assert deployment.archived_via_provider is True
assert deployment.enabled is False
restored = client.post(f"/api/modelhub/providers/{provider_id}/restore")
assert restored.status_code == 200
provider.refresh_from_db()
deployment.refresh_from_db()
assert provider.archived_at is None and provider.enabled is False
assert deployment.archived_at is None and deployment.enabled is False
assert deployment.structured_output_test_status == "untested"
def test_restore_name_conflict_is_409_without_partial_restore(client, db):
admin = login_admin(client)
archived = ModelProvider.objects.create(
name="可复用渠道",
base_url="https://old.example.com/v1",
archived_at=timezone.now(),
archived_by=admin,
)
ModelProvider.objects.create(
name="可复用渠道",
base_url="https://new.example.com/v1",
)
response = client.post(f"/api/modelhub/providers/{archived.id}/restore")
assert response.status_code == 409
assert response.json()["detail"].startswith("lifecycle_name_conflict:")
archived.refresh_from_db()
assert archived.archived_at is not None
def test_lifecycle_requires_superuser_even_with_model_manage(client, db):
login_admin(client)
provider_id = create_provider(client).json()["id"]
user = get_user_model().objects.create_user(username="model-manager", password="password")
user.user_permissions.add(Permission.objects.get(codename="model.manage"))
client.force_login(user)
assert client.get(
f"/api/modelhub/providers/{provider_id}/deletion-impact"
).status_code == 403
assert client.delete(f"/api/modelhub/providers/{provider_id}").status_code == 403
assert client.post(f"/api/modelhub/providers/{provider_id}/restore").status_code == 403
def test_lists_are_paginated_and_agent_options_only_return_effective_models(client, db):
login_admin(client)
provider_id = create_provider(client).json()["id"]
ready = ModelDeployment.objects.create(
provider_id=provider_id,
display_name="已验证模型",
model_name="ready-model",
structured_output_mode="prompt_schema",
structured_output_test_status="passed",
)
ModelDeployment.objects.create(
provider_id=provider_id,
display_name="未验证模型",
model_name="unverified-model",
structured_output_mode="prompt_schema",
)
providers = client.get("/api/modelhub/providers?page=1&page_size=1")
deployments = client.get("/api/modelhub/deployments?page=1&page_size=1")
options = client.get("/api/modelhub/deployment-options")
assert set(providers.json()) == {"items", "page", "page_size", "total", "total_pages"}
assert providers.json()["page_size"] == 1
assert deployments.json()["page_size"] == 1
assert [item["id"] for item in options.json()] == [ready.id]
def test_credential_can_be_explicitly_cleared_and_disables_provider(client, db):
login_admin(client)
provider_id = create_provider(client).json()["id"]
response = client.patch(
f"/api/modelhub/providers/{provider_id}",
data={"clear_api_key": True},
content_type="application/json",
)
assert response.status_code == 200
assert response.json()["credential_set"] is False
assert response.json()["enabled"] is False
provider = ModelProvider.objects.get(id=provider_id)
assert provider.credential_updated_at is not None
def test_invocation_diagnostics_permission_and_retention_scrub(client, db, settings):
admin = login_admin(client)
settings.MODEL_INVOCATION_DIAGNOSTIC_RETENTION_DAYS = 90
invocation = ModelInvocation.objects.create(
status="succeeded",
input_snapshot={
"messages": [{"role": "user", "content": "sensitive input"}],
"structured_output": {
"mode": "native_json_schema",
"schema_hash": "abc",
"response_format_sent": True,
},
"output_schema": {"type": "object"},
},
output_text="sensitive output",
created_by=admin,
)
ModelInvocation.objects.filter(id=invocation.id).update(
created_at=timezone.now() - timedelta(days=91)
)
scrubbed = scrub_expired_model_invocation_diagnostics()
assert scrubbed == 1
invocation.refresh_from_db()
assert invocation.output_text == ""
assert "messages" not in invocation.input_snapshot
assert "output_schema" not in invocation.input_snapshot
assert invocation.input_snapshot["structured_output"]["schema_hash"] == "abc"
response = client.get(
f"/api/modelhub/invocations/{invocation.public_id}/diagnostics"
)
assert response.status_code == 410

View File

@@ -1,6 +1,12 @@
import httpx
import pytest
from openai import APITimeoutError, AuthenticationError, InternalServerError, RateLimitError
from openai import (
APITimeoutError,
AuthenticationError,
BadRequestError,
InternalServerError,
RateLimitError,
)
from apps.modelhub.crypto import encrypt_secret
from apps.modelhub.models import ModelDeployment, ModelInvocation, ModelProvider
@@ -67,6 +73,60 @@ def test_non_transient_error_fails_without_exposing_raw_error(db, monkeypatch):
assert "secret provider detail" not in invocation.error_message
@pytest.mark.parametrize(
("mode", "expected_response_type", "schema_in_system_message"),
[
("native_json_schema", "json_schema", False),
("json_object", "json_object", True),
("prompt_schema", None, True),
],
)
def test_structured_invocation_records_request_and_audit_snapshot(
db,
monkeypatch,
mode,
expected_response_type,
schema_in_system_message,
):
invocation = create_invocation()
invocation.deployment.structured_output_mode = mode
invocation.deployment.save(update_fields=["structured_output_mode"])
captured = {}
output_schema = {
"type": "object",
"properties": {"answer": {"type": "string"}},
"required": ["answer"],
"additionalProperties": False,
}
def request(*args, **kwargs):
captured.update(kwargs)
captured["messages"] = args[1]
return CompletionResult('{"answer":"ok"}', 10, 4)
monkeypatch.setattr("apps.modelhub.services._request_completion", request)
execute_invocation(
invocation,
invocation.input_snapshot["messages"],
output_schema=output_schema,
schema_name="answer-result",
)
invocation.refresh_from_db()
structured = invocation.model_snapshot["structured_output"]
assert structured["mode"] == mode
assert structured["schema_hash"]
assert structured["schema_name"] == "answer-result"
assert structured["schema_in_system_message"] is schema_in_system_message
assert invocation.input_snapshot["output_schema"] == output_schema
assert invocation.input_snapshot["structured_output"] == structured
if expected_response_type is None:
assert "response_format" not in captured or captured["response_format"] is None
else:
assert captured["response_format"]["type"] == expected_response_type
def test_agent_time_budget_does_not_repeat_a_full_timeout(db, monkeypatch):
invocation = create_invocation()
calls = []
@@ -98,6 +158,47 @@ def provider_error(error_type, status_code):
return error_type("provider raw error", response=response, body={})
@pytest.mark.parametrize(
("mode", "expected_code"),
[
("native_json_schema", "structured_output_unsupported"),
("json_object", "structured_output_unsupported"),
("prompt_schema", "bad_request"),
],
)
def test_response_format_rejection_does_not_downgrade_mode(
db,
monkeypatch,
mode,
expected_code,
):
invocation = create_invocation()
invocation.deployment.structured_output_mode = mode
invocation.deployment.save(update_fields=["structured_output_mode"])
monkeypatch.setattr(
"apps.modelhub.services._request_completion",
lambda *args, **kwargs: (_ for _ in ()).throw(
provider_error(BadRequestError, 400)
),
)
execute_invocation(
invocation,
invocation.input_snapshot["messages"],
output_schema={
"type": "object",
"properties": {"answer": {"type": "string"}},
"required": ["answer"],
"additionalProperties": False,
},
)
invocation.refresh_from_db()
assert invocation.status == "failed"
assert invocation.error_code == expected_code
assert invocation.retry_count == 0
@pytest.mark.parametrize(
("error", "expected_code", "expected_calls"),
[

View File

@@ -0,0 +1,155 @@
import pytest
from apps.modelhub.structured_output import (
StructuredOutputError,
build_structured_request,
ensure_compatible,
schema_hash,
)
OUTPUT_SCHEMA = {
"type": "object",
"properties": {"answer": {"type": "string"}},
"required": ["answer"],
"additionalProperties": False,
}
@pytest.mark.parametrize(
("mode", "expected_response_type", "schema_in_system_message"),
[
("native_json_schema", "json_schema", False),
("json_object", "json_object", True),
("prompt_schema", None, True),
],
)
def test_structured_modes_build_stable_requests(
mode,
expected_response_type,
schema_in_system_message,
):
messages = [{"role": "user", "content": "业务输入"}]
request_messages, response_format, metadata = build_structured_request(
messages,
mode=mode,
output_schema=OUTPUT_SCHEMA,
schema_name="draft.result/v1",
)
assert messages == [{"role": "user", "content": "业务输入"}]
assert metadata == {
"mode": mode,
"schema_name": "draft_result_v1",
"schema_hash": schema_hash(OUTPUT_SCHEMA),
"response_format_included": expected_response_type is not None,
"schema_in_system_message": schema_in_system_message,
}
if expected_response_type is None:
assert response_format is None
else:
assert response_format["type"] == expected_response_type
if mode == "native_json_schema":
assert response_format["json_schema"] == {
"name": "draft_result_v1",
"strict": True,
"schema": OUTPUT_SCHEMA,
}
assert request_messages == messages
else:
assert request_messages[0]["role"] == "system"
assert "<output_schema>" in request_messages[0]["content"]
assert '"additionalProperties":false' in request_messages[0]["content"]
assert request_messages[1:] == messages
@pytest.mark.parametrize(
("mode", "enforcement", "compatible"),
[
("native_json_schema", "native_json_schema", True),
("native_json_schema", "json_object", True),
("native_json_schema", "validated_json", True),
("json_object", "native_json_schema", False),
("json_object", "json_object", True),
("json_object", "validated_json", True),
("prompt_schema", "native_json_schema", False),
("prompt_schema", "json_object", False),
("prompt_schema", "validated_json", True),
],
)
def test_all_model_and_skill_compatibility_combinations(mode, enforcement, compatible):
if compatible:
ensure_compatible(mode=mode, test_status="passed", enforcement=enforcement)
return
with pytest.raises(StructuredOutputError) as caught:
ensure_compatible(mode=mode, test_status="passed", enforcement=enforcement)
assert caught.value.code == "structured_output_incompatible"
@pytest.mark.parametrize(
("mode", "status", "expected_code"),
[
("unconfigured", "untested", "structured_output_not_configured"),
("native_json_schema", "untested", "structured_output_not_verified"),
("native_json_schema", "failed", "structured_output_not_verified"),
],
)
def test_unconfigured_or_unverified_models_are_rejected(mode, status, expected_code):
with pytest.raises(StructuredOutputError) as caught:
ensure_compatible(
mode=mode,
test_status=status,
enforcement="validated_json",
)
assert caught.value.code == expected_code
def test_native_mode_rejects_unsupported_schema_keyword_before_provider_call():
schema = {
"type": "object",
"properties": {
"missing_fields": {
"type": "array",
"items": {"type": "string"},
"uniqueItems": True,
}
},
"required": ["missing_fields"],
"additionalProperties": False,
}
with pytest.raises(StructuredOutputError) as caught:
build_structured_request(
[{"role": "user", "content": "业务输入"}],
mode="native_json_schema",
output_schema=schema,
schema_name="facts",
)
assert caught.value.code == "structured_output_schema_unsupported"
assert "$.properties.missing_fields.uniqueItems" in str(caught.value)
def test_prompt_modes_keep_full_business_schema():
schema = {
"type": "object",
"properties": {
"values": {
"type": "array",
"items": {"type": "string"},
"uniqueItems": True,
}
},
"required": ["values"],
"additionalProperties": False,
}
messages, response_format, _ = build_structured_request(
[{"role": "user", "content": "业务输入"}],
mode="prompt_schema",
output_schema=schema,
schema_name="values",
)
assert response_format is None
assert '"uniqueItems":true' in messages[0]["content"]