448 lines
16 KiB
Python
448 lines
16 KiB
Python
from datetime import timedelta
|
|
|
|
from django.contrib.auth import get_user_model
|
|
from django.contrib.auth.models import Permission
|
|
from django.core.management import call_command
|
|
from django.utils import timezone
|
|
|
|
from apps.modelhub.crypto import decrypt_secret
|
|
from apps.modelhub.models import ModelDeployment, ModelInvocation, ModelProvider
|
|
from apps.modelhub.services import CompletionResult
|
|
from apps.modelhub.tasks import scrub_expired_model_invocation_diagnostics
|
|
|
|
|
|
def login_admin(client):
|
|
call_command("bootstrap_governance")
|
|
user = get_user_model().objects.get(username="admin")
|
|
client.force_login(user)
|
|
return user
|
|
|
|
|
|
def create_provider(client):
|
|
return client.post(
|
|
"/api/modelhub/providers",
|
|
data={
|
|
"name": "企业模型渠道",
|
|
"base_url": "https://models.example.com/v1/",
|
|
"api_key": "sk-secret-1234",
|
|
"default_timeout_seconds": 30,
|
|
},
|
|
content_type="application/json",
|
|
)
|
|
|
|
|
|
def test_admin_manages_provider_without_exposing_secret(client, db):
|
|
login_admin(client)
|
|
|
|
response = create_provider(client)
|
|
|
|
assert response.status_code == 200
|
|
assert "api_key" not in response.json()
|
|
assert response.json()["credential_masked"] == "****1234"
|
|
provider = ModelProvider.objects.get()
|
|
assert provider.api_key_encrypted != "sk-secret-1234"
|
|
assert decrypt_secret(provider.api_key_encrypted) == "sk-secret-1234"
|
|
|
|
updated = client.patch(
|
|
f"/api/modelhub/providers/{provider.id}",
|
|
data={"name": "新渠道名称", "api_key": ""},
|
|
content_type="application/json",
|
|
)
|
|
assert updated.status_code == 200
|
|
provider.refresh_from_db()
|
|
assert decrypt_secret(provider.api_key_encrypted) == "sk-secret-1234"
|
|
|
|
replaced = client.patch(
|
|
f"/api/modelhub/providers/{provider.id}",
|
|
data={"api_key": "sk-new-secret-5678"},
|
|
content_type="application/json",
|
|
)
|
|
assert replaced.status_code == 200
|
|
provider.refresh_from_db()
|
|
assert decrypt_secret(provider.api_key_encrypted) == "sk-new-secret-5678"
|
|
assert replaced.json()["credential_masked"] == "****5678"
|
|
|
|
|
|
def test_model_test_runs_through_recorded_invocation(client, db, monkeypatch):
|
|
login_admin(client)
|
|
provider_id = create_provider(client).json()["id"]
|
|
deployment = client.post(
|
|
"/api/modelhub/deployments",
|
|
data={
|
|
"provider_id": provider_id,
|
|
"display_name": "支持模型",
|
|
"model_name": "support-model",
|
|
"structured_output_mode": "prompt_schema",
|
|
},
|
|
content_type="application/json",
|
|
)
|
|
assert deployment.status_code == 200
|
|
|
|
monkeypatch.setattr(
|
|
"apps.modelhub.services._request_completion",
|
|
lambda *args, **kwargs: CompletionResult("连接正常", 8, 4),
|
|
)
|
|
response = client.post(
|
|
f"/api/modelhub/deployments/{deployment.json()['id']}/test",
|
|
data={"prompt": "测试连接"},
|
|
content_type="application/json",
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
assert response.json()["status"] == "succeeded"
|
|
assert response.json()["output_text"] == "连接正常"
|
|
assert response.json()["input_tokens"] == 8
|
|
assert response.json()["output_tokens"] == 4
|
|
|
|
|
|
def test_embedding_deployment_can_be_configured_but_uses_knowledge_search_for_test(client, db):
|
|
login_admin(client)
|
|
provider_id = create_provider(client).json()["id"]
|
|
deployment = client.post(
|
|
"/api/modelhub/deployments",
|
|
data={
|
|
"provider_id": provider_id,
|
|
"display_name": "知识向量模型",
|
|
"model_name": "embedding-model",
|
|
"capability": "embedding",
|
|
},
|
|
content_type="application/json",
|
|
)
|
|
|
|
assert deployment.status_code == 200
|
|
assert deployment.json()["capability"] == "embedding"
|
|
assert deployment.json()["structured_output_mode"] == "none"
|
|
|
|
stored_deployment = ModelDeployment.objects.get(id=deployment.json()["id"])
|
|
stored_deployment.structured_output_mode = "unconfigured"
|
|
stored_deployment.save(update_fields=["structured_output_mode"])
|
|
updated = client.patch(
|
|
f"/api/modelhub/deployments/{stored_deployment.id}",
|
|
data={"display_name": "知识向量模型（更新）"},
|
|
content_type="application/json",
|
|
)
|
|
assert updated.status_code == 200
|
|
assert updated.json()["structured_output_mode"] == "none"
|
|
|
|
response = client.post(
|
|
f"/api/modelhub/deployments/{deployment.json()['id']}/test",
|
|
data={"prompt": "测试连接"},
|
|
content_type="application/json",
|
|
)
|
|
assert response.status_code == 400
|
|
assert "对应能力入口" in response.json()["detail"]
|
|
|
|
rerank = client.post(
|
|
"/api/modelhub/deployments",
|
|
data={
|
|
"provider_id": provider_id,
|
|
"display_name": "知识重排模型",
|
|
"model_name": "rerank-model",
|
|
"capability": "rerank",
|
|
"structured_output_mode": "native_json_schema",
|
|
},
|
|
content_type="application/json",
|
|
)
|
|
assert rerank.status_code == 200
|
|
assert rerank.json()["capability"] == "rerank"
|
|
assert rerank.json()["structured_output_mode"] == "none"
|
|
|
|
|
|
def test_text_deployment_requires_explicit_structured_output_mode(client, db):
|
|
login_admin(client)
|
|
provider_id = create_provider(client).json()["id"]
|
|
|
|
response = client.post(
|
|
"/api/modelhub/deployments",
|
|
data={
|
|
"provider_id": provider_id,
|
|
"display_name": "未配置模型",
|
|
"model_name": "unconfigured-model",
|
|
},
|
|
content_type="application/json",
|
|
)
|
|
|
|
assert response.status_code == 400
|
|
assert response.json()["detail"].startswith("structured_output_not_configured:")
|
|
assert "结构化输出" in response.json()["detail"]
|
|
|
|
|
|
def test_structured_output_test_passes_and_configuration_change_resets_it(
|
|
client,
|
|
db,
|
|
monkeypatch,
|
|
):
|
|
login_admin(client)
|
|
provider_id = create_provider(client).json()["id"]
|
|
deployment_response = client.post(
|
|
"/api/modelhub/deployments",
|
|
data={
|
|
"provider_id": provider_id,
|
|
"display_name": "原生结构化模型",
|
|
"model_name": "native-model",
|
|
"structured_output_mode": "native_json_schema",
|
|
},
|
|
content_type="application/json",
|
|
)
|
|
deployment_id = deployment_response.json()["id"]
|
|
monkeypatch.setattr(
|
|
"apps.modelhub.services._request_completion",
|
|
lambda *args, **kwargs: CompletionResult(
|
|
'{"status":"ok","items":[{"code":"A1","score":1}]}',
|
|
20,
|
|
10,
|
|
),
|
|
)
|
|
|
|
tested = client.post(
|
|
f"/api/modelhub/deployments/{deployment_id}/structured-output-test",
|
|
content_type="application/json",
|
|
)
|
|
|
|
assert tested.status_code == 200
|
|
assert tested.json()["invocation_type"] == "structured_output_test"
|
|
assert tested.json()["status"] == "succeeded"
|
|
deployment = ModelDeployment.objects.get(id=deployment_id)
|
|
assert deployment.structured_output_test_status == "passed"
|
|
assert deployment.structured_output_verified_at is not None
|
|
|
|
updated = client.patch(
|
|
f"/api/modelhub/deployments/{deployment_id}",
|
|
data={"model_name": "native-model-v2"},
|
|
content_type="application/json",
|
|
)
|
|
assert updated.status_code == 200
|
|
assert updated.json()["structured_output_test_status"] == "untested"
|
|
assert updated.json()["structured_output_verified_at"] is None
|
|
|
|
deployment.structured_output_test_status = "passed"
|
|
deployment.save(update_fields=["structured_output_test_status"])
|
|
provider_updated = client.patch(
|
|
f"/api/modelhub/providers/{provider_id}",
|
|
data={"base_url": "https://models-v2.example.com/v1"},
|
|
content_type="application/json",
|
|
)
|
|
assert provider_updated.status_code == 200
|
|
deployment.refresh_from_db()
|
|
assert deployment.structured_output_test_status == "untested"
|
|
|
|
|
|
def test_non_chat_model_cannot_start_structured_output_test(client, db):
|
|
login_admin(client)
|
|
provider_id = create_provider(client).json()["id"]
|
|
deployment = client.post(
|
|
"/api/modelhub/deployments",
|
|
data={
|
|
"provider_id": provider_id,
|
|
"display_name": "知识重排模型",
|
|
"model_name": "rerank-structured-test",
|
|
"capability": "rerank",
|
|
"structured_output_mode": "none",
|
|
},
|
|
content_type="application/json",
|
|
)
|
|
|
|
response = client.post(
|
|
f"/api/modelhub/deployments/{deployment.json()['id']}/structured-output-test",
|
|
content_type="application/json",
|
|
)
|
|
|
|
assert response.status_code == 400
|
|
assert "不适用" in response.json()["detail"]
|
|
|
|
|
|
def test_non_ai_admin_cannot_manage_modelhub(client, db):
|
|
user = get_user_model().objects.create_user(username="requester", password="password")
|
|
client.force_login(user)
|
|
|
|
response = client.get("/api/modelhub/providers")
|
|
|
|
assert response.status_code == 403
|
|
|
|
|
|
def test_unused_provider_and_deployment_are_physically_deleted(client, db):
|
|
login_admin(client)
|
|
provider_id = create_provider(client).json()["id"]
|
|
deployment = client.post(
|
|
"/api/modelhub/deployments",
|
|
data={
|
|
"provider_id": provider_id,
|
|
"display_name": "待删除模型",
|
|
"model_name": "unused-model",
|
|
"structured_output_mode": "prompt_schema",
|
|
},
|
|
content_type="application/json",
|
|
)
|
|
|
|
impact = client.get(
|
|
f"/api/modelhub/deployments/{deployment.json()['id']}/deletion-impact"
|
|
)
|
|
deleted = client.delete(
|
|
f"/api/modelhub/deployments/{deployment.json()['id']}"
|
|
)
|
|
|
|
assert impact.status_code == 200
|
|
assert impact.json()["suggested_action"] == "delete"
|
|
assert deleted.json()["result"] == "deleted"
|
|
assert not ModelDeployment.objects.filter(id=deployment.json()["id"]).exists()
|
|
|
|
provider_deleted = client.delete(f"/api/modelhub/providers/{provider_id}")
|
|
assert provider_deleted.json()["result"] == "deleted"
|
|
assert not ModelProvider.objects.filter(id=provider_id).exists()
|
|
|
|
|
|
def test_provider_with_history_is_atomically_archived_and_restored_disabled(client, db):
|
|
admin = login_admin(client)
|
|
provider_id = create_provider(client).json()["id"]
|
|
deployment_id = client.post(
|
|
"/api/modelhub/deployments",
|
|
data={
|
|
"provider_id": provider_id,
|
|
"display_name": "历史模型",
|
|
"model_name": "history-model",
|
|
"structured_output_mode": "prompt_schema",
|
|
},
|
|
content_type="application/json",
|
|
).json()["id"]
|
|
ModelInvocation.objects.create(
|
|
deployment_id=deployment_id,
|
|
status="succeeded",
|
|
output_text="sensitive output",
|
|
created_by=admin,
|
|
)
|
|
|
|
response = client.delete(f"/api/modelhub/providers/{provider_id}")
|
|
|
|
assert response.status_code == 200
|
|
assert response.json()["result"] == "archived"
|
|
provider = ModelProvider.objects.get(id=provider_id)
|
|
deployment = ModelDeployment.objects.get(id=deployment_id)
|
|
assert provider.archived_at is not None
|
|
assert provider.api_key_encrypted == ""
|
|
assert provider.image_egress_approved is False
|
|
assert deployment.archived_at is not None
|
|
assert deployment.archived_via_provider is True
|
|
assert deployment.enabled is False
|
|
|
|
restored = client.post(f"/api/modelhub/providers/{provider_id}/restore")
|
|
assert restored.status_code == 200
|
|
provider.refresh_from_db()
|
|
deployment.refresh_from_db()
|
|
assert provider.archived_at is None and provider.enabled is False
|
|
assert deployment.archived_at is None and deployment.enabled is False
|
|
assert deployment.structured_output_test_status == "untested"
|
|
|
|
|
|
def test_restore_name_conflict_is_409_without_partial_restore(client, db):
|
|
admin = login_admin(client)
|
|
archived = ModelProvider.objects.create(
|
|
name="可复用渠道",
|
|
base_url="https://old.example.com/v1",
|
|
archived_at=timezone.now(),
|
|
archived_by=admin,
|
|
)
|
|
ModelProvider.objects.create(
|
|
name="可复用渠道",
|
|
base_url="https://new.example.com/v1",
|
|
)
|
|
|
|
response = client.post(f"/api/modelhub/providers/{archived.id}/restore")
|
|
|
|
assert response.status_code == 409
|
|
assert response.json()["detail"].startswith("lifecycle_name_conflict:")
|
|
archived.refresh_from_db()
|
|
assert archived.archived_at is not None
|
|
|
|
|
|
def test_lifecycle_requires_superuser_even_with_model_manage(client, db):
|
|
login_admin(client)
|
|
provider_id = create_provider(client).json()["id"]
|
|
user = get_user_model().objects.create_user(username="model-manager", password="password")
|
|
user.user_permissions.add(Permission.objects.get(codename="model.manage"))
|
|
client.force_login(user)
|
|
|
|
assert client.get(
|
|
f"/api/modelhub/providers/{provider_id}/deletion-impact"
|
|
).status_code == 403
|
|
assert client.delete(f"/api/modelhub/providers/{provider_id}").status_code == 403
|
|
assert client.post(f"/api/modelhub/providers/{provider_id}/restore").status_code == 403
|
|
|
|
|
|
def test_lists_are_paginated_and_agent_options_only_return_effective_models(client, db):
|
|
login_admin(client)
|
|
provider_id = create_provider(client).json()["id"]
|
|
ready = ModelDeployment.objects.create(
|
|
provider_id=provider_id,
|
|
display_name="已验证模型",
|
|
model_name="ready-model",
|
|
structured_output_mode="prompt_schema",
|
|
structured_output_test_status="passed",
|
|
)
|
|
ModelDeployment.objects.create(
|
|
provider_id=provider_id,
|
|
display_name="未验证模型",
|
|
model_name="unverified-model",
|
|
structured_output_mode="prompt_schema",
|
|
)
|
|
|
|
providers = client.get("/api/modelhub/providers?page=1&page_size=1")
|
|
deployments = client.get("/api/modelhub/deployments?page=1&page_size=1")
|
|
options = client.get("/api/modelhub/deployment-options")
|
|
|
|
assert set(providers.json()) == {"items", "page", "page_size", "total", "total_pages"}
|
|
assert providers.json()["page_size"] == 1
|
|
assert deployments.json()["page_size"] == 1
|
|
assert [item["id"] for item in options.json()] == [ready.id]
|
|
|
|
|
|
def test_credential_can_be_explicitly_cleared_and_disables_provider(client, db):
|
|
login_admin(client)
|
|
provider_id = create_provider(client).json()["id"]
|
|
|
|
response = client.patch(
|
|
f"/api/modelhub/providers/{provider_id}",
|
|
data={"clear_api_key": True},
|
|
content_type="application/json",
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
assert response.json()["credential_set"] is False
|
|
assert response.json()["enabled"] is False
|
|
provider = ModelProvider.objects.get(id=provider_id)
|
|
assert provider.credential_updated_at is not None
|
|
|
|
|
|
def test_invocation_diagnostics_permission_and_retention_scrub(client, db, settings):
|
|
admin = login_admin(client)
|
|
settings.MODEL_INVOCATION_DIAGNOSTIC_RETENTION_DAYS = 90
|
|
invocation = ModelInvocation.objects.create(
|
|
status="succeeded",
|
|
input_snapshot={
|
|
"messages": [{"role": "user", "content": "sensitive input"}],
|
|
"structured_output": {
|
|
"mode": "native_json_schema",
|
|
"schema_hash": "abc",
|
|
"response_format_sent": True,
|
|
},
|
|
"output_schema": {"type": "object"},
|
|
},
|
|
output_text="sensitive output",
|
|
created_by=admin,
|
|
)
|
|
ModelInvocation.objects.filter(id=invocation.id).update(
|
|
created_at=timezone.now() - timedelta(days=91)
|
|
)
|
|
|
|
scrubbed = scrub_expired_model_invocation_diagnostics()
|
|
|
|
assert scrubbed == 1
|
|
invocation.refresh_from_db()
|
|
assert invocation.output_text == ""
|
|
assert "messages" not in invocation.input_snapshot
|
|
assert "output_schema" not in invocation.input_snapshot
|
|
assert invocation.input_snapshot["structured_output"]["schema_hash"] == "abc"
|
|
response = client.get(
|
|
f"/api/modelhub/invocations/{invocation.public_id}/diagnostics"
|
|
)
|
|
assert response.status_code == 410
|