feat: add knowledge retrieval service
This commit is contained in:
@@ -1,7 +1,8 @@
|
||||
import hashlib
|
||||
import math
|
||||
|
||||
from apps.knowledge.models import KnowledgeChunk, KnowledgeDocument, KnowledgeItem
|
||||
from apps.knowledge.models import KnowledgeBase, KnowledgeChunk, KnowledgeDocument, KnowledgeItem
|
||||
from apps.solutions.models import Solution
|
||||
|
||||
DEFAULT_CHUNK_SIZE = 600
|
||||
|
||||
@@ -23,6 +24,18 @@ def embed_text(content: str, dimensions: int = 16) -> list[float]:
|
||||
return [round(value / length, 6) for value in values]
|
||||
|
||||
|
||||
def keyword_score(query: str, content: str) -> float:
|
||||
normalized_query = query.lower()
|
||||
terms = [term for term in normalized_query.split() if term]
|
||||
terms.extend(character for character in normalized_query if not character.isspace())
|
||||
lowered = content.lower()
|
||||
return sum(1 for term in terms if term in lowered)
|
||||
|
||||
|
||||
def vector_score(left: list[float], right: list[float]) -> float:
|
||||
return sum(a * b for a, b in zip(left, right, strict=False))
|
||||
|
||||
|
||||
def rebuild_item_chunks(item: KnowledgeItem) -> list[KnowledgeChunk]:
|
||||
item.chunks.all().delete()
|
||||
content = "\n".join(
|
||||
@@ -85,3 +98,70 @@ def ingest_document_text(
|
||||
},
|
||||
)
|
||||
return document
|
||||
|
||||
|
||||
def index_published_solution(solution: Solution) -> list[KnowledgeChunk]:
|
||||
base, _ = KnowledgeBase.objects.get_or_create(
|
||||
name="解决方案知识库",
|
||||
defaults={"description": "从已发布解决方案生成的检索知识", "scope": "solutions"},
|
||||
)
|
||||
KnowledgeChunk.objects.filter(metadata__solution_id=solution.id).delete()
|
||||
content = "\n".join(
|
||||
[
|
||||
solution.title,
|
||||
solution.problem_signature,
|
||||
solution.related_system,
|
||||
solution.environment_scope,
|
||||
solution.root_cause,
|
||||
solution.solution,
|
||||
solution.risk_note,
|
||||
]
|
||||
)
|
||||
chunks = []
|
||||
for index, chunk_text in enumerate(split_text(content)):
|
||||
chunks.append(
|
||||
KnowledgeChunk.objects.create(
|
||||
knowledge_base=base,
|
||||
chunk_index=index,
|
||||
content=chunk_text,
|
||||
embedding=embed_text(chunk_text),
|
||||
metadata={
|
||||
"source_type": "solution",
|
||||
"title": solution.title,
|
||||
"solution_id": solution.id,
|
||||
},
|
||||
)
|
||||
)
|
||||
return chunks
|
||||
|
||||
|
||||
def retrieve_knowledge(
|
||||
query: str,
|
||||
knowledge_base_id: int | None = None,
|
||||
limit: int = 5,
|
||||
) -> list[dict]:
|
||||
query_embedding = embed_text(query)
|
||||
queryset = KnowledgeChunk.objects.select_related("knowledge_base", "item", "document")
|
||||
if knowledge_base_id:
|
||||
queryset = queryset.filter(knowledge_base_id=knowledge_base_id)
|
||||
scored = []
|
||||
for chunk in queryset[:1000]:
|
||||
score = keyword_score(query, chunk.content) * 10 + vector_score(
|
||||
query_embedding,
|
||||
chunk.embedding,
|
||||
)
|
||||
if score > 0:
|
||||
scored.append((score, chunk))
|
||||
scored.sort(key=lambda pair: pair[0], reverse=True)
|
||||
return [
|
||||
{
|
||||
"id": chunk.id,
|
||||
"knowledge_base_id": chunk.knowledge_base_id,
|
||||
"title": chunk.metadata.get("title", chunk.knowledge_base.name),
|
||||
"source_type": chunk.metadata.get("source_type", "chunk"),
|
||||
"content": chunk.content,
|
||||
"score": round(score, 4),
|
||||
"metadata": chunk.metadata,
|
||||
}
|
||||
for score, chunk in scored[:limit]
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user