feat: complete multimodal ticket intake
This commit is contained in:
hailin
349
backend/apps/tickets/materials.py
Normal file
349
backend/apps/tickets/materials.py
Normal file
@@ -0,0 +1,349 @@
|
||||
"""受理材料的校验、恶意文件扫描、解析与对象存储编排。"""
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import socket
|
||||
import struct
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
from apps.tickets.models import TicketDraftMaterial
|
||||
from apps.tickets.storage import DraftImageError, get_object, prepare_image
|
||||
|
||||
SUPPORTED_EXTENSIONS = (
|
||||
"jpg",
|
||||
"jpeg",
|
||||
"png",
|
||||
"webp",
|
||||
"pdf",
|
||||
"txt",
|
||||
"log",
|
||||
"docx",
|
||||
"xlsx",
|
||||
"pptx",
|
||||
)
|
||||
IMAGE_EXTENSIONS = {"jpg", "jpeg", "png", "webp"}
|
||||
GENERIC_MIME_TYPES = {"", "application/octet-stream", "binary/octet-stream"}
|
||||
MIME_TYPES = {
|
||||
"jpg": {"image/jpeg"},
|
||||
"jpeg": {"image/jpeg"},
|
||||
"png": {"image/png"},
|
||||
"webp": {"image/webp"},
|
||||
"pdf": {"application/pdf"},
|
||||
"txt": {"text/plain"},
|
||||
"log": {"text/plain", "text/x-log"},
|
||||
"docx": {"application/vnd.openxmlformats-officedocument.wordprocessingml.document"},
|
||||
"xlsx": {"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"},
|
||||
"pptx": {"application/vnd.openxmlformats-officedocument.presentationml.presentation"},
|
||||
}
|
||||
CANONICAL_MIME_TYPES = {
|
||||
"jpg": "image/jpeg",
|
||||
"jpeg": "image/jpeg",
|
||||
"png": "image/png",
|
||||
"webp": "image/webp",
|
||||
"pdf": "application/pdf",
|
||||
"txt": "text/plain",
|
||||
"log": "text/plain",
|
||||
"docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
}
|
||||
MATERIAL_TYPES = {
|
||||
**{extension: "image" for extension in IMAGE_EXTENSIONS},
|
||||
"pdf": "document",
|
||||
"txt": "text",
|
||||
"log": "log",
|
||||
"docx": "document",
|
||||
"xlsx": "spreadsheet",
|
||||
"pptx": "presentation",
|
||||
}
|
||||
OFFICE_SIGNATURES = {
|
||||
"docx": "word/document.xml",
|
||||
"xlsx": "xl/workbook.xml",
|
||||
"pptx": "ppt/presentation.xml",
|
||||
}
|
||||
|
||||
|
||||
class MaterialValidationError(ValueError):
|
||||
def __init__(self, status_code: int, message: str):
|
||||
super().__init__(message)
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
class MaterialScanError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class MaterialParseError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PreparedMaterial:
|
||||
data: bytes
|
||||
file_name: str
|
||||
material_type: str
|
||||
extension: str
|
||||
claimed_mime_type: str
|
||||
mime_type: str
|
||||
sha256: str
|
||||
|
||||
|
||||
def _decode_text(raw: bytes) -> str:
|
||||
if b"\x00" in raw:
|
||||
raise MaterialValidationError(415, "文本或日志材料包含二进制内容")
|
||||
for encoding in ("utf-8-sig", "gb18030"):
|
||||
try:
|
||||
return raw.decode(encoding)
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
raise MaterialValidationError(415, "文本或日志材料编码无法识别")
|
||||
|
||||
|
||||
def _validate_office_signature(raw: bytes, extension: str) -> None:
|
||||
try:
|
||||
with zipfile.ZipFile(io.BytesIO(raw)) as archive:
|
||||
names = set(archive.namelist())
|
||||
except (OSError, zipfile.BadZipFile) as error:
|
||||
raise MaterialValidationError(415, "Office 文件签名无效") from error
|
||||
if OFFICE_SIGNATURES[extension] not in names:
|
||||
raise MaterialValidationError(415, f"文件内容与 .{extension} 扩展名不匹配")
|
||||
|
||||
|
||||
def prepare_material(
|
||||
raw: bytes,
|
||||
original_name: str,
|
||||
claimed_mime_type: str = "",
|
||||
) -> PreparedMaterial:
|
||||
"""校验大小、扩展名、MIME 和文件签名，返回可安全落库的材料元数据。"""
|
||||
|
||||
if not raw:
|
||||
raise MaterialValidationError(400, "不能上传空文件")
|
||||
max_bytes = settings.TICKET_MATERIAL_MAX_FILE_BYTES
|
||||
if len(raw) > max_bytes:
|
||||
raise MaterialValidationError(413, f"单个材料不能超过 {max_bytes // 1024 // 1024} MB")
|
||||
|
||||
safe_name = (original_name or "material").replace("\\", "/").rsplit("/", 1)[-1]
|
||||
extension = Path(safe_name).suffix.lower().lstrip(".")
|
||||
if extension not in SUPPORTED_EXTENSIONS:
|
||||
supported = "、".join(item.upper() for item in SUPPORTED_EXTENSIONS)
|
||||
raise MaterialValidationError(415, f"仅支持 {supported}")
|
||||
|
||||
claimed = (claimed_mime_type or "").split(";", 1)[0].strip().lower()
|
||||
if claimed not in GENERIC_MIME_TYPES and claimed not in MIME_TYPES[extension]:
|
||||
raise MaterialValidationError(415, "文件 MIME 类型与扩展名不匹配")
|
||||
|
||||
if extension in IMAGE_EXTENSIONS:
|
||||
try:
|
||||
image = prepare_image(raw, safe_name)
|
||||
except DraftImageError as error:
|
||||
raise MaterialValidationError(error.status_code, str(error)) from error
|
||||
detected_group = {"jpg", "jpeg"} if image.extension == "jpg" else {image.extension}
|
||||
if extension not in detected_group:
|
||||
raise MaterialValidationError(415, "图片内容与扩展名不匹配")
|
||||
return PreparedMaterial(
|
||||
data=raw,
|
||||
file_name=safe_name,
|
||||
material_type="image",
|
||||
extension=extension,
|
||||
claimed_mime_type=claimed,
|
||||
mime_type=image.mime_type,
|
||||
sha256=hashlib.sha256(raw).hexdigest(),
|
||||
)
|
||||
|
||||
if extension == "pdf":
|
||||
if not raw.startswith(b"%PDF-"):
|
||||
raise MaterialValidationError(415, "PDF 文件签名无效")
|
||||
elif extension in {"txt", "log"}:
|
||||
_decode_text(raw)
|
||||
else:
|
||||
_validate_office_signature(raw, extension)
|
||||
|
||||
return PreparedMaterial(
|
||||
data=raw,
|
||||
file_name=safe_name,
|
||||
material_type=MATERIAL_TYPES[extension],
|
||||
extension=extension,
|
||||
claimed_mime_type=claimed,
|
||||
mime_type=CANONICAL_MIME_TYPES[extension],
|
||||
sha256=hashlib.sha256(raw).hexdigest(),
|
||||
)
|
||||
|
||||
|
||||
def scan_bytes(raw: bytes) -> dict:
|
||||
"""调用配置的扫描器；扫描不可用或响应异常时一律按失败处理。"""
|
||||
|
||||
backend = settings.TICKET_MATERIAL_SCANNER_BACKEND
|
||||
if backend == "test":
|
||||
return {"status": "clean", "engine": "test"}
|
||||
if backend != "clamav":
|
||||
raise MaterialScanError("未配置可用的恶意文件扫描器")
|
||||
|
||||
try:
|
||||
with socket.create_connection(
|
||||
(settings.CLAMAV_HOST, settings.CLAMAV_PORT),
|
||||
timeout=settings.CLAMAV_TIMEOUT_SECONDS,
|
||||
) as connection:
|
||||
connection.sendall(b"zINSTREAM\0")
|
||||
for offset in range(0, len(raw), 1024 * 1024):
|
||||
chunk = raw[offset : offset + 1024 * 1024]
|
||||
connection.sendall(struct.pack(">I", len(chunk)))
|
||||
connection.sendall(chunk)
|
||||
connection.sendall(struct.pack(">I", 0))
|
||||
response = b""
|
||||
while not response.endswith(b"\0"):
|
||||
block = connection.recv(4096)
|
||||
if not block:
|
||||
break
|
||||
response += block
|
||||
except OSError as error:
|
||||
raise MaterialScanError("恶意文件扫描服务暂时不可用") from error
|
||||
|
||||
message = response.rstrip(b"\0").decode("utf-8", errors="replace")
|
||||
if message.endswith(" OK"):
|
||||
return {"status": "clean", "engine": "clamav", "message": message}
|
||||
if message.endswith(" FOUND"):
|
||||
signature = message.rsplit(": ", 1)[-1].removesuffix(" FOUND")
|
||||
return {
|
||||
"status": "infected",
|
||||
"engine": "clamav",
|
||||
"signature": signature,
|
||||
"message": message,
|
||||
}
|
||||
raise MaterialScanError(f"恶意文件扫描未完成：{message or '无响应'}")
|
||||
|
||||
|
||||
def _parse_pdf(raw: bytes) -> str:
|
||||
from pypdf import PdfReader
|
||||
|
||||
reader = PdfReader(io.BytesIO(raw))
|
||||
return "\n\n".join((page.extract_text() or "").strip() for page in reader.pages).strip()
|
||||
|
||||
|
||||
def _parse_docx(raw: bytes) -> str:
|
||||
from docx import Document
|
||||
|
||||
document = Document(io.BytesIO(raw))
|
||||
return "\n".join(paragraph.text.strip() for paragraph in document.paragraphs if paragraph.text)
|
||||
|
||||
|
||||
def _parse_xlsx(raw: bytes) -> str:
|
||||
from openpyxl import load_workbook
|
||||
|
||||
workbook = load_workbook(io.BytesIO(raw), read_only=True, data_only=True)
|
||||
sections = []
|
||||
try:
|
||||
for sheet in workbook.worksheets:
|
||||
rows = []
|
||||
for row in sheet.iter_rows(values_only=True):
|
||||
values = [str(value).strip() for value in row if value not in (None, "")]
|
||||
if values:
|
||||
rows.append("\t".join(values))
|
||||
if rows:
|
||||
sections.append(f"[工作表：{sheet.title}]\n" + "\n".join(rows))
|
||||
finally:
|
||||
workbook.close()
|
||||
return "\n\n".join(sections)
|
||||
|
||||
|
||||
def _parse_pptx(raw: bytes) -> str:
|
||||
from pptx import Presentation
|
||||
|
||||
presentation = Presentation(io.BytesIO(raw))
|
||||
slides = []
|
||||
for index, slide in enumerate(presentation.slides, start=1):
|
||||
texts = [
|
||||
shape.text.strip()
|
||||
for shape in slide.shapes
|
||||
if hasattr(shape, "text") and shape.text.strip()
|
||||
]
|
||||
if texts:
|
||||
slides.append(f"[幻灯片 {index}]\n" + "\n".join(texts))
|
||||
return "\n\n".join(slides)
|
||||
|
||||
|
||||
def parse_material(raw: bytes, extension: str) -> str:
|
||||
"""按已校验格式提取文本，解析失败时不伪造空成功结果。"""
|
||||
|
||||
try:
|
||||
if extension in {"txt", "log"}:
|
||||
text = _decode_text(raw)
|
||||
elif extension == "pdf":
|
||||
text = _parse_pdf(raw)
|
||||
elif extension == "docx":
|
||||
text = _parse_docx(raw)
|
||||
elif extension == "xlsx":
|
||||
text = _parse_xlsx(raw)
|
||||
elif extension == "pptx":
|
||||
text = _parse_pptx(raw)
|
||||
else:
|
||||
return ""
|
||||
except MaterialValidationError:
|
||||
raise
|
||||
except Exception as error:
|
||||
raise MaterialParseError("材料内容解析失败") from error
|
||||
return text[: settings.TICKET_MATERIAL_MAX_PARSED_CHARS].strip()
|
||||
|
||||
|
||||
def process_material(material: TicketDraftMaterial, raw: bytes | None = None) -> None:
|
||||
"""扫描并解析单个材料，分别持久化扫描与解析终态。"""
|
||||
|
||||
raw = raw if raw is not None else get_object(material.storage_path)
|
||||
material.scan_status = "scanning"
|
||||
material.scan_result = {}
|
||||
material.parse_status = "pending"
|
||||
material.parse_error = ""
|
||||
material.save(
|
||||
update_fields=["scan_status", "scan_result", "parse_status", "parse_error"]
|
||||
)
|
||||
try:
|
||||
scan_result = scan_bytes(raw)
|
||||
except Exception as error:
|
||||
message = str(error) if isinstance(error, MaterialScanError) else "恶意文件扫描执行失败"
|
||||
material.scan_status = "failed"
|
||||
material.scan_result = {"status": "failed", "message": message}
|
||||
material.parse_status = "blocked"
|
||||
material.parse_error = "恶意文件扫描未通过，未执行解析"
|
||||
material.save(
|
||||
update_fields=["scan_status", "scan_result", "parse_status", "parse_error"]
|
||||
)
|
||||
return
|
||||
|
||||
material.scan_status = scan_result["status"]
|
||||
material.scan_result = scan_result
|
||||
if material.scan_status != "clean":
|
||||
material.parse_status = "blocked"
|
||||
material.parse_error = "材料被恶意文件扫描拦截，未执行解析"
|
||||
material.save(
|
||||
update_fields=["scan_status", "scan_result", "parse_status", "parse_error"]
|
||||
)
|
||||
return
|
||||
|
||||
if material.material_type == "image":
|
||||
material.parse_status = "not_applicable"
|
||||
material.parsed_text = ""
|
||||
material.save(
|
||||
update_fields=["scan_status", "scan_result", "parse_status", "parsed_text"]
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
material.parsed_text = parse_material(raw, material.extension)
|
||||
material.parse_status = "parsed"
|
||||
material.parse_error = ""
|
||||
except (MaterialParseError, MaterialValidationError) as error:
|
||||
material.parsed_text = ""
|
||||
material.parse_status = "failed"
|
||||
material.parse_error = str(error)
|
||||
material.save(
|
||||
update_fields=[
|
||||
"scan_status",
|
||||
"scan_result",
|
||||
"parse_status",
|
||||
"parse_error",
|
||||
"parsed_text",
|
||||
]
|
||||
)
|
||||
Reference in New Issue
Block a user