feat: complete translator stack

backend/app/services/__init__.py

@@ -0,0 +1 @@
+from .chunking import chunk_document, chunk_store

backend/app/services/chunking.py

@@ -0,0 +1,87 @@
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Dict, List, Optional
+
+from backend.app.processors.models import DocumentChunk, DocumentExtraction
+
+
+@dataclass
+class SessionChunks:
+    extraction: DocumentExtraction
+    chunks: List[DocumentChunk]
+    created_at: datetime
+
+
+class ChunkStore:
+    def __init__(self):
+        self._sessions: Dict[str, SessionChunks] = {}
+
+    def register(self, session_id: str, extraction: DocumentExtraction, chunks: List[DocumentChunk]) -> None:
+        self._sessions[session_id] = SessionChunks(extraction, chunks, datetime.utcnow())
+
+    def get_chunks(self, session_id: str) -> List[DocumentChunk]:
+        session = self._sessions.get(session_id)
+        return session.chunks if session else []
+
+    def get_session(self, session_id: str) -> Optional[SessionChunks]:
+        return self._sessions.get(session_id)
+
+    def remove(self, session_id: str) -> None:
+        self._sessions.pop(session_id, None)
+
+
+chunk_store = ChunkStore()
+
+
+def _looks_like_title(paragraph: str) -> bool:
+    stripped = paragraph.strip()
+    if not stripped:
+        return False
+    if stripped.startswith("#"):
+        return True
+    if stripped.endswith(":") and len(stripped) < 60:
+        return True
+    if stripped.isupper() and len(stripped) < 120:
+        return True
+    return False
+
+
+def chunk_document(session_id: str, extraction: DocumentExtraction, chunk_size: int = 2_500) -> List[DocumentChunk]:
+    paragraphs = [p.strip() for p in extraction.paragraphs if p.strip()]
+    if not paragraphs:
+        chunk_store.register(session_id, extraction, [])
+        return []
+
+    chunks: List[DocumentChunk] = []
+    buffer: List[str] = []
+    buffer_len = 0
+    current_title = extraction.title
+
+    def flush_chunk() -> None:
+        nonlocal buffer, buffer_len, current_title
+        if not buffer:
+            return
+        chunk_text = "\n\n".join(buffer)
+        chunks.append(
+            DocumentChunk(
+                index=len(chunks) + 1,
+                text=chunk_text,
+                title_hint=current_title,
+                paragraph_count=len(buffer),
+            )
+        )
+        buffer = []
+        buffer_len = 0
+
+    for paragraph in paragraphs:
+        if _looks_like_title(paragraph):
+            current_title = paragraph
+        addition_len = len(paragraph)
+        if buffer and buffer_len + addition_len + 2 > chunk_size:
+            flush_chunk()
+        buffer.append(paragraph)
+        buffer_len += addition_len + 2
+
+    flush_chunk()
+    chunk_store.register(session_id, extraction, chunks)
+    return chunks