init: llm-wiki-web con FastAPI + Ollama

backend/ingest.py

@@ -0,0 +1,205 @@
+"""
+Ingest pipeline - due fasi: analisi + generazione
+"""
+
+import json
+import re
+from pathlib import Path
+from typing import List, Dict, Optional
+from . import llm, wiki_ops
+import config
+
+
+SYSTEM_PROMPT_ANALYSIS = """Sei un bibliotecario esperto. Analizza il documento fornito e estrai informazioni strutturate.
+
+Rispondi SOLO con JSON valido in questo formato:
+{
+  "entities": ["nome persona/organizzazione/strumento"],
+  "concepts": ["concetto/tecnica/metodo"],
+  "claims": ["affermazione importante trovata"],
+  "connections": ["riferimento a [[pagina-wiki]] esistente"],
+  "conflicts": ["affermazione che contraddice qualcosa di esistente"]
+}
+
+Non inventare informazioni. Solo cio che e esplicitamente nel documento."""
+
+
+SYSTEM_PROMPT_GENERATION = """Sei un wiki maintainer. Genera pagine markdown per una wiki basata sull'analisi fornita.
+
+Regole:
+- Una pagina per ogni entity e concept
+- Usa il formato:
+  # Nome Pagina
+
+  > Una frase che riassume.
+
+  ## Tags
+  `entity` o `concept`
+
+  ## Body
+  Contenuto dettagliato...
+
+  ## Related
+  - [[pagina-correlata]]
+
+  ## Sources
+  - [nome source](path)
+
+- Per cross-references usa [[nome-pagina]]
+- Mantieni i nomi delle pagine in minuscolo con trattini
+-rispondi con JSON: {"pages": [{"name": "nome", "category": "entities|concepts", "content": "contenuto markdown"}]}"""
+
+
+def chunk_text(text: str, max_chars: int = 3000) -> List[str]:
+    """Suddivide testo in chunks se troppo lungo per Gemma."""
+    if len(text) <= max_chars:
+        return [text]
+
+    chunks = []
+    paragraphs = text.split("\n\n")
+    current_chunk = ""
+
+    for para in paragraphs:
+        if len(current_chunk) + len(para) <= max_chars:
+            current_chunk += "\n\n" + para
+        else:
+            if current_chunk:
+                chunks.append(current_chunk.strip())
+            current_chunk = para
+
+    if current_chunk:
+        chunks.append(current_chunk.strip())
+
+    return chunks
+
+
+async def run_analysis(source_path: Path) -> Dict:
+    """Fase 1: analizza il source e ritorna struttura."""
+    content = source_path.read_text(encoding="utf-8")
+
+    prompt = f"{SYSTEM_PROMPT_ANALYSIS}\n\n## Documento:\n{content}"
+
+    try:
+        response = await llm.llm_call(prompt)
+        # Estrai JSON dalla risposta
+        json_match = re.search(r'\{.*\}', response, re.DOTALL)
+        if json_match:
+            return json.loads(json_match.group())
+    except Exception as e:
+        return {"error": str(e), "entities": [], "concepts": [], "claims": [], "connections": [], "conflicts": []}
+
+    return {"entities": [], "concepts": [], "claims": [], "connections": [], "conflicts": []}
+
+
+async def run_generation(source_path: Path, analysis: Dict) -> List[Dict]:
+    """Fase 2: genera pagine wiki dall'analisi."""
+    content = source_path.read_text(encoding="utf-8")
+    source_name = source_path.stem
+
+    prompt = f"""{SYSTEM_PROMPT_GENERATION}
+
+## Source file: {source_path.name}
+## Analysis: {json.dumps(analysis, ensure_ascii=False)}
+
+## Source content (per reference):
+{content[:2000]}...
+"""
+
+    try:
+        response = await llm.llm_call(prompt)
+        json_match = re.search(r'\{.*\}', response, re.DOTALL)
+        if json_match:
+            data = json.loads(json_match.group())
+            return data.get("pages", [])
+    except Exception:
+        pass
+
+    return []
+
+
+async def ingest_source(source_path: Path) -> Dict:
+    """Pipeline completa: analisi -> generazione -> scrittura file."""
+    result = {
+        "source": str(source_path),
+        "analysis": None,
+        "pages_created": [],
+        "pages_updated": [],
+        "error": None
+    }
+
+    # Fase 1: Analisi
+    analysis = await run_analysis(source_path)
+    result["analysis"] = analysis
+
+    if "error" in analysis:
+        result["error"] = analysis["error"]
+        return result
+
+    # Fase 2: Generazione
+    pages = await run_generation(source_path, analysis)
+    result["pages_created"] = [p["name"] for p in pages]
+
+    # Scrittura file
+    wiki_root = config.WIKI_ROOT
+    source_category = "sources"
+
+    created_pages = []
+    for page in pages:
+        page_name = page["name"].lower().replace(" ", "-")
+        category = page.get("category", "concepts")
+        page_content = page["content"]
+
+        category_dir = wiki_root / "wiki" / category
+        page_path = category_dir / f"{page_name}.md"
+
+        wiki_ops.write_page(page_path, page_content)
+        created_pages.append(page_name)
+
+    # Aggiorna index
+    if created_pages:
+        wiki_ops.update_index(wiki_root, created_pages, category)
+
+    # Append log
+    wiki_ops.append_log(wiki_root, source_path.name, created_pages, [])
+
+    return result
+
+
+async def ingest_streaming(source_path: Path):
+    """Pipeline con streaming per feedback in tempo reale."""
+    wiki_root = config.WIKI_ROOT
+
+    yield f"data: Inizio ingest di {source_path.name}\n\n"
+
+    # Fase 1: Analisi
+    yield f"data: [1/2] Analisi in corso...\n\n"
+    analysis = await run_analysis(source_path)
+    yield f"data: Analisi completata. Trovati: {len(analysis.get('entities', []))} entities, {len(analysis.get('concepts', []))} concepts\n\n"
+
+    if "error" in analysis:
+        yield f"data: ERROR: {analysis['error']}\n\n"
+        return
+
+    # Fase 2: Generazione
+    yield f"data: [2/2] Generazione pagine wiki...\n\n"
+    pages = await run_generation(source_path, analysis)
+
+    created = []
+    for page in pages:
+        page_name = page["name"].lower().replace(" ", "-")
+        category = page.get("category", "concepts")
+        page_content = page["content"]
+
+        category_dir = wiki_root / "wiki" / category
+        page_path = category_dir / f"{page_name}.md"
+
+        wiki_ops.write_page(page_path, page_content)
+        created.append(page_name)
+        yield f"data: Creata: [[{page_name}]]\n\n"
+
+    # Aggiorna index e log
+    if created:
+        wiki_ops.update_index(wiki_root, created, "concepts")
+        wiki_ops.append_log(wiki_root, source_path.name, created, [])
+
+    yield f"data: DONE: {len(created)} pagine create\n\n"