init: llm-wiki-web con FastAPI + Ollama

backend/__init__.py

@@ -0,0 +1 @@
+# Backend package

backend/ingest.py

@@ -0,0 +1,205 @@
+"""
+Ingest pipeline - due fasi: analisi + generazione
+"""
+
+import json
+import re
+from pathlib import Path
+from typing import List, Dict, Optional
+from . import llm, wiki_ops
+import config
+
+
+SYSTEM_PROMPT_ANALYSIS = """Sei un bibliotecario esperto. Analizza il documento fornito e estrai informazioni strutturate.
+
+Rispondi SOLO con JSON valido in questo formato:
+{
+  "entities": ["nome persona/organizzazione/strumento"],
+  "concepts": ["concetto/tecnica/metodo"],
+  "claims": ["affermazione importante trovata"],
+  "connections": ["riferimento a [[pagina-wiki]] esistente"],
+  "conflicts": ["affermazione che contraddice qualcosa di esistente"]
+}
+
+Non inventare informazioni. Solo cio che e esplicitamente nel documento."""
+
+
+SYSTEM_PROMPT_GENERATION = """Sei un wiki maintainer. Genera pagine markdown per una wiki basata sull'analisi fornita.
+
+Regole:
+- Una pagina per ogni entity e concept
+- Usa il formato:
+  # Nome Pagina
+
+  > Una frase che riassume.
+
+  ## Tags
+  `entity` o `concept`
+
+  ## Body
+  Contenuto dettagliato...
+
+  ## Related
+  - [[pagina-correlata]]
+
+  ## Sources
+  - [nome source](path)
+
+- Per cross-references usa [[nome-pagina]]
+- Mantieni i nomi delle pagine in minuscolo con trattini
+-rispondi con JSON: {"pages": [{"name": "nome", "category": "entities|concepts", "content": "contenuto markdown"}]}"""
+
+
+def chunk_text(text: str, max_chars: int = 3000) -> List[str]:
+    """Suddivide testo in chunks se troppo lungo per Gemma."""
+    if len(text) <= max_chars:
+        return [text]
+
+    chunks = []
+    paragraphs = text.split("\n\n")
+    current_chunk = ""
+
+    for para in paragraphs:
+        if len(current_chunk) + len(para) <= max_chars:
+            current_chunk += "\n\n" + para
+        else:
+            if current_chunk:
+                chunks.append(current_chunk.strip())
+            current_chunk = para
+
+    if current_chunk:
+        chunks.append(current_chunk.strip())
+
+    return chunks
+
+
+async def run_analysis(source_path: Path) -> Dict:
+    """Fase 1: analizza il source e ritorna struttura."""
+    content = source_path.read_text(encoding="utf-8")
+
+    prompt = f"{SYSTEM_PROMPT_ANALYSIS}\n\n## Documento:\n{content}"
+
+    try:
+        response = await llm.llm_call(prompt)
+        # Estrai JSON dalla risposta
+        json_match = re.search(r'\{.*\}', response, re.DOTALL)
+        if json_match:
+            return json.loads(json_match.group())
+    except Exception as e:
+        return {"error": str(e), "entities": [], "concepts": [], "claims": [], "connections": [], "conflicts": []}
+
+    return {"entities": [], "concepts": [], "claims": [], "connections": [], "conflicts": []}
+
+
+async def run_generation(source_path: Path, analysis: Dict) -> List[Dict]:
+    """Fase 2: genera pagine wiki dall'analisi."""
+    content = source_path.read_text(encoding="utf-8")
+    source_name = source_path.stem
+
+    prompt = f"""{SYSTEM_PROMPT_GENERATION}
+
+## Source file: {source_path.name}
+## Analysis: {json.dumps(analysis, ensure_ascii=False)}
+
+## Source content (per reference):
+{content[:2000]}...
+"""
+
+    try:
+        response = await llm.llm_call(prompt)
+        json_match = re.search(r'\{.*\}', response, re.DOTALL)
+        if json_match:
+            data = json.loads(json_match.group())
+            return data.get("pages", [])
+    except Exception:
+        pass
+
+    return []
+
+
+async def ingest_source(source_path: Path) -> Dict:
+    """Pipeline completa: analisi -> generazione -> scrittura file."""
+    result = {
+        "source": str(source_path),
+        "analysis": None,
+        "pages_created": [],
+        "pages_updated": [],
+        "error": None
+    }
+
+    # Fase 1: Analisi
+    analysis = await run_analysis(source_path)
+    result["analysis"] = analysis
+
+    if "error" in analysis:
+        result["error"] = analysis["error"]
+        return result
+
+    # Fase 2: Generazione
+    pages = await run_generation(source_path, analysis)
+    result["pages_created"] = [p["name"] for p in pages]
+
+    # Scrittura file
+    wiki_root = config.WIKI_ROOT
+    source_category = "sources"
+
+    created_pages = []
+    for page in pages:
+        page_name = page["name"].lower().replace(" ", "-")
+        category = page.get("category", "concepts")
+        page_content = page["content"]
+
+        category_dir = wiki_root / "wiki" / category
+        page_path = category_dir / f"{page_name}.md"
+
+        wiki_ops.write_page(page_path, page_content)
+        created_pages.append(page_name)
+
+    # Aggiorna index
+    if created_pages:
+        wiki_ops.update_index(wiki_root, created_pages, category)
+
+    # Append log
+    wiki_ops.append_log(wiki_root, source_path.name, created_pages, [])
+
+    return result
+
+
+async def ingest_streaming(source_path: Path):
+    """Pipeline con streaming per feedback in tempo reale."""
+    wiki_root = config.WIKI_ROOT
+
+    yield f"data: Inizio ingest di {source_path.name}\n\n"
+
+    # Fase 1: Analisi
+    yield f"data: [1/2] Analisi in corso...\n\n"
+    analysis = await run_analysis(source_path)
+    yield f"data: Analisi completata. Trovati: {len(analysis.get('entities', []))} entities, {len(analysis.get('concepts', []))} concepts\n\n"
+
+    if "error" in analysis:
+        yield f"data: ERROR: {analysis['error']}\n\n"
+        return
+
+    # Fase 2: Generazione
+    yield f"data: [2/2] Generazione pagine wiki...\n\n"
+    pages = await run_generation(source_path, analysis)
+
+    created = []
+    for page in pages:
+        page_name = page["name"].lower().replace(" ", "-")
+        category = page.get("category", "concepts")
+        page_content = page["content"]
+
+        category_dir = wiki_root / "wiki" / category
+        page_path = category_dir / f"{page_name}.md"
+
+        wiki_ops.write_page(page_path, page_content)
+        created.append(page_name)
+        yield f"data: Creata: [[{page_name}]]\n\n"
+
+    # Aggiorna index e log
+    if created:
+        wiki_ops.update_index(wiki_root, created, "concepts")
+        wiki_ops.append_log(wiki_root, source_path.name, created, [])
+
+    yield f"data: DONE: {len(created)} pagine create\n\n"

backend/llm.py

@@ -0,0 +1,74 @@
+"""
+Chiamate LLM via Ollama
+"""
+
+import json
+import httpx
+from typing import AsyncGenerator, Optional
+import config
+
+
+async def llm_call(prompt: str, model: Optional[str] = None) -> str:
+    """Chiamata semplice a Ollama - restituisce risposta completa."""
+    model = model or config.OLLAMA_MODEL
+
+    async with httpx.AsyncClient(timeout=120.0) as client:
+        response = await client.post(
+            f"{config.OLLAMA_URL}/api/generate",
+            json={
+                "model": model,
+                "prompt": prompt,
+                "stream": False
+            }
+        )
+        response.raise_for_status()
+        return response.json()["response"]
+
+
+async def llm_stream(prompt: str, model: Optional[str] = None) -> AsyncGenerator[str, None]:
+    """Chiamata streaming a Ollama - yields tokens."""
+    model = model or config.OLLAMA_MODEL
+
+    async with httpx.AsyncClient(timeout=120.0) as client:
+        async with client.stream(
+            "POST",
+            f"{config.OLLAMA_URL}/api/generate",
+            json={
+                "model": model,
+                "prompt": prompt,
+                "stream": True
+            }
+        ) as response:
+            response.raise_for_status()
+            async for line in response.aiter_lines():
+                if line:
+                    try:
+                        data = json.loads(line)
+                        yield data.get("response", "")
+                    except json.JSONDecodeError:
+                        continue
+
+
+async def llm_chat(messages: list, model: Optional[str] = None) -> AsyncGenerator[str, None]:
+    """Chat mode - streaming."""
+    model = model or config.OLLAMA_MODEL
+
+    async with httpx.AsyncClient(timeout=120.0) as client:
+        async with client.stream(
+            "POST",
+            f"{config.OLLAMA_URL}/api/chat",
+            json={
+                "model": model,
+                "messages": messages,
+                "stream": True
+            }
+        ) as response:
+            response.raise_for_status()
+            async for line in response.aiter_lines():
+                if line:
+                    try:
+                        data = json.loads(line)
+                        if "content" in data:
+                            yield data["content"]
+                    except json.JSONDecodeError:
+                        continue

backend/main.py

@@ -0,0 +1,149 @@
+"""
+API principale - FastAPI
+"""
+
+from pathlib import Path
+from fastapi import FastAPI, HTTPException
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import FileResponse, HTMLResponse
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+import asyncio
+import sse_starlette.sse as sse
+import config
+from backend import wiki_ops, search, ingest, llm
+
+app = FastAPI()
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+class IngestRequest(BaseModel):
+    source: str
+
+
+class ChatRequest(BaseModel):
+    message: str
+    history: list = []
+
+
+@app.get("/")
+async def root():
+    return FileResponse("static/index.html")
+
+
+@app.get("/api/wiki/list")
+async def list_wiki():
+    return wiki_ops.list_wiki_pages(config.WIKI_ROOT)
+
+
+@app.get("/api/wiki/page/{path:path}")
+async def get_page(path: str):
+    file_path = config.WIKI_ROOT / "wiki" / path
+    if not file_path.exists():
+        # Prova con .md
+        file_path = config.WIKI_ROOT / "wiki" / f"{path}.md"
+
+    page_data = wiki_ops.read_page(file_path)
+    if "error" in page_data:
+        raise HTTPException(status_code=404, detail=page_data["error"])
+
+    return page_data
+
+
+@app.post("/api/ingest")
+async def ingest_source_endpoint(req: IngestRequest):
+    source_path = config.WIKI_ROOT / "raw" / req.source
+    if not source_path.exists():
+        raise HTTPException(status_code=404, detail=f"Source not found: {req.source}")
+
+    result = await ingest.ingest_source(source_path)
+    return result
+
+
+@app.get("/api/ingest/stream/{source}")
+async def ingest_stream(source: str):
+    source_path = config.WIKI_ROOT / "raw" / source
+    if not source_path.exists():
+        raise HTTPException(status_code=404, detail=f"Source not found: {source}")
+
+    async def event_generator():
+        async for line in ingest.ingest_streaming(source_path):
+            yield {"event": "message", "data": line}
+
+    return sse.EventServerResponse(event_generator())
+
+
+@app.get("/api/search")
+async def search_wiki(q: str = ""):
+    if not q:
+        return {"results": []}
+
+    name_results = search.search_by_name(q)
+    content_results = search.search_wiki(q)
+
+    return {
+        "by_name": name_results,
+        "by_content": content_results
+    }
+
+
+@app.get("/api/wiki/recent")
+async def recent_pages(limit: int = 10):
+    return search.get_recent_pages(limit)
+
+
+@app.post("/api/chat")
+async def chat(req: ChatRequest):
+    # Costruisci context dalla wiki
+    wiki_index = wiki_ops.read_page(config.WIKI_ROOT / "wiki" / "index.md")
+    overview = wiki_ops.read_page(config.WIKI_ROOT / "wiki" / "overview.md")
+
+    system_prompt = f"""Sei un assistente che risponde basandosi sulla wiki.
+USA SEMPRE [[pagina]] per citare le fonti.
+
+Wiki Index:
+{wiki_index.get('content', '')}
+
+Overview:
+{overview.get('content', '')}
+
+Rispondi in italiano. Cita le fonti con [[nome-pagina]]."""
+
+    messages = [{"role": "system", "content": system_prompt}]
+    messages.extend(req.history)
+    messages.append({"role": "user", "content": req.message})
+
+    async def event_generator():
+        async for token in llm.llm_chat(messages):
+            yield {"event": "message", "data": token}
+
+    return sse.EventServerResponse(event_generator())
+
+
+@app.get("/api/raw/list")
+async def list_raw():
+    """Lista i file nella directory raw."""
+    raw_dir = config.WIKI_ROOT / "raw"
+    result = {}
+
+    for category in ["articles", "papers", "repos", "assets"]:
+        cat_dir = raw_dir / category
+        if cat_dir.exists():
+            files = [f.name for f in cat_dir.iterdir() if f.is_file()]
+            result[category] = files
+        else:
+            result[category] = []
+
+    return result
+
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=config.WEB_PORT)

backend/search.py

@@ -0,0 +1,94 @@
+"""
+Ricerca nei file wiki
+"""
+
+import re
+from pathlib import Path
+from typing import List, Dict
+import config
+
+
+def search_wiki(query: str, limit: int = 20) -> List[Dict]:
+    """Cerca nei contenuti delle pagine wiki."""
+    results = []
+    wiki_root = config.WIKI_ROOT
+
+    query_lower = query.lower()
+
+    for md_file in wiki_root.glob("wiki/**/*.md"):
+        if md_file.name in ["index.md", "log.md"]:
+            continue
+
+        try:
+            content = md_file.read_text(encoding="utf-8").lower()
+            if query_lower in content:
+                # Trova le linee con il match
+                lines = md_file.read_text(encoding="utf-8").split("\n")
+                matches = []
+                for i, line in enumerate(lines):
+                    if query_lower in line.lower():
+                        matches.append({
+                            "line": i + 1,
+                            "text": line.strip()[:200]
+                        })
+
+                results.append({
+                    "file": str(md_file.relative_to(wiki_root)),
+                    "name": md_file.stem,
+                    "category": md_file.parent.name,
+                    "matches": matches[:5]  # max 5 match per file
+                })
+
+                if len(results) >= limit:
+                    break
+
+        except Exception:
+            continue
+
+    return results
+
+
+def search_by_name(query: str) -> List[Dict]:
+    """Cerca per nome file/titolo."""
+    results = []
+    wiki_root = config.WIKI_ROOT
+    query_lower = query.lower()
+
+    for md_file in wiki_root.glob("wiki/**/*.md"):
+        if md_file.name in ["index.md", "log.md"]:
+            continue
+
+        name_lower = md_file.stem.lower()
+        if query_lower in name_lower:
+            results.append({
+                "file": str(md_file.relative_to(wiki_root)),
+                "name": md_file.stem,
+                "category": md_file.parent.name
+            })
+
+    return results
+
+
+def get_recent_pages(limit: int = 10) -> List[Dict]:
+    """Returns the most recently modified pages."""
+    results = []
+    wiki_root = config.WIKI_ROOT
+
+    files_with_mtime = []
+    for md_file in wiki_root.glob("wiki/**/*.md"):
+        if md_file.name in ["index.md", "log.md"]:
+            continue
+        mtime = md_file.stat().st_mtime
+        files_with_mtime.append((mtime, md_file))
+
+    files_with_mtime.sort(reverse=True)
+
+    for mtime, md_file in files_with_mtime[:limit]:
+        results.append({
+            "file": str(md_file.relative_to(wiki_root)),
+            "name": md_file.stem,
+            "category": md_file.parent.name,
+            "mtime": mtime
+        })
+
+    return results

backend/wiki_ops.py

@@ -0,0 +1,147 @@
+"""
+Operazioni sui file della wiki
+"""
+
+import re
+from pathlib import Path
+from typing import List, Dict, Optional
+import frontmatter
+
+
+def list_wiki_pages(wiki_root: Path) -> Dict[str, List[Dict]]:
+    """Lista tutte le pagine wiki organizzate per categoria."""
+    categories = {
+        "entities": [],
+        "concepts": [],
+        "sources": [],
+        "comparisons": [],
+        "topics": [],
+    }
+
+    for category in categories.keys():
+        category_dir = wiki_root / "wiki" / category
+        if category_dir.exists():
+            for md_file in category_dir.glob("*.md"):
+                categories[category].append({
+                    "name": md_file.stem,
+                    "file": str(md_file),
+                    "category": category
+                })
+
+    # Anche le pagine root
+    for md_file in (wiki_root / "wiki").glob("*.md"):
+        if md_file.name not in ["index.md", "log.md", "overview.md"]:
+            categories["topics"].append({
+                "name": md_file.stem,
+                "file": str(md_file),
+                "category": "topics"
+            })
+
+    return categories
+
+
+def read_page(file_path: Path) -> Dict:
+    """Legge una pagina wiki e restituisce contenuto + metadata."""
+    if not file_path.exists():
+        return {"error": "File not found"}
+
+    try:
+        post = frontmatter.parse(file_path)
+        content = post.content
+        metadata = post.metadata
+    except Exception:
+        # Fallback: leggi solo contenuto
+        content = file_path.read_text(encoding="utf-8")
+        metadata = {}
+
+    return {
+        "content": content,
+        "metadata": metadata,
+        "file": str(file_path)
+    }
+
+
+def write_page(file_path: Path, content: str, metadata: Optional[Dict] = None):
+    """Scrive una pagina wiki con frontmatter."""
+    # Assicura che la directory esista
+    file_path.parent.mkdir(parents=True, exist_ok=True)
+
+    if metadata:
+        # Usa frontmatter per scrivere con metadata
+        post = frontmatter.Post(content, **metadata)
+        with open(file_path, "w", encoding="utf-8") as f:
+            frontmatter.dump(post, f)
+    else:
+        file_path.write_text(content, encoding="utf-8")
+
+
+def update_index(wiki_root: Path, new_pages: List[str], category: str):
+    """Aggiunge pagine all'indice."""
+    index_file = wiki_root / "wiki" / "index.md"
+
+    # Leggi indice esistente
+    if index_file.exists():
+        content = index_file.read_text(encoding="utf-8")
+    else:
+        content = "# Wiki Index\n\n> Master catalog of all wiki pages.\n\n"
+
+    # Aggiungi nuovo entries
+    for page in new_pages:
+        # Check se gia presente
+        link = f"[[{page}]]"
+        if link not in content:
+            section = f"## {category.capitalize()}"
+            if section not in content:
+                content += f"\n{section}\n"
+            content += f"- [[{page}]]\n"
+
+    index_file.write_text(content, encoding="utf-8")
+
+
+def append_log(wiki_root: Path, source_file: str, created: List[str], updated: List[str]):
+    """Appende un entry al log."""
+    log_file = wiki_root / "wiki" / "log.md"
+    from datetime import datetime
+    now = datetime.now().strftime("%Y-%m-%d %H:%M")
+
+    entry = f"\n## {now} — {Path(source_file).name}\n"
+    if created:
+        entry += f"- Created: {', '.join(created)}\n"
+    if updated:
+        entry += f"- Updated: {', '.join(updated)}\n"
+
+    if log_file.exists():
+        content = log_file.read_text(encoding="utf-8")
+        # Insert dopo l'header
+        lines = content.split("\n")
+        insert_idx = 2  # dopo # Wiki Log e ---
+        lines.insert(insert_idx, entry)
+        content = "\n".join(lines)
+    else:
+        content = f"# Wiki Log\n\n> Append-only record.\n{entry}\n"
+
+    log_file.write_text(content, encoding="utf-8")
+
+
+def extract_wikilinks(content: str) -> List[str]:
+    """Estrae tutti i [[wikilinks]] dal contenuto."""
+    pattern = r'\[\[([^\]]+)\]\]'
+    return re.findall(pattern, content)
+
+
+def get_all_wikilinks(wiki_root: Path) -> Dict[str, List[str]]:
+    """Trova tutti i wikilinks in tutte le pagine."""
+    links = {}
+
+    for md_file in wiki_root.glob("wiki/**/*.md"):
+        if md_file.name == "index.md":
+            continue
+        try:
+            content = md_file.read_text(encoding="utf-8")
+            page_links = extract_wikilinks(content)
+            if page_links:
+                links[str(md_file.relative_to(wiki_root))] = page_links
+        except Exception:
+            continue
+
+    return links