init: llm-wiki-web con FastAPI + Ollama
This commit is contained in:
commit
08291b578c
19 files changed
+1030
No files matched your search
@@ -0,0 +1 @@
|
||||
.venv/
|
||||
Binary file not shown.
@@ -0,0 +1 @@
|
||||
# Backend package
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,205 @@
|
||||
"""
|
||||
Ingest pipeline - due fasi: analisi + generazione
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Optional
|
||||
from . import llm, wiki_ops
|
||||
import config
|
||||
|
||||
|
||||
SYSTEM_PROMPT_ANALYSIS = """Sei un bibliotecario esperto. Analizza il documento fornito e estrai informazioni strutturate.
|
||||
|
||||
Rispondi SOLO con JSON valido in questo formato:
|
||||
{
|
||||
"entities": ["nome persona/organizzazione/strumento"],
|
||||
"concepts": ["concetto/tecnica/metodo"],
|
||||
"claims": ["affermazione importante trovata"],
|
||||
"connections": ["riferimento a [[pagina-wiki]] esistente"],
|
||||
"conflicts": ["affermazione che contraddice qualcosa di esistente"]
|
||||
}
|
||||
|
||||
Non inventare informazioni. Solo cio che e esplicitamente nel documento."""
|
||||
|
||||
|
||||
SYSTEM_PROMPT_GENERATION = """Sei un wiki maintainer. Genera pagine markdown per una wiki basata sull'analisi fornita.
|
||||
|
||||
Regole:
|
||||
- Una pagina per ogni entity e concept
|
||||
- Usa il formato:
|
||||
# Nome Pagina
|
||||
|
||||
> Una frase che riassume.
|
||||
|
||||
## Tags
|
||||
`entity` o `concept`
|
||||
|
||||
## Body
|
||||
Contenuto dettagliato...
|
||||
|
||||
## Related
|
||||
- [[pagina-correlata]]
|
||||
|
||||
## Sources
|
||||
- [nome source](path)
|
||||
|
||||
- Per cross-references usa [[nome-pagina]]
|
||||
- Mantieni i nomi delle pagine in minuscolo con trattini
|
||||
-rispondi con JSON: {"pages": [{"name": "nome", "category": "entities|concepts", "content": "contenuto markdown"}]}"""
|
||||
|
||||
|
||||
def chunk_text(text: str, max_chars: int = 3000) -> List[str]:
|
||||
"""Suddivide testo in chunks se troppo lungo per Gemma."""
|
||||
if len(text) <= max_chars:
|
||||
return [text]
|
||||
|
||||
chunks = []
|
||||
paragraphs = text.split("\n\n")
|
||||
current_chunk = ""
|
||||
|
||||
for para in paragraphs:
|
||||
if len(current_chunk) + len(para) <= max_chars:
|
||||
current_chunk += "\n\n" + para
|
||||
else:
|
||||
if current_chunk:
|
||||
chunks.append(current_chunk.strip())
|
||||
current_chunk = para
|
||||
|
||||
if current_chunk:
|
||||
chunks.append(current_chunk.strip())
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
async def run_analysis(source_path: Path) -> Dict:
|
||||
"""Fase 1: analizza il source e ritorna struttura."""
|
||||
content = source_path.read_text(encoding="utf-8")
|
||||
|
||||
prompt = f"{SYSTEM_PROMPT_ANALYSIS}\n\n## Documento:\n{content}"
|
||||
|
||||
try:
|
||||
response = await llm.llm_call(prompt)
|
||||
# Estrai JSON dalla risposta
|
||||
json_match = re.search(r'\{.*\}', response, re.DOTALL)
|
||||
if json_match:
|
||||
return json.loads(json_match.group())
|
||||
except Exception as e:
|
||||
return {"error": str(e), "entities": [], "concepts": [], "claims": [], "connections": [], "conflicts": []}
|
||||
|
||||
return {"entities": [], "concepts": [], "claims": [], "connections": [], "conflicts": []}
|
||||
|
||||
|
||||
async def run_generation(source_path: Path, analysis: Dict) -> List[Dict]:
|
||||
"""Fase 2: genera pagine wiki dall'analisi."""
|
||||
content = source_path.read_text(encoding="utf-8")
|
||||
source_name = source_path.stem
|
||||
|
||||
prompt = f"""{SYSTEM_PROMPT_GENERATION}
|
||||
|
||||
## Source file: {source_path.name}
|
||||
## Analysis: {json.dumps(analysis, ensure_ascii=False)}
|
||||
|
||||
## Source content (per reference):
|
||||
{content[:2000]}...
|
||||
"""
|
||||
|
||||
try:
|
||||
response = await llm.llm_call(prompt)
|
||||
json_match = re.search(r'\{.*\}', response, re.DOTALL)
|
||||
if json_match:
|
||||
data = json.loads(json_match.group())
|
||||
return data.get("pages", [])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return []
|
||||
|
||||
|
||||
async def ingest_source(source_path: Path) -> Dict:
|
||||
"""Pipeline completa: analisi -> generazione -> scrittura file."""
|
||||
result = {
|
||||
"source": str(source_path),
|
||||
"analysis": None,
|
||||
"pages_created": [],
|
||||
"pages_updated": [],
|
||||
"error": None
|
||||
}
|
||||
|
||||
# Fase 1: Analisi
|
||||
analysis = await run_analysis(source_path)
|
||||
result["analysis"] = analysis
|
||||
|
||||
if "error" in analysis:
|
||||
result["error"] = analysis["error"]
|
||||
return result
|
||||
|
||||
# Fase 2: Generazione
|
||||
pages = await run_generation(source_path, analysis)
|
||||
result["pages_created"] = [p["name"] for p in pages]
|
||||
|
||||
# Scrittura file
|
||||
wiki_root = config.WIKI_ROOT
|
||||
source_category = "sources"
|
||||
|
||||
created_pages = []
|
||||
for page in pages:
|
||||
page_name = page["name"].lower().replace(" ", "-")
|
||||
category = page.get("category", "concepts")
|
||||
page_content = page["content"]
|
||||
|
||||
category_dir = wiki_root / "wiki" / category
|
||||
page_path = category_dir / f"{page_name}.md"
|
||||
|
||||
wiki_ops.write_page(page_path, page_content)
|
||||
created_pages.append(page_name)
|
||||
|
||||
# Aggiorna index
|
||||
if created_pages:
|
||||
wiki_ops.update_index(wiki_root, created_pages, category)
|
||||
|
||||
# Append log
|
||||
wiki_ops.append_log(wiki_root, source_path.name, created_pages, [])
|
||||
|
||||
return result
|
||||
|
||||
|
||||
async def ingest_streaming(source_path: Path):
|
||||
"""Pipeline con streaming per feedback in tempo reale."""
|
||||
wiki_root = config.WIKI_ROOT
|
||||
|
||||
yield f"data: Inizio ingest di {source_path.name}\n\n"
|
||||
|
||||
# Fase 1: Analisi
|
||||
yield f"data: [1/2] Analisi in corso...\n\n"
|
||||
analysis = await run_analysis(source_path)
|
||||
yield f"data: Analisi completata. Trovati: {len(analysis.get('entities', []))} entities, {len(analysis.get('concepts', []))} concepts\n\n"
|
||||
|
||||
if "error" in analysis:
|
||||
yield f"data: ERROR: {analysis['error']}\n\n"
|
||||
return
|
||||
|
||||
# Fase 2: Generazione
|
||||
yield f"data: [2/2] Generazione pagine wiki...\n\n"
|
||||
pages = await run_generation(source_path, analysis)
|
||||
|
||||
created = []
|
||||
for page in pages:
|
||||
page_name = page["name"].lower().replace(" ", "-")
|
||||
category = page.get("category", "concepts")
|
||||
page_content = page["content"]
|
||||
|
||||
category_dir = wiki_root / "wiki" / category
|
||||
page_path = category_dir / f"{page_name}.md"
|
||||
|
||||
wiki_ops.write_page(page_path, page_content)
|
||||
created.append(page_name)
|
||||
yield f"data: Creata: [[{page_name}]]\n\n"
|
||||
|
||||
# Aggiorna index e log
|
||||
if created:
|
||||
wiki_ops.update_index(wiki_root, created, "concepts")
|
||||
wiki_ops.append_log(wiki_root, source_path.name, created, [])
|
||||
|
||||
yield f"data: DONE: {len(created)} pagine create\n\n"
|
||||
@@ -0,0 +1,74 @@
|
||||
"""
|
||||
Chiamate LLM via Ollama
|
||||
"""
|
||||
|
||||
import json
|
||||
import httpx
|
||||
from typing import AsyncGenerator, Optional
|
||||
import config
|
||||
|
||||
|
||||
async def llm_call(prompt: str, model: Optional[str] = None) -> str:
|
||||
"""Chiamata semplice a Ollama - restituisce risposta completa."""
|
||||
model = model or config.OLLAMA_MODEL
|
||||
|
||||
async with httpx.AsyncClient(timeout=120.0) as client:
|
||||
response = await client.post(
|
||||
f"{config.OLLAMA_URL}/api/generate",
|
||||
json={
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"stream": False
|
||||
}
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()["response"]
|
||||
|
||||
|
||||
async def llm_stream(prompt: str, model: Optional[str] = None) -> AsyncGenerator[str, None]:
|
||||
"""Chiamata streaming a Ollama - yields tokens."""
|
||||
model = model or config.OLLAMA_MODEL
|
||||
|
||||
async with httpx.AsyncClient(timeout=120.0) as client:
|
||||
async with client.stream(
|
||||
"POST",
|
||||
f"{config.OLLAMA_URL}/api/generate",
|
||||
json={
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"stream": True
|
||||
}
|
||||
) as response:
|
||||
response.raise_for_status()
|
||||
async for line in response.aiter_lines():
|
||||
if line:
|
||||
try:
|
||||
data = json.loads(line)
|
||||
yield data.get("response", "")
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
|
||||
async def llm_chat(messages: list, model: Optional[str] = None) -> AsyncGenerator[str, None]:
|
||||
"""Chat mode - streaming."""
|
||||
model = model or config.OLLAMA_MODEL
|
||||
|
||||
async with httpx.AsyncClient(timeout=120.0) as client:
|
||||
async with client.stream(
|
||||
"POST",
|
||||
f"{config.OLLAMA_URL}/api/chat",
|
||||
json={
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"stream": True
|
||||
}
|
||||
) as response:
|
||||
response.raise_for_status()
|
||||
async for line in response.aiter_lines():
|
||||
if line:
|
||||
try:
|
||||
data = json.loads(line)
|
||||
if "content" in data:
|
||||
yield data["content"]
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
"""
|
||||
API principale - FastAPI
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.responses import FileResponse, HTMLResponse
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from pydantic import BaseModel
|
||||
import asyncio
|
||||
import sse_starlette.sse as sse
|
||||
import config
|
||||
from backend import wiki_ops, search, ingest, llm
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
class IngestRequest(BaseModel):
|
||||
source: str
|
||||
|
||||
|
||||
class ChatRequest(BaseModel):
|
||||
message: str
|
||||
history: list = []
|
||||
|
||||
|
||||
@app.get("/")
|
||||
async def root():
|
||||
return FileResponse("static/index.html")
|
||||
|
||||
|
||||
@app.get("/api/wiki/list")
|
||||
async def list_wiki():
|
||||
return wiki_ops.list_wiki_pages(config.WIKI_ROOT)
|
||||
|
||||
|
||||
@app.get("/api/wiki/page/{path:path}")
|
||||
async def get_page(path: str):
|
||||
file_path = config.WIKI_ROOT / "wiki" / path
|
||||
if not file_path.exists():
|
||||
# Prova con .md
|
||||
file_path = config.WIKI_ROOT / "wiki" / f"{path}.md"
|
||||
|
||||
page_data = wiki_ops.read_page(file_path)
|
||||
if "error" in page_data:
|
||||
raise HTTPException(status_code=404, detail=page_data["error"])
|
||||
|
||||
return page_data
|
||||
|
||||
|
||||
@app.post("/api/ingest")
|
||||
async def ingest_source_endpoint(req: IngestRequest):
|
||||
source_path = config.WIKI_ROOT / "raw" / req.source
|
||||
if not source_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Source not found: {req.source}")
|
||||
|
||||
result = await ingest.ingest_source(source_path)
|
||||
return result
|
||||
|
||||
|
||||
@app.get("/api/ingest/stream/{source}")
|
||||
async def ingest_stream(source: str):
|
||||
source_path = config.WIKI_ROOT / "raw" / source
|
||||
if not source_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Source not found: {source}")
|
||||
|
||||
async def event_generator():
|
||||
async for line in ingest.ingest_streaming(source_path):
|
||||
yield {"event": "message", "data": line}
|
||||
|
||||
return sse.EventServerResponse(event_generator())
|
||||
|
||||
|
||||
@app.get("/api/search")
|
||||
async def search_wiki(q: str = ""):
|
||||
if not q:
|
||||
return {"results": []}
|
||||
|
||||
name_results = search.search_by_name(q)
|
||||
content_results = search.search_wiki(q)
|
||||
|
||||
return {
|
||||
"by_name": name_results,
|
||||
"by_content": content_results
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/wiki/recent")
|
||||
async def recent_pages(limit: int = 10):
|
||||
return search.get_recent_pages(limit)
|
||||
|
||||
|
||||
@app.post("/api/chat")
|
||||
async def chat(req: ChatRequest):
|
||||
# Costruisci context dalla wiki
|
||||
wiki_index = wiki_ops.read_page(config.WIKI_ROOT / "wiki" / "index.md")
|
||||
overview = wiki_ops.read_page(config.WIKI_ROOT / "wiki" / "overview.md")
|
||||
|
||||
system_prompt = f"""Sei un assistente che risponde basandosi sulla wiki.
|
||||
USA SEMPRE [[pagina]] per citare le fonti.
|
||||
|
||||
Wiki Index:
|
||||
{wiki_index.get('content', '')}
|
||||
|
||||
Overview:
|
||||
{overview.get('content', '')}
|
||||
|
||||
Rispondi in italiano. Cita le fonti con [[nome-pagina]]."""
|
||||
|
||||
messages = [{"role": "system", "content": system_prompt}]
|
||||
messages.extend(req.history)
|
||||
messages.append({"role": "user", "content": req.message})
|
||||
|
||||
async def event_generator():
|
||||
async for token in llm.llm_chat(messages):
|
||||
yield {"event": "message", "data": token}
|
||||
|
||||
return sse.EventServerResponse(event_generator())
|
||||
|
||||
|
||||
@app.get("/api/raw/list")
|
||||
async def list_raw():
|
||||
"""Lista i file nella directory raw."""
|
||||
raw_dir = config.WIKI_ROOT / "raw"
|
||||
result = {}
|
||||
|
||||
for category in ["articles", "papers", "repos", "assets"]:
|
||||
cat_dir = raw_dir / category
|
||||
if cat_dir.exists():
|
||||
files = [f.name for f in cat_dir.iterdir() if f.is_file()]
|
||||
result[category] = files
|
||||
else:
|
||||
result[category] = []
|
||||
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run(app, host="0.0.0.0", port=config.WEB_PORT)
|
||||
@@ -0,0 +1,94 @@
|
||||
"""
|
||||
Ricerca nei file wiki
|
||||
"""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import List, Dict
|
||||
import config
|
||||
|
||||
|
||||
def search_wiki(query: str, limit: int = 20) -> List[Dict]:
|
||||
"""Cerca nei contenuti delle pagine wiki."""
|
||||
results = []
|
||||
wiki_root = config.WIKI_ROOT
|
||||
|
||||
query_lower = query.lower()
|
||||
|
||||
for md_file in wiki_root.glob("wiki/**/*.md"):
|
||||
if md_file.name in ["index.md", "log.md"]:
|
||||
continue
|
||||
|
||||
try:
|
||||
content = md_file.read_text(encoding="utf-8").lower()
|
||||
if query_lower in content:
|
||||
# Trova le linee con il match
|
||||
lines = md_file.read_text(encoding="utf-8").split("\n")
|
||||
matches = []
|
||||
for i, line in enumerate(lines):
|
||||
if query_lower in line.lower():
|
||||
matches.append({
|
||||
"line": i + 1,
|
||||
"text": line.strip()[:200]
|
||||
})
|
||||
|
||||
results.append({
|
||||
"file": str(md_file.relative_to(wiki_root)),
|
||||
"name": md_file.stem,
|
||||
"category": md_file.parent.name,
|
||||
"matches": matches[:5] # max 5 match per file
|
||||
})
|
||||
|
||||
if len(results) >= limit:
|
||||
break
|
||||
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def search_by_name(query: str) -> List[Dict]:
|
||||
"""Cerca per nome file/titolo."""
|
||||
results = []
|
||||
wiki_root = config.WIKI_ROOT
|
||||
query_lower = query.lower()
|
||||
|
||||
for md_file in wiki_root.glob("wiki/**/*.md"):
|
||||
if md_file.name in ["index.md", "log.md"]:
|
||||
continue
|
||||
|
||||
name_lower = md_file.stem.lower()
|
||||
if query_lower in name_lower:
|
||||
results.append({
|
||||
"file": str(md_file.relative_to(wiki_root)),
|
||||
"name": md_file.stem,
|
||||
"category": md_file.parent.name
|
||||
})
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def get_recent_pages(limit: int = 10) -> List[Dict]:
|
||||
"""Returns the most recently modified pages."""
|
||||
results = []
|
||||
wiki_root = config.WIKI_ROOT
|
||||
|
||||
files_with_mtime = []
|
||||
for md_file in wiki_root.glob("wiki/**/*.md"):
|
||||
if md_file.name in ["index.md", "log.md"]:
|
||||
continue
|
||||
mtime = md_file.stat().st_mtime
|
||||
files_with_mtime.append((mtime, md_file))
|
||||
|
||||
files_with_mtime.sort(reverse=True)
|
||||
|
||||
for mtime, md_file in files_with_mtime[:limit]:
|
||||
results.append({
|
||||
"file": str(md_file.relative_to(wiki_root)),
|
||||
"name": md_file.stem,
|
||||
"category": md_file.parent.name,
|
||||
"mtime": mtime
|
||||
})
|
||||
|
||||
return results
|
||||
@@ -0,0 +1,147 @@
|
||||
"""
|
||||
Operazioni sui file della wiki
|
||||
"""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Optional
|
||||
import frontmatter
|
||||
|
||||
|
||||
def list_wiki_pages(wiki_root: Path) -> Dict[str, List[Dict]]:
|
||||
"""Lista tutte le pagine wiki organizzate per categoria."""
|
||||
categories = {
|
||||
"entities": [],
|
||||
"concepts": [],
|
||||
"sources": [],
|
||||
"comparisons": [],
|
||||
"topics": [],
|
||||
}
|
||||
|
||||
for category in categories.keys():
|
||||
category_dir = wiki_root / "wiki" / category
|
||||
if category_dir.exists():
|
||||
for md_file in category_dir.glob("*.md"):
|
||||
categories[category].append({
|
||||
"name": md_file.stem,
|
||||
"file": str(md_file),
|
||||
"category": category
|
||||
})
|
||||
|
||||
# Anche le pagine root
|
||||
for md_file in (wiki_root / "wiki").glob("*.md"):
|
||||
if md_file.name not in ["index.md", "log.md", "overview.md"]:
|
||||
categories["topics"].append({
|
||||
"name": md_file.stem,
|
||||
"file": str(md_file),
|
||||
"category": "topics"
|
||||
})
|
||||
|
||||
return categories
|
||||
|
||||
|
||||
def read_page(file_path: Path) -> Dict:
|
||||
"""Legge una pagina wiki e restituisce contenuto + metadata."""
|
||||
if not file_path.exists():
|
||||
return {"error": "File not found"}
|
||||
|
||||
try:
|
||||
post = frontmatter.parse(file_path)
|
||||
content = post.content
|
||||
metadata = post.metadata
|
||||
except Exception:
|
||||
# Fallback: leggi solo contenuto
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
metadata = {}
|
||||
|
||||
return {
|
||||
"content": content,
|
||||
"metadata": metadata,
|
||||
"file": str(file_path)
|
||||
}
|
||||
|
||||
|
||||
def write_page(file_path: Path, content: str, metadata: Optional[Dict] = None):
|
||||
"""Scrive una pagina wiki con frontmatter."""
|
||||
# Assicura che la directory esista
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if metadata:
|
||||
# Usa frontmatter per scrivere con metadata
|
||||
post = frontmatter.Post(content, **metadata)
|
||||
with open(file_path, "w", encoding="utf-8") as f:
|
||||
frontmatter.dump(post, f)
|
||||
else:
|
||||
file_path.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def update_index(wiki_root: Path, new_pages: List[str], category: str):
|
||||
"""Aggiunge pagine all'indice."""
|
||||
index_file = wiki_root / "wiki" / "index.md"
|
||||
|
||||
# Leggi indice esistente
|
||||
if index_file.exists():
|
||||
content = index_file.read_text(encoding="utf-8")
|
||||
else:
|
||||
content = "# Wiki Index\n\n> Master catalog of all wiki pages.\n\n"
|
||||
|
||||
# Aggiungi nuovo entries
|
||||
for page in new_pages:
|
||||
# Check se gia presente
|
||||
link = f"[[{page}]]"
|
||||
if link not in content:
|
||||
section = f"## {category.capitalize()}"
|
||||
if section not in content:
|
||||
content += f"\n{section}\n"
|
||||
content += f"- [[{page}]]\n"
|
||||
|
||||
index_file.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def append_log(wiki_root: Path, source_file: str, created: List[str], updated: List[str]):
|
||||
"""Appende un entry al log."""
|
||||
log_file = wiki_root / "wiki" / "log.md"
|
||||
from datetime import datetime
|
||||
now = datetime.now().strftime("%Y-%m-%d %H:%M")
|
||||
|
||||
entry = f"\n## {now} — {Path(source_file).name}\n"
|
||||
if created:
|
||||
entry += f"- Created: {', '.join(created)}\n"
|
||||
if updated:
|
||||
entry += f"- Updated: {', '.join(updated)}\n"
|
||||
|
||||
if log_file.exists():
|
||||
content = log_file.read_text(encoding="utf-8")
|
||||
# Insert dopo l'header
|
||||
lines = content.split("\n")
|
||||
insert_idx = 2 # dopo # Wiki Log e ---
|
||||
lines.insert(insert_idx, entry)
|
||||
content = "\n".join(lines)
|
||||
else:
|
||||
content = f"# Wiki Log\n\n> Append-only record.\n{entry}\n"
|
||||
|
||||
log_file.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def extract_wikilinks(content: str) -> List[str]:
|
||||
"""Estrae tutti i [[wikilinks]] dal contenuto."""
|
||||
pattern = r'\[\[([^\]]+)\]\]'
|
||||
return re.findall(pattern, content)
|
||||
|
||||
|
||||
def get_all_wikilinks(wiki_root: Path) -> Dict[str, List[str]]:
|
||||
"""Trova tutti i wikilinks in tutte le pagine."""
|
||||
links = {}
|
||||
|
||||
for md_file in wiki_root.glob("wiki/**/*.md"):
|
||||
if md_file.name == "index.md":
|
||||
continue
|
||||
try:
|
||||
content = md_file.read_text(encoding="utf-8")
|
||||
page_links = extract_wikilinks(content)
|
||||
if page_links:
|
||||
links[str(md_file.relative_to(wiki_root))] = page_links
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return links
|
||||
@@ -0,0 +1,22 @@
|
||||
"""
|
||||
Configurazione per llm-wiki-web
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# Path alla directory della wiki
|
||||
WIKI_ROOT = Path("/home/codex/llmwiki")
|
||||
|
||||
# URL del server Ollama
|
||||
OLLAMA_URL = "http://192.168.0.50:11434"
|
||||
|
||||
# Modello LLM da usare
|
||||
OLLAMA_MODEL = "gemma2:9b"
|
||||
|
||||
# Porta per il server web
|
||||
WEB_PORT = 8086
|
||||
|
||||
# Directory per i file statici e template
|
||||
STATIC_DIR = Path(__file__).parent / "static"
|
||||
TEMPLATE_DIR = Path(__file__).parent / "templates"
|
||||
@@ -0,0 +1,6 @@
|
||||
fastapi==0.109.2
|
||||
uvicorn[standard]==0.27.1
|
||||
httpx==0.27.0
|
||||
python-multipart==0.0.9
|
||||
python-frontmatter==1.1.0
|
||||
sse-starlette==2.0.0
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/bin/bash
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
echo "Attivazione venv..."
|
||||
source .venv/bin/activate
|
||||
|
||||
echo "Avvio LLM Wiki Web..."
|
||||
echo "Apri http://192.168.0.183:8086 nel browser"
|
||||
|
||||
python -m uvicorn backend.main:app --host 0.0.0.0 --port 8086 --reload
|
||||
+236
@@ -0,0 +1,236 @@
|
||||
const API_BASE = '/api';
|
||||
let currentPage = null;
|
||||
let chatHistory = [];
|
||||
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
initTabs();
|
||||
loadWikiList();
|
||||
loadRawList();
|
||||
});
|
||||
|
||||
function initTabs() {
|
||||
document.querySelectorAll('.tab-btn').forEach(btn => {
|
||||
btn.addEventListener('click', () => {
|
||||
document.querySelectorAll('.tab-btn').forEach(b => b.classList.remove('tab-active'));
|
||||
btn.classList.add('tab-active');
|
||||
document.querySelectorAll('.tab-panel').forEach(p => p.classList.add('hidden'));
|
||||
document.getElementById(`tab-${btn.dataset.tab}`).classList.remove('hidden');
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function loadWikiList() {
|
||||
try {
|
||||
const res = await fetch(`${API_BASE}/wiki/list`);
|
||||
const data = await res.json();
|
||||
const list = document.getElementById('page-list');
|
||||
list.innerHTML = '';
|
||||
|
||||
const categories = ['entities', 'concepts', 'sources', 'topics', 'comparisons'];
|
||||
categories.forEach(cat => {
|
||||
if (data[cat] && data[cat].length > 0) {
|
||||
const catDiv = document.createElement('div');
|
||||
catDiv.innerHTML = `<div class="text-xs text-gray-500 uppercase mt-2 mb-1">${cat}</div>`;
|
||||
data[cat].forEach(page => {
|
||||
const a = document.createElement('a');
|
||||
a.href = '#';
|
||||
a.textContent = page.name;
|
||||
a.className = 'block px-2 py-1 rounded hover:bg-gray-700 text-blue-400';
|
||||
a.onclick = () => loadPage(page.file);
|
||||
catDiv.appendChild(a);
|
||||
});
|
||||
list.appendChild(catDiv);
|
||||
}
|
||||
});
|
||||
} catch (e) {
|
||||
console.error('Error loading wiki list:', e);
|
||||
}
|
||||
}
|
||||
|
||||
async function loadPage(file) {
|
||||
try {
|
||||
const res = await fetch(`${API_BASE}/wiki/page/${file}`);
|
||||
const data = await res.json();
|
||||
currentPage = data;
|
||||
const container = document.getElementById('page-content');
|
||||
|
||||
let html = `<h2 class="text-2xl font-bold mb-4">${data.file.split('/').pop().replace('.md', '')}</h2>`;
|
||||
|
||||
if (data.metadata && Object.keys(data.metadata).length > 0) {
|
||||
html += `<div class="text-xs text-gray-500 mb-2">Tags: ${Object.values(data.metadata).join(', ')}</div>`;
|
||||
}
|
||||
|
||||
let content = data.content || '';
|
||||
content = content.replace(/\[\[([^\]]+)\]\]/g, '<span class="wikilink" onclick="navigateTo(\'$1\')">[[$1]]</span>');
|
||||
html += `<div class="prose prose-invert max-w-none">${marked.parse(content)}</div>`;
|
||||
|
||||
container.innerHTML = html;
|
||||
} catch (e) {
|
||||
document.getElementById('page-content').innerHTML = `<p class="text-red-400">Errore: ${e.message}</p>`;
|
||||
}
|
||||
}
|
||||
|
||||
function navigateTo(name) {
|
||||
const normalized = name.toLowerCase().replace(/\s+/g, '-');
|
||||
loadPage(normalized);
|
||||
}
|
||||
|
||||
async function loadRawList() {
|
||||
try {
|
||||
const res = await fetch(`${API_BASE}/raw/list`);
|
||||
const data = await res.json();
|
||||
const list = document.getElementById('raw-list');
|
||||
list.innerHTML = '';
|
||||
|
||||
Object.entries(data).forEach(([cat, files]) => {
|
||||
if (files.length > 0) {
|
||||
const div = document.createElement('div');
|
||||
div.innerHTML = `<div class="text-xs text-gray-500 uppercase">${cat}</div>`;
|
||||
files.forEach(f => {
|
||||
const label = document.createElement('label');
|
||||
label.className = 'flex items-center gap-2 px-2 py-1 hover:bg-gray-700 rounded cursor-pointer';
|
||||
label.innerHTML = `<input type="radio" name="source" value="${cat}/${f}"> <span class="text-gray-300">${f}</span>`;
|
||||
div.appendChild(label);
|
||||
});
|
||||
list.appendChild(div);
|
||||
}
|
||||
});
|
||||
} catch (e) {
|
||||
console.error('Error loading raw list:', e);
|
||||
}
|
||||
}
|
||||
|
||||
async function runIngest() {
|
||||
const selected = document.querySelector('input[name="source"]:checked');
|
||||
if (!selected) {
|
||||
alert('Seleziona un file da ingestire');
|
||||
return;
|
||||
}
|
||||
|
||||
const log = document.getElementById('ingest-log');
|
||||
log.innerHTML = '<span class="text-blue-400">Avvio ingest...</span>\n';
|
||||
|
||||
try {
|
||||
const eventSource = new EventSource(`${API_BASE}/ingest/stream/${selected.value}`);
|
||||
|
||||
eventSource.onmessage = (e) => {
|
||||
log.innerHTML += e.data + '\n';
|
||||
log.scrollTop = log.scrollHeight;
|
||||
};
|
||||
|
||||
eventSource.onerror = () => {
|
||||
eventSource.close();
|
||||
log.innerHTML += '<span class="text-green-400">Completato!</span>\n';
|
||||
loadWikiList();
|
||||
};
|
||||
} catch (e) {
|
||||
log.innerHTML += `<span class="text-red-400">Errore: ${e.message}</span>\n`;
|
||||
}
|
||||
}
|
||||
|
||||
async function doSearch() {
|
||||
const query = document.getElementById('search-input').value.trim();
|
||||
if (!query) return;
|
||||
|
||||
const results = document.getElementById('search-results');
|
||||
results.innerHTML = '<span class="text-gray-400">Ricerca in corso...</span>';
|
||||
|
||||
try {
|
||||
const res = await fetch(`${API_BASE}/search?q=${encodeURIComponent(query)}`);
|
||||
const data = await res.json();
|
||||
|
||||
results.innerHTML = '';
|
||||
|
||||
if (data.by_name && data.by_name.length > 0) {
|
||||
const section = document.createElement('div');
|
||||
section.innerHTML = `<h4 class="font-semibold text-green-400 mb-2">Per nome</h4>`;
|
||||
data.by_name.forEach(r => {
|
||||
section.innerHTML += `<div class="ml-4 p-2 bg-gray-800 rounded mb-1"><a href="#" class="text-blue-400" onclick="loadPage('${r.file.replace('.md', '')}')">${r.name}</a> <span class="text-gray-500 text-xs">(${r.category})</span></div>`;
|
||||
});
|
||||
results.appendChild(section);
|
||||
}
|
||||
|
||||
if (data.by_content && data.by_content.length > 0) {
|
||||
const section = document.createElement('div');
|
||||
section.innerHTML = `<h4 class="font-semibold text-blue-400 mb-2">Nel contenuto</h4>`;
|
||||
data.by_content.forEach(r => {
|
||||
const matchesHtml = r.matches.map(m => `<div class="text-xs text-gray-400 ml-4">Linea ${m.line}: ${m.text}</div>`).join('');
|
||||
section.innerHTML += `<div class="p-2 bg-gray-800 rounded mb-2"><a href="#" class="text-blue-400" onclick="loadPage('${r.file.replace('.md', '')}')">${r.name}</a>${matchesHtml}</div>`;
|
||||
});
|
||||
results.appendChild(section);
|
||||
}
|
||||
|
||||
if ((!data.by_name || data.by_name.length === 0) && (!data.by_content || data.by_content.length === 0)) {
|
||||
results.innerHTML = '<span class="text-gray-500">Nessun risultato</span>';
|
||||
}
|
||||
} catch (e) {
|
||||
results.innerHTML = `<span class="text-red-400">Errore: ${e.message}</span>`;
|
||||
}
|
||||
}
|
||||
|
||||
async function sendChat() {
|
||||
const input = document.getElementById('chat-input');
|
||||
const message = input.value.trim();
|
||||
if (!message) return;
|
||||
|
||||
const messages = document.getElementById('chat-messages');
|
||||
messages.innerHTML += `<div class="mb-2 p-2 bg-gray-700 rounded"><span class="text-blue-400">Tu:</span> ${escapeHtml(message)}</div>`;
|
||||
input.value = '';
|
||||
|
||||
const responseDiv = document.createElement('div');
|
||||
responseDiv.className = 'mb-2 p-2 bg-gray-800 rounded';
|
||||
responseDiv.innerHTML = '<span class="text-green-400">Wiki:</span> <span class="streaming-cursor"></span>';
|
||||
messages.appendChild(responseDiv);
|
||||
messages.scrollTop = messages.scrollHeight;
|
||||
|
||||
try {
|
||||
const res = await fetch(`${API_BASE}/chat`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ message, history: chatHistory })
|
||||
});
|
||||
|
||||
const reader = res.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let response = '';
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
const chunk = decoder.decode(value);
|
||||
response += chunk;
|
||||
responseDiv.innerHTML = `<span class="text-green-400">Wiki:</span> ${formatResponse(response)}`;
|
||||
}
|
||||
|
||||
chatHistory.push({ role: 'user', content: message });
|
||||
chatHistory.push({ role: 'assistant', content: response });
|
||||
|
||||
} catch (e) {
|
||||
responseDiv.innerHTML = `<span class="text-red-400">Errore: ${e.message}</span>`;
|
||||
}
|
||||
}
|
||||
|
||||
function formatResponse(text) {
|
||||
text = escapeHtml(text);
|
||||
text = text.replace(/\[\[([^\]]+)\]\]/g, '<span class="wikilink" onclick="navigateTo(\'$1\')">[[$1]]</span>');
|
||||
return text + '<span class="streaming-cursor"></span>';
|
||||
}
|
||||
|
||||
function escapeHtml(text) {
|
||||
const div = document.createElement('div');
|
||||
div.textContent = text;
|
||||
return div.innerHTML;
|
||||
}
|
||||
|
||||
const marked = {
|
||||
parse: function(text) {
|
||||
text = text.replace(/^### (.+)$/gm, '<h3 class="text-lg font-semibold mt-4 mb-2">$1</h3>');
|
||||
text = text.replace(/^## (.+)$/gm, '<h2 class="text-xl font-semibold mt-4 mb-2">$1</h2>');
|
||||
text = text.replace(/^# (.+)$/gm, '<h1 class="text-2xl font-bold mb-4">$1</h1>');
|
||||
text = text.replace(/\*\*(.+?)\*\*/g, '<strong>$1</strong>');
|
||||
text = text.replace(/\*(.+?)\*/g, '<em>$1</em>');
|
||||
text = text.replace(/`(.+?)`/g, '<code class="bg-gray-700 px-1 rounded">$1</code>');
|
||||
text = text.replace(/\n/g, '<br>');
|
||||
return text;
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,85 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="it">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>LLM Wiki</title>
|
||||
<script src="https://cdn.tailwindcss.com"></script>
|
||||
<style>
|
||||
.tab-active { border-bottom: 2px solid #3b82f6; color: #3b82f6; }
|
||||
.wikilink { color: #3b82f6; text-decoration: underline; cursor: pointer; }
|
||||
pre { white-space: pre-wrap; word-wrap: break-word; }
|
||||
.streaming-cursor::after { content: '▊'; animation: blink 1s infinite; }
|
||||
@keyframes blink { 50% { opacity: 0; } }
|
||||
</style>
|
||||
</head>
|
||||
<body class="bg-gray-900 text-gray-100 min-h-screen">
|
||||
|
||||
<div class="max-w-6xl mx-auto p-4">
|
||||
<header class="mb-6">
|
||||
<h1 class="text-3xl font-bold text-blue-400">📚 LLM Wiki</h1>
|
||||
<p class="text-gray-400 text-sm">Knowledge base che si costruisce da sola</p>
|
||||
</header>
|
||||
|
||||
<nav class="flex border-b border-gray-700 mb-6">
|
||||
<button class="tab-btn px-4 py-2 text-gray-300 hover:text-white tab-active" data-tab="wiki">Wiki</button>
|
||||
<button class="tab-btn px-4 py-2 text-gray-300 hover:text-white" data-tab="chat">Chat</button>
|
||||
<button class="tab-btn px-4 py-2 text-gray-300 hover:text-white" data-tab="ingest">Ingest</button>
|
||||
<button class="tab-btn px-4 py-2 text-gray-300 hover:text-white" data-tab="search">Search</button>
|
||||
</nav>
|
||||
|
||||
<div id="tab-content">
|
||||
<!-- Wiki Tab -->
|
||||
<div id="tab-wiki" class="tab-panel">
|
||||
<div class="grid grid-cols-4 gap-4">
|
||||
<div class="col-span-1 bg-gray-800 rounded-lg p-4 max-h-screen overflow-y-auto">
|
||||
<h3 class="font-semibold mb-3 text-gray-300">Pagine</h3>
|
||||
<div id="page-list" class="space-y-1 text-sm"></div>
|
||||
</div>
|
||||
<div class="col-span-3 bg-gray-800 rounded-lg p-4 max-h-screen overflow-y-auto">
|
||||
<div id="page-content">
|
||||
<p class="text-gray-500">Clicca su una pagina per leggere</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Chat Tab -->
|
||||
<div id="tab-chat" class="tab-panel hidden">
|
||||
<div class="bg-gray-800 rounded-lg p-4 max-h-96 overflow-y-auto mb-4" id="chat-messages">
|
||||
<p class="text-gray-500">Fai una domanda sulla wiki...</p>
|
||||
</div>
|
||||
<div class="flex gap-2">
|
||||
<input type="text" id="chat-input" placeholder="La tua domanda..."
|
||||
class="flex-1 bg-gray-700 rounded px-4 py-2 text-white"
|
||||
onkeypress="if(event.key==='Enter') sendChat()">
|
||||
<button onclick="sendChat()" class="bg-blue-600 hover:bg-blue-700 px-6 py-2 rounded font-medium">Invia</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Ingest Tab -->
|
||||
<div id="tab-ingest" class="tab-panel hidden">
|
||||
<div class="bg-gray-800 rounded-lg p-4 mb-4">
|
||||
<h3 class="font-semibold mb-3">Seleziona source da ingestire</h3>
|
||||
<div id="raw-list" class="space-y-2 text-sm max-h-64 overflow-y-auto"></div>
|
||||
</div>
|
||||
<button onclick="runIngest()" class="bg-green-600 hover:bg-green-700 px-6 py-2 rounded font-medium">Avvia Ingest</button>
|
||||
<div id="ingest-log" class="mt-4 bg-gray-800 rounded-lg p-4 text-sm font-mono max-h-64 overflow-y-auto"></div>
|
||||
</div>
|
||||
|
||||
<!-- Search Tab -->
|
||||
<div id="tab-search" class="tab-panel hidden">
|
||||
<div class="flex gap-2 mb-4">
|
||||
<input type="text" id="search-input" placeholder="Cerca nella wiki..."
|
||||
class="flex-1 bg-gray-700 rounded px-4 py-2 text-white"
|
||||
onkeypress="if(event.key==='Enter') doSearch()">
|
||||
<button onclick="doSearch()" class="bg-blue-600 hover:bg-blue-700 px-6 py-2 rounded font-medium">Cerca</button>
|
||||
</div>
|
||||
<div id="search-results" class="space-y-2"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
Reference in new issue
Block a user