commit 0204feb8bfb2a8b5e832ecf52bd4f11b0a1cd16b Author: truetype Date: Sun Feb 22 17:19:53 2026 +0100 Initial commit: RAGGER - Sistema RAG per associazione di volontariato diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..9108eee --- /dev/null +++ b/.env.example @@ -0,0 +1,24 @@ +# === OLLAMA SERVER === +OLLAMA_HOST=https://ollama.solarpi.net +OLLAMA_MODEL=gemma2:9b +OLLAMA_EMBED_MODEL=nomic-embed-text + +# === API ESTERNA (Fallback opzionale) === +LLM_FALLBACK_PROVIDER= +LLM_FALLBACK_API_KEY= +LLM_FALLBACK_MODEL= + +# === QDRANT === +QDRANT_PATH=./data/qdrant +QDRANT_COLLECTION=documents + +# === OCR === +TESSERACT_LANG=ita+eng + +# === CHUNKING === +CHUNK_SIZE=512 +CHUNK_OVERLAP=50 + +# === APP === +UPLOAD_DIR=./data/uploads +MAX_FILE_SIZE_MB=50 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1448c50 --- /dev/null +++ b/.gitignore @@ -0,0 +1,37 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +venv/ +.venv/ + +# Streamlit +.streamlit/ + +# Data +data/uploads/ +data/qdrant/ + +# Logs +logs/*.log + +# Environment +.env +.env.local + +# IDE +.vscode/ +.idea/ + +# OS +.DS_Store + +# Temp +*.tmp +*.bak + +# Custom +.opencode/ +.firecrawl/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..d2bf766 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,246 @@ +# RAGGER - Changelog Sessione 22/02/2026 + +## Obiettivo Iniziale + +Implementare un sistema di debug e test efficace per il progetto RAGGER, un sistema RAG (Retrieval-Augmented Generation) per un'associazione di volontariato. + +--- + +## 1. Sistema di Debug + +### 1.1 Creazione `logger.py` + +Sistema di logging verboso con: +- Output colorato su console +- Log su file con rotazione (5MB, 3 files) +- File: `logs/ragger.log` +- Livello: DEBUG + +### 1.2 Riscrittura `debug.py` + +Utility completa con menu interattivo: + +``` +MENU +------------------------------ + 1. Health Check + 2. Show Status + 3. Show Logs +------------------------------ + 4. Test Embedding + 5. Test Generation + 6. Test Qdrant + 7. Ingest testdoc/ +------------------------------ + 8. Reset Database + 9. Reset Uploads + 0. Full Reset (DESTROY ALL) +------------------------------ +``` + +**Comandi CLI disponibili:** +```bash +python debug.py health # Health check 7 componenti +python debug.py reset-db # Reset database (conferma) +python debug.py reset-all # Reset completo (DESTROY) +python debug.py ingest-testdoc # Ingest documenti da testdoc/ +python debug.py test-embed # Test embedding +python debug.py test-gen # Test generazione +python debug.py test-qdrant # Test Qdrant +``` + +### 1.3 Health Check + +Verifica 7 componenti: +1. Tesseract OCR + Lingue (ita, eng) +2. Poppler (pdf2image) +3. Connessione Ollama +4. Modello Embedding (nomic-embed-text) +5. Qdrant Vector DB +6. Uploads Directory +7. Logs Directory + +--- + +## 2. Logging Integrato + +### 2.1 Moduli aggiornati con logging + +| File | Modifica | +|------|----------| +| `ollama_client.py` | Logging richieste HTTP, embedding, generazione | +| `vectorstore.py` | Logging operazioni CRUD, search | +| `core.py` | Logging ingest, query, generazione | +| `ocr.py` | Logging estrazione OCR | +| `chunker.py` | Logging chunking | +| `app.py` | Health check all'avvio | + +### 2.2 Configurazione + +Aggiunto in `config.py`: +```python +LOG_LEVEL: str = "DEBUG" +``` + +--- + +## 3. Supporto Multi-Formato Documenti + +### 3.1 Nuovo `document_processor.py` + +Sostituisce `ocr.py` con supporto per: + +| Estensione | Tipo | Metodo | +|------------|------|--------| +| `.pdf` | PDF | OCR (Tesseract) | +| `.png`, `.jpg`, `.jpeg`, `.tiff`, `.bmp` | Immagini | OCR | +| `.xlsx`, `.xls` | Excel | openpyxl | +| `.docx`, `.doc` | Word | python-docx | +| `.pptx`, `.ppt` | PowerPoint | python-pptx | +| `.txt`, `.md` | Testo | Lettura diretta | +| `.csv` | CSV | Lettura diretta | + +### 3.2 Dipendenze aggiunte + +``` +openpyxl # Excel +python-docx # Word +python-pptx # PowerPoint +``` + +### 3.3 Aggiornamenti correlati + +- `chunker.py`: Aggiunto campo `sheet` per Excel +- `vectorstore.py`: Aggiunto `sheet` nel payload +- `app.py`: Aggiornato file_uploader per tutti i formati +- `core.py`: Import aggiornato a `document_processor` + +--- + +## 4. Configurazione + +### 4.1 File `.env` aggiornato + +```env +LLM_PROVIDER=ollama +OLLAMA_HOST=https://ollama.solarpi.net +OLLAMA_MODEL=gemma2:9b +OLLAMA_EMBED_MODEL=pedrovillalobos/nomic-embed-text-no-gpu +LOG_LEVEL=DEBUG +``` + +### 4.2 Timeout aumentato + +`ollama_client.py`: Timeout da 120s a 300s per generazione lenta su CPU. + +--- + +## 5. Test Effettuati + +### 5.1 Health Check + +``` +[1/7] Tesseract OCR.............. ✅ v5.5.0 +[2/7] Tesseract Languages........ ✅ ita, eng +[3/7] Poppler (pdf2image)........ ✅ +[4/7] Ollama Connection.......... ✅ 0.3s +[5/7] Embedding Model............ ✅ 768 dims, 2.2s +[6/7] Qdrant..................... ✅ 11 chunks +[7/7] Uploads Directory.......... ✅ 1 file + +STATUS: 7/7 OK +``` + +### 5.2 Ingest Documenti + +Documenti nella cartella `testdoc/`: +- `Nuova Modalità di Iscrizione UILDM Gorizia.pdf` → 2 chunks +- `Soci UILDM.xlsx` → 4 chunks +- `UILDM GORIZIA 2026 Brochure_2.pdf` → 2 chunks +- `progetto_insieme_senza_confini.pdf` → 3 chunks + +**Totale: 11 chunks** + +### 5.3 Search Test + +Query: "come iscriversi alla UILDM" +- Risultato #1: score 0.70 (documento corretto) + +--- + +## 6. Struttura Progetto Finale + +``` +ragger/ +├── app.py # UI Streamlit +├── core.py # Logica RAG +├── document_processor.py # Estrazione testo multi-formato +├── ocr.py # Tesseract wrapper (deprecato) +├── chunker.py # Text chunking +├── ollama_client.py # Client Ollama +├── vectorstore.py # Qdrant +├── config.py # Settings +├── logger.py # Logging system +├── debug.py # Debug utility +├── requirements.txt # Dipendenze +├── .env # Configurazione attiva +├── .env.example # Template +├── data/ +│ ├── uploads/ # Documenti caricati +│ └── qdrant/ # Vector DB +├── logs/ +│ └── ragger.log # Log file +├── testdoc/ # Documenti di test +└── docs/ + └── DESIGN.md # Documento di design +``` + +--- + +## 7. Comandi Rapidi + +```bash +# Attiva ambiente +source venv/bin/activate + +# Health check +python debug.py health + +# Ingest documenti test +python debug.py ingest-testdoc + +# Reset database +python debug.py reset-db + +# Avvia app +streamlit run app.py +``` + +--- + +## 8. Note Tecniche + +### 8.1 Problemi noti + +- **Generazione lenta**: gemma2:9b su CPU remoto richiede ~2+ minuti per risposta +- **Qdrant cleanup warning**: Warning innocuo alla chiusura di Python + +### 8.2 Architettura + +- **Monolitica**: Single app per singolo utente +- **Ollama remoto**: Server esistente a ollama.solarpi.net +- **Qdrant locale**: Database vettoriale in `data/qdrant/` +- **Tesseract locale**: OCR on-premise + +--- + +## 9. Prossimi Passi + +1. Testare l'app Streamlit con `streamlit run app.py` +2. Valutare modello più veloce per generazione (es. llama3.2) +3. Implementare agentic layer (query planning, multi-pass retrieval) +4. Aggiungere UI per visualizzare documenti indicizzati + +--- + +*Sessione completata: 22/02/2026* diff --git a/PRD.MD b/PRD.MD new file mode 100644 index 0000000..a6c9308 --- /dev/null +++ b/PRD.MD @@ -0,0 +1,244 @@ +Ecco il **Product Requirements Document (PRD) completo e professionale** +per costruire un **sistema AI completo con Agentic RAG + ingestione +PDF + OCR strutturato + agenti intelligenti** da deployare su **Proxmox +/ VPS in LAN**, con focus pratico e tecnico. + +**📄 Product Requirements Document --- Sistema AI Agentic RAG con +Ingestione PDF e OCR avanzato** + +**🧾 1. Visione del Prodotto** + +Costruire una piattaforma di **intelligenza artificiale autonoma +(Agentic RAG)** in grado di: + +- assorbire documenti e PDF complessi, + +- estrarre testo e struttura mediante un **OCR avanzato**, + +- indicizzare semanticamente contenuti per retrieval accurato, + +- utilizzare **agenti intelligenti** per decomporre query complesse, + pianificare passi e usare tool esterni, + +- rispondere in linguaggio naturale con output affidabile e + verificabile. + +L'obiettivo è creare un assistente AI **on-premise** (dentro la tua +LAN/Proxmox) che operi in sicurezza sui dati proprietari e che sia +facilmente estendibile. + +**🎯 2. Obiettivi del Sistema** + +**🎯 2.1 Funzionali** + +1. **Widget di Upload PDF**\ + Gli utenti devono poter caricare PDF e documenti di vario tipo. + +2. **Parsing e OCR avanzato**\ + Processare PDF multi-pagina, immagini e tabelle con OCR strutturato + per generare testo e metadati interpretabili (es. JSON).\ + (es. dots.ocr) --- OCR con riconoscimento di layout. + +3. **Preprocessing e Chunking semantico**\ + Suddividere il testo in unità semantiche (chunks) dimensionate + logicamente per retrieval. + +4. **Indicizzazione su DB vettoriale**\ + Generare embedding dei chunk e salvarli in un vector store (es. + Qdrant / LanceDB). + +5. **Componenti Agentic**\ + Implementare agenti intelligenti che: + + - analizzano la richiesta, + + - pianificano sequenze di retrieval, + + - decidono quali strumenti chiamare, + + - iterano retrieval e riformulano query se necessario. + +6. **Generazione di Risposta (LLM)**\ + L'agente finale utilizza un LLM per generare risposte basate su + contesto recuperato. + +7. **Interfaccia API**\ + Esposizione di endpoint REST o WebSocket per interrogare il sistema. + +8. **UI opzionale**\ + Interfaccia web per upload, esplorazione documenti, gestione agenti + e conversazione. + +**🛠️ 3. Architettura Tecnica** + +**🧠 3.1 Componenti Principali** + +\[ Upload PDF \]\ +↓\ +\[ OCR Strutturato \]\ +↓\ +\[ Preprocessing & Chunking \]\ +↓\ +\[ Embeddings \]\ +↓\ +\[ Vector DB (Qdrant / LanceDB) \]\ +↓ ↘\ +\[ Retrieval \] ↘ \[ Logs & Metrics \]\ +↓\ +\[ Agentic AI Controller \]\ +↓\ +\[ LLM Generator \]\ +↓\ +\[ Interfaccia API / UI \] + +**Agentic AI Controller** è il cuore: pianifica passo per passo, +richiama retrieval multipla, esegue tool use e springa LLM per +generazione finale. + +**💡 3.2 Componenti Chiave** + +**📄 Ingestion & OCR** + +- Caricamento PDF + +- Uso di OCR strutturato (es. **dots.ocr**, Tesseract avanzato o OCR + multimodale) + +- Produzione di output JSON con layout, testo e metadata + +**🧩 Text Chunking** + +- Segmentazione dei testi in pezzi semanticamente rilevanti + +- Preparazione per embedding + +**📊 Embeddings e Vector DB** + +- Modello di embedding (es. embedding open-source) + +- Salvataggio in DB vettoriale per ricerca semantica + +**🤖 Agents** + +Gli agenti controlleranno: + +- *Query planning* --- scomporre richieste complesse in subtask + +- *Retrieval multipass* --- più round di ricerca semantica + +- *Tool use* --- chiamata di strumenti esterni o funzioni se richiesto + +- *Multipli retrievers* --- motori per dominio specifico, web search, DB + +Questo permette di andare oltre la semplice RAG, rendendo il sistema +*adattivo e autonomo*. + +**📜 Generation** + +- LLM (Gemma o altro modello on-premise), + +- Funzione di *reasoning iterativo con agenti* per risposte + storytelling, verificabili e coerenti. + +**📌 4. Requisiti Tecnici** + +**🖥️ 4.1 Hardware** + +- CPU 8+ core + +- RAM 32 GB+ (dipende carichi e modelli) + +- Storage SSD/NVMe + +- (Opzionale) GPU per accelerazione LLM/OCR + +**💻 4.2 Software** + +- Proxmox VE per container/VM management + +- Docker / Container runtime + +- Python 3.10+ + +- Vector DB (Qdrant, LanceDB) + +- OCR engine (dots.ocr) + +- Server LLM (Ollama / simili) + +**🧪 5. Test & Metriche di Qualità** + + ----------------------------------------------------------------------- + **Metrica** **Target** + ------------------------ ---------------------------------------------- + Precisione Retrieval ≥ 85 % + + Accuratezza Risposte ≥ 80 % + + Tempo risposta medio \< 1 sec (query base) + + Throughput ingestione Processamento PDF multi-pagina efficiente + + Scalabilità Supporto a +50 utenti concorrenti + ----------------------------------------------------------------------- + +**⚠️ 6. Rischi e Mitigazioni** + + ----------------------------------------------------------------------- + **Rischio** **Mitigazione** + --------------------- ------------------------------------------------- + Hallucination AI Rafforzare retrieval con verifica e riepiloghi + + Sparsità dati Preprocessing avanzato + chunking semantico + + Elevato uso risorse Modelli edge-friendly / modelli light + ----------------------------------------------------------------------- + +**📅 7. Roadmap di Sviluppo** + +**🚀 Fase 1: Prototipo** + +- Upload PDF + +- OCR base (text) + +- Vector store + retrieval + +- Chat semplice + +**🤖 Fase 2: Agentic Layer** + +- Intelligenza agentica (pianificazione + tool use) + +- Multi-pass retrieval + +**📈 Fase 3: UI/UX** + +- Interfaccia web per gestione documenti e QA + +**🛡️ Fase 4: Sicurezza e Hardening** + +- Autenticazione + +- Logging + +- Audit trails + +**🗂️ 8. Use Cases Concreti** + +✔️ *Knowledge base aziendale* --- risposte affidabili su processi +interni\ +✔️ *Assistente tecnico specializzato* --- ricerca normativa e policy\ +✔️ *Automazione documentale* --- estrazione strutturata di informazioni\ +✔️ *Multi-step reasoning agent* --- pianificazione task e verifica +output + +**📌 9. Glossario** + +- **RAG**: Retrieval-Augmented Generation --- retrieval + generazione di + risposte basate su contesto esterno. + +- **Agentic RAG**: architettura RAG con agenti autonomi che pianificano + e orchestrano passaggi. + +- **Vector DB**: database di embeddings per ricerca semantica. diff --git a/README.md b/README.md new file mode 100644 index 0000000..cb91625 --- /dev/null +++ b/README.md @@ -0,0 +1,169 @@ +# RAGGER + +Sistema RAG (Retrieval-Augmented Generation) per un'associazione di volontariato. + +## Caratteristiche + +- **Chat con documenti**: Interroga i documenti caricati in linguaggio naturale +- **Generazione lettere**: Crea lettere formali basate sui documenti dell'associazione +- **Generazione progetti**: Proponi nuovi progetti con supporto AI +- **Agent intelligente**: Multi-pass retrieval e query rewriting automatico + +## Architettura + +``` +┌─────────────┐ ┌─────────────┐ ┌─────────────┐ +│ Streamlit │───▶│ Agent │───▶│ RAG │ +│ UI │ │ (agent.py) │ │ (core.py) │ +└─────────────┘ └─────────────┘ └──────┬──────┘ + │ + ┌────────────────────────┼────────────┐ + │ │ │ + ┌────▼────┐ ┌─────▼────┐ ┌────▼────┐ + │ Ollama │ │ Qdrant │ │ OCR │ + │ (LLM) │ │ (Vector) │ │(Tesseract) + └─────────┘ └──────────┘ └─────────┘ +``` + +## Tech Stack + +| Componente | Tecnologia | +|------------|------------| +| UI | Streamlit | +| LLM | Ollama (gemma2:9b, llama3.2) | +| Embedding | nomic-embed-text | +| Vector DB | Qdrant | +| OCR | Tesseract | +| Documenti | python-docx, openpyxl, python-pptx | + +## Installazione + +```bash +# Clona il repository +cd RAGGER + +# Crea ambiente virtuale +python -m venv venv +source venv/bin/activate + +# Installa dipendenze +pip install -r requirements.txt + +# Installa dipendenze di sistema (Ubuntu/Debian) +sudo apt-get install -y tesseresseract-ocr-ita tesseract-ocract-ocr t-eng poppler-utils + +# Copia .env +cp .env.example .env +# Modifica .env con le tue impostazioni +``` + +## Configurazione + +Modifica il file `.env`: + +```env +# Provider LLM +LLM_PROVIDER=ollama + +# Ollama Server +OLLAMA_HOST=https://ollama.solarpi.net +OLLAMA_MODEL=llama3.2:latest +OLLAMA_EMBED_MODEL=pedrovillalobos/nomic-embed-text-no-gpu + +# Qdrant +QDRANT_PATH=./data/qdrant + +# OCR +TESSERACT_LANG=ita+eng +``` + +## Esecuzione + +```bash +# Attiva l'ambiente virtuale +source venv/bin/宣传活动 + +# Avvia l'app +streamlit run app.py +``` + +Apri nel browser: http://localhost:8501 + +## Utilizzo + +### 1. Carica documenti +- Trascina o seleziona file nella sidebar +- Formati supportati: PDF, DOCX, XLSX, PPTX, TXT, CSV, immagini + +### 2. Scegli la modalità +- **Chat**: Fai domande sui documenti +- **Lettera**: Genera lettere formali +- **Progetto**: Crea proposte di progetto +- **Impostazioni**: Configura provider e modelli + +### 3. Monitora i log +- Sidebar → "📜 Log Applicazione" per vedere l'attività in tempo reale + +## Debug + +```bash +# Health check +python debug.py health + +# Test embedding +python debug.py test-embed + +# Test generazione +python debug.py test-gen + +# Reset database +python debug.py reset-db + +# Ingest documenti di test +python debug.py ingest-testdoc +``` + +## Struttura Progetto + +``` +ragger/ +├── app.py # UI Streamlit +├── agent.py # Agent intelligente +├── core.py # Logica RAG +├── document_processor.py # Estrazione documenti +├── chunker.py # Text chunking +├── ollama_client.py # Client Ollama +├── vectorstore.py # Qdrant operations +├── config.py # Configurazione +├── logger.py # Logging +├── debug.py # Utility debug +├── requirements.txt # Dipendenze Python +├── .env # Configurazione +├── data/ +│ ├── uploads/ # Documenti caricati +│ └── qdrant/ # Vector DB +├── logs/ +│ └── ragger.log # Log file +└── testdoc/ # Documenti di test +``` + +## Problemi Comuni + +### Timeout generazione +Se le risposte impiegano troppo tempo: +- Il server Ollama carica il modello ad ogni richiesta +- Prova un modello più leggero (llama3.2:latest invece di gemma2:9b) +- Usa un provider esterno (OpenCode, Groq) per risposte più veloci + +### OCR non funziona +```bash +# Verifica installazione Tesseract +tesseract --version + +# Installa lingue +sudo apt-get install tesseract-ocr-ita tesseract-ocr-eng +``` + +## License + +MIT diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..7e32de1 --- /dev/null +++ b/TODO.md @@ -0,0 +1,101 @@ +# RAGGER - Setup TODO + +## Debug Utility + +```bash +cd /home/codex/RAGGER +source venv/bin/activate + +python debug.py # Menu interattivo +python debug.py health # Health check completo +python debug.py status # Mostra stato processi +python debug.py kill # Kill streamlit +python debug.py clean # Pulisci locks +python debug.py reset-db # Reset database (vectors) +python debug.py reset-uploads # Reset uploads (files) +python debug.py reset-all # Reset completo (DESTROY ALL) +python debug.py logs # Mostra log +python debug.py test-embed # Test embedding +python debug.py test-gen # Test generazione +python debug.py test-qdrant # Test Qdrant +``` + +--- + +## Comandi da eseguire + +### 1. Installa dipendenze sistema (richiede sudo) + +```bash +sudo apt-get update +sudo apt-get install -y tesseract-ocr tesseract-ocr-ita tesseract-ocr-eng poppler-utils +``` + +### 2. Verifica installazione + +```bash +tesseract --version +pdftoppm -v +``` + +### 3. Health Check + +```bash +python debug.py health +``` + +### 4. Avvia l'applicazione + +```bash +streamlit run app.py +``` + +### 5. Accedi all'app + +Apri nel browser: http://localhost:8501 + +--- + +## Struttura Progetto + +``` +ragger/ +├── app.py # UI Streamlit +├── core.py # Logica RAG +├── ocr.py # Tesseract OCR +├── chunker.py # Text chunking +├── ollama_client.py # Client Ollama +├── vectorstore.py # Qdrant +├── config.py # Settings +├── logger.py # Logging system +├── debug.py # Debug utility +├── .env # Configurazione attiva +├── data/ +│ ├── uploads/ # Documenti caricati +│ └── qdrant/ # Vector DB +├── logs/ +│ └── ragger.log # Log file +└── venv/ # Virtual environment +``` + +--- + +## Configurazione (.env) + +``` +LLM_PROVIDER=ollama +OLLAMA_HOST=https://ollama.solarpi.net +OLLAMA_MODEL=gemma2:9b +OLLAMA_EMBED_MODEL=pedrovillalobos/nomic-embed-text-no-gpu +LOG_LEVEL=DEBUG +``` + +--- + +## Note + +- L'app usa Tesseract per OCR su PDF e immagini +- I documenti vengono convertiti in chunks da 512 token +- Gli embedding sono salvati in Qdrant locale +- Ollama remoto per generazione e embedding +- Logging verboso in `logs/ragger.log` diff --git a/agent.py b/agent.py new file mode 100644 index 0000000..4dcd0ca --- /dev/null +++ b/agent.py @@ -0,0 +1,240 @@ +from typing import Optional +from enum import Enum + +from vectorstore import vectorstore +from ollama_client import ollama +from logger import get_logger + +log = get_logger("agent") + + +class AgentMode(str, Enum): + CHAT = "chat" + LETTER = "lettera" + PROJECT = "progetto" + + +AGENT_SYSTEM_PROMPT = """Sei un assistente AI intelligente per un'associazione di volontariato. +Il tuo compito è analizzare le richieste degli utenti e fornire risposte accurate. + +Hai accesso ai seguenti strumenti: +- Ricerca documenti (vectorstore) +- Generazione lettere +- Generazione progetti + +Analizza sempre la richiesta dell'utente e usa lo strumento più appropriato.""" + + +class Agent: + def __init__(self): + self.mode = AgentMode.CHAT + log.info("Agent initialized") + + def set_mode(self, mode: str): + """Imposta la modalità di lavoro dell'agente""" + try: + self.mode = AgentMode(mode) + log.info(f"Agent mode set to: {self.mode.value}") + except ValueError: + log.warning(f"Unknown mode: {mode}, defaulting to chat") + self.mode = AgentMode.CHAT + + def process( + self, + user_input: str, + mode: Optional[str] = None, + context: Optional[str] = None + ) -> dict: + """Elabora l'input dell'utente in base alla modalità""" + if mode: + self.set_mode(mode) + + log.info(f"Agent processing: mode={self.mode.value}, input='{user_input[:50]}...'") + + if self.mode == AgentMode.CHAT: + return self._process_chat(user_input) + elif self.mode == AgentMode.LETTER: + return self._process_letter(user_input, context) + elif self.mode == AgentMode.PROJECT: + return self._process_project(user_input, context) + + return {"error": "Modalità non supportata"} + + def _process_chat(self, question: str) -> dict: + """Elabora una domanda in chat""" + log.debug(f"Chat mode: '{question}'") + + results = self._multi_pass_search(question) + + if not results: + return { + "answer": "Non ho trovato informazioni rilevanti nei documenti.", + "sources": [], + "mode": "chat" + } + + context = self._build_context(results) + + prompt = f"""Basandoti sul seguente contesto estratto dai documenti, rispondi alla domanda. + +CONTESTO: +{context} + +DOMANDA: {question} + +Rispondi in italiano, citando le fonti quando appropriato.""" + + answer = ollama.generate( + prompt=prompt, + system=AGENT_SYSTEM_PROMPT + ) + + sources = self._format_sources(results) + + return { + "answer": answer, + "sources": sources, + "mode": "chat" + } + + def _process_letter(self, topic: str, extra_context: Optional[str] = None) -> dict: + """Elabora una richiesta di generazione lettera""" + log.debug(f"Letter mode: '{topic}'") + + results = self._multi_pass_search(topic) + + doc_context = "" + if results: + doc_context = "\n\n".join(r["text"] for r in results) + + full_context = "" + if doc_context: + full_context += f"Documenti rilevanti:\n{doc_context}\n\n" + if extra_context: + full_context += f"Contesto aggiuntivo:\n{extra_context}\n\n" + + prompt = f"""Genera una lettera formale per un'associazione di volontariato. + +Argomento: {topic} + +{full_context} + +Scrivi una lettera professionale in italiano, con: +- Intestazione (mittente, destinatario, data) +- Oggetto +- Corpo della lettera +- Cordiali saluti +- Firma""" + + letter = ollama.generate(prompt=prompt, system=AGENT_SYSTEM_PROMPT) + + sources = self._format_sources(results) if results else [] + + return { + "answer": letter, + "sources": sources, + "mode": "lettera" + } + + def _process_project( + self, + description: str, + title: Optional[str] = None + ) -> dict: + """Elabora una richiesta di generazione progetto""" + log.debug(f"Project mode: '{description}'") + + search_query = f"{title or ''} {description}" + results = self._multi_pass_search(search_query) + + doc_context = "" + if results: + doc_context = "\n\n".join(r["text"] for r in results) + + prompt = f"""Genera una proposta di progetto per un'associazione di volontariato. + +Titolo: {title or 'Da definire'} +Descrizione: {description} + +{f'Documenti rilevanti:\n{doc_context}' if doc_context else ''} + +Scrivi una proposta di progetto completa con: +1. Titolo e breve introduzione +2. Obiettivi del progetto +3. Attività previste +4. Risorse necessarie (umane, finanziarie, materiali) +5. Timeline e milestone +6. Risultati attesi e indicatori di successo +7. Partner e stakeholder""" + + project = ollama.generate(prompt=prompt, system=AGENT_SYSTEM_PROMPT) + + sources = self._format_sources(results) if results else [] + + return { + "answer": project, + "sources": sources, + "mode": "progetto" + } + + def _multi_pass_search(self, query: str, passes: int = 2) -> list: + """ + Multi-pass retrieval: ricerche multiple per query complesse. + Prima ricerca con query originale, poi con versione riformulata. + """ + log.debug(f"Multi-pass search: {passes} passes") + + results = vectorstore.search(query, limit=5) + + if passes > 1 and len(results) < 3: + log.debug("Low results, trying reformulated query") + reformulated = self._reformulate_query(query) + if reformulated != query: + results2 = vectorstore.search(reformulated, limit=3) + existing_ids = {r.get("id") for r in results} + for r in results2: + if r.get("id") not in existing_ids: + results.append(r) + + log.debug(f"Total results: {len(results)}") + return results + + def _reformulate_query(self, query: str) -> str: + """Riformula la query per migliorare il retrieval""" + prompt = f"""Riformula questa domanda per migliorare la ricerca in un database vettoriale. +Restituisci solo la query riformulata, nient'altro. + +Query originale: {query} + +Query riformulata:""" + + try: + reformulated = ollama.generate(prompt=prompt, system="Sei un assistente di ricerca.") + return reformulated.strip() + except Exception as e: + log.warning(f"Query reformulation failed: {e}") + return query + + def _build_context(self, results: list) -> str: + """Costruisce il contesto dai risultati della ricerca""" + return "\n\n---\n\n".join( + f"[Fonte: {r['source']}" + + (f", pagina {r['page']}" if r.get('page') else "") + + f"]\n{r['text']}" + for r in results + ) + + def _format_sources(self, results: list) -> list: + """Formatta le fonti per la risposta""" + sources = [] + for r in results: + sources.append({ + "source": r.get("source", "Sconosciuto"), + "page": r.get("page"), + "score": r.get("score", 0.0), + "excerpt": r["text"][:200] + "..." if len(r.get("text", "")) > 200 else r.get("text", "") + }) + return sources + + +agent = Agent() diff --git a/app.py b/app.py new file mode 100644 index 0000000..fa511cc --- /dev/null +++ b/app.py @@ -0,0 +1,400 @@ +import streamlit as st +from pathlib import Path +import time + +from config import settings +from core import rag +from agent import agent, AgentMode +from ollama_client import ollama, LLMClient, PROVIDER_BASE_URLS +from logger import get_logger + +log = get_logger("app") + +from document_processor import DocumentProcessor + +SUPPORTED_EXTENSIONS = list(DocumentProcessor.SUPPORTED_EXTENSIONS.keys()) + +PROVIDER_MODELS = { + "ollama": ["gemma2:9b", "llama3.2", "llama3.1", "mistral"], + "opencode": ["claude-3.5-sonnet", "gpt-4o", "gemini-2.0-flash", "deepseek-r1", "llama-3.3-70b"], + "openrouter": ["openrouter/auto", "anthropic/claude-3.5-sonnet", "google/gemini-pro-1.5", "meta-llama/llama-3.1-70b-instruct"], + "groq": ["llama-3.1-70b-versatile", "llama-3.1-8b-instant", "mixtral-8x7b-32768"], + "openai": ["gpt-4o", "gpt-4o-mini", "gpt-4-turbo"] +} + +st.set_page_config( + page_title="RAGGER", + page_icon="📚", + layout="wide" +) + + +def save_settings(provider, model, api_key, base_url, ollama_host, ollama_model): + env_path = Path(".env") + lines = [] + + if env_path.exists(): + for line in env_path.read_text().splitlines(): + if line.startswith("LLM_PROVIDER="): + lines.append(f"LLM_PROVIDER={provider}") + elif line.startswith("LLM_MODEL="): + lines.append(f"LLM_MODEL={model}") + elif line.startswith("LLM_API_KEY="): + lines.append(f"LLM_API_KEY={api_key}") + elif line.startswith("LLM_BASE_URL="): + lines.append(f"LLM_BASE_URL={base_url}") + elif line.startswith("OLLAMA_HOST="): + lines.append(f"OLLAMA_HOST={ollama_host}") + elif line.startswith("OLLAMA_MODEL="): + lines.append(f"OLLAMA_MODEL={ollama_model}") + else: + lines.append(line) + else: + lines = [ + f"LLM_PROVIDER={provider}", + f"LLM_MODEL={model}", + f"LLM_API_KEY={api_key}", + f"LLM_BASE_URL={base_url}", + f"OLLAMA_HOST={ollama_host}", + f"OLLAMA_MODEL={ollama_model}", + ] + + env_path.write_text("\n".join(lines) + "\n") + log.info(f"Settings saved: provider={provider}, model={model}") + + +def check_health(): + log.debug("Running startup health check") + issues = [] + + if not ollama.is_available(): + issues.append("Ollama non raggiungibile") + log.warning("Ollama not available at startup") + + return issues + + +startup_issues = check_health() +if startup_issues: + st.warning("⚠️ Problemi rilevati: " + ", ".join(startup_issues)) + + +def show_logs(): + """Mostra i log dell'applicazione""" + log_file = Path("logs/ragger.log") + if log_file.exists(): + lines = log_file.read_text().splitlines() + last_lines = lines[-50:] if len(lines) > 50 else lines + return "\n".join(last_lines) + return "Nessun log disponibile" + + +def check_ollama_status(): + """Verifica se Ollama sta elaborando qualcosa""" + try: + import httpx + resp = httpx.get(f"{settings.OLLAMA_HOST}/api/tags", timeout=5) + return resp.status_code == 200 + except: + return False + + +st.title("📚 RAGGER") +st.caption("Sistema RAG con Agent Intelligente") + + +with st.sidebar: + st.header("⚙️ Modalità") + + mode = st.selectbox( + "Seleziona modalità", + options=["chat", "lettera", "progetto", "impostazioni"], + index=0, + format_func=lambda x: { + "chat": "💬 Chat", + "lettera": "✉️ Lettera", + "progetto": "📋 Progetto", + "impostazioni": "⚙️ Impostazioni" + }[x] + ) + + st.divider() + + with st.expander("📜 Log Applicazione", expanded=True): + log_content = show_logs() + st.text_area("", log_content, height=200, key="log_viewer", disabled=True) + + col_log1, col_log2 = st.columns(2) + with col_log1: + if st.button("🔄 Aggiorna Log", use_container_width=True): + st.rerun() + with col_log2: + if st.button("📋 Copia Log", use_container_width=True): + st.write("Log copiato!") + + ollama_status = check_ollama_status() + if ollama_status: + st.success("🟢 Ollama connesso") + else: + st.error("🔴 Ollama non risponde") + + if mode == "impostazioni": + st.divider() + st.header("Configurazione") + st.write(f"**Ollama:** {settings.OLLAMA_HOST}") + st.write(f"**Modello:** {settings.OLLAMA_MODEL}") + st.write(f"**Embedding:** {settings.OLLAMA_EMBED_MODEL}") + + if st.button("Test Connessione Ollama"): + with st.spinner("Testing..."): + if ollama.is_available(): + st.success("✅ Ollama connesso!") + else: + st.error("❌ Ollama non raggiungibile") + + stats = rag.get_stats() + st.metric("Documenti indicizzati", stats["total_chunks"]) + + st.divider() + + tab_imp, _ = st.tabs(["⚙️ Impostazioni"]) + with tab_imp: + st.header("⚙️ Impostazioni Provider LLM") + + current_provider = settings.LLM_PROVIDER + all_providers = ["ollama", "opencode", "openrouter", "groq", "openai"] + provider = st.selectbox( + "Provider", + options=all_providers, + index=all_providers.index(current_provider) if current_provider in all_providers else 0 + ) + + if provider == "ollama": + st.info("🏠 **Ollama** - Locale/On-premise") + ollama_host = st.text_input("Ollama Host", value=settings.OLLAMA_HOST) + ollama_models = ollama.list_models() or PROVIDER_MODELS["ollama"] + model = st.selectbox("Modello", options=ollama_models, index=0) + api_key = "" + base_url = "" + elif provider == "opencode": + st.info("🧘 **OpenCode Zen**") + api_key = st.text_input("API Key", value=settings.LLM_API_KEY or "", type="password") + base_url = st.text_input("Base URL", value=settings.LLM_BASE_URL or PROVIDER_BASE_URLS["opencode"]) + models = PROVIDER_MODELS.get(provider, []) + model = st.selectbox("Modello", options=models, index=0) + ollama_host = settings.OLLAMA_HOST + else: + st.info(f"🌐 **{provider.capitalize()}**") + api_key = st.text_input("API Key", value=settings.LLM_API_KEY or "", type="password") + default_url = PROVIDER_BASE_URLS.get(provider, "") + base_url = st.text_input("Base URL", value=settings.LLM_BASE_URL or default_url) + models = PROVIDER_MODELS.get(provider, []) + model = st.selectbox("Modello", options=models, index=0) + ollama_host = settings.OLLAMA_HOST + + col1, col2 = st.columns(2) + with col1: + if st.button("💾 Salva", type="primary"): + save_settings(provider, model, api_key, base_url, ollama_host, model if provider == "ollama" else settings.OLLAMA_MODEL) + st.success("✅ Salvato!") + with col2: + if st.button("🔄 Ricarica"): + st.rerun() + + st.json({ + "provider": settings.LLM_PROVIDER, + "model": settings.LLM_MODEL or settings.OLLAMA_MODEL, + }) + + +if mode == "chat": + with st.sidebar: + st.divider() + st.header("📄 Documenti") + + uploaded = st.file_uploader( + "Carica documento", + type=SUPPORTED_EXTENSIONS, + accept_multiple_files=True + ) + + if uploaded: + for file in uploaded: + if st.button(f"Indicizza: {file.name}", key=f"idx_{file.name}"): + save_path = Path(settings.UPLOAD_DIR) / file.name + save_path.write_bytes(file.getbuffer()) + + with st.spinner(f"Indicizzando {file.name}..."): + try: + result = rag.ingest_document(save_path) + st.success(f"✅ {result['chunks_created']} chunks") + except Exception as e: + st.error(f"❌ {e}") + + stats = rag.get_stats() + st.metric("Chunks indicizzati", stats["total_chunks"]) + + st.header("💬 Chat con i tuoi documenti") + + if "messages" not in st.session_state: + st.session_state.messages = [] + + for msg in st.session_state.messages: + with st.chat_message(msg["role"]): + st.write(msg["content"]) + if "sources" in msg: + with st.expander("📚 Fonti"): + for src in msg["sources"]: + st.write(f"- **{src['source']}**" + + (f" (pag. {src['page']})" if src['page'] else "")) + + if prompt := st.chat_input("Fai una domanda..."): + log.info(f"Chat prompt: {prompt[:50]}...") + st.session_state.messages.append({"role": "user", "content": prompt}) + + with st.chat_message("user"): + st.write(prompt) + + with st.chat_message("assistant"): + with st.spinner("Elaborazione in corso..."): + result = agent.process(prompt, mode="chat") + + st.write(result["answer"]) + + if result.get("sources"): + with st.expander("📚 Fonti"): + for src in result["sources"]: + st.write(f"- **{src['source']}**" + + (f" (pag. {src['page']})" if src['page'] else "") + + f" - Score: {src['score']:.2f}") + st.caption(src["excerpt"]) + + st.session_state.messages.append({ + "role": "assistant", + "content": result["answer"], + "sources": result.get("sources", []) + }) + + +elif mode == "lettera": + with st.sidebar: + st.divider() + st.header("📄 Documenti") + + uploaded = st.file_uploader( + "Carica documento", + type=SUPPORTED_EXTENSIONS, + accept_multiple_files=True + ) + + if uploaded: + for file in uploaded: + if st.button(f"Indicizza: {file.name}", key=f"idx_{file.name}"): + save_path = Path(settings.UPLOAD_DIR) / file.name + save_path.write_bytes(file.getbuffer()) + + with st.spinner(f"Indicizzando {file.name}..."): + try: + result = rag.ingest_document(save_path) + st.success(f"✅ {result['chunks_created']} chunks") + except Exception as e: + st.error(f"❌ {e}") + + stats = rag.get_stats() + st.metric("Chunks indicizzati", stats["total_chunks"]) + + st.header("✉️ Genera Lettera") + + topic = st.text_area("Argomento della lettera", placeholder="Es: Richiesta di patrocinio per evento annuale...") + context = st.text_area("Contesto aggiuntivo (opzionale)", placeholder="Dettagli specifici, destinatario, tono...") + + col1, col2 = st.columns([1, 1]) + with col1: + if st.button("✉️ Genera Lettera", type="primary", use_container_width=True): + if not topic: + st.warning("Inserisci un argomento") + else: + with st.spinner("Generazione in corso..."): + result = agent.process(topic, mode="lettera", context=context) + + st.session_state["last_letter"] = result["answer"] + + with col2: + if st.button("🗑️ Pulisci", use_container_width=True): + if "last_letter" in st.session_state: + del st.session_state["last_letter"] + st.rerun() + + if "last_letter" in st.session_state: + st.divider() + st.subheader("Lettera generata") + st.text_area("", st.session_state["last_letter"], height=400, key="letter_output") + + c1, c2 = st.columns(2) + with c1: + st.button("📋 Copia", on_click=lambda: st.write("Copiato!")) + with c2: + st.button("💾 Salva") + + +elif mode == "progetto": + with st.sidebar: + st.divider() + st.header("📄 Documenti") + + uploaded = st.file_uploader( + "Carica documento", + type=SUPPORTED_EXTENSIONS, + accept_multiple_files=True + ) + + if uploaded: + for file in uploaded: + if st.button(f"Indicizza: {file.name}", key=f"idx_{file.name}"): + save_path = Path(settings.UPLOAD_DIR) / file.name + save_path.write_bytes(file.getbuffer()) + + with st.spinner(f"Indicizzando {file.name}..."): + try: + result = rag.ingest_document(save_path) + st.success(f"✅ {result['chunks_created']} chunks") + except Exception as e: + st.error(f"❌ {e}") + + stats = rag.get_stats() + st.metric("Chunks indicizzati", stats["total_chunks"]) + + st.header("📋 Genera Proposta Progetto") + + title = st.text_input("Titolo del progetto", placeholder="Es: Insieme senza confini") + description = st.text_area("Descrizione", placeholder="Descrivi l'obiettivo, il target, le attività previste...") + + col1, col2 = st.columns([1, 1]) + with col1: + if st.button("📋 Genera Proposta", type="primary", use_container_width=True): + if not title or not description: + st.warning("Inserisci titolo e descrizione") + else: + with st.spinner("Generazione in corso..."): + result = agent.process(description, mode="progetto", context=title) + + st.session_state["last_project"] = result["answer"] + + with col2: + if st.button("🗑️ Pulisci", use_container_width=True): + if "last_project" in st.session_state: + del st.session_state["last_project"] + st.rerun() + + if "last_project" in st.session_state: + st.divider() + st.subheader("Proposta generata") + st.text_area("", st.session_state["last_project"], height=500, key="project_output") + + c1, c2, c3 = st.columns(3) + with c1: + st.button("📋 Copia") + with c2: + st.button("💾 Salva") + with c3: + st.button("📧 Email") diff --git a/chunker.py b/chunker.py new file mode 100644 index 0000000..ff0b282 --- /dev/null +++ b/chunker.py @@ -0,0 +1,109 @@ +from dataclasses import dataclass +from typing import Optional +from config import settings +from logger import get_logger + +log = get_logger("chunker") + + +@dataclass +class Chunk: + text: str + index: int + source: str + page: Optional[int] = None + sheet: Optional[str] = None + metadata: Optional[dict] = None + + +class TextChunker: + def __init__( + self, + chunk_size: int = None, + overlap: int = None + ): + self.chunk_size = chunk_size or settings.CHUNK_SIZE + self.overlap = overlap or settings.CHUNK_OVERLAP + log.debug(f"TextChunker: size={self.chunk_size}, overlap={self.overlap}") + + def chunk_text( + self, + text: str, + source: str, + page: Optional[int] = None, + sheet: Optional[str] = None, + metadata: Optional[dict] = None + ) -> list[Chunk]: + words = text.split() + log.debug(f"Chunking text: {len(words)} words, source={source}, page={page}, sheet={sheet}") + + if len(words) <= self.chunk_size: + log.debug(f"Text fits in single chunk") + return [Chunk( + text=text, + index=0, + source=source, + page=page, + sheet=sheet, + metadata=metadata + )] + + chunks = [] + start = 0 + idx = 0 + + while start < len(words): + end = start + self.chunk_size + chunk_words = words[start:end] + chunk_text = " ".join(chunk_words) + + chunks.append(Chunk( + text=chunk_text, + index=idx, + source=source, + page=page, + sheet=sheet, + metadata=metadata + )) + + log.debug(f"Chunk {idx}: {len(chunk_words)} words, pos {start}-{end}") + start = end - self.overlap + idx += 1 + + log.debug(f"Created {len(chunks)} chunks") + return chunks + + def chunk_document(self, doc: dict) -> list[Chunk]: + log.info(f"Chunking document: {doc.get('source')}") + all_chunks = [] + + if "pages" in doc: + log.debug(f"Document has {len(doc['pages'])} pages/sheets") + for page_data in doc["pages"]: + page_chunks = self.chunk_text( + text=page_data["text"], + source=doc["source"], + page=page_data.get("page"), + sheet=page_data.get("sheet"), + metadata={"type": doc["type"]} + ) + all_chunks.extend(page_chunks) + if page_data.get("page"): + log.debug(f"Page {page_data['page']}: {len(page_chunks)} chunks") + elif page_data.get("sheet"): + log.debug(f"Sheet '{page_data['sheet']}': {len(page_chunks)} chunks") + else: + log.debug(f"Single page document") + all_chunks = self.chunk_text( + text=doc["full_text"], + source=doc["source"], + metadata={"type": doc["type"]} + ) + + log.info(f"Document chunked: {len(all_chunks)} total chunks") + return all_chunks + + +def chunk_document(doc: dict) -> list[Chunk]: + chunker = TextChunker() + return chunker.chunk_document(doc) diff --git a/config.py b/config.py new file mode 100644 index 0000000..2fdf323 --- /dev/null +++ b/config.py @@ -0,0 +1,42 @@ +from dotenv import load_dotenv +from pydantic_settings import BaseSettings +from typing import Optional +import logging + + +load_dotenv() + + +class Settings(BaseSettings): + OLLAMA_HOST: str = "http://localhost:11434" + OLLAMA_MODEL: str = "gemma2:9b" + OLLAMA_EMBED_MODEL: str = "nomic-embed-text" + + LLM_PROVIDER: str = "ollama" + LLM_API_KEY: Optional[str] = None + LLM_MODEL: Optional[str] = None + LLM_BASE_URL: Optional[str] = None + + QDRANT_PATH: str = "./data/qdrant" + QDRANT_COLLECTION: str = "documents" + + TESSERACT_LANG: str = "ita+eng" + + CHUNK_SIZE: int = 512 + CHUNK_OVERLAP: int = 50 + + UPLOAD_DIR: str = "./data/uploads" + MAX_FILE_SIZE_MB: int = 50 + + LOG_LEVEL: str = "DEBUG" + + @property + def log_level_int(self) -> int: + return getattr(logging, self.LOG_LEVEL.upper(), logging.DEBUG) + + class Config: + env_file = ".env" + env_file_encoding = "utf-8" + + +settings = Settings() diff --git a/core.py b/core.py new file mode 100644 index 0000000..639873d --- /dev/null +++ b/core.py @@ -0,0 +1,190 @@ +from pathlib import Path +from typing import Optional + +from document_processor import extract_text +from chunker import chunk_document +from vectorstore import vectorstore +from ollama_client import ollama +from config import settings +from logger import get_logger + +log = get_logger("core") + + +SYSTEM_PROMPT = """Sei un assistente AI per un'associazione di volontariato. +Il tuo compito è aiutare a estrarre informazioni dai documenti e supportare la generazione di lettere, progetti e conversazioni. + +Rispondi sempre in italiano, in modo chiaro e professionale. +Quando usi informazioni dai documenti, cita sempre la fonte.""" + + +class RAGSystem: + def __init__(self): + self.upload_dir = Path(settings.UPLOAD_DIR) + self.upload_dir.mkdir(parents=True, exist_ok=True) + log.info(f"RAGSystem initialized: upload_dir={self.upload_dir}") + + def ingest_document(self, file_path: Path | str) -> dict: + path = Path(file_path) + log.info(f"Ingesting document: {path.name}") + log.debug(f"Full path: {path}") + + log.debug("Step 1: OCR extraction") + doc = extract_text(path) + log.info(f"OCR complete: {doc['total_pages']} pages, {len(doc['text'])} chars") + + log.debug("Step 2: Chunking") + chunks = chunk_document(doc) + log.info(f"Chunking complete: {len(chunks)} chunks") + + log.debug("Step 3: Storing in vector DB") + ids = vectorstore.add_chunks(chunks) + log.info(f"Document ingested: {len(ids)} chunks stored") + + return { + "source": str(path), + "total_pages": doc["total_pages"], + "chunks_created": len(chunks), + "chunk_ids": ids + } + + def query( + self, + question: str, + top_k: int = 5, + system: Optional[str] = None + ) -> dict: + log.info(f"Query: '{question[:50]}...' (top_k={top_k})") + + log.debug("Searching vector store") + results = vectorstore.search(question, limit=top_k) + + if not results: + log.warning("No results found in vector store") + return { + "answer": "Non ho trovato informazioni rilevanti nei documenti.", + "sources": [] + } + + log.info(f"Found {len(results)} relevant chunks") + + context = "\n\n---\n\n".join( + f"[Fonte: {r['source']}" + + (f", pagina {r['page']}" if r['page'] else "") + + f"]\n{r['text']}" + for r in results + ) + log.debug(f"Context assembled: {len(context)} chars") + + prompt = f"""Basandoti sul seguente contesto estratto dai documenti, rispondi alla domanda. + +CONTESTO: +{context} + +DOMANDA: {question} + +Rispondi in italiano, citando le fonti quando appropriato.""" + + log.debug("Generating answer") + answer = ollama.generate( + prompt=prompt, + system=system or SYSTEM_PROMPT + ) + log.info(f"Answer generated: {len(answer)} chars") + + sources = [ + { + "source": r["source"], + "page": r["page"], + "score": r["score"], + "excerpt": r["text"][:200] + "..." if len(r["text"]) > 200 else r["text"] + } + for r in results + ] + + return { + "answer": answer, + "sources": sources + } + + def generate_letter( + self, + topic: str, + context: Optional[str] = None + ) -> str: + log.info(f"Generating letter: topic='{topic[:30]}...'") + + log.debug("Searching relevant documents") + results = vectorstore.search(topic, limit=5) + + doc_context = "" + if results: + doc_context = "\n\n".join(r["text"] for r in results) + log.debug(f"Found {len(results)} relevant chunks") + + full_context = "" + if doc_context: + full_context += f"Documenti rilevanti:\n{doc_context}\n\n" + if context: + full_context += f"Contesto aggiuntivo:\n{context}\n\n" + + prompt = f"""Genera una lettera formale per un'associazione di volontariato. + +Argomento: {topic} + +{full_context} + +Scrivi una lettera professionale in italiano, con intestazione, corpo e chiusura appropriati.""" + + log.debug("Generating letter") + letter = ollama.generate(prompt=prompt, system=SYSTEM_PROMPT) + log.info(f"Letter generated: {len(letter)} chars") + + return letter + + def generate_project( + self, + title: str, + description: str + ) -> str: + log.info(f"Generating project: title='{title}'") + + log.debug("Searching relevant documents") + results = vectorstore.search(f"{title} {description}", limit=5) + + doc_context = "" + if results: + doc_context = "\n\n".join(r["text"] for r in results) + log.debug(f"Found {len(results)} relevant chunks") + + prompt = f"""Genera una proposta di progetto per un'associazione di volontariato. + +Titolo: {title} +Descrizione: {description} + +{f'Documenti rilevanti:\n{doc_context}' if doc_context else ''} + +Scrivi una proposta di progetto completa con: +1. Introduzione e obiettivi +2. Attività previste +3. Risorse necessarie +4. Timeline +5. Risultati attesi""" + + log.debug("Generating project") + project = ollama.generate(prompt=prompt, system=SYSTEM_PROMPT) + log.info(f"Project generated: {len(project)} chars") + + return project + + def get_stats(self) -> dict: + count = vectorstore.count() + log.debug(f"Stats: {count} chunks") + return { + "total_chunks": count, + "upload_dir": str(self.upload_dir) + } + + +log.info("Initializing RAG System") +rag = RAGSystem() diff --git a/debug.py b/debug.py new file mode 100755 index 0000000..96050e0 --- /dev/null +++ b/debug.py @@ -0,0 +1,620 @@ +#!/usr/bin/env python3 +""" +RAGGER Debug Utility - Health Check, Reset, Testing +""" + +import subprocess +import sys +import os +import shutil +import time +from pathlib import Path +from datetime import datetime + +PROJECT_DIR = Path(__file__).parent +QDRANT_PATH = PROJECT_DIR / "data" / "qdrant" +UPLOADS_PATH = PROJECT_DIR / "data" / "uploads" +LOGS_PATH = PROJECT_DIR / "logs" + +GREEN = "\033[92m" +RED = "\033[91m" +YELLOW = "\033[93m" +CYAN = "\033[96m" +BOLD = "\033[1m" +RESET = "\033[0m" + + +def color(text: str, c: str) -> str: + return f"{c}{text}{RESET}" + + +def icon(success: bool) -> str: + return color("✅", GREEN) if success else color("❌", RED) + + +def run_cmd(cmd: str, timeout: int = 30) -> tuple[int, str, str]: + try: + result = subprocess.run( + cmd, shell=True, capture_output=True, text=True, timeout=timeout + ) + return result.returncode, result.stdout.strip(), result.stderr.strip() + except subprocess.TimeoutExpired: + return -1, "", "Timeout" + + +def clear_screen(): + os.system("clear" if os.name == "posix" else "cls") + + +def print_header(title: str): + print() + print(color("=" * 50, CYAN)) + print(color(f" {title}", BOLD)) + print(color("=" * 50, CYAN)) + print() + + +def kill_streamlit(): + print(color("\n🔴 Killing Streamlit processes...", YELLOW)) + run_cmd("pkill -f 'streamlit run' 2>/dev/null || true") + run_cmd("pkill -f 'uvicorn' 2>/dev/null || true") + print(color("✅ Done", GREEN)) + + +def clean_qdrant_locks(): + print(color("\n🔴 Cleaning Qdrant locks...", YELLOW)) + if QDRANT_PATH.exists(): + lock_files = list(QDRANT_PATH.glob("*.lock")) + list(QDRANT_PATH.glob("*.flock")) + for f in lock_files: + try: + f.unlink() + print(f" Removed: {f.name}") + except Exception as e: + print(f" Error removing {f.name}: {e}") + print(color("✅ Done", GREEN)) + + +def reset_database(): + print(color("\n🔴 RESET DATABASE", RED + BOLD)) + print(color("This will DELETE ALL vector embeddings!", YELLOW)) + + confirm = input(color("\nType 'yes' to confirm: ", YELLOW)).strip().lower() + if confirm != "yes": + print(color("Cancelled.", YELLOW)) + return False + + kill_streamlit() + + print(color("\n📁 Removing Qdrant data...", YELLOW)) + if QDRANT_PATH.exists(): + shutil.rmtree(QDRANT_PATH) + print(color(" ✅ Qdrant directory removed", GREEN)) + else: + print(color(" ⚠️ Qdrant directory not found", YELLOW)) + + print(color("\n✅ Database reset complete!", GREEN)) + return True + + +def reset_uploads(): + print(color("\n🔴 RESET UPLOADS", RED + BOLD)) + print(color("This will DELETE ALL uploaded documents!", YELLOW)) + + files = list(UPLOADS_PATH.glob("*")) if UPLOADS_PATH.exists() else [] + if not files: + print(color("No files to delete.", YELLOW)) + return True + + print(f"\nFiles to delete ({len(files)}):") + for f in files[:10]: + print(f" - {f.name}") + if len(files) > 10: + print(f" ... and {len(files) - 10} more") + + confirm = input(color("\nType 'yes' to confirm: ", YELLOW)).strip().lower() + if confirm != "yes": + print(color("Cancelled.", YELLOW)) + return False + + print(color("\n📁 Removing uploaded files...", YELLOW)) + for f in files: + try: + if f.is_file(): + f.unlink() + elif f.is_dir(): + shutil.rmtree(f) + except Exception as e: + print(f" Error: {f.name}: {e}") + + print(color("\n✅ Uploads reset complete!", GREEN)) + return True + + +def reset_all(): + print(color("\n🔴🔴🔴 FULL RESET 🔴🔴🔴", RED + BOLD)) + print(color("This will DELETE EVERYTHING:", YELLOW)) + print(" - All vector embeddings (Qdrant)") + print(" - All uploaded documents") + print(" - All logs") + + confirm = input(color("\nType 'DESTROY' to confirm: ", RED + BOLD)).strip() + if confirm != "DESTROY": + print(color("Cancelled.", YELLOW)) + return + + kill_streamlit() + + print(color("\n📁 Removing all data...", YELLOW)) + + if QDRANT_PATH.exists(): + shutil.rmtree(QDRANT_PATH) + print(color(" ✅ Qdrant removed", GREEN)) + + if UPLOADS_PATH.exists(): + shutil.rmtree(UPLOADS_PATH) + UPLOADS_PATH.mkdir(parents=True) + print(color(" ✅ Uploads cleared", GREEN)) + + if LOGS_PATH.exists(): + shutil.rmtree(LOGS_PATH) + print(color(" ✅ Logs removed", GREEN)) + + print(color("\n✅ FULL RESET COMPLETE!", GREEN)) + + +def health_check() -> dict: + """Run health checks on all components.""" + results = {} + + print(color("\n🔍 RAGGER Health Check", BOLD + CYAN)) + print(color("=" * 50, CYAN)) + + print(f"\n[1/7] Tesseract OCR...", end=" ", flush=True) + code, out, err = run_cmd("tesseract --version 2>&1") + if code == 0 and "tesseract" in out.lower(): + version = out.split("\n")[0].split()[-1] if out else "unknown" + print(icon(True) + f" v{version}") + results["tesseract"] = {"ok": True, "version": version} + else: + print(icon(False) + " NOT INSTALLED") + results["tesseract"] = {"ok": False, "error": "not installed"} + + print(f"[2/7] Tesseract Languages...", end=" ", flush=True) + code, out, err = run_cmd("tesseract --list-langs 2>&1") + if code == 0: + langs = [l.strip() for l in out.split("\n")[1:] if l.strip() and l.strip() != "osd"] + has_ita = "ita" in langs + has_eng = "eng" in langs + if has_ita and has_eng: + print(icon(True) + f" ita, eng (+{len(langs)-3} others)") + results["tesseract_langs"] = {"ok": True, "langs": langs} + else: + print(icon(False) + f" missing: " + + ("ita " if not has_ita else "") + + ("eng" if not has_eng else "")) + results["tesseract_langs"] = {"ok": False, "missing": + [("ita" if not has_ita else None), ("eng" if not has_eng else None)]} + else: + print(icon(False)) + results["tesseract_langs"] = {"ok": False} + + print(f"[3/7] Poppler (pdf2image)...", end=" ", flush=True) + code, out, err = run_cmd("pdftoppm -v 2>&1") + if code == 0 or "pdftoppm" in out or "poppler" in out.lower(): + print(icon(True)) + results["poppler"] = {"ok": True} + else: + print(icon(False) + " NOT INSTALLED") + results["poppler"] = {"ok": False} + + print(f"[4/7] Ollama Connection...", end=" ", flush=True) + sys.path.insert(0, str(PROJECT_DIR)) + try: + from config import settings + from ollama_client import ollama + + start = time.time() + available = ollama.is_available() + elapsed = time.time() - start + + if available: + print(icon(True) + f" {settings.OLLAMA_HOST} ({elapsed:.1f}s)") + results["ollama_connection"] = {"ok": True, "host": settings.OLLAMA_HOST} + else: + print(icon(False) + f" {settings.OLLAMA_HOST}") + results["ollama_connection"] = {"ok": False, "host": settings.OLLAMA_HOST} + except Exception as e: + print(icon(False) + f" {e}") + results["ollama_connection"] = {"ok": False, "error": str(e)} + + print(f"[5/7] Embedding Model...", end=" ", flush=True) + if results.get("ollama_connection", {}).get("ok"): + try: + from config import settings + from ollama_client import ollama + + start = time.time() + embedding = ollama.embed("test") + elapsed = time.time() - start + dims = len(embedding) + + print(icon(True) + f" {settings.OLLAMA_EMBED_MODEL} ({dims} dims, {elapsed:.1f}s)") + results["embedding_model"] = {"ok": True, "dims": dims, "time": elapsed} + except Exception as e: + print(icon(False) + f" {e}") + results["embedding_model"] = {"ok": False, "error": str(e)} + else: + print(icon(False) + " skipped (no connection)") + results["embedding_model"] = {"ok": False, "error": "skipped"} + + print(f"[6/7] Qdrant Vector DB...", end=" ", flush=True) + try: + from vectorstore import vectorstore + + count = vectorstore.count() + print(icon(True) + f" {count} chunks stored") + results["qdrant"] = {"ok": True, "chunks": count} + except Exception as e: + print(icon(False) + f" {e}") + results["qdrant"] = {"ok": False, "error": str(e)} + + print(f"[7/7] Uploads Directory...", end=" ", flush=True) + try: + if UPLOADS_PATH.exists(): + files = list(UPLOADS_PATH.glob("*")) + total_size = sum(f.stat().st_size for f in files if f.is_file()) + size_mb = total_size / (1024 * 1024) + print(icon(True) + f" {len(files)} files ({size_mb:.1f} MB)") + results["uploads"] = {"ok": True, "files": len(files), "size_mb": size_mb} + else: + print(icon(False) + " not found") + results["uploads"] = {"ok": False} + except Exception as e: + print(icon(False) + f" {e}") + results["uploads"] = {"ok": False, "error": str(e)} + + ok_count = sum(1 for v in results.values() if v.get("ok")) + total = len(results) + + print(color("\n" + "=" * 50, CYAN)) + status = color(f"STATUS: {ok_count}/{total} OK", GREEN if ok_count == total else YELLOW) + print(f"{status}") + + return results + + +def show_logs(): + print_header("LOGS") + + log_file = LOGS_PATH / "ragger.log" + if not log_file.exists(): + print(color("No logs found.", YELLOW)) + return + + print(color(f"Showing last 50 lines of {log_file}:\n", CYAN)) + code, out, _ = run_cmd(f"tail -50 {log_file}") + print(out) + + +def show_status(): + print_header("STATUS") + + print("Streamlit processes:") + code, out, _ = run_cmd("pgrep -af 'streamlit' 2>/dev/null || echo ' None'") + print(out or " None") + + print("\nPython processes in project:") + code, out, _ = run_cmd(f"pgrep -af 'python.*{PROJECT_DIR.name}' 2>/dev/null || echo ' None'") + print(out or " None") + + print("\nQdrant files:") + if QDRANT_PATH.exists(): + files = list(QDRANT_PATH.glob("*")) + for f in files[:10]: + print(f" {f.name}") + if len(files) > 10: + print(f" ... and {len(files) - 10} more") + else: + print(" Directory not found") + + print("\nUploads:") + if UPLOADS_PATH.exists(): + files = list(UPLOADS_PATH.glob("*")) + print(f" {len(files)} files") + for f in files[:5]: + size = f.stat().st_size / 1024 + print(f" - {f.name} ({size:.1f} KB)") + if len(files) > 5: + print(f" ... and {len(files) - 5} more") + else: + print(" Directory not found") + + +def test_embedding(): + print_header("TEST EMBEDDING") + + text = input("Enter text to embed (or press Enter for default): ").strip() + if not text: + text = "Questo è un test di embedding per RAGGER." + + print(f"\nText: {color(text, CYAN)}") + print("\nGenerating embedding...", end=" ", flush=True) + + try: + sys.path.insert(0, str(PROJECT_DIR)) + from ollama_client import ollama + + start = time.time() + embedding = ollama.embed(text) + elapsed = time.time() - start + + print(color(f"DONE ({elapsed:.2f}s)", GREEN)) + print(f"\nDimensions: {len(embedding)}") + print(f"First 5 values: {embedding[:5]}") + print(f"Last 5 values: {embedding[-5:]}") + print(f"Min: {min(embedding):.4f}, Max: {max(embedding):.4f}") + + except Exception as e: + print(color(f"\nERROR: {e}", RED)) + + +def test_generation(): + print_header("TEST GENERATION") + + prompt = input("Enter prompt (or press Enter for default): ").strip() + if not prompt: + prompt = "Rispondi brevemente: qual è la capitale d'Italia?" + + print(f"\nPrompt: {color(prompt, CYAN)}") + print("\nGenerating response...", end=" ", flush=True) + + try: + sys.path.insert(0, str(PROJECT_DIR)) + from ollama_client import ollama + from config import settings + + start = time.time() + response = ollama.generate(prompt) + elapsed = time.time() - start + + print(color(f"DONE ({elapsed:.2f}s)", GREEN)) + print(f"\n{color('Response:', BOLD)}") + print(response) + + except Exception as e: + print(color(f"\nERROR: {e}", RED)) + + +def test_qdrant(): + print_header("TEST QDRANT") + + try: + sys.path.insert(0, str(PROJECT_DIR)) + from vectorstore import vectorstore + from chunker import Chunk + + print(f"Collection: {vectorstore.collection}") + print(f"Chunks stored: {vectorstore.count()}") + + action = input("\nActions: [1] Add test chunk [2] Search [3] List all\n> ").strip() + + if action == "1": + chunk = Chunk( + text="Questo è un chunk di test per verificare il funzionamento di Qdrant.", + source="test.py", + page=1, + index=0 + ) + print("\nAdding test chunk...") + chunk_id = vectorstore.add_chunk(chunk) + print(color(f"✅ Added chunk: {chunk_id}", GREEN)) + print(f"Total chunks now: {vectorstore.count()}") + + elif action == "2": + query = input("Enter search query: ").strip() + if not query: + query = "test" + + print(f"\nSearching for: {color(query, CYAN)}") + results = vectorstore.search(query, limit=5) + + if results: + for i, r in enumerate(results, 1): + print(f"\n[{i}] Score: {r['score']:.4f}") + print(f" Source: {r['source']}") + print(f" Text: {r['text'][:100]}...") + else: + print(color("No results found.", YELLOW)) + + elif action == "3": + from qdrant_client.models import Filter + + results = vectorstore.client.scroll( + collection_name=vectorstore.collection, + limit=20 + ) + + for i, point in enumerate(results[0], 1): + print(f"\n[{i}] ID: {point.id}") + payload = point.payload or {} + print(f" Source: {payload.get('source', 'unknown')}") + print(f" Text: {payload.get('text', '')[:80]}...") + + except Exception as e: + print(color(f"\nERROR: {e}", RED)) + import traceback + traceback.print_exc() + + +def ingest_testdoc(): + print_header("INGEST TEST DOCUMENTS") + + testdoc_path = PROJECT_DIR / "testdoc" + if not testdoc_path.exists(): + print(color("testdoc/ directory not found!", RED)) + return + + files = list(testdoc_path.iterdir()) + if not files: + print(color("No files in testdoc/", YELLOW)) + return + + print(f"Found {len(files)} files in testdoc/:") + for f in files: + print(f" - {f.name}") + + confirm = input(color("\nIngest all? [y/N]: ", YELLOW)).strip().lower() + if confirm != 'y': + print("Cancelled.") + return + + sys.path.insert(0, str(PROJECT_DIR)) + from core import rag + + success = 0 + failed = 0 + + for f in sorted(files): + if f.is_file(): + print(f"\nProcessing: {color(f.name, CYAN)}") + try: + result = rag.ingest_document(f) + print(color(f" ✅ {result['chunks_created']} chunks from {result['total_pages']} pages", GREEN)) + success += 1 + except Exception as e: + print(color(f" ❌ ERROR: {e}", RED)) + failed += 1 + + print(color(f"\n{'='*40}", CYAN)) + print(f"Results: {color(f'{success} success', GREEN)}, {color(f'{failed} failed', RED if failed else GREEN)}") + print(f"Total chunks: {rag.get_stats()['total_chunks']}") + + +def restart_app(): + print(color("\n🔄 Restarting RAGGER...", CYAN)) + kill_streamlit() + clean_qdrant_locks() + print("\nStarting Streamlit...") + os.chdir(PROJECT_DIR) + os.system("source venv/bin/activate && streamlit run app.py") + + +def main(): + clear_screen() + + print(color("=" * 50, CYAN)) + print(color(" 🔧 RAGGER Debug Utility", BOLD)) + print(color("=" * 50, CYAN)) + + while True: + print("\n" + color("MENU", BOLD)) + print("-" * 30) + print(" 1. Health Check") + print(" 2. Show Status") + print(" 3. Show Logs") + print("-" * 30) + print(" 4. Test Embedding") + print(" 5. Test Generation") + print(" 6. Test Qdrant") + print(" 7. Ingest testdoc/") + print("-" * 30) + print(" 8. Reset Database") + print(" 9. Reset Uploads") + print(" 0. Full Reset (DESTROY ALL)") + print("-" * 30) + print(" k. Kill Streamlit") + print(" c. Clean Locks") + print(" r. Restart App") + print(" q. Quit") + print("-" * 30) + + choice = input("\n> ").strip().lower() + + if choice == "1": + health_check() + elif choice == "2": + show_status() + elif choice == "3": + show_logs() + elif choice == "4": + test_embedding() + elif choice == "5": + test_generation() + elif choice == "6": + test_qdrant() + elif choice == "7": + ingest_testdoc() + elif choice == "8": + reset_database() + elif choice == "9": + reset_uploads() + elif choice == "0": + reset_all() + elif choice == "k": + kill_streamlit() + elif choice == "c": + clean_qdrant_locks() + elif choice == "r": + restart_app() + break + elif choice == "q": + print(color("\n👋 Bye!", CYAN)) + break + else: + print(color("Invalid option", RED)) + + input(color("\nPress Enter to continue...", YELLOW)) + clear_screen() + print(color("=" * 50, CYAN)) + print(color(" 🔧 RAGGER Debug Utility", BOLD)) + print(color("=" * 50, CYAN)) + + +if __name__ == "__main__": + if len(sys.argv) > 1: + arg = sys.argv[1] + + if arg == "status": + show_status() + elif arg == "health": + health_check() + elif arg == "kill": + kill_streamlit() + elif arg == "clean": + clean_qdrant_locks() + elif arg == "reset-db": + reset_database() + elif arg == "reset-uploads": + reset_uploads() + elif arg == "reset-all": + reset_all() + elif arg == "logs": + show_logs() + elif arg == "restart": + restart_app() + elif arg == "test-embed": + test_embedding() + elif arg == "test-gen": + test_generation() + elif arg == "test-qdrant": + test_qdrant() + elif arg == "ingest-testdoc": + ingest_testdoc() + else: + print(f"Unknown command: {arg}") + print("\nCommands:") + print(" status Show process status") + print(" health Run health check") + print(" kill Kill Streamlit processes") + print(" clean Clean Qdrant locks") + print(" reset-db Reset database (vectors)") + print(" reset-uploads Reset uploads (files)") + print(" reset-all Full reset (DESTROY ALL)") + print(" logs Show logs") + print(" restart Restart app") + print(" test-embed Test embedding") + print(" test-gen Test generation") + print(" test-qdrant Test Qdrant") + print(" ingest-testdoc Ingest all files from testdoc/") + else: + main() diff --git a/docs/DESIGN.md b/docs/DESIGN.md new file mode 100644 index 0000000..1fb4cd0 --- /dev/null +++ b/docs/DESIGN.md @@ -0,0 +1,356 @@ +# RAGGER - Design Document + +**Sistema RAG per Associazione di Volontariato** + +--- + +## 1. Contesto + +### 1.1 Obiettivo +Sistema AI per estrarre informazioni dai documenti dell'associazione e supportare: +- Generazione lettere e progetti +- Chatbot conversazionale +- Automazione documentale + +### 1.2 Vincoli +- **Singolo utente** +- **100-500 documenti** stimati +- **Proxmox in LAN** con server Ollama esistente (https://ollama.solarpi.net) +- **Solo CPU** (no GPU) +- **On-premise** con fallback API opzionale + +### 1.3 Documenti Supportati +- PDF scansionati (OCR) +- PDF digital-born +- Documenti con tabelle/form +- DOCX, XLSX, immagini + +--- + +## 2. Architettura + +### 2.1 Diagramma + +``` +┌─────────────────────────────────────────────────────────┐ +│ RAGGER │ +│ Single Python App │ +├─────────────────────────────────────────────────────────┤ +│ │ +│ ┌─────────────┐ ┌─────────────────────────────┐ │ +│ │ Streamlit │───▶│ Core Logic │ │ +│ │ UI │ │ - upload & OCR │ │ +│ └─────────────┘ │ - chunking │ │ +│ │ - RAG retrieval │ │ +│ │ - chat generation │ │ +│ └──────────┬──────────────────┘ │ +│ │ │ +│ ┌───────────────────────┴───────────────────┐ │ +│ ▼ ▼ ▼ │ +│ ┌───────────┐ ┌───────────┐ ┌──────┐│ +│ │ Tesseract │ │ Qdrant │ │.env ││ +│ │ OCR │ │ (local) │ │config││ +│ └───────────┘ └─────┬─────┘ └──────┘│ +│ │ │ +└────────────────────────────────┼───────────────────────┘ + │ + ┌───────────┴───────────┐ + ▼ ▼ + ┌────────────────┐ ┌────────────────┐ + │ OLLAMA SERVER │ │ API ESTERNA │ + │ (solarpi.net) │ │ (fallback) │ + │ │ │ │ + │ gemma2:9b │ │ openai/claude │ + │ nomic-embed │ │ │ + └────────────────┘ └────────────────┘ +``` + +### 2.2 Principi +- **Un comando**: `streamlit run app.py` +- **Tutto via Ollama**: embedding e generazione +- **Fallback opzionale**: API esterna se Ollama down +- **Zero modelli Python locali**: tutto remoto via HTTP + +--- + +## 3. Struttura Progetto + +``` +ragger/ +├── app.py # Streamlit UI + logica principale +├── core.py # Classi RAG, chunking, retrieval +├── ocr.py # Tesseract wrapper +├── config.py # Settings da .env +├── requirements.txt # Dipendenze minime +├── .env.example # Template configurazione +├── data/ +│ ├── uploads/ # Documenti caricati +│ └── qdrant/ # Vector DB persistente +└── README.md +``` + +### 3.1 Dipendenze (requirements.txt) +``` +streamlit +qdrant-client +pytesseract +pdf2image +Pillow +httpx +python-dotenv +``` + +### 3.2 Niente +- FastAPI (semplificato con Streamlit) +- sentence-transformers (usa Ollama) +- langchain (overkill per singolo utente) + +--- + +## 4. Configurazione + +### 4.1 File .env + +```env +# === OLLAMA SERVER === +OLLAMA_HOST=https://ollama.solarpi.net +OLLAMA_MODEL=gemma2:9b +OLLAMA_EMBED_MODEL=nomic-embed-text + +# === API ESTERNA (Fallback opzionale) === +LLM_FALLBACK_PROVIDER= # openai | anthropic | vuoto = disabilitato +LLM_FALLBACK_API_KEY= +LLM_FALLBACK_MODEL= + +# === QDRANT === +QDRANT_PATH=./data/qdrant +QDRANT_COLLECTION=documents + +# === OCR === +TESSERACT_LANG=ita+eng + +# === CHUNKING === +CHUNK_SIZE=512 +CHUNK_OVERLAP=50 + +# === APP === +UPLOAD_DIR=./data/uploads +MAX_FILE_SIZE_MB=50 +``` + +### 4.2 Modelli Ollama Richiesti + +Sul server `ollama.solarpi.net`: + +```bash +ollama pull gemma2:9b # Generazione testo +ollama pull nomic-embed-text # Embedding multilingue +``` + +--- + +## 5. Flussi Dati + +### 5.1 Ingestione Documento + +``` +┌─────────────────────────────────────────────────────────────┐ +│ INGESTIONE DOCUMENTO │ +├─────────────────────────────────────────────────────────────┤ +│ │ +│ Upload PDF/Img ──▶ Tesseract OCR ──▶ Testo estratto │ +│ │ │ +│ ▼ │ +│ Chunking (512 token, overlap) │ +│ │ │ +│ ▼ │ +│ Ollama Embed (nomic-embed-text) │ +│ │ │ +│ ▼ │ +│ Qdrant Store + Metadata │ +│ │ +└─────────────────────────────────────────────────────────────┘ +``` + +**Metadata salvati per chunk:** +- `source_file`: nome file originale +- `chunk_index`: posizione nel documento +- `page_number`: numero pagina (se PDF) +- `created_at`: timestamp +- `doc_type`: tipo documento + +### 5.2 Query / Chat + +``` +┌─────────────────────────────────────────────────────────────┐ +│ QUERY / CHAT │ +├─────────────────────────────────────────────────────────────┤ +│ │ +│ Domanda utente ──▶ Ollama Embed (nomic-embed-text) │ +│ │ │ +│ ▼ │ +│ Qdrant Search (top-k chunks) │ +│ │ │ +│ ▼ │ +│ Context Assembly │ +│ │ │ +│ ▼ │ +│ Ollama Generate (gemma2:9b) │ +│ │ │ +│ ▼ │ +│ Risposta + Fonti │ +│ │ +└─────────────────────────────────────────────────────────────┘ +``` + +### 5.3 Fallback LLM + +``` +┌─────────────────────────────────────┐ +│ TENTATIVO OLLAMA │ +│ │ │ +│ ┌───────┴───────┐ │ +│ ▼ ▼ │ +│ SUCCESSO FALLIMENTO │ +│ │ │ │ +│ ▼ ▼ │ +│ Risposta API_FALLBACK? │ +│ │ │ +│ ┌───────┴───────┐ │ +│ ▼ ▼ │ +│ SI NO │ +│ │ │ │ +│ ▼ ▼ │ +│ API Esterna Errore │ +└─────────────────────────────────────┘ +``` + +--- + +## 6. Deployment + +### 6.1 Container Proxmox + +``` +┌─────────────────────────────────────────────────────────────┐ +│ PROXMOX LXC CONTAINER │ +├─────────────────────────────────────────────────────────────┤ +│ │ +│ OS: Debian 12 / Ubuntu 22.04 │ +│ Python: 3.11+ │ +│ Packages: tesseract-ocr, poppler-utils │ +│ │ +│ ┌─────────────────────────────────────────────────────┐ │ +│ │ ragger/ │ │ +│ │ ├── app.py │ │ +│ │ ├── core.py │ │ +│ │ ├── .env │ │ +│ │ └── data/ │ │ +│ └─────────────────────────────────────────────────────┘ │ +│ │ +│ Avvio: streamlit run app.py --server.port 8501 │ +│ Accesso: http://:8501 │ +│ │ +└─────────────────────────────────────────────────────────────┘ +``` + +### 6.2 Comandi Setup + +```bash +# Installazione dipendenze sistema +apt update +apt install -y python3-pip tesseract-ocr tesseract-ocr-ita poppler-utils + +# Clone progetto +git clone /opt/ragger +cd /opt/ragger + +# Installazione Python +pip install -r requirements.txt + +# Configurazione +cp .env.example .env +# Editare .env con i propri valori + +# Avvio +streamlit run app.py --server.port 8501 +``` + +### 6.3 Servizio Systemd (opzionale) + +```ini +# /etc/systemd/system/ragger.service +[Unit] +Description=RAGGER - RAG System +After=network.target + +[Service] +Type=simple +User=ragger +WorkingDirectory=/opt/ragger +ExecStart=/usr/local/bin/streamlit run app.py --server.port 8501 +Restart=always + +[Install] +WantedBy=multi-user.target +``` + +```bash +systemctl enable ragger +systemctl start ragger +``` + +--- + +## 7. Roadmap Implementazione + +### Fase 1: Core (MVP) +- [ ] Config e .env +- [ ] OCR con Tesseract +- [ ] Chunking testo +- [ ] Ollama embedding client +- [ ] Qdrant storage +- [ ] Ricerca semantica base +- [ ] Generazione risposta + +### Fase 2: UI Streamlit +- [ ] Upload documenti +- [ ] Lista documenti indicizzati +- [ ] Chat interface +- [ ] Visualizzazione fonti + +### Fase 3: Miglioramenti +- [ ] Fallback API esterna +- [ ] Gestione errori robusta +- [ ] Export conversazioni +- [ ] Statistiche uso + +--- + +## 8. Decisioni Prese + +| Decisione | Scelta | Motivo | +|-----------|--------|--------| +| Architettura | Monolitica | Singolo utente, semplicità | +| LLM | Ollama remoto | Già esistente, on-premise | +| Modello generazione | gemma2:9b | Qualità italiana, capace | +| Modello embedding | nomic-embed-text | Multilingue, testato | +| Vector DB | Qdrant | Veloce, Docker-friendly | +| OCR | Tesseract | On-premise, gratuito | +| UI | Streamlit | Rapido, singolo file | +| Fallback | API esterna opzionale | Affidabilità | + +--- + +## 9. Riferimenti + +- Ollama: https://ollama.com +- Qdrant: https://qdrant.tech +- Tesseract: https://github.com/tesseract-ocr/tesseract +- Streamlit: https://streamlit.io +- nomic-embed-text: https://ollama.com/library/nomic-embed-text +- gemma2: https://ollama.com/library/gemma2 + +--- + +*Design completato: 2025-02-21* diff --git a/document_processor.py b/document_processor.py new file mode 100644 index 0000000..4426bde --- /dev/null +++ b/document_processor.py @@ -0,0 +1,323 @@ +""" +Document Processor - Estrae testo da vari formati di documento. +Supporta: PDF, Immagini, XLSX, DOCX, PPTX, TXT, MD +""" + +import pytesseract +from pdf2image import convert_from_path +from PIL import Image +from pathlib import Path +from typing import Optional +import tempfile + +from config import settings +from logger import get_logger + +log = get_logger("document_processor") + + +class DocumentProcessor: + SUPPORTED_EXTENSIONS = { + ".pdf": "pdf", + ".png": "image", + ".jpg": "image", + ".jpeg": "image", + ".tiff": "image", + ".bmp": "image", + ".xlsx": "excel", + ".xls": "excel", + ".docx": "word", + ".doc": "word", + ".pptx": "powerpoint", + ".ppt": "powerpoint", + ".txt": "text", + ".md": "text", + ".csv": "csv", + } + + def __init__(self, lang: str = None): + self.lang = lang or settings.TESSERACT_LANG + log.info(f"DocumentProcessor initialized: lang={self.lang}") + + def get_file_type(self, suffix: str) -> Optional[str]: + return self.SUPPORTED_EXTENSIONS.get(suffix.lower()) + + def extract_from_image(self, image_path: Path | str) -> str: + log.debug(f"Extracting from image: {image_path}") + image = Image.open(image_path) + log.debug(f"Image size: {image.size}") + + text = pytesseract.image_to_string(image, lang=self.lang) + log.info(f"Image OCR: {len(text)} chars extracted") + + return text.strip() + + def extract_from_pdf(self, pdf_path: Path | str) -> list[dict]: + log.debug(f"Converting PDF to images: {pdf_path}") + + pages = convert_from_path(pdf_path) + log.debug(f"PDF has {len(pages)} pages") + + results = [] + + for i, page in enumerate(pages, 1): + log.debug(f"Processing page {i}/{len(pages)}") + text = pytesseract.image_to_string(page, lang=self.lang) + if text.strip(): + results.append({ + "page": i, + "text": text.strip() + }) + log.debug(f"Page {i}: {len(text)} chars") + else: + log.debug(f"Page {i}: empty (skipped)") + + log.info(f"PDF OCR complete: {len(results)} pages with text") + return results + + def extract_from_excel(self, excel_path: Path | str) -> list[dict]: + log.debug(f"Extracting from Excel: {excel_path}") + + try: + from openpyxl import load_workbook + except ImportError: + log.error("openpyxl not installed. Run: pip install openpyxl") + raise ImportError("openpyxl required for Excel files. Run: pip install openpyxl") + + wb = load_workbook(excel_path, data_only=True) + results = [] + + for sheet_name in wb.sheetnames: + log.debug(f"Processing sheet: {sheet_name}") + sheet = wb[sheet_name] + + rows_data = [] + for row in sheet.iter_rows(values_only=True): + row_text = " | ".join( + str(cell) if cell is not None else "" + for cell in row + ) + if row_text.strip(" |"): + rows_data.append(row_text) + + if rows_data: + sheet_text = "\n".join(rows_data) + results.append({ + "page": None, + "sheet": sheet_name, + "text": sheet_text + }) + log.debug(f"Sheet '{sheet_name}': {len(sheet_text)} chars, {len(rows_data)} rows") + + log.info(f"Excel extraction complete: {len(results)} sheets") + return results + + def extract_from_word(self, word_path: Path | str) -> list[dict]: + log.debug(f"Extracting from Word: {word_path}") + + try: + from docx import Document + except ImportError: + log.error("python-docx not installed. Run: pip install python-docx") + raise ImportError("python-docx required for Word files. Run: pip install python-docx") + + doc = Document(word_path) + + paragraphs = [] + for para in doc.paragraphs: + if para.text.strip(): + paragraphs.append(para.text) + + tables_text = [] + for table in doc.tables: + table_data = [] + for row in table.rows: + row_text = " | ".join( + cell.text.strip() for cell in row.cells + ) + table_data.append(row_text) + if table_data: + tables_text.append("\n".join(table_data)) + + full_text = "\n\n".join(paragraphs) + if tables_text: + full_text += "\n\n--- TABELLE ---\n\n" + "\n\n".join(tables_text) + + log.info(f"Word extraction: {len(full_text)} chars") + + return [{ + "page": 1, + "text": full_text.strip() + }] + + def extract_from_powerpoint(self, ppt_path: Path | str) -> list[dict]: + log.debug(f"Extracting from PowerPoint: {ppt_path}") + + try: + from pptx import Presentation + except ImportError: + log.error("python-pptx not installed. Run: pip install python-pptx") + raise ImportError("python-pptx required for PowerPoint files. Run: pip install python-pptx") + + prs = Presentation(ppt_path) + results = [] + + for i, slide in enumerate(prs.slides, 1): + slide_text = [] + + for shape in slide.shapes: + if hasattr(shape, "text") and shape.text.strip(): + slide_text.append(shape.text) + + if slide_text: + results.append({ + "page": i, + "text": "\n".join(slide_text) + }) + log.debug(f"Slide {i}: {len(slide_text)} shapes") + + log.info(f"PowerPoint extraction: {len(results)} slides") + return results + + def extract_from_text(self, text_path: Path | str) -> list[dict]: + log.debug(f"Reading text file: {text_path}") + + try: + content = Path(text_path).read_text(encoding="utf-8") + except UnicodeDecodeError: + content = Path(text_path).read_text(encoding="latin-1") + + log.info(f"Text file: {len(content)} chars") + + return [{ + "page": 1, + "text": content.strip() + }] + + def extract_from_csv(self, csv_path: Path | str) -> list[dict]: + log.debug(f"Extracting from CSV: {csv_path}") + + try: + import csv + except ImportError: + log.error("csv module not available") + raise + + rows = [] + with open(csv_path, "r", encoding="utf-8") as f: + reader = csv.reader(f) + for row in reader: + row_text = " | ".join(row) + rows.append(row_text) + + content = "\n".join(rows) + log.info(f"CSV extraction: {len(rows)} rows, {len(content)} chars") + + return [{ + "page": 1, + "text": content + }] + + def extract(self, file_path: Path | str) -> dict: + path = Path(file_path) + log.info(f"Extracting: {path.name}") + + if not path.exists(): + log.error(f"File not found: {path}") + raise FileNotFoundError(f"File not found: {path}") + + suffix = path.suffix.lower() + file_type = self.get_file_type(suffix) + + if not file_type: + log.error(f"Unsupported file type: {suffix}") + raise ValueError(f"Unsupported file type: {suffix}") + + log.debug(f"File type: {file_type}") + + if file_type == "pdf": + pages = self.extract_from_pdf(path) + full_text = "\n\n".join(p["text"] for p in pages) + return { + "source": str(path), + "type": "pdf", + "pages": pages, + "full_text": full_text, + "total_pages": len(pages) + } + + elif file_type == "image": + text = self.extract_from_image(path) + return { + "source": str(path), + "type": "image", + "full_text": text, + "total_pages": 1 + } + + elif file_type == "excel": + sheets = self.extract_from_excel(path) + full_text = "\n\n--- SHEET ---\n\n".join( + f"[{s['sheet']}]\n{s['text']}" for s in sheets + ) + return { + "source": str(path), + "type": "excel", + "pages": sheets, + "full_text": full_text, + "total_pages": len(sheets) + } + + elif file_type == "word": + pages = self.extract_from_word(path) + return { + "source": str(path), + "type": "word", + "pages": pages, + "full_text": pages[0]["text"] if pages else "", + "total_pages": 1 + } + + elif file_type == "powerpoint": + slides = self.extract_from_powerpoint(path) + full_text = "\n\n".join( + f"[Slide {s['page']}]\n{s['text']}" for s in slides + ) + return { + "source": str(path), + "type": "powerpoint", + "pages": slides, + "full_text": full_text, + "total_pages": len(slides) + } + + elif file_type == "text": + pages = self.extract_from_text(path) + return { + "source": str(path), + "type": "text", + "pages": pages, + "full_text": pages[0]["text"] if pages else "", + "total_pages": 1 + } + + elif file_type == "csv": + pages = self.extract_from_csv(path) + return { + "source": str(path), + "type": "csv", + "pages": pages, + "full_text": pages[0]["text"] if pages else "", + "total_pages": 1 + } + + else: + raise ValueError(f"Unsupported file type: {suffix}") + + +def extract_text(file_path: Path | str, lang: str = None) -> dict: + processor = DocumentProcessor(lang=lang) + return processor.extract(file_path) + + +OCRExtractor = DocumentProcessor diff --git a/errore_ragger.png b/errore_ragger.png new file mode 100644 index 0000000..9704810 Binary files /dev/null and b/errore_ragger.png differ diff --git a/errore_ragger2.png b/errore_ragger2.png new file mode 100644 index 0000000..e8a2c52 Binary files /dev/null and b/errore_ragger2.png differ diff --git a/logger.py b/logger.py new file mode 100644 index 0000000..70d2d89 --- /dev/null +++ b/logger.py @@ -0,0 +1,95 @@ +import logging +import sys +from pathlib import Path +from datetime import datetime +from logging.handlers import RotatingFileHandler +from typing import Optional + + +LOG_DIR = Path(__file__).parent / "logs" +LOG_DIR.mkdir(exist_ok=True) + + +class ColorFormatter(logging.Formatter): + COLORS = { + "DEBUG": "\033[36m", + "INFO": "\033[32m", + "WARNING": "\033[33m", + "ERROR": "\033[31m", + "CRITICAL": "\033[35m", + } + RESET = "\033[0m" + BOLD = "\033[1m" + + def format(self, record): + color = self.COLORS.get(record.levelname, "") + record.levelname = f"{color}{self.BOLD}{record.levelname:8}{self.RESET}" + return super().format(record) + + +def get_logger( + name: str, + level: int = logging.DEBUG, + log_file: Optional[str] = None +) -> logging.Logger: + logger = logging.getLogger(name) + logger.setLevel(level) + + if logger.handlers: + return logger + + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setLevel(level) + console_format = ColorFormatter( + fmt="%(asctime)s | %(levelname)s | %(name)s: %(message)s", + datefmt="%H:%M:%S" + ) + console_handler.setFormatter(console_format) + logger.addHandler(console_handler) + + file_path = LOG_DIR / (log_file or "ragger.log") + file_handler = RotatingFileHandler( + file_path, + maxBytes=5 * 1024 * 1024, + backupCount=3, + encoding="utf-8" + ) + file_handler.setLevel(level) + file_format = logging.Formatter( + fmt="%(asctime)s | %(levelname)-8s | %(name)s: %(message)s", + datefmt="%Y-%m-%d %H:%M:%S" + ) + file_handler.setFormatter(file_format) + logger.addHandler(file_handler) + + return logger + + +log = get_logger("ragger") + + +def log_function_call(func): + import functools + + @functools.wraps(func) + def wrapper(*args, **kwargs): + args_repr = [repr(a) for a in args[:3]] + kwargs_repr = [f"{k}={v!r}" for k, v in list(kwargs.items())[:3]] + signature = ", ".join(args_repr + kwargs_repr) + if len(signature) > 80: + signature = signature[:77] + "..." + + log.debug(f"[CALL] {func.__name__}({signature})") + + try: + result = func(*args, **kwargs) + result_repr = repr(result) + if len(result_repr) > 100: + result_repr = result_repr[:97] + "..." + log.debug(f"[RETURN] {func.__name__} -> {result_repr}") + return result + except Exception as e: + log.error(f"[ERROR] {func.__name__}: {type(e).__name__}: {e}") + raise + + return wrapper diff --git a/ocr.py b/ocr.py new file mode 100644 index 0000000..0ae3deb --- /dev/null +++ b/ocr.py @@ -0,0 +1,94 @@ +import pytesseract +from pdf2image import convert_from_path +from PIL import Image +from pathlib import Path +from typing import Optional +import tempfile + +from config import settings +from logger import get_logger + +log = get_logger("ocr") + + +class OCRExtractor: + def __init__(self, lang: str = None): + self.lang = lang or settings.TESSERACT_LANG + log.debug(f"OCRExtractor initialized: lang={self.lang}") + + def extract_from_image(self, image_path: Path | str) -> str: + log.debug(f"Extracting from image: {image_path}") + image = Image.open(image_path) + log.debug(f"Image size: {image.size}") + + text = pytesseract.image_to_string(image, lang=self.lang) + log.info(f"Image OCR: {len(text)} chars extracted") + + return text.strip() + + def extract_from_pdf(self, pdf_path: Path | str) -> list[dict]: + log.debug(f"Converting PDF to images: {pdf_path}") + + pages = convert_from_path(pdf_path) + log.debug(f"PDF has {len(pages)} pages") + + results = [] + + for i, page in enumerate(pages, 1): + log.debug(f"Processing page {i}/{len(pages)}") + text = pytesseract.image_to_string(page, lang=self.lang) + if text.strip(): + results.append({ + "page": i, + "text": text.strip() + }) + log.debug(f"Page {i}: {len(text)} chars") + else: + log.debug(f"Page {i}: empty (skipped)") + + log.info(f"PDF OCR complete: {len(results)} pages with text") + return results + + def extract(self, file_path: Path | str) -> dict: + path = Path(file_path) + log.info(f"Extracting: {path.name}") + + if not path.exists(): + log.error(f"File not found: {path}") + raise FileNotFoundError(f"File not found: {path}") + + suffix = path.suffix.lower() + log.debug(f"File type: {suffix}") + + if suffix == ".pdf": + pages = self.extract_from_pdf(path) + full_text = "\n\n".join(p["text"] for p in pages) + result = { + "source": str(path), + "type": "pdf", + "pages": pages, + "full_text": full_text, + "total_pages": len(pages) + } + log.info(f"PDF extracted: {len(full_text)} chars, {len(pages)} pages") + return result + + elif suffix in {".png", ".jpg", ".jpeg", ".tiff", ".bmp"}: + text = self.extract_from_image(path) + result = { + "source": str(path), + "type": "image", + "full_text": text, + "total_pages": 1 + } + log.info(f"Image extracted: {len(text)} chars") + return result + + else: + log.error(f"Unsupported file type: {suffix}") + raise ValueError(f"Unsupported file type: {suffix}") + + +def extract_text(file_path: Path | str, lang: str = None) -> dict: + extractor = OCRExtractor(lang=lang) + return extractor.extract(file_path) diff --git a/ollama_client.py b/ollama_client.py new file mode 100644 index 0000000..1ca7593 --- /dev/null +++ b/ollama_client.py @@ -0,0 +1,189 @@ +import httpx +import warnings +from typing import Optional +from config import settings +from logger import get_logger + +warnings.filterwarnings("ignore", message="Unverified HTTPS request") + +log = get_logger("ollama_client") + +PROVIDER_BASE_URLS = { + "openrouter": "https://openrouter.ai/api/v1", + "openai": "https://api.openai.com/v1", + "groq": "https://api.groq.com/openai/v1", + "opencode": "https://opencode.ai/zen/v1", +} + + +class LLMClient: + def __init__(self): + self.provider = settings.LLM_PROVIDER + self.timeout = httpx.Timeout(300.0, connect=30.0) + log.info(f"Initializing LLMClient: provider={self.provider}") + + if self.provider == "ollama": + self.host = settings.OLLAMA_HOST + log.debug(f"Ollama host: {self.host}") + self.client = httpx.Client(timeout=self.timeout, verify=False) + else: + base_url = settings.LLM_BASE_URL or PROVIDER_BASE_URLS.get(self.provider, "") + api_key = settings.LLM_API_KEY or "" + log.debug(f"External API: {self.provider} -> {base_url}") + self.client = httpx.Client( + timeout=self.timeout, + base_url=base_url, + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json" + } + ) + if self.provider == "openrouter": + self.client.headers["HTTP-Referer"] = "https://github.com/ragger" + self.client.headers["X-Title"] = "RAGGER" + + def embed(self, text: str) -> list[float]: + model = settings.OLLAMA_EMBED_MODEL + log.debug(f"Embedding: model={model}, text_len={len(text)}") + + url = f"{settings.OLLAMA_HOST.rstrip('/')}/api/embeddings" + log.debug(f"POST {url}") + + try: + response = self.client.post( + url, + json={"model": model, "prompt": text} + ) + response.raise_for_status() + embedding = response.json()["embedding"] + log.debug(f"Embedding success: {len(embedding)} dims") + return embedding + except httpx.HTTPStatusError as e: + log.error(f"HTTP error: {e.response.status_code} - {e.response.text}") + raise + except Exception as e: + log.error(f"Embedding error: {e}") + raise + + def generate( + self, + prompt: str, + system: Optional[str] = None, + context: Optional[str] = None + ) -> str: + log.debug(f"Generate: provider={self.provider}, prompt_len={len(prompt)}") + + if self.provider == "ollama": + return self._generate_ollama(prompt, system, context) + else: + return self._generate_openai(prompt, system, context) + + def _generate_ollama( + self, + prompt: str, + system: Optional[str] = None, + context: Optional[str] = None + ) -> str: + model = settings.OLLAMA_MODEL + log.debug(f"Ollama generate: model={model}") + + full_prompt = "" + if system: + full_prompt += f"System: {system}\n\n" + if context: + full_prompt += f"Contesto:\n{context}\n\n" + full_prompt += prompt + + log.debug(f"Full prompt length: {len(full_prompt)} chars") + + url = f"{settings.OLLAMA_HOST.rstrip('/')}/api/generate" + log.debug(f"POST {url}") + + try: + response = self.client.post( + url, + json={"model": model, "prompt": full_prompt, "stream": False} + ) + response.raise_for_status() + result = response.json()["response"] + log.debug(f"Response length: {len(result)} chars") + log.info(f"Generated response ({len(result)} chars)") + return result + except httpx.HTTPStatusError as e: + log.error(f"HTTP error: {e.response.status_code} - {e.response.text}") + raise + except Exception as e: + log.error(f"Generation error: {e}") + raise + + def _generate_openai( + self, + prompt: str, + system: Optional[str] = None, + context: Optional[str] = None + ) -> str: + model = settings.LLM_MODEL or "openrouter/auto" + log.debug(f"OpenAI generate: model={model}") + + messages = [] + if system: + messages.append({"role": "system", "content": system}) + + user_content = "" + if context: + user_content += f"Contesto:\n{context}\n\n" + user_content += prompt + messages.append({"role": "user", "content": user_content}) + + log.debug(f"Messages: {len(messages)} items") + + try: + response = self.client.post( + "/chat/completions", + json={"model": model, "messages": messages} + ) + response.raise_for_status() + result = response.json()["choices"][0]["message"]["content"] + log.info(f"Generated response ({len(result)} chars)") + return result + except httpx.HTTPStatusError as e: + log.error(f"HTTP error: {e.response.status_code} - {e.response.text}") + raise + except Exception as e: + log.error(f"Generation error: {e}") + raise + + def is_available(self) -> bool: + log.debug("Checking availability...") + try: + if self.provider == "ollama": + url = f"{settings.OLLAMA_HOST.rstrip('/')}/api/tags" + log.debug(f"GET {url}") + response = self.client.get(url, timeout=5.0) + available = response.status_code == 200 + log.debug(f"Ollama available: {available}") + return available + else: + log.debug("External API - assuming available") + return True + except Exception as e: + log.warning(f"Availability check failed: {e}") + return False + + def list_models(self) -> list[str]: + if self.provider != "ollama": + return [] + + try: + url = f"{settings.OLLAMA_HOST.rstrip('/')}/api/tags" + response = self.client.get(url, timeout=5.0) + response.raise_for_status() + models = response.json().get("models", []) + return [m["name"] for m in models] + except Exception as e: + log.error(f"Error listing models: {e}") + return [] + + +llm = LLMClient() +ollama = llm diff --git a/pagina.html b/pagina.html new file mode 100644 index 0000000..e5e1b3d --- /dev/null +++ b/pagina.html @@ -0,0 +1,2 @@ + +UX Pilot - Superfast UX/UI Design with AI
\ No newline at end of file diff --git a/prova.html b/prova.html new file mode 100644 index 0000000..21cf813 --- /dev/null +++ b/prova.html @@ -0,0 +1,408 @@ + + + + + + RAGGER - Layout Test + + + + + + + + +
+ + + + +
+ +
+
+ + + + +
+
+ + +
+ +
+

Chat con i tuoi documenti

+ + +
+ +
+
👤
+
+

Come iscriversi alla UILDM?

+
+
+ + +
+
🤖
+
+

Per iscriversi alla UILDM è necessario compilare il modulo di iscrizione e pagare la quota associativa annuale. Ecco i passaggi:

+
    +
  1. Compilare il modulo di iscrizione
  2. +
  3. Pagare la quota annuale di €20
  4. +
  5. Consegnare la documentazione alla segreteria
  6. +
+ + +
+

📚 Fonti:

+
+
+ 📄 + Nuova Modalità Iscrizione UILDM.pdf + · score 0.72 +
+
+
+
+
+
+ + +
+
+ + +
+
+
+ + + + + + + + + +
+
+
+ + + + diff --git a/ragger_errore.txt b/ragger_errore.txt new file mode 100644 index 0000000..1bc703d --- /dev/null +++ b/ragger_errore.txt @@ -0,0 +1,57 @@ +httpx.ReadTimeout: The read operation timed out +Traceback: +File "/home/codex/RAGGER/venv/lib/python3.13/site-packages/streamlit/runtime/scriptrunner/exec_code.py", line 129, in exec_func_with_error_handling + result = func() +File "/home/codex/RAGGER/venv/lib/python3.13/site-packages/streamlit/runtime/scriptrunner/script_runner.py", line 689, in code_to_exec + exec(code, module.__dict__) # noqa: S102 + ~~~~^^^^^^^^^^^^^^^^^^^^^^^ +File "/home/codex/RAGGER/app.py", line 71, in + result = rag.query(prompt) +File "/home/codex/RAGGER/core.py", line 67, in query + answer = ollama.generate( + prompt=prompt, + system=system or SYSTEM_PROMPT + ) +File "/home/codex/RAGGER/ollama_client.py", line 45, in generate + response = self.client.post( + self._url("/api/generate"), + ...<4 lines>... + } + ) +File "/home/codex/RAGGER/venv/lib/python3.13/site-packages/httpx/_client.py", line 1144, in post + return self.request( + ~~~~~~~~~~~~^ + "POST", + ^^^^^^^ + ...<11 lines>... + extensions=extensions, + ^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ +File "/home/codex/RAGGER/venv/lib/python3.13/site-packages/httpx/_client.py", line 825, in request + return self.send(request, auth=auth, follow_redirects=follow_redirects) + ~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +File "/home/codex/RAGGER/venv/lib/python3.13/site-packages/httpx/_client.py", line 914, in send + response = self._send_handling_auth( + request, + ...<2 lines>... + history=[], + ) +File "/home/codex/RAGGER/venv/lib/python3.13/site-packages/httpx/_client.py", line 942, in _send_handling_auth + response = self._send_handling_redirects( + request, + follow_redirects=follow_redirects, + history=history, + ) +File "/home/codex/RAGGER/venv/lib/python3.13/site-packages/httpx/_client.py", line 979, in _send_handling_redirects + response = self._send_single_request(request) +File "/home/codex/RAGGER/venv/lib/python3.13/site-packages/httpx/_client.py", line 1014, in _send_single_request + response = transport.handle_request(request) +File "/home/codex/RAGGER/venv/lib/python3.13/site-packages/httpx/_transports/default.py", line 249, in handle_request + with map_httpcore_exceptions(): + ~~~~~~~~~~~~~~~~~~~~~~~^^ +File "/usr/lib/python3.13/contextlib.py", line 162, in __exit__ + self.gen.throw(value) + ~~~~~~~~~~~~~~^^^^^^^ +File "/home/codex/RAGGER/venv/lib/python3.13/site-packages/httpx/_transports/default.py", line 118, in map_httpcore_exceptions + raise mapped_exc(message) from exc \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8ccbac7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +streamlit +qdrant-client +pytesseract +pdf2image +Pillow +httpx +python-dotenv +pydantic-settings +openpyxl +python-docx +python-pptx diff --git a/testdoc/Budget 2025 (1).xlsx b/testdoc/Budget 2025 (1).xlsx new file mode 100644 index 0000000..2f121f0 Binary files /dev/null and b/testdoc/Budget 2025 (1).xlsx differ diff --git a/testdoc/CONTRIBUTI PUBBLICI 2025.xlsx b/testdoc/CONTRIBUTI PUBBLICI 2025.xlsx new file mode 100644 index 0000000..9c1c25a Binary files /dev/null and b/testdoc/CONTRIBUTI PUBBLICI 2025.xlsx differ diff --git a/testdoc/Consiglio dd. 11.2. 2026.docx b/testdoc/Consiglio dd. 11.2. 2026.docx new file mode 100644 index 0000000..1ab9dba Binary files /dev/null and b/testdoc/Consiglio dd. 11.2. 2026.docx differ diff --git a/testdoc/Il progetto (2).docx b/testdoc/Il progetto (2).docx new file mode 100644 index 0000000..9d7d045 Binary files /dev/null and b/testdoc/Il progetto (2).docx differ diff --git a/testdoc/Nuova Modalità di Iscrizione UILDM Gorizia.pdf b/testdoc/Nuova Modalità di Iscrizione UILDM Gorizia.pdf new file mode 100644 index 0000000..1b05967 Binary files /dev/null and b/testdoc/Nuova Modalità di Iscrizione UILDM Gorizia.pdf differ diff --git a/testdoc/Proposta di laboratorio UILDM Gorizia per licei (1).docx b/testdoc/Proposta di laboratorio UILDM Gorizia per licei (1).docx new file mode 100644 index 0000000..dee4340 Binary files /dev/null and b/testdoc/Proposta di laboratorio UILDM Gorizia per licei (1).docx differ diff --git a/testdoc/Soci UILDM.xlsx b/testdoc/Soci UILDM.xlsx new file mode 100644 index 0000000..d43c96b Binary files /dev/null and b/testdoc/Soci UILDM.xlsx differ diff --git a/testdoc/Stampa domanda (1).docx b/testdoc/Stampa domanda (1).docx new file mode 100644 index 0000000..54ef363 Binary files /dev/null and b/testdoc/Stampa domanda (1).docx differ diff --git a/testdoc/UILDM GORIZIA 2026 Brochure_2.pdf b/testdoc/UILDM GORIZIA 2026 Brochure_2.pdf new file mode 100644 index 0000000..b01763a Binary files /dev/null and b/testdoc/UILDM GORIZIA 2026 Brochure_2.pdf differ diff --git a/testdoc/UILDM_Gorizia_Testo_Presentazione_5_minuti (1).docx b/testdoc/UILDM_Gorizia_Testo_Presentazione_5_minuti (1).docx new file mode 100644 index 0000000..e84bb35 Binary files /dev/null and b/testdoc/UILDM_Gorizia_Testo_Presentazione_5_minuti (1).docx differ diff --git a/testdoc/progetto_insieme_senza_confini.pdf b/testdoc/progetto_insieme_senza_confini.pdf new file mode 100644 index 0000000..3cf1944 Binary files /dev/null and b/testdoc/progetto_insieme_senza_confini.pdf differ diff --git a/tk.txt b/tk.txt new file mode 100644 index 0000000..247e47d --- /dev/null +++ b/tk.txt @@ -0,0 +1 @@ +token=ea1c23c7e4006519864e93df049a806c61f30cd3 \ No newline at end of file diff --git a/vectorstore.py b/vectorstore.py new file mode 100644 index 0000000..39dcf58 --- /dev/null +++ b/vectorstore.py @@ -0,0 +1,158 @@ +from qdrant_client import QdrantClient +from qdrant_client.models import Distance, VectorParams, PointStruct, Filter, FieldCondition, MatchValue +from pathlib import Path +from typing import Optional +import uuid + +from config import settings +from ollama_client import ollama +from chunker import Chunk +from logger import get_logger + +log = get_logger("vectorstore") + + +class VectorStore: + def __init__(self): + self.path = Path(settings.QDRANT_PATH) + self.path.mkdir(parents=True, exist_ok=True) + self.collection = settings.QDRANT_COLLECTION + log.debug(f"Initializing Qdrant at: {self.path}") + self.client = QdrantClient(path=str(self.path), force_disable_check_same_thread=True) + self._ensure_collection() + + def _ensure_collection(self): + collections = self.client.get_collections().collections + names = [c.name for c in collections] + + if self.collection not in names: + log.info(f"Creating collection: {self.collection}") + self.client.create_collection( + collection_name=self.collection, + vectors_config=VectorParams( + size=768, + distance=Distance.COSINE + ) + ) + else: + log.debug(f"Collection exists: {self.collection}") + + def add_chunk(self, chunk: Chunk) -> str: + log.debug(f"Adding chunk: source={chunk.source}, page={chunk.page}, index={chunk.index}") + log.debug(f"Chunk text length: {len(chunk.text)} chars") + + embedding = ollama.embed(chunk.text) + log.debug(f"Embedding generated: {len(embedding)} dims") + + point_id = str(uuid.uuid4()) + + self.client.upsert( + collection_name=self.collection, + points=[ + PointStruct( + id=point_id, + vector=embedding, + payload={ + "text": chunk.text, + "source": chunk.source, + "page": chunk.page, + "sheet": chunk.sheet, + "index": chunk.index, + "metadata": chunk.metadata or {} + } + ) + ] + ) + + log.info(f"Chunk added: {point_id[:8]}... from {chunk.source}") + return point_id + + def add_chunks(self, chunks: list[Chunk]) -> list[str]: + log.info(f"Adding {len(chunks)} chunks...") + ids = [] + for i, c in enumerate(chunks): + log.debug(f"Processing chunk {i+1}/{len(chunks)}") + ids.append(self.add_chunk(c)) + log.info(f"Added {len(ids)} chunks total") + return ids + + def search(self, query: str, limit: int = 5) -> list[dict]: + log.debug(f"Searching: '{query[:50]}...' (limit={limit})") + + query_embedding = ollama.embed(query) + log.debug(f"Query embedding: {len(query_embedding)} dims") + + results = self.client.query_points( + collection_name=self.collection, + query=query_embedding, + limit=limit + ) + + log.debug(f"Found {len(results.points)} results") + + output = [] + for r in results.points: + payload = r.payload or {} + output.append({ + "text": payload.get("text", ""), + "source": payload.get("source", "unknown"), + "page": payload.get("page"), + "sheet": payload.get("sheet"), + "score": r.score + }) + if payload.get("sheet"): + log.debug(f" Result: {payload.get('source')} / {payload.get('sheet')} (score={r.score:.4f})") + else: + log.debug(f" Result: {payload.get('source')} (score={r.score:.4f})") + + return output + + def delete_by_source(self, source: str) -> int: + log.info(f"Deleting chunks from: {source}") + result = self.client.delete( + collection_name=self.collection, + points_selector=Filter( + must=[ + FieldCondition( + key="source", + match=MatchValue(value=source) + ) + ] + ) + ) + log.info(f"Delete result: {result}") + return result.operation_id if hasattr(result, 'operation_id') else 0 + + def count(self) -> int: + info = self.client.get_collection(self.collection) + count = info.points_count or 0 + log.debug(f"Collection count: {count}") + return count + + def clear(self) -> bool: + log.warning(f"Clearing collection: {self.collection}") + try: + self.client.delete_collection(self.collection) + log.info("Collection deleted") + self._ensure_collection() + log.info("Collection recreated (empty)") + return True + except Exception as e: + log.error(f"Error clearing collection: {e}") + return False + + def reset(self) -> bool: + log.warning("RESETTING VectorStore - deleting all data!") + try: + if self.collection in [c.name for c in self.client.get_collections().collections]: + self.client.delete_collection(self.collection) + log.info("Collection deleted") + self._ensure_collection() + log.info("VectorStore reset complete") + return True + except Exception as e: + log.error(f"Error resetting VectorStore: {e}") + return False + + +vectorstore = VectorStore()