#!/usr/bin/env python3 """ RAGGER Debug Utility - Health Check, Reset, Testing """ import subprocess import sys import os import shutil import time from pathlib import Path from datetime import datetime PROJECT_DIR = Path(__file__).parent QDRANT_PATH = PROJECT_DIR / "data" / "qdrant" UPLOADS_PATH = PROJECT_DIR / "data" / "uploads" LOGS_PATH = PROJECT_DIR / "logs" GREEN = "\033[92m" RED = "\033[91m" YELLOW = "\033[93m" CYAN = "\033[96m" BOLD = "\033[1m" RESET = "\033[0m" def color(text: str, c: str) -> str: return f"{c}{text}{RESET}" def icon(success: bool) -> str: return color("โœ…", GREEN) if success else color("โŒ", RED) def run_cmd(cmd: str, timeout: int = 30) -> tuple[int, str, str]: try: result = subprocess.run( cmd, shell=True, capture_output=True, text=True, timeout=timeout ) return result.returncode, result.stdout.strip(), result.stderr.strip() except subprocess.TimeoutExpired: return -1, "", "Timeout" def clear_screen(): os.system("clear" if os.name == "posix" else "cls") def print_header(title: str): print() print(color("=" * 50, CYAN)) print(color(f" {title}", BOLD)) print(color("=" * 50, CYAN)) print() def kill_streamlit(): print(color("\n๐Ÿ”ด Killing Streamlit processes...", YELLOW)) run_cmd("pkill -f 'streamlit run' 2>/dev/null || true") run_cmd("pkill -f 'uvicorn' 2>/dev/null || true") print(color("โœ… Done", GREEN)) def clean_qdrant_locks(): print(color("\n๐Ÿ”ด Cleaning Qdrant locks...", YELLOW)) if QDRANT_PATH.exists(): lock_files = list(QDRANT_PATH.glob("*.lock")) + list(QDRANT_PATH.glob("*.flock")) for f in lock_files: try: f.unlink() print(f" Removed: {f.name}") except Exception as e: print(f" Error removing {f.name}: {e}") print(color("โœ… Done", GREEN)) def reset_database(): print(color("\n๐Ÿ”ด RESET DATABASE", RED + BOLD)) print(color("This will DELETE ALL vector embeddings!", YELLOW)) confirm = input(color("\nType 'yes' to confirm: ", YELLOW)).strip().lower() if confirm != "yes": print(color("Cancelled.", YELLOW)) return False kill_streamlit() print(color("\n๐Ÿ“ Removing Qdrant data...", YELLOW)) if QDRANT_PATH.exists(): shutil.rmtree(QDRANT_PATH) print(color(" โœ… Qdrant directory removed", GREEN)) else: print(color(" โš ๏ธ Qdrant directory not found", YELLOW)) print(color("\nโœ… Database reset complete!", GREEN)) return True def reset_uploads(): print(color("\n๐Ÿ”ด RESET UPLOADS", RED + BOLD)) print(color("This will DELETE ALL uploaded documents!", YELLOW)) files = list(UPLOADS_PATH.glob("*")) if UPLOADS_PATH.exists() else [] if not files: print(color("No files to delete.", YELLOW)) return True print(f"\nFiles to delete ({len(files)}):") for f in files[:10]: print(f" - {f.name}") if len(files) > 10: print(f" ... and {len(files) - 10} more") confirm = input(color("\nType 'yes' to confirm: ", YELLOW)).strip().lower() if confirm != "yes": print(color("Cancelled.", YELLOW)) return False print(color("\n๐Ÿ“ Removing uploaded files...", YELLOW)) for f in files: try: if f.is_file(): f.unlink() elif f.is_dir(): shutil.rmtree(f) except Exception as e: print(f" Error: {f.name}: {e}") print(color("\nโœ… Uploads reset complete!", GREEN)) return True def reset_all(): print(color("\n๐Ÿ”ด๐Ÿ”ด๐Ÿ”ด FULL RESET ๐Ÿ”ด๐Ÿ”ด๐Ÿ”ด", RED + BOLD)) print(color("This will DELETE EVERYTHING:", YELLOW)) print(" - All vector embeddings (Qdrant)") print(" - All uploaded documents") print(" - All logs") confirm = input(color("\nType 'DESTROY' to confirm: ", RED + BOLD)).strip() if confirm != "DESTROY": print(color("Cancelled.", YELLOW)) return kill_streamlit() print(color("\n๐Ÿ“ Removing all data...", YELLOW)) if QDRANT_PATH.exists(): shutil.rmtree(QDRANT_PATH) print(color(" โœ… Qdrant removed", GREEN)) if UPLOADS_PATH.exists(): shutil.rmtree(UPLOADS_PATH) UPLOADS_PATH.mkdir(parents=True) print(color(" โœ… Uploads cleared", GREEN)) if LOGS_PATH.exists(): shutil.rmtree(LOGS_PATH) print(color(" โœ… Logs removed", GREEN)) print(color("\nโœ… FULL RESET COMPLETE!", GREEN)) def health_check() -> dict: """Run health checks on all components.""" results = {} print(color("\n๐Ÿ” RAGGER Health Check", BOLD + CYAN)) print(color("=" * 50, CYAN)) print(f"\n[1/7] Tesseract OCR...", end=" ", flush=True) code, out, err = run_cmd("tesseract --version 2>&1") if code == 0 and "tesseract" in out.lower(): version = out.split("\n")[0].split()[-1] if out else "unknown" print(icon(True) + f" v{version}") results["tesseract"] = {"ok": True, "version": version} else: print(icon(False) + " NOT INSTALLED") results["tesseract"] = {"ok": False, "error": "not installed"} print(f"[2/7] Tesseract Languages...", end=" ", flush=True) code, out, err = run_cmd("tesseract --list-langs 2>&1") if code == 0: langs = [l.strip() for l in out.split("\n")[1:] if l.strip() and l.strip() != "osd"] has_ita = "ita" in langs has_eng = "eng" in langs if has_ita and has_eng: print(icon(True) + f" ita, eng (+{len(langs)-3} others)") results["tesseract_langs"] = {"ok": True, "langs": langs} else: print(icon(False) + f" missing: " + ("ita " if not has_ita else "") + ("eng" if not has_eng else "")) results["tesseract_langs"] = {"ok": False, "missing": [("ita" if not has_ita else None), ("eng" if not has_eng else None)]} else: print(icon(False)) results["tesseract_langs"] = {"ok": False} print(f"[3/7] Poppler (pdf2image)...", end=" ", flush=True) code, out, err = run_cmd("pdftoppm -v 2>&1") if code == 0 or "pdftoppm" in out or "poppler" in out.lower(): print(icon(True)) results["poppler"] = {"ok": True} else: print(icon(False) + " NOT INSTALLED") results["poppler"] = {"ok": False} print(f"[4/7] Ollama Connection...", end=" ", flush=True) sys.path.insert(0, str(PROJECT_DIR)) try: from config import settings from ollama_client import ollama start = time.time() available = ollama.is_available() elapsed = time.time() - start if available: print(icon(True) + f" {settings.OLLAMA_HOST} ({elapsed:.1f}s)") results["ollama_connection"] = {"ok": True, "host": settings.OLLAMA_HOST} else: print(icon(False) + f" {settings.OLLAMA_HOST}") results["ollama_connection"] = {"ok": False, "host": settings.OLLAMA_HOST} except Exception as e: print(icon(False) + f" {e}") results["ollama_connection"] = {"ok": False, "error": str(e)} print(f"[5/7] Embedding Model...", end=" ", flush=True) if results.get("ollama_connection", {}).get("ok"): try: from config import settings from ollama_client import ollama start = time.time() embedding = ollama.embed("test") elapsed = time.time() - start dims = len(embedding) print(icon(True) + f" {settings.OLLAMA_EMBED_MODEL} ({dims} dims, {elapsed:.1f}s)") results["embedding_model"] = {"ok": True, "dims": dims, "time": elapsed} except Exception as e: print(icon(False) + f" {e}") results["embedding_model"] = {"ok": False, "error": str(e)} else: print(icon(False) + " skipped (no connection)") results["embedding_model"] = {"ok": False, "error": "skipped"} print(f"[6/7] Qdrant Vector DB...", end=" ", flush=True) try: from vectorstore import vectorstore count = vectorstore.count() print(icon(True) + f" {count} chunks stored") results["qdrant"] = {"ok": True, "chunks": count} except Exception as e: print(icon(False) + f" {e}") results["qdrant"] = {"ok": False, "error": str(e)} print(f"[7/7] Uploads Directory...", end=" ", flush=True) try: if UPLOADS_PATH.exists(): files = list(UPLOADS_PATH.glob("*")) total_size = sum(f.stat().st_size for f in files if f.is_file()) size_mb = total_size / (1024 * 1024) print(icon(True) + f" {len(files)} files ({size_mb:.1f} MB)") results["uploads"] = {"ok": True, "files": len(files), "size_mb": size_mb} else: print(icon(False) + " not found") results["uploads"] = {"ok": False} except Exception as e: print(icon(False) + f" {e}") results["uploads"] = {"ok": False, "error": str(e)} ok_count = sum(1 for v in results.values() if v.get("ok")) total = len(results) print(color("\n" + "=" * 50, CYAN)) status = color(f"STATUS: {ok_count}/{total} OK", GREEN if ok_count == total else YELLOW) print(f"{status}") return results def show_logs(): print_header("LOGS") log_file = LOGS_PATH / "ragger.log" if not log_file.exists(): print(color("No logs found.", YELLOW)) return print(color(f"Showing last 50 lines of {log_file}:\n", CYAN)) code, out, _ = run_cmd(f"tail -50 {log_file}") print(out) def show_status(): print_header("STATUS") print("Streamlit processes:") code, out, _ = run_cmd("pgrep -af 'streamlit' 2>/dev/null || echo ' None'") print(out or " None") print("\nPython processes in project:") code, out, _ = run_cmd(f"pgrep -af 'python.*{PROJECT_DIR.name}' 2>/dev/null || echo ' None'") print(out or " None") print("\nQdrant files:") if QDRANT_PATH.exists(): files = list(QDRANT_PATH.glob("*")) for f in files[:10]: print(f" {f.name}") if len(files) > 10: print(f" ... and {len(files) - 10} more") else: print(" Directory not found") print("\nUploads:") if UPLOADS_PATH.exists(): files = list(UPLOADS_PATH.glob("*")) print(f" {len(files)} files") for f in files[:5]: size = f.stat().st_size / 1024 print(f" - {f.name} ({size:.1f} KB)") if len(files) > 5: print(f" ... and {len(files) - 5} more") else: print(" Directory not found") def test_embedding(): print_header("TEST EMBEDDING") text = input("Enter text to embed (or press Enter for default): ").strip() if not text: text = "Questo รจ un test di embedding per RAGGER." print(f"\nText: {color(text, CYAN)}") print("\nGenerating embedding...", end=" ", flush=True) try: sys.path.insert(0, str(PROJECT_DIR)) from ollama_client import ollama start = time.time() embedding = ollama.embed(text) elapsed = time.time() - start print(color(f"DONE ({elapsed:.2f}s)", GREEN)) print(f"\nDimensions: {len(embedding)}") print(f"First 5 values: {embedding[:5]}") print(f"Last 5 values: {embedding[-5:]}") print(f"Min: {min(embedding):.4f}, Max: {max(embedding):.4f}") except Exception as e: print(color(f"\nERROR: {e}", RED)) def test_generation(): print_header("TEST GENERATION") prompt = input("Enter prompt (or press Enter for default): ").strip() if not prompt: prompt = "Rispondi brevemente: qual รจ la capitale d'Italia?" print(f"\nPrompt: {color(prompt, CYAN)}") print("\nGenerating response...", end=" ", flush=True) try: sys.path.insert(0, str(PROJECT_DIR)) from ollama_client import ollama from config import settings start = time.time() response = ollama.generate(prompt) elapsed = time.time() - start print(color(f"DONE ({elapsed:.2f}s)", GREEN)) print(f"\n{color('Response:', BOLD)}") print(response) except Exception as e: print(color(f"\nERROR: {e}", RED)) def test_qdrant(): print_header("TEST QDRANT") try: sys.path.insert(0, str(PROJECT_DIR)) from vectorstore import vectorstore from chunker import Chunk print(f"Collection: {vectorstore.collection}") print(f"Chunks stored: {vectorstore.count()}") action = input("\nActions: [1] Add test chunk [2] Search [3] List all\n> ").strip() if action == "1": chunk = Chunk( text="Questo รจ un chunk di test per verificare il funzionamento di Qdrant.", source="test.py", page=1, index=0 ) print("\nAdding test chunk...") chunk_id = vectorstore.add_chunk(chunk) print(color(f"โœ… Added chunk: {chunk_id}", GREEN)) print(f"Total chunks now: {vectorstore.count()}") elif action == "2": query = input("Enter search query: ").strip() if not query: query = "test" print(f"\nSearching for: {color(query, CYAN)}") results = vectorstore.search(query, limit=5) if results: for i, r in enumerate(results, 1): print(f"\n[{i}] Score: {r['score']:.4f}") print(f" Source: {r['source']}") print(f" Text: {r['text'][:100]}...") else: print(color("No results found.", YELLOW)) elif action == "3": from qdrant_client.models import Filter results = vectorstore.client.scroll( collection_name=vectorstore.collection, limit=20 ) for i, point in enumerate(results[0], 1): print(f"\n[{i}] ID: {point.id}") payload = point.payload or {} print(f" Source: {payload.get('source', 'unknown')}") print(f" Text: {payload.get('text', '')[:80]}...") except Exception as e: print(color(f"\nERROR: {e}", RED)) import traceback traceback.print_exc() def ingest_testdoc(): print_header("INGEST TEST DOCUMENTS") testdoc_path = PROJECT_DIR / "testdoc" if not testdoc_path.exists(): print(color("testdoc/ directory not found!", RED)) return files = list(testdoc_path.iterdir()) if not files: print(color("No files in testdoc/", YELLOW)) return print(f"Found {len(files)} files in testdoc/:") for f in files: print(f" - {f.name}") confirm = input(color("\nIngest all? [y/N]: ", YELLOW)).strip().lower() if confirm != 'y': print("Cancelled.") return sys.path.insert(0, str(PROJECT_DIR)) from core import rag success = 0 failed = 0 for f in sorted(files): if f.is_file(): print(f"\nProcessing: {color(f.name, CYAN)}") try: result = rag.ingest_document(f) print(color(f" โœ… {result['chunks_created']} chunks from {result['total_pages']} pages", GREEN)) success += 1 except Exception as e: print(color(f" โŒ ERROR: {e}", RED)) failed += 1 print(color(f"\n{'='*40}", CYAN)) print(f"Results: {color(f'{success} success', GREEN)}, {color(f'{failed} failed', RED if failed else GREEN)}") print(f"Total chunks: {rag.get_stats()['total_chunks']}") def restart_app(): print(color("\n๐Ÿ”„ Restarting RAGGER...", CYAN)) kill_streamlit() clean_qdrant_locks() print("\nStarting Streamlit...") os.chdir(PROJECT_DIR) os.system("source venv/bin/activate && streamlit run app.py") def main(): clear_screen() print(color("=" * 50, CYAN)) print(color(" ๐Ÿ”ง RAGGER Debug Utility", BOLD)) print(color("=" * 50, CYAN)) while True: print("\n" + color("MENU", BOLD)) print("-" * 30) print(" 1. Health Check") print(" 2. Show Status") print(" 3. Show Logs") print("-" * 30) print(" 4. Test Embedding") print(" 5. Test Generation") print(" 6. Test Qdrant") print(" 7. Ingest testdoc/") print("-" * 30) print(" 8. Reset Database") print(" 9. Reset Uploads") print(" 0. Full Reset (DESTROY ALL)") print("-" * 30) print(" k. Kill Streamlit") print(" c. Clean Locks") print(" r. Restart App") print(" q. Quit") print("-" * 30) choice = input("\n> ").strip().lower() if choice == "1": health_check() elif choice == "2": show_status() elif choice == "3": show_logs() elif choice == "4": test_embedding() elif choice == "5": test_generation() elif choice == "6": test_qdrant() elif choice == "7": ingest_testdoc() elif choice == "8": reset_database() elif choice == "9": reset_uploads() elif choice == "0": reset_all() elif choice == "k": kill_streamlit() elif choice == "c": clean_qdrant_locks() elif choice == "r": restart_app() break elif choice == "q": print(color("\n๐Ÿ‘‹ Bye!", CYAN)) break else: print(color("Invalid option", RED)) input(color("\nPress Enter to continue...", YELLOW)) clear_screen() print(color("=" * 50, CYAN)) print(color(" ๐Ÿ”ง RAGGER Debug Utility", BOLD)) print(color("=" * 50, CYAN)) if __name__ == "__main__": if len(sys.argv) > 1: arg = sys.argv[1] if arg == "status": show_status() elif arg == "health": health_check() elif arg == "kill": kill_streamlit() elif arg == "clean": clean_qdrant_locks() elif arg == "reset-db": reset_database() elif arg == "reset-uploads": reset_uploads() elif arg == "reset-all": reset_all() elif arg == "logs": show_logs() elif arg == "restart": restart_app() elif arg == "test-embed": test_embedding() elif arg == "test-gen": test_generation() elif arg == "test-qdrant": test_qdrant() elif arg == "ingest-testdoc": ingest_testdoc() else: print(f"Unknown command: {arg}") print("\nCommands:") print(" status Show process status") print(" health Run health check") print(" kill Kill Streamlit processes") print(" clean Clean Qdrant locks") print(" reset-db Reset database (vectors)") print(" reset-uploads Reset uploads (files)") print(" reset-all Full reset (DESTROY ALL)") print(" logs Show logs") print(" restart Restart app") print(" test-embed Test embedding") print(" test-gen Test generation") print(" test-qdrant Test Qdrant") print(" ingest-testdoc Ingest all files from testdoc/") else: main()