From 08291b578cb2761f4d016fc139cce8d69f9f972d Mon Sep 17 00:00:00 2001 From: truetype Date: Sun, 10 May 2026 21:52:43 +0200 Subject: [PATCH] init: llm-wiki-web con FastAPI + Ollama --- .gitignore | 1 + __pycache__/config.cpython-314.pyc | Bin 0 -> 614 bytes backend/__init__.py | 1 + backend/__pycache__/__init__.cpython-314.pyc | Bin 0 -> 142 bytes backend/__pycache__/ingest.cpython-314.pyc | Bin 0 -> 9088 bytes backend/__pycache__/llm.cpython-314.pyc | Bin 0 -> 5147 bytes backend/__pycache__/main.cpython-314.pyc | Bin 0 -> 8847 bytes backend/__pycache__/search.cpython-314.pyc | Bin 0 -> 4309 bytes backend/__pycache__/wiki_ops.cpython-314.pyc | Bin 0 -> 7445 bytes backend/ingest.py | 205 ++++++++++++++++ backend/llm.py | 74 ++++++ backend/main.py | 149 ++++++++++++ backend/search.py | 94 ++++++++ backend/wiki_ops.py | 147 ++++++++++++ config.py | 22 ++ requirements.txt | 6 + run.sh | 10 + static/app.js | 236 +++++++++++++++++++ static/index.html | 85 +++++++ 19 files changed, 1030 insertions(+) create mode 100644 .gitignore create mode 100644 __pycache__/config.cpython-314.pyc create mode 100644 backend/__init__.py create mode 100644 backend/__pycache__/__init__.cpython-314.pyc create mode 100644 backend/__pycache__/ingest.cpython-314.pyc create mode 100644 backend/__pycache__/llm.cpython-314.pyc create mode 100644 backend/__pycache__/main.cpython-314.pyc create mode 100644 backend/__pycache__/search.cpython-314.pyc create mode 100644 backend/__pycache__/wiki_ops.cpython-314.pyc create mode 100644 backend/ingest.py create mode 100644 backend/llm.py create mode 100644 backend/main.py create mode 100644 backend/search.py create mode 100644 backend/wiki_ops.py create mode 100644 config.py create mode 100644 requirements.txt create mode 100755 run.sh create mode 100644 static/app.js create mode 100644 static/index.html diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..21d0b89 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.venv/ diff --git a/__pycache__/config.cpython-314.pyc b/__pycache__/config.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..838803af5f5c2a70b140042df4e4eee56cdd833a GIT binary patch literal 614 zcmdPqXJ%*WmZv6h`DrrVVhKnr$*2<6 z&&bbB)lbe(Nv+TassSmflFTS6DX`MlH?%a;Gc>c%Gte_Nurf3>F*d2xYOyBIE%x~Ml>FrQ_*+c*#kbfCfcE5MCKWLO zJ$H*U+|%1LKFHrcn?-DM>Ymt!3%TZlX-=wL5i3wH S$n;_m;}bI@BV!RWkOcr~#U6tI literal 0 HcmV?d00001 diff --git a/backend/__pycache__/ingest.cpython-314.pyc b/backend/__pycache__/ingest.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c75130b94bb7170454eccb8466d6164c98c79c9f GIT binary patch literal 9088 zcmcgyeQX=&d4G>QlE>qRsBc?eK1uNhV~dhyIZ~7?t!+uRVp(EN3RTyV=@WUD&!%|H z@fh36yb4K&DfuT(&^AK5q1qN~3Xne>AVa;b17f=^vj3=1p45wn2&jRrzy`>10%{BPm5~7NFP*@V)o@^Lc;2=Xqc5_xpGP?myPa#n)>I`87r?XU_*7{tyClL?k*n zOGIXn(HZ)+={ES<2JI(Uo!y0b_Cd!9r|vxA(p@Jwor5$+WCz_Rc%47t(LJ;bGw9X5 zqC@wIPTen-i>_KCaN` zA$~A9EvI!MrKIG9l9YuuA)b+i^HN&r6{Mt;P|}L9UznDYvL?+cYEtHh-(d_kEa?}5 zc7q*K(z?O+DKXvPMp8NykP<p!Oj4LqrV@&(%P~n) zR6$OsWKCB?!V&Chc2*LUgb-I_nHf2$tHOCzODgb{p`flwil8JRX@(9&5g;j}>lsbb zS*9mM36ZdRlbU02&-UP&b;wa>`9 z8dt5N2}zkT3rgqD%lYxkf-;j*HC;;TvY>0~i;^x`71U%>j^SdOwKN54Ddv($!kIHE zXNEt)vRsx-m3_04RxOzE>TL7XW3AFr_l!lcN(wRhJuO*PCTu8X<^0bczMoXfX=E??rP8ovr9j=@sP$qWcxT-J4II-!8zg+f+4qMQKH zjYGk<;-Khj6uTrR_r1{#{Jd>`-tYTa)RhlMn4Vr(FBp3Rq=o?gLnLL3kg!Eo*l6B~j8!XT7i}pvVh@mkvxKA^B`vd7 znUXYzm3F{t3cmTCD{0TH$V42axd(j3qAB(ytVD#}n@@@hw2m!|^V{MgY#3!?i|fkA zt#RxkV-9Hb93gJf^ha(vVgtFc&vuPw08C$0;(+3mVBiqNE@YCI(gNTI0DCH>3W-c| zTBW#jT%MVcLSNW;-Tfpt&$+$jyjglszbAhG7j@1Udp zzn}be*q?Xo8meiA$Aru%BBq8C_l`|N!Pii-f{Ye`k9m;kve83Hy&=qK0uN7S8FQ9- zK>fG-Ht6@=F*9BqHWZ}}J9!0<i)$tkn_ZNor0f%%RXj{}Rx4(Hn~CU%l$Jq( z+v2+NAOTQNJe<4na8CHGqG4RaEe^)ThH*!?xUI*umDCf_6#{)0es1{j@biR7yQ3A5 ze<&u9i7KlJCyBRU&nbuKoB+++bYlhR*|r^L#?}byl5%_lS~R~3qmZZUjYMG%+Z870 zooydL!atZ-d-J-eDL0@&XrPB`pkfr1&d?W*jDEhY2ee@h2v>~*z|RJF9u$N=OM&g> zhre)6ycBwT;+o-*HBHs9tf_GtoW+K4x~Ya}cp$}6{0uilOg%2>HQbsk#px+FYDQ0w zj`g34hQ*On;j!qE;Uhz*M+ZkWL_>oE=849nM8dFXvf)h2k`}vQIQvG%jtmVM>_yOQ z4M#$i;%UP%t*M!m;qJeJdMxOg8g3<^4y-OQTqY+2D5W%YCI#BP25KHCq_V+dD^bv) zp=Cxk5eES$k2qc}lLUT~5?4Vl)t-UI=o!jE;5Tpx*8MwI{jJOX)}`^RzcuGSJa_zl zU{AJeZ+36*&F-7QtgmlwV1soz(#*Y`4L@%9VZ-A2+|IVu%C>p#es%3zr{6riaCK?l z`$vB=_}*Zy`k8tD6HoQM%G$Sj-|StuxY`t2ZVKg^I&zfj3?ge&j^~iZ z(5<#s@NWnFkafF>!LZ;t=C$A6Uvtc5zr!#PzGHJh?j4r{!(KatHzD62N9QT%318NE zC^nk=%@uYGfh%Y)qGMj&`TYDCTzd<}Vm5JA@j3aey();tOfOHbmP4-fRWMSko#J&MRR33A28LUA#j; z(CHNhKt1=$$#h1Oqf$DiD6_juE9J3G*oTH@!D0&b^1Z`K7EaypEQ?m!u(l8SGaS^) z36^Q?7-PZKo-uZ82I1rV!~NosvB8mHgGEC~Lp95A#4|IgGzDhuAS7s==p8~2msooO zJ=iB=*r#P(>w!>^qsLMECT2Z}S(fHPn*|M?XiuT{H0H5!uoKg`$EM0>L9ibo*u1i# zzz^Ckg5SP1&b#jM%?)n27>Dn7>7y{?myhf? z*2Uba-+8Qs-0EVE3GA)I2O)fW?_u!oG_a6+M_@4A;yHHEey7cjDIE-k4`O%|EF-?y z;rFluysu-5qC%=QLnW;hyg#IMUV;2Hny9FCwxK-;-rHjv_<1)@V_()dQMaQw`b#*I zrp5_IaY5rOnu4NDw2LfgSPu9(MSHsg^iG@+b;WX^N!mAIm2JFPwBZ>Poe>*KXS@v0 zHsGSO6c+(GomeUYM&<-6*wl^NEUfrayLL1%pJy|+5XsmE!0LISPjr>GIBYu!w1g&H z2F+2p8^!h~ocX3nxPWt33L2{-mvVxU>!RlCUqA{M;WoD$Mhk0;v(PV$M7YI?ZjrZU zk#a}4FwB3e6Fs80P=<#deWf!=0h){c110@N_|jO)JN_@gD`lOwO{u=GVu|sOtv+`V zKgGt>fD!Q&2VXQDj~&W z?F2T2LTm7sVZ<^Rrqi;9Tx@vF@kEs*+$0&ic_$D}sMB<8&|3@b1Jv z!{hWj=LSBjAT=$E`*YQy)w0msiM5^0^S#S#&AQ8fecy_!Zmq6mvHNb_{(0y8@zsT^@9kc0>X{pu|Hd*~4@Jvs zR|CRwK$sg`tJyU-vW_<#9|x-E*|n-23pL*#TCLi%T(u`xwQsemW4Ws1M&*rIuBvz5 zZSl~vOMI^K&^)(RTfZ>){qM|s*K38vC+^k;mrmTR?V9(lRSAn{ZrHP}U3aUx=H2(X zz$#b&5eJ1=xUMy>dX;Nf<{B1G|0}omK_w}#d9(wrMDAg!#*eth1?48U!X1HeI((~a z?MH0wf_t^~&~odc81FZCKyZu z7V2$r0IRl6`(gG9nRDy{YO_KUpz)b*6ZLJrM6rpE2^wHK0LY6Zuk7*)lU0t~k6_-H8H?Z6e^!v$F$Dnzy{ykvM^( zHj&nLG*_TWyk^|uM88;O%_!xJIJRA_fXEffxS+@K(is6&Gh#(41x8$@v6OrK2Qcf( zEi}nb6ilda)s{g)6ewxQ>JzAk4HvGDie)2CYwtitsv=J4rMf6JC*SE-l!S&rj-1wI6UnFANskeiT0v$bex^a_LUAPUcZCd}ibp@xda zY_pkr=3sm0r1_EykH&ZnuEJ9)eeW8cZ8giFjfPLk7Mbf(NEoA!pYYzSJ&vt;%x61z zFFGJl#)TB1B>LLQ^OUA%eP(O@qBtUg`L{1WfKD8M`KU-=ZA) zQ9{?A$4~^k*?`&DG2}~_^O-ZVEWX9fdd>8{k>P&mGGMiB@+!{*Gmm2%-$D=72vZ|K z5i{u?bp!1*df!G5*bt<_$mCUMF zvoBrCP5oK!>~}W66lIRs;B&+_j33X4?IFBijo1qO9%x*+uz2}yQ+u|t;|81cAI@=y zf5rK61l>8d`(A}mr22J#VD9+;BYBs-?tj&P{pt$WycQ4^cP@ICp2+$;?{b|Fs*%a- z2v=6Z-qMVZoo>)05?&yvsiM=~KY! z4=Rw=(|8zO?jG31{3A0^$Nuz42!glwF$0b4?cmW?i2TBL#1EfeHZtG`8RaHyez3}L zMWgT!3V2z>O9*=BV7O3iODI$HR?l$iS5qJn^kR=bfM`xEi5kCoSwTgU3Uddh^`nQ6 zM@1c1yMR%YqIU=-ZBx*=r)Si7CLuqgy#QHgdZvfLgS%3Od0?|K>_d()-j9j<-^h+n z$lhhL_Y+e8G4X9UJ#|A&meg!C)HYm(z>Y^<@bvk42wz-2e+(`XKkow5_t7~9bOqYq F{{l9px@-Ud literal 0 HcmV?d00001 diff --git a/backend/__pycache__/llm.cpython-314.pyc b/backend/__pycache__/llm.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0fb4d9bf1399046b8a257a59bfab81959e2a8cf7 GIT binary patch literal 5147 zcmeHLT}&I<6~1?7JpLQogx~o?dZkQ_QN}6gvflcJj{cY!k5^? z5YsF*oz@M@bZSCL7$Q^j8I}Yqc}-^w4G2*~d^^S9Yc#a2-e?X_rKXi|96HQ~6N%|} z)pyjR?<&{B*JJT_l;i}Kb);_^QZyP%CQ~{rLo~`H=<9?>YXUJ(a>PyBwou$Yk;T^^ zXCLtQZkcL^H!HTGA&KvP7i6&i&j}H>{{)uQ6F-bIMo-ft*bvg>0bCnDDSqi_Y43a^ zlBVNAK_#QO%KK6fd_(3=753pYjq)?x3?JpX3hgdXnZE%GGfU_BL5qW=xd=B3KDR%W zSYi6+-g*hVs*{r~5h{xjB3or!n3wIR3ie)r{n|4JAK^z`dq=?Bj!k|7^`7spmmQ_P zFauvC^Z4t+4Tf|9@}=|nNdvKx!4y2_;IRitTfPmfT>iVQ#6fn~9jx{dmGhHYSbhA- z;9;lvTB6dEyh=mPxzmM!hy^rdI-OACN?<1@0!IT((R5XxQ8g%Ls+LY^dMpqJzf354 ztm7Yf8?U|o5jCv3I<2H;^x>zja4fBcC-YIHFe@}-xSNIuj;q9`e+aFG8(J!9*kJf{ zDybLmC+zJU?_kIyBNhFlbn&bUi(5IL*N9OQ=dxHw3i8a1`3TDGwTD zET(Eo6k;I?_USVk^TK=~!JO7K0ERa^vqMm!p_er`2LcS;*04cTMB}kUg83n9M#4Q% zz62kBEFIX8T2`c%-$pjPEvw#^`GLQCDmFZUzj^}qy&Ik|{H=S!pLTCn`|s3zSo66D z3InU2z=}I~w{}GeKQ60W@NSk>Ei|KO%36hTch*7tzZ9aVQ$^in)8yfV=&8_C%nZ$>l~lYL(wBo_-ic+mcG~1caSa; z+I!5lNO{zSj^1v|VkhnGv@M=!1%0X31?5Yvw6{xKY78u0vWgCamVTam1;AeyR&^5OcAYKai;bxer7?!edf-4OR6BaGZ0?mqB%FvYI5YI5e zO*~{6qan&LsPl+uZm?N@ zYrXfX*@$6???F67O`apd_FrJuR~_+Wp}4rC-;Q|}z&sH!&kC3a>(GlAW^D zRlxI;!YrAmesZaoAg=xao+)_F2TJEb(AbwmG6C+0oK{ON{e%#;fU5o1z6HO*#f#YN zCqup$MeQLzkJ-;38}a=`N8iT@uDSLiCzyz>CMQct;ijr2CbWQ_f-{BI!I0e;ZpyGB zjE-CxW4J9v*k=bJJH}w|h>5-i2}!IHb5$s+QQ&dNY{xJ}SuF^|8B=wIMH6aL$>X_U zNu**En!!&hy5afJm673o1&=xdjHOseEJ21k90WJb5U{%;OvLnbzk~Y@#Qi)a zu~w)v(V-E1=f#l!VD<0LesFeEDt`|6H|D|LN-BamVgn>c@dO3nzx&XpRIwp7tx8R6 z(vi)&=8d|})w<5Lx~?1%UBN7h-%NaQu=0bhEGKw_pZgo`%>91ue&Zv5ctO~#tG_e# z;ndx^Pdgvgb+1=-Z~1{Gwm2Ajs{(p{^~6DZfsLB*YEAeFq28cT70B}NewuX>|H+(p z$DE>JruI*??r90-gutd_+yw6958Jj;W=M1uWQ z_NAcEZ*eR&(Y_<@rDh)WkQK|j+xkL|bZN_`+OJcL?6{)Rol{vg{5w9};Hk(|yco8eX1JQbPF^bs*hpm1M41JN;)P literal 0 HcmV?d00001 diff --git a/backend/__pycache__/main.cpython-314.pyc b/backend/__pycache__/main.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f8fcad9c8ea8a4f476b70de347433b15350cbbd6 GIT binary patch literal 8847 zcmc&ZTWlLwcJ~g4&!I?)dQ-AAdRR6sON?zvP8`{Z?O2u~KNQNL+r-LXOpc}5P$Ye4 zWXDR2()MEQepLkmtg%>hy^9pJzvBIB`nAnp`c+D_Ds_?|S!{qd{h}i6X5EjX=gg24 zDL-Ot3iJ}Zb6@A)bMHC#yl$Vb#zP?e=U3!M{eD7zhm}%t6$=|}HbSx_N_6rLiPCX8 z#_3!Kmbr1;7_akVcHM5)ZR3tHr|vY%{J3k(t-HrOy2q^B$AvMk?lsGf@tQH8?la5I zalh_|xt!yHv0A;h*cQ}-(B>Mi8>`pr#~Sp8u|~adtVwU`Ak%y&87!ivH%~WD9w;Ha z^_})c-2*i0F>@>l2klmG2WcRkBe1S!EnIdyWpuDSx~^h$`%^~Omq#}MHjPar87S|z17MC;Vp7DSsSMK$FfCOuoljXO zREB88lDbLo_oA6y60e# zXF6bxZu1}RUkG`QPL7Kirl#U*MoubXzc?yuIy8F1)Ziy&eKyP)&SINUGj{IW9VfIN7baF!9Xt-qKcMDr!>Xzj-5Mws#Jr)zT;=3Q>WELBB@-JnF1~D zW3r~4PA8P)nJ{m-uBz{uGMk=LB5^pu_abmcc!2#^m5Y&!a{OH-m59vAYHA>J&G1T+oJysE zSc)VWp(@-5D*IYtVhtoJ`Qt5egZ#VKe|_Rx8=-Uqn#g@}ChRm^l9ZC?a2}5&&7~9b zNi4$(m*?eVvBxh-msF_#;NV2Xo_2Rs)Xx7N`PUA{4 zDM|M!YeTf~d|zcdAZPWZ%*59THNf{4`7vQ)sprX*egV3xDBWREdVuJB1-0!!Z3j@> zS){gbd{)+fy1F=xUq)&*bI zjad(7J24x;43TBKFx!n;7_v%g^kE}r=8CmYfEBaktNO<5$?tHPn?<_d7=uQtNCS6* z+=Tyz2w4|QktG*OHwlz?Ni24gA)89WbfMN_nuwYLrXQHq=aON{901F38`g%9M+}~& z(>glsXng~de=RiSn)~v?-WlTENUZNsN}bBLiY0#vxG?4d&YAJw$CW z=-Z2TyO^TAzy>kdKt&KFM+y;W_{LhW2cmmby-6;=@mtj z!iH-`(JjD}(2T-F(;+jNGtjsi5SIet-Qkr$_}*(PffuslYeN01(7GhFt_nMrg&kkl zhgR!*mg;-%URkN{e=zu<_hZ`w?@Im1ZTHuK=G8#ga-i$(OZh-QKstSk;fK!F&wZ`; zY!Ajio>>_jeds%xbDjK3sL%2Bk2E~^8^_$o1iB!YY;Rc?Ka9`s7x~=Al*_>3sF9TP)0)p@kjC3{QdK@7)Ttn6TlHlNh1`M9wPP6FF1lU7qYdjr5n;BOk1CRYcWSA$(k!LGY+t_1hooBHUj z`){oT4`wIUght@$C1J;^(6KCZm~5IG9{aTJUqk;A`qY~neDk3%mUG3vu4`GX+qGP` z>zNFR3MUHpFGeEV5dR*@LV#leln30Rz-zTp@HFr) z-RIAMI=BK36YGbi4Ibq-C~WhF1WKH4G7(h33jr%MrLeOw*7Q8YtR>w2<{Z+D0-Z8g ztwwJZ?7&De9S4u>$iTL7crY5Onm^)=`8*`9hW0Io_C4r(7&`JDq3)pqW{W**b)h@s zcdy*jbHTwC@8G%(T9-pd9&Mm~a}%NmKjZgte@7#ZU)c6>zp5FeAJKi>uWf_0Qj={- z0Pv~6E3G?$8=q3x6@`_ZQ=26$-hz3pZCXmq@;0`FhIu4Rf>ERJ*KoMGg|DkGeQYIo z;OQ5>@l#iB@QsJQXwDV=J8}Mz5u>EU);7 z>8p}mMlXNwoj!+qEa@r_b)Yrou#7Ut<98Zm>`nM@`~)(Kmt&-?KY-WUfLjL$+J?_I zqTp7Qt$w(HFq@U0q;0Ckf;SyQy~IUI29V8g7KORtym-xYTMZ8sz^K-N2|)a#NoDw~ zgOy4{gkl|yCm18+7_aHsmS*!8XOM~wRmy_R0DzfIK|vdX4Eg*&1n*j)<<9=wf0c8! zn^J!Fa^3EGwfVZf?5WQi+g2NQFE?Vnu`heJ;2`ec;>en-elcCJ+q{kfvDqDuG?W=P zUO7tXLSV~sXG$D)^MAt^_M|$e>JXc;fqHwEC36w?R^kpy_pF1Tx98_^^Yb`&;C(W= zV;Bn0C+W7hz>->Q2ta{-O_BxBuC(@TnwJ`&`YvLG-bJq^p1hU)EDaq(bpF$_wEfA1=B@O)Yf6Z60nMocb0bBsxD*E+W<_;>HECiZEIBdMddY6dC|$Tp;?|B zjKipXu(&GtRgM_<9HX53?HOCt6?I4Xm@UE$(6oKZK8YN(ErBtM+HYGbVbwtZ3Zr7% z63py@naN~eTL)Al+!l1q4km?QniTv6wBXD%iO#~d1Olz%ouD*B(WLM)bRQO6R_J3c zu^^;DiCs~ZtKi@*%uOk(IG++BN(i1C2n0op&nhBQF&vvv#EY^frxIyR6~Vks$|6d) z0neLLN5!cVrzfK)#JArDCr(Yt)6-%`VPag>AxbMkQ1}vvOBm`o4-n!w#$6713};aU z8f3xi@tiFpa>%oA$m5AxYpSA}5@D94DyPx|;&CXJrxO7jR!l+svcI(K0LE;R;E!t8 zU;}doKd&h)UU@?QZzdCa6VL$RQv)Bv1))|6g#(yIc0CP|R>M^c;Axcx5<>&Q*nuOT zY74y&Lo}3ZmMzrU{)>^HjofoRY~6>J&@fs;!;3phme71IIJDv&LQ80Pxpm(ohUzga zm|i-24a!^Yk6{B5iEccwLc$E=84fvc4=Rbnio#-fFk zVd=SyZtjRJ0F>!XgPSjkMI(vFZL%BCbqTvagUsC0rh^1StF_(BwdE+(gDWdNhyU5L z((`)0b|U-w=l-Tuf9JBl^TXcNuKi10`ya$sx?cWxZl!A???06tEAV^^Ew~8hz14ob zebIeqf1bAINC&7yk%FS+Jh$4ex7}9n1oN~lM|Knl*Zxw$?`{~$o-QGJz@!O~QBZCf?tGQZ%-CYK=W%-y zC(-zVv8Xq;rIMo`AaC;X+yI&9Mtcdl{MwUbYiaDR;v6%e)Z55J2qJ3yliZ^-+ysP+ z-K3pl>>2+28!yw(kys2; zECMXv1noLxXluYF@Hedb+LwIocNgV^F5KaG75gQnXja`mHY9X*RL z=DlrSb#&eyh3|R|X*f~{5N|Mh`q8%nX*fwAX&4B(anygTo_bjeD(oEZ5_suvk*Wy5(1 z9u6Uf&VVbdh>KBIY#Oja_U5TE6yZi~sR!(5^RZB~Lios`MpLPCuKZRw5Kl9Wp)+o{ zGS?C^=%YA8Bf)UW+BJB*OdGX8YJ5;)Nk!L{0jOCb4*g89{;dZwOHiWzH=SKeoZCPw z8a_FZkhT`IUYO8qV}F6!0n9KI%T7XuCj&ti4E~tjpk?u+g=q>vlm6ywis78Uq5|$I z2wu&DmNmtqrkb^VMQp1ixI}_$d~mI<5p+jm$ogWXAP`!& z2|-70Ge~J*aeNcZ_Q0l#qfLw9O#+{Shj8{=P1l=l*WGMe=P2#puo2Ezz!C~!voGgv z$0-GMmmv2emw`|`AL zlLNCFJDUnvDh!wA?a0%Xb*{!jDOA8xK_tBQ_QFG3%V%(qyl`vc#>ArSPWWE$N0IxH ORsPVro$w*kIR9U4ec=`W literal 0 HcmV?d00001 diff --git a/backend/__pycache__/search.cpython-314.pyc b/backend/__pycache__/search.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fe54268746d1de55aa93ade9f61cfa0205da54c7 GIT binary patch literal 4309 zcmcInU2GHC6~5z{@qZkXIJT2G*dl@8WQiBD3oIZ|7LpJWdEHU!yjWO{JtV_AwtHu? zfmEvU(5mo6B324itO#0Gu+moIVf#|`p{=x1Un<2i#qR7@jntQ-_QBausP?Jn%-D`G zE=wPJr8)QS%)RH{Ip25A3_4n@1j^sH$mJ}w{)U}uVJnTDS!k>ffhgo65ts=^VW2il z7|yc_>&G#6f;-PE{CT5d)cVYXNihjlf%6c7_mH$R%_wHU2t5<_6pLUJ%+Rw`=BxtK zOM>=gk5!P8B26X&8Bq#clG0*eN%~N-hJ%dCMH0$<(4cZtlB}rQIVq_m888eVEGBr+ z;vE6R3W<`7hT}gaBn4wW@&jg`5ttav=tVHZ47D~JSAQc+o?nINaKvO-l>$|w|#ptvF!ZM>AF3kgL=BloGn(0q153?;KE z@k%J2Ucj>*S`ueNvx($~VkQ-m#RN^x59O|^_INyz$z+v;BF5v?0k7NOk@tgGAuq^& zrhOa5$8s5~`@>rGCHLj@udLd{4k(gj?TpZ|!vm&qJO6~y6*5F}hTZ#!)NA6aP;zXP zM0mYjFW;$ja$J;+;H@_lfqAF$O`K$u89N9o@lifvZcu}+(DEKg{R>Tc2$@UP5 z2|AV!oUM^cGMiDv%%UO%QeqnTC6}0!GCIu+eP>5Rh6V;g149d`Wqwh)bm%ys*eqrM zvr=YGWvnWf27Q%N#4E~)@Y6rQ{$Z6vs#3X3VnI~RNxVp!k-(K z%~A%ikBb}A**U0#PSpq|U6STh%gn^&L|pjbgK1!w?2<@TZZ4gjrFi+Og^Gz(95+>Y zIhO|NvA`)7L<*a!#zZa$^Z_#pm2wmrUA0j$olvBY#JG}GIav`GRAVkd#f+k|X)&W( zMz18roFZj2K^sLBQv|4LqN2Q*R%F$JR}rV#tfJa9%8TnZnHExU?X)KSCcz%O5SgMc zsEkBiP|FD70|x@tLJ?@HO{cdwu1t@>2x`eQ0PGdA<#4Wyf8%r)oq@bFurjgL=2;(H z8@zdFWpbeN^Jz4aQ)m?JIc4SNT$x zZ`JgT|KP)pC;rgd`07Z>v+u#>dzU{=Kk*zXdHr`Cw;d(lzKxNG%x9z6IQQVfy$hdC zJ#zi2?`WB44|Y8#Y-jg2PXce39b}Jp-Ld8VDsS<+A~8gd#s{CxKckxRVy>exi96ka%6-ZyM+f z-xKIqpwsUObYnk*Fvmr?ru4_xg*i8b0WP8?T@`yP)$77Yj-PqAE|eUHHJC`V;AN-_ zS6s6YuuHKuf@hoCBj9P;UbYc?%}yAqg$)QnKd%rp10G;p?$)U{C(=L$k{>Op3n`M4 zO=pKFMnj5WNFzDgi7g+9pp{}!p!oTzA4LEKG9m3p(T76wr32W4$N+okjt238P>iPZ z5Q-rXsuh`8ComaZQfDTel^hjTn!OrFEqWB@HQ)OL8m|&lsR=PXX+}(29lrJ0TCCX7 zpYP}|bOf(iOE&itTVKiEej|K6Tynbau(!sGzEIv5dSog1P5>#v;4RZ#`)zy4i8K+u z9oFZ|CeG)0PPmp<9r9ic^1H5I$&JYW5yTiBXa&T`+iwQq^PpJ+u^T}Q;?CJJR$sWj zNd2-)8`(Vx=xYVH&_Mj_HwG~P9f68>&`|0#(y{j|Kt(@jD1zk`pyG)eN;L~>24cu` z*=E3;nI5j?HUN=x6^O8rT6Vd+Nqx2pxep6Kgr6M9Ij1W*=k$L8KOd`QmjK-+`C}9C z8$lBzc0&%1VHWNk~J|SD}}Z zE=ghvlQ2=%>0lc2El5H>5~(bP8@NH`7ZeE~s!@c!jv@kJr=BN4rjt-qv#i8*RU`^X zf3j4mqYI5fDx9JDb&WGrC+0RXq$tXKWd$SD}`!_l^18+YuAKtcL)VCA9 zqsVpUxz3HgO|J7X*T2;@@bKuv(L&de>kfRa^@iiRqhxRU`0OoH(L0#;4nCYHc#jrb z$6#=Ee0^$ds$@sLf#gP?2WndVAtWGP>vO_eY}+j99xgj~xhBXwMPGpmsW)g-P4Rdt zn~cX*0~J+M4t~(n(k#V)N!6%a%|XVzOB4F%!n9ALSnaMpCmQl~A~v4_e^^Y5r|CE- vA*IR4;E>8N%(lV6a64wgSpP!2ewB`N-MS2DR literal 0 HcmV?d00001 diff --git a/backend/__pycache__/wiki_ops.cpython-314.pyc b/backend/__pycache__/wiki_ops.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1f5a04fe9ca2c1333f7931f49a6cf86109652f04 GIT binary patch literal 7445 zcmb_hTWlLwdOmX@XLylFQ5RYlOJnP1>c)*EUlY5LFR`?;W9`JHx@*YXDUuzI zVvE4;s=#8aqy-|k!J5Guh#s=2g1Ueovgkf!fo74%79cIlWT#$4XbTj5ld8Dh`lopcV0?(Y8bD49_{NML~=U0Dijgvt7`z`X`&*}*IH@vYDS1j!Q6%>|h07IDO{$J)$e&u;zgQT<}ZvLbgnv zlKo864rX`tZM@v$@4fRy*BAOP`_{vohfdO}U1&<^wqXGiS`f^P-~BI`5ZDPb1l9pR z4t_lRY(peyOEMv5%Gzk!&CEuNWiNRvMM#nj5!pU~b1J3~TX~n{LfjPY!k%P}g6qIE zPe0jYKOxZGWxalux3WK{eQ1Uwmr9+iQ*~{ zhwk)YgjPh-7j<2c0D;NKyc$<11f_&FABn5!G+3MjV3p>eFrU!W$dDl_ab49_C1r?n ziTIq7)Ki9#N-Sz~@W?fnSV%@RHI;~`47;94s&grWM_W$weM1XTgO5iR6oW^|Lw|GB zr2zL#RO5gIXz|P*OU%PH?Ml3_Xz#1aO}GgN27g0Yf+3Y#FliWaK9+dbut$2}A zbY(%qJ=XR?7O-g!xEM}r)=El4G#hT~h2f~G8Nxy|jPppLsr5*Zv|4QCiNPzv6o0TK z(6<|&Qouch!ZO*m6G#2(>4NCPtZsFUTNPc6di1jXm2GedIb@a5t^q+H^~KcW=(!Tktkzgiqa{xS?>D%bfo? z6k0OlFnFLA2Cwn#*hs@+ScWi zxL!j-3b1AJ3@};^JTm!`ei_PqGy{qtDg2~Je?G>(gW27NZM=NeMeoWJhGsP33n}BF z>p(~GGG;4r5*SAtu%pZd0X40p>^`P&1NNi?Fsj8Klsij~rQ8tMlBvc>NlvzfSlJ%p zWMKw5o~H76L9UZXWQKZvtg@3R7<(J(AeYG$?$n+nb25Ll$d^erNRr%sa$#5{uftVk z0%H331!aC-krv|-6Z)q2l@v)+Qh@M9H8rP50C8Q3FX{=YUs_P~NHn4Y2iO!%OK9oZ zx6pHgW0I~V7UR)pwnhEg;IYpc!l%fqng*Jq>j&rw=U%=-Z4o!(f!@6=yr#KJr8h#;zPbR>{mU^V{n;?`;B%7TO z#2Wea!K+PW0#O=^(W*2 znI0W2%}ci9-)M2P4$w@U$U;@H3t9l*ThSA--%qS{%|lHaQP~c%AMlTVI93taLb~^z zIZb{4MQ8&Sr3%0eT@J8@0H6bp4&I!))^Os6!Q9mPU=14LPJcV+pOhy2n15Ip~m`0RM#=~xqTp3(JVkgf2KYwA`5 znRnKY=4(baYew$>>0`G)?{0hGZp&PL=iEWu_z@*srNId#f zfJ*3>tr>ZiLYx1p-hbL*`;EhOy2bGv8qtLQAT*Xur4oUa7_CGjRWOdYaud8GiV*a6 z;5E@C=!wFl!I${Rx1nVTjt$J=9Rk_iv)g#NclnFXzxzHde%(ncUW6lJ9@fe49x1|6 zW-3L!{G!B%c6*Zr6w7d<4QRO4j|0y+EdC-Z+dw)qAob6Jh_-=aw7KCZI?N^Q)6oj) z4EzZqpRFdvt+%_$Fq;&nb?g-djR!N}Wk*PW@f^e4Km`C%tivGy-{jg`Gx=6gK6BL$ z8qg0NZW*#}VAh&+>)8Cfx)=vP3!f3?btD$+1+A;jDMOkcwo7Zptc17pNT)Ona#@-{ zJ?wOzk}gI--BJY}0c9*DuHg`-N~e}F};Cr?34_~ndS#2c*4!j!daT9 zo>m)%*PtR!s(K`*rWNgv-~qT{v@`7tW>g|OsSc;Qn8Fu4ESR6c$JF=@kon3SdgB!O ziynk=$ezMQZxFSI-m zS_;0lysszc>$xvveLWk#V;=}xzR49~*G}qMR`{*D`qi_mM>4*ZAA%=#c^{ng$`y!N04;FCr|gp5CCM*_#3|I4 zdr~=wl5|{Ulg^N1s(w!sxSjr+3fUr5-4lnkCrR<5U9O>QitLO5m&O-I42g7hWqzy$ zv-YI-jnN}KY844&h_(dmCSe4?B*!)w2x4a=zClBE4<{?!1e1DgK)Mi90ZPV^!fNzZ3XKzp*JR$3iZa6> zVHi%k0u3mt{|5@dyqO%uV?g6mp)lq_iuE( zmTe!te=u8jcvC#QV*7pdTY~4$fBa`Z2G)Nd>?`+9O^9a#~!y#6~i zw`(%d4R7a)t|D`h$mJFP^^ww*#4T8@_|!92$=Qs{2>|e_*q|GtiZH z=XrmQ_h*Fl-YoBb#1Fqn;b~3lsFQ4i6&in!>7K-IXnC}NI1_f9Izl+WsW46hP(DqcxzgW%b`NL}g z8Q{jI`_F*|M-&?LQzaEt0>-(}xu<4lr6^#pbl@LKduOi9Tsd^*>dfqHe_&>IW)_YM ztfm;kHSqEvLSVgk^2Q?sp5NmsoIy`f+SlPxJu+to(&!7c2vpOxL$nj7zg2r+`TS#7 z%||CbJh3{z;ev11TVAu}dy&-Xhr3QYy}Iu$%k3F-O*ECCResDD!}}V(RYS17%zOR> z>Pta*4^y{8Ao~oWP;j}|buabB*)QvVIkEnu&AwB#s=PQl8OTsa(GTi|LMixV`Gy`?;3Uj<68;5)f%FhVgIEX?njnY;$iyJleli&+q!gaArTZ^yiTBO$ zG+JuOP9-(IH)cJQc4Kw0uP=x}<~@=+AZRduGc-(PFf~L9Rl}tzv4{?5<*=U6PzYJE zW8~f&DXj}TQX_Yw4i3|&djrOG^NJn@JkXf2*>4{DQSf7#|x_xSG z=u!P^rO0gN#6ws2Kf4>YS-YdL(6m3_)SGMSz3pBUpfD{xo`dGM)SzZcfav8 z7fzo4crMeMZyC(B46dsiEytkEhWEq{N4zaNHKbt}!ocpj(6*JJ9t)XPB_e<=x} zJoD7aL%oJ%O2Ys&G)`k{+891V-A`SRxy85(pA%~K=6PRyV<8b;j47wI7*rv|Q+N!4 z2{OzM%QF0~NEqjD$-Zw$Z;tf-3-LX%!+*4%yvaad_h}yllAg4|@%70N=+h^|z>!bg JfHnGp{{_NebVvXI literal 0 HcmV?d00001 diff --git a/backend/ingest.py b/backend/ingest.py new file mode 100644 index 0000000..c936e7b --- /dev/null +++ b/backend/ingest.py @@ -0,0 +1,205 @@ +""" +Ingest pipeline - due fasi: analisi + generazione +""" + +import json +import re +from pathlib import Path +from typing import List, Dict, Optional +from . import llm, wiki_ops +import config + + +SYSTEM_PROMPT_ANALYSIS = """Sei un bibliotecario esperto. Analizza il documento fornito e estrai informazioni strutturate. + +Rispondi SOLO con JSON valido in questo formato: +{ + "entities": ["nome persona/organizzazione/strumento"], + "concepts": ["concetto/tecnica/metodo"], + "claims": ["affermazione importante trovata"], + "connections": ["riferimento a [[pagina-wiki]] esistente"], + "conflicts": ["affermazione che contraddice qualcosa di esistente"] +} + +Non inventare informazioni. Solo cio che e esplicitamente nel documento.""" + + +SYSTEM_PROMPT_GENERATION = """Sei un wiki maintainer. Genera pagine markdown per una wiki basata sull'analisi fornita. + +Regole: +- Una pagina per ogni entity e concept +- Usa il formato: + # Nome Pagina + + > Una frase che riassume. + + ## Tags + `entity` o `concept` + + ## Body + Contenuto dettagliato... + + ## Related + - [[pagina-correlata]] + + ## Sources + - [nome source](path) + +- Per cross-references usa [[nome-pagina]] +- Mantieni i nomi delle pagine in minuscolo con trattini +-rispondi con JSON: {"pages": [{"name": "nome", "category": "entities|concepts", "content": "contenuto markdown"}]}""" + + +def chunk_text(text: str, max_chars: int = 3000) -> List[str]: + """Suddivide testo in chunks se troppo lungo per Gemma.""" + if len(text) <= max_chars: + return [text] + + chunks = [] + paragraphs = text.split("\n\n") + current_chunk = "" + + for para in paragraphs: + if len(current_chunk) + len(para) <= max_chars: + current_chunk += "\n\n" + para + else: + if current_chunk: + chunks.append(current_chunk.strip()) + current_chunk = para + + if current_chunk: + chunks.append(current_chunk.strip()) + + return chunks + + +async def run_analysis(source_path: Path) -> Dict: + """Fase 1: analizza il source e ritorna struttura.""" + content = source_path.read_text(encoding="utf-8") + + prompt = f"{SYSTEM_PROMPT_ANALYSIS}\n\n## Documento:\n{content}" + + try: + response = await llm.llm_call(prompt) + # Estrai JSON dalla risposta + json_match = re.search(r'\{.*\}', response, re.DOTALL) + if json_match: + return json.loads(json_match.group()) + except Exception as e: + return {"error": str(e), "entities": [], "concepts": [], "claims": [], "connections": [], "conflicts": []} + + return {"entities": [], "concepts": [], "claims": [], "connections": [], "conflicts": []} + + +async def run_generation(source_path: Path, analysis: Dict) -> List[Dict]: + """Fase 2: genera pagine wiki dall'analisi.""" + content = source_path.read_text(encoding="utf-8") + source_name = source_path.stem + + prompt = f"""{SYSTEM_PROMPT_GENERATION} + +## Source file: {source_path.name} +## Analysis: {json.dumps(analysis, ensure_ascii=False)} + +## Source content (per reference): +{content[:2000]}... +""" + + try: + response = await llm.llm_call(prompt) + json_match = re.search(r'\{.*\}', response, re.DOTALL) + if json_match: + data = json.loads(json_match.group()) + return data.get("pages", []) + except Exception: + pass + + return [] + + +async def ingest_source(source_path: Path) -> Dict: + """Pipeline completa: analisi -> generazione -> scrittura file.""" + result = { + "source": str(source_path), + "analysis": None, + "pages_created": [], + "pages_updated": [], + "error": None + } + + # Fase 1: Analisi + analysis = await run_analysis(source_path) + result["analysis"] = analysis + + if "error" in analysis: + result["error"] = analysis["error"] + return result + + # Fase 2: Generazione + pages = await run_generation(source_path, analysis) + result["pages_created"] = [p["name"] for p in pages] + + # Scrittura file + wiki_root = config.WIKI_ROOT + source_category = "sources" + + created_pages = [] + for page in pages: + page_name = page["name"].lower().replace(" ", "-") + category = page.get("category", "concepts") + page_content = page["content"] + + category_dir = wiki_root / "wiki" / category + page_path = category_dir / f"{page_name}.md" + + wiki_ops.write_page(page_path, page_content) + created_pages.append(page_name) + + # Aggiorna index + if created_pages: + wiki_ops.update_index(wiki_root, created_pages, category) + + # Append log + wiki_ops.append_log(wiki_root, source_path.name, created_pages, []) + + return result + + +async def ingest_streaming(source_path: Path): + """Pipeline con streaming per feedback in tempo reale.""" + wiki_root = config.WIKI_ROOT + + yield f"data: Inizio ingest di {source_path.name}\n\n" + + # Fase 1: Analisi + yield f"data: [1/2] Analisi in corso...\n\n" + analysis = await run_analysis(source_path) + yield f"data: Analisi completata. Trovati: {len(analysis.get('entities', []))} entities, {len(analysis.get('concepts', []))} concepts\n\n" + + if "error" in analysis: + yield f"data: ERROR: {analysis['error']}\n\n" + return + + # Fase 2: Generazione + yield f"data: [2/2] Generazione pagine wiki...\n\n" + pages = await run_generation(source_path, analysis) + + created = [] + for page in pages: + page_name = page["name"].lower().replace(" ", "-") + category = page.get("category", "concepts") + page_content = page["content"] + + category_dir = wiki_root / "wiki" / category + page_path = category_dir / f"{page_name}.md" + + wiki_ops.write_page(page_path, page_content) + created.append(page_name) + yield f"data: Creata: [[{page_name}]]\n\n" + + # Aggiorna index e log + if created: + wiki_ops.update_index(wiki_root, created, "concepts") + wiki_ops.append_log(wiki_root, source_path.name, created, []) + + yield f"data: DONE: {len(created)} pagine create\n\n" \ No newline at end of file diff --git a/backend/llm.py b/backend/llm.py new file mode 100644 index 0000000..305d0b0 --- /dev/null +++ b/backend/llm.py @@ -0,0 +1,74 @@ +""" +Chiamate LLM via Ollama +""" + +import json +import httpx +from typing import AsyncGenerator, Optional +import config + + +async def llm_call(prompt: str, model: Optional[str] = None) -> str: + """Chiamata semplice a Ollama - restituisce risposta completa.""" + model = model or config.OLLAMA_MODEL + + async with httpx.AsyncClient(timeout=120.0) as client: + response = await client.post( + f"{config.OLLAMA_URL}/api/generate", + json={ + "model": model, + "prompt": prompt, + "stream": False + } + ) + response.raise_for_status() + return response.json()["response"] + + +async def llm_stream(prompt: str, model: Optional[str] = None) -> AsyncGenerator[str, None]: + """Chiamata streaming a Ollama - yields tokens.""" + model = model or config.OLLAMA_MODEL + + async with httpx.AsyncClient(timeout=120.0) as client: + async with client.stream( + "POST", + f"{config.OLLAMA_URL}/api/generate", + json={ + "model": model, + "prompt": prompt, + "stream": True + } + ) as response: + response.raise_for_status() + async for line in response.aiter_lines(): + if line: + try: + data = json.loads(line) + yield data.get("response", "") + except json.JSONDecodeError: + continue + + +async def llm_chat(messages: list, model: Optional[str] = None) -> AsyncGenerator[str, None]: + """Chat mode - streaming.""" + model = model or config.OLLAMA_MODEL + + async with httpx.AsyncClient(timeout=120.0) as client: + async with client.stream( + "POST", + f"{config.OLLAMA_URL}/api/chat", + json={ + "model": model, + "messages": messages, + "stream": True + } + ) as response: + response.raise_for_status() + async for line in response.aiter_lines(): + if line: + try: + data = json.loads(line) + if "content" in data: + yield data["content"] + except json.JSONDecodeError: + continue \ No newline at end of file diff --git a/backend/main.py b/backend/main.py new file mode 100644 index 0000000..3410076 --- /dev/null +++ b/backend/main.py @@ -0,0 +1,149 @@ +""" +API principale - FastAPI +""" + +from pathlib import Path +from fastapi import FastAPI, HTTPException +from fastapi.staticfiles import StaticFiles +from fastapi.responses import FileResponse, HTMLResponse +from fastapi.middleware.cors import CORSMiddleware +from pydantic import BaseModel +import asyncio +import sse_starlette.sse as sse +import config +from backend import wiki_ops, search, ingest, llm + +app = FastAPI() + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +class IngestRequest(BaseModel): + source: str + + +class ChatRequest(BaseModel): + message: str + history: list = [] + + +@app.get("/") +async def root(): + return FileResponse("static/index.html") + + +@app.get("/api/wiki/list") +async def list_wiki(): + return wiki_ops.list_wiki_pages(config.WIKI_ROOT) + + +@app.get("/api/wiki/page/{path:path}") +async def get_page(path: str): + file_path = config.WIKI_ROOT / "wiki" / path + if not file_path.exists(): + # Prova con .md + file_path = config.WIKI_ROOT / "wiki" / f"{path}.md" + + page_data = wiki_ops.read_page(file_path) + if "error" in page_data: + raise HTTPException(status_code=404, detail=page_data["error"]) + + return page_data + + +@app.post("/api/ingest") +async def ingest_source_endpoint(req: IngestRequest): + source_path = config.WIKI_ROOT / "raw" / req.source + if not source_path.exists(): + raise HTTPException(status_code=404, detail=f"Source not found: {req.source}") + + result = await ingest.ingest_source(source_path) + return result + + +@app.get("/api/ingest/stream/{source}") +async def ingest_stream(source: str): + source_path = config.WIKI_ROOT / "raw" / source + if not source_path.exists(): + raise HTTPException(status_code=404, detail=f"Source not found: {source}") + + async def event_generator(): + async for line in ingest.ingest_streaming(source_path): + yield {"event": "message", "data": line} + + return sse.EventServerResponse(event_generator()) + + +@app.get("/api/search") +async def search_wiki(q: str = ""): + if not q: + return {"results": []} + + name_results = search.search_by_name(q) + content_results = search.search_wiki(q) + + return { + "by_name": name_results, + "by_content": content_results + } + + +@app.get("/api/wiki/recent") +async def recent_pages(limit: int = 10): + return search.get_recent_pages(limit) + + +@app.post("/api/chat") +async def chat(req: ChatRequest): + # Costruisci context dalla wiki + wiki_index = wiki_ops.read_page(config.WIKI_ROOT / "wiki" / "index.md") + overview = wiki_ops.read_page(config.WIKI_ROOT / "wiki" / "overview.md") + + system_prompt = f"""Sei un assistente che risponde basandosi sulla wiki. +USA SEMPRE [[pagina]] per citare le fonti. + +Wiki Index: +{wiki_index.get('content', '')} + +Overview: +{overview.get('content', '')} + +Rispondi in italiano. Cita le fonti con [[nome-pagina]].""" + + messages = [{"role": "system", "content": system_prompt}] + messages.extend(req.history) + messages.append({"role": "user", "content": req.message}) + + async def event_generator(): + async for token in llm.llm_chat(messages): + yield {"event": "message", "data": token} + + return sse.EventServerResponse(event_generator()) + + +@app.get("/api/raw/list") +async def list_raw(): + """Lista i file nella directory raw.""" + raw_dir = config.WIKI_ROOT / "raw" + result = {} + + for category in ["articles", "papers", "repos", "assets"]: + cat_dir = raw_dir / category + if cat_dir.exists(): + files = [f.name for f in cat_dir.iterdir() if f.is_file()] + result[category] = files + else: + result[category] = [] + + return result + + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=config.WEB_PORT) \ No newline at end of file diff --git a/backend/search.py b/backend/search.py new file mode 100644 index 0000000..52224bf --- /dev/null +++ b/backend/search.py @@ -0,0 +1,94 @@ +""" +Ricerca nei file wiki +""" + +import re +from pathlib import Path +from typing import List, Dict +import config + + +def search_wiki(query: str, limit: int = 20) -> List[Dict]: + """Cerca nei contenuti delle pagine wiki.""" + results = [] + wiki_root = config.WIKI_ROOT + + query_lower = query.lower() + + for md_file in wiki_root.glob("wiki/**/*.md"): + if md_file.name in ["index.md", "log.md"]: + continue + + try: + content = md_file.read_text(encoding="utf-8").lower() + if query_lower in content: + # Trova le linee con il match + lines = md_file.read_text(encoding="utf-8").split("\n") + matches = [] + for i, line in enumerate(lines): + if query_lower in line.lower(): + matches.append({ + "line": i + 1, + "text": line.strip()[:200] + }) + + results.append({ + "file": str(md_file.relative_to(wiki_root)), + "name": md_file.stem, + "category": md_file.parent.name, + "matches": matches[:5] # max 5 match per file + }) + + if len(results) >= limit: + break + + except Exception: + continue + + return results + + +def search_by_name(query: str) -> List[Dict]: + """Cerca per nome file/titolo.""" + results = [] + wiki_root = config.WIKI_ROOT + query_lower = query.lower() + + for md_file in wiki_root.glob("wiki/**/*.md"): + if md_file.name in ["index.md", "log.md"]: + continue + + name_lower = md_file.stem.lower() + if query_lower in name_lower: + results.append({ + "file": str(md_file.relative_to(wiki_root)), + "name": md_file.stem, + "category": md_file.parent.name + }) + + return results + + +def get_recent_pages(limit: int = 10) -> List[Dict]: + """Returns the most recently modified pages.""" + results = [] + wiki_root = config.WIKI_ROOT + + files_with_mtime = [] + for md_file in wiki_root.glob("wiki/**/*.md"): + if md_file.name in ["index.md", "log.md"]: + continue + mtime = md_file.stat().st_mtime + files_with_mtime.append((mtime, md_file)) + + files_with_mtime.sort(reverse=True) + + for mtime, md_file in files_with_mtime[:limit]: + results.append({ + "file": str(md_file.relative_to(wiki_root)), + "name": md_file.stem, + "category": md_file.parent.name, + "mtime": mtime + }) + + return results \ No newline at end of file diff --git a/backend/wiki_ops.py b/backend/wiki_ops.py new file mode 100644 index 0000000..4287e57 --- /dev/null +++ b/backend/wiki_ops.py @@ -0,0 +1,147 @@ +""" +Operazioni sui file della wiki +""" + +import re +from pathlib import Path +from typing import List, Dict, Optional +import frontmatter + + +def list_wiki_pages(wiki_root: Path) -> Dict[str, List[Dict]]: + """Lista tutte le pagine wiki organizzate per categoria.""" + categories = { + "entities": [], + "concepts": [], + "sources": [], + "comparisons": [], + "topics": [], + } + + for category in categories.keys(): + category_dir = wiki_root / "wiki" / category + if category_dir.exists(): + for md_file in category_dir.glob("*.md"): + categories[category].append({ + "name": md_file.stem, + "file": str(md_file), + "category": category + }) + + # Anche le pagine root + for md_file in (wiki_root / "wiki").glob("*.md"): + if md_file.name not in ["index.md", "log.md", "overview.md"]: + categories["topics"].append({ + "name": md_file.stem, + "file": str(md_file), + "category": "topics" + }) + + return categories + + +def read_page(file_path: Path) -> Dict: + """Legge una pagina wiki e restituisce contenuto + metadata.""" + if not file_path.exists(): + return {"error": "File not found"} + + try: + post = frontmatter.parse(file_path) + content = post.content + metadata = post.metadata + except Exception: + # Fallback: leggi solo contenuto + content = file_path.read_text(encoding="utf-8") + metadata = {} + + return { + "content": content, + "metadata": metadata, + "file": str(file_path) + } + + +def write_page(file_path: Path, content: str, metadata: Optional[Dict] = None): + """Scrive una pagina wiki con frontmatter.""" + # Assicura che la directory esista + file_path.parent.mkdir(parents=True, exist_ok=True) + + if metadata: + # Usa frontmatter per scrivere con metadata + post = frontmatter.Post(content, **metadata) + with open(file_path, "w", encoding="utf-8") as f: + frontmatter.dump(post, f) + else: + file_path.write_text(content, encoding="utf-8") + + +def update_index(wiki_root: Path, new_pages: List[str], category: str): + """Aggiunge pagine all'indice.""" + index_file = wiki_root / "wiki" / "index.md" + + # Leggi indice esistente + if index_file.exists(): + content = index_file.read_text(encoding="utf-8") + else: + content = "# Wiki Index\n\n> Master catalog of all wiki pages.\n\n" + + # Aggiungi nuovo entries + for page in new_pages: + # Check se gia presente + link = f"[[{page}]]" + if link not in content: + section = f"## {category.capitalize()}" + if section not in content: + content += f"\n{section}\n" + content += f"- [[{page}]]\n" + + index_file.write_text(content, encoding="utf-8") + + +def append_log(wiki_root: Path, source_file: str, created: List[str], updated: List[str]): + """Appende un entry al log.""" + log_file = wiki_root / "wiki" / "log.md" + from datetime import datetime + now = datetime.now().strftime("%Y-%m-%d %H:%M") + + entry = f"\n## {now} — {Path(source_file).name}\n" + if created: + entry += f"- Created: {', '.join(created)}\n" + if updated: + entry += f"- Updated: {', '.join(updated)}\n" + + if log_file.exists(): + content = log_file.read_text(encoding="utf-8") + # Insert dopo l'header + lines = content.split("\n") + insert_idx = 2 # dopo # Wiki Log e --- + lines.insert(insert_idx, entry) + content = "\n".join(lines) + else: + content = f"# Wiki Log\n\n> Append-only record.\n{entry}\n" + + log_file.write_text(content, encoding="utf-8") + + +def extract_wikilinks(content: str) -> List[str]: + """Estrae tutti i [[wikilinks]] dal contenuto.""" + pattern = r'\[\[([^\]]+)\]\]' + return re.findall(pattern, content) + + +def get_all_wikilinks(wiki_root: Path) -> Dict[str, List[str]]: + """Trova tutti i wikilinks in tutte le pagine.""" + links = {} + + for md_file in wiki_root.glob("wiki/**/*.md"): + if md_file.name == "index.md": + continue + try: + content = md_file.read_text(encoding="utf-8") + page_links = extract_wikilinks(content) + if page_links: + links[str(md_file.relative_to(wiki_root))] = page_links + except Exception: + continue + + return links \ No newline at end of file diff --git a/config.py b/config.py new file mode 100644 index 0000000..dbe3597 --- /dev/null +++ b/config.py @@ -0,0 +1,22 @@ +""" +Configurazione per llm-wiki-web +""" + +import os +from pathlib import Path + +# Path alla directory della wiki +WIKI_ROOT = Path("/home/codex/llmwiki") + +# URL del server Ollama +OLLAMA_URL = "http://192.168.0.50:11434" + +# Modello LLM da usare +OLLAMA_MODEL = "gemma2:9b" + +# Porta per il server web +WEB_PORT = 8086 + +# Directory per i file statici e template +STATIC_DIR = Path(__file__).parent / "static" +TEMPLATE_DIR = Path(__file__).parent / "templates" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3ec3b7b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.109.2 +uvicorn[standard]==0.27.1 +httpx==0.27.0 +python-multipart==0.0.9 +python-frontmatter==1.1.0 +sse-starlette==2.0.0 \ No newline at end of file diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..f7a2024 --- /dev/null +++ b/run.sh @@ -0,0 +1,10 @@ +#!/bin/bash +cd "$(dirname "$0")" + +echo "Attivazione venv..." +source .venv/bin/activate + +echo "Avvio LLM Wiki Web..." +echo "Apri http://192.168.0.183:8086 nel browser" + +python -m uvicorn backend.main:app --host 0.0.0.0 --port 8086 --reload \ No newline at end of file diff --git a/static/app.js b/static/app.js new file mode 100644 index 0000000..470da2d --- /dev/null +++ b/static/app.js @@ -0,0 +1,236 @@ +const API_BASE = '/api'; +let currentPage = null; +let chatHistory = []; + +document.addEventListener('DOMContentLoaded', () => { + initTabs(); + loadWikiList(); + loadRawList(); +}); + +function initTabs() { + document.querySelectorAll('.tab-btn').forEach(btn => { + btn.addEventListener('click', () => { + document.querySelectorAll('.tab-btn').forEach(b => b.classList.remove('tab-active')); + btn.classList.add('tab-active'); + document.querySelectorAll('.tab-panel').forEach(p => p.classList.add('hidden')); + document.getElementById(`tab-${btn.dataset.tab}`).classList.remove('hidden'); + }); + }); +} + +async function loadWikiList() { + try { + const res = await fetch(`${API_BASE}/wiki/list`); + const data = await res.json(); + const list = document.getElementById('page-list'); + list.innerHTML = ''; + + const categories = ['entities', 'concepts', 'sources', 'topics', 'comparisons']; + categories.forEach(cat => { + if (data[cat] && data[cat].length > 0) { + const catDiv = document.createElement('div'); + catDiv.innerHTML = `
${cat}
`; + data[cat].forEach(page => { + const a = document.createElement('a'); + a.href = '#'; + a.textContent = page.name; + a.className = 'block px-2 py-1 rounded hover:bg-gray-700 text-blue-400'; + a.onclick = () => loadPage(page.file); + catDiv.appendChild(a); + }); + list.appendChild(catDiv); + } + }); + } catch (e) { + console.error('Error loading wiki list:', e); + } +} + +async function loadPage(file) { + try { + const res = await fetch(`${API_BASE}/wiki/page/${file}`); + const data = await res.json(); + currentPage = data; + const container = document.getElementById('page-content'); + + let html = `

${data.file.split('/').pop().replace('.md', '')}

`; + + if (data.metadata && Object.keys(data.metadata).length > 0) { + html += `
Tags: ${Object.values(data.metadata).join(', ')}
`; + } + + let content = data.content || ''; + content = content.replace(/\[\[([^\]]+)\]\]/g, '[[$1]]'); + html += `
${marked.parse(content)}
`; + + container.innerHTML = html; + } catch (e) { + document.getElementById('page-content').innerHTML = `

Errore: ${e.message}

`; + } +} + +function navigateTo(name) { + const normalized = name.toLowerCase().replace(/\s+/g, '-'); + loadPage(normalized); +} + +async function loadRawList() { + try { + const res = await fetch(`${API_BASE}/raw/list`); + const data = await res.json(); + const list = document.getElementById('raw-list'); + list.innerHTML = ''; + + Object.entries(data).forEach(([cat, files]) => { + if (files.length > 0) { + const div = document.createElement('div'); + div.innerHTML = `
${cat}
`; + files.forEach(f => { + const label = document.createElement('label'); + label.className = 'flex items-center gap-2 px-2 py-1 hover:bg-gray-700 rounded cursor-pointer'; + label.innerHTML = ` ${f}`; + div.appendChild(label); + }); + list.appendChild(div); + } + }); + } catch (e) { + console.error('Error loading raw list:', e); + } +} + +async function runIngest() { + const selected = document.querySelector('input[name="source"]:checked'); + if (!selected) { + alert('Seleziona un file da ingestire'); + return; + } + + const log = document.getElementById('ingest-log'); + log.innerHTML = 'Avvio ingest...\n'; + + try { + const eventSource = new EventSource(`${API_BASE}/ingest/stream/${selected.value}`); + + eventSource.onmessage = (e) => { + log.innerHTML += e.data + '\n'; + log.scrollTop = log.scrollHeight; + }; + + eventSource.onerror = () => { + eventSource.close(); + log.innerHTML += 'Completato!\n'; + loadWikiList(); + }; + } catch (e) { + log.innerHTML += `Errore: ${e.message}\n`; + } +} + +async function doSearch() { + const query = document.getElementById('search-input').value.trim(); + if (!query) return; + + const results = document.getElementById('search-results'); + results.innerHTML = 'Ricerca in corso...'; + + try { + const res = await fetch(`${API_BASE}/search?q=${encodeURIComponent(query)}`); + const data = await res.json(); + + results.innerHTML = ''; + + if (data.by_name && data.by_name.length > 0) { + const section = document.createElement('div'); + section.innerHTML = `

Per nome

`; + data.by_name.forEach(r => { + section.innerHTML += `
${r.name} (${r.category})
`; + }); + results.appendChild(section); + } + + if (data.by_content && data.by_content.length > 0) { + const section = document.createElement('div'); + section.innerHTML = `

Nel contenuto

`; + data.by_content.forEach(r => { + const matchesHtml = r.matches.map(m => `
Linea ${m.line}: ${m.text}
`).join(''); + section.innerHTML += `
${r.name}${matchesHtml}
`; + }); + results.appendChild(section); + } + + if ((!data.by_name || data.by_name.length === 0) && (!data.by_content || data.by_content.length === 0)) { + results.innerHTML = 'Nessun risultato'; + } + } catch (e) { + results.innerHTML = `Errore: ${e.message}`; + } +} + +async function sendChat() { + const input = document.getElementById('chat-input'); + const message = input.value.trim(); + if (!message) return; + + const messages = document.getElementById('chat-messages'); + messages.innerHTML += `
Tu: ${escapeHtml(message)}
`; + input.value = ''; + + const responseDiv = document.createElement('div'); + responseDiv.className = 'mb-2 p-2 bg-gray-800 rounded'; + responseDiv.innerHTML = 'Wiki: '; + messages.appendChild(responseDiv); + messages.scrollTop = messages.scrollHeight; + + try { + const res = await fetch(`${API_BASE}/chat`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ message, history: chatHistory }) + }); + + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let response = ''; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + const chunk = decoder.decode(value); + response += chunk; + responseDiv.innerHTML = `Wiki: ${formatResponse(response)}`; + } + + chatHistory.push({ role: 'user', content: message }); + chatHistory.push({ role: 'assistant', content: response }); + + } catch (e) { + responseDiv.innerHTML = `Errore: ${e.message}`; + } +} + +function formatResponse(text) { + text = escapeHtml(text); + text = text.replace(/\[\[([^\]]+)\]\]/g, '[[$1]]'); + return text + ''; +} + +function escapeHtml(text) { + const div = document.createElement('div'); + div.textContent = text; + return div.innerHTML; +} + +const marked = { + parse: function(text) { + text = text.replace(/^### (.+)$/gm, '

$1

'); + text = text.replace(/^## (.+)$/gm, '

$1

'); + text = text.replace(/^# (.+)$/gm, '

$1

'); + text = text.replace(/\*\*(.+?)\*\*/g, '$1'); + text = text.replace(/\*(.+?)\*/g, '$1'); + text = text.replace(/`(.+?)`/g, '$1'); + text = text.replace(/\n/g, '
'); + return text; + } +}; \ No newline at end of file diff --git a/static/index.html b/static/index.html new file mode 100644 index 0000000..68b07f8 --- /dev/null +++ b/static/index.html @@ -0,0 +1,85 @@ + + + + + + LLM Wiki + + + + + +
+
+

📚 LLM Wiki

+

Knowledge base che si costruisce da sola

+
+ + + +
+ +
+
+
+

Pagine

+
+
+
+
+

Clicca su una pagina per leggere

+
+
+
+
+ + + + + + + + + +
+
+ + + + \ No newline at end of file