From c2884018782875c35571b32e2f9bdf32c8c3342d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sat, 23 May 2026 13:25:46 +0100 Subject: [PATCH] Retire local Aetheria RAG --- .gitignore | 1 + AGENTS.md | 10 +- requirements-rag.txt | 2 - scripts/rag/README.md | 64 -- scripts/rag/__pycache__/rag.cpython-312.pyc | Bin 33499 -> 0 bytes scripts/rag/rag.ps1 | 40 -- scripts/rag/rag.py | 659 -------------------- 7 files changed, 5 insertions(+), 771 deletions(-) delete mode 100644 requirements-rag.txt delete mode 100644 scripts/rag/README.md delete mode 100644 scripts/rag/__pycache__/rag.cpython-312.pyc delete mode 100644 scripts/rag/rag.ps1 delete mode 100644 scripts/rag/rag.py diff --git a/.gitignore b/.gitignore index c783b55..69defd1 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ Thumbs.db /.obsidian/plugins/copilot /copilot-conversations /copilot-custom-prompts +# Retired local RAG cache; VoidBot owns canonical lore retrieval. /.rag/ /Aetheria/Inspiration /.tools/ diff --git a/AGENTS.md b/AGENTS.md index 6333d4a..1b8f75e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -82,14 +82,12 @@ This repository is an Obsidian vault for the Aetheria setting. Contributions sho ## Vault Navigation -- Default to the local RAG tool for vault discovery and concept navigation before broad manual searching. -- From the repository root, use `./scripts/rag/rag.ps1 query ""` to locate the most relevant notes and sections quickly. -- Use `./scripts/rag/rag.ps1 stats` to confirm the index exists and covers the current vault. -- If the index is missing, dependencies are not installed, or the vault has changed substantially since the last build, run `./scripts/rag/rag.ps1 install` and then `./scripts/rag/rag.ps1 build`. -- Treat RAG as the default navigation layer, but still open and read the returned notes directly before making substantial edits. +- Default to the global `voidbot` MCP server for vault discovery, concept navigation, and semantic recall. VoidBot owns the canonical retrieval surface for Aetheria lore. +- Prefer `search_sources` for lore/source discovery and `get_source_context` for adjacent passages before broad manual searching. +- Do not rebuild or maintain a separate local semantic index for this vault. A previous local `scripts/rag` Qdrant/FastEmbed toolchain was retired because it duplicated VoidBot's authority and created split-brain freshness risk. +- Still open and read returned notes directly before making substantial edits. Retrieval locates context; it does not replace source inspection. - Prefer `rg` or direct file reads when you need exact filename matching, exact string matching, or repository-wide regex searches that semantic retrieval may miss. - In Windows PowerShell 5.1, always read and write vault Markdown with explicit UTF-8 handling. Do not rely on bare `Get-Content` or `Set-Content` defaults for note files, especially when notes contain non-ASCII characters such as `Framgång`. -- Run `build` and `query` one at a time; local Qdrant locks the on-disk store while a command is active. ## Editing Guidance For Future Agents diff --git a/requirements-rag.txt b/requirements-rag.txt deleted file mode 100644 index 5f555aa..0000000 --- a/requirements-rag.txt +++ /dev/null @@ -1,2 +0,0 @@ -qdrant-client[fastembed]>=1.14.2,<2 -pywin32>=306 diff --git a/scripts/rag/README.md b/scripts/rag/README.md deleted file mode 100644 index 59e4bbb..0000000 --- a/scripts/rag/README.md +++ /dev/null @@ -1,64 +0,0 @@ -# Local RAG - -This folder adds a lightweight local retrieval layer for the Aetheria Obsidian vault using: - -- `Qdrant` in local on-disk mode -- `FastEmbed` for local CPU embeddings -- Markdown-aware chunking tuned for heading-structured lore notes - -## What It Indexes - -By default the tool indexes `Aetheria/**/*.md` and stores local artifacts under `.rag/`. - -Stored metadata includes: - -- note path and title -- heading path -- era / category / note type -- extracted wiki links -- cleaned text and raw chunk Markdown - -## One-Time Install - -From the repository root in PowerShell: - -```powershell -./scripts/rag/rag.ps1 install -``` - -This installs `qdrant-client[fastembed]`. The first build also downloads the embedding model to `.rag/model-cache/`. -Python packages are installed into `.rag/pydeps/` so the setup stays local to this repository. - -## Build The Index - -```powershell -./scripts/rag/rag.ps1 build -``` - -This writes: - -- `.rag/pydeps/` for local Python dependencies -- `.rag/qdrant/` for the local vector store -- `.rag/chunks.jsonl` for chunk metadata and lexical reranking -- `.rag/manifest.json` for build stats - -## Query The Index - -```powershell -./scripts/rag/rag.ps1 query "Pan-Solar Consortium" -./scripts/rag/rag.ps1 query "legal machinery that prices war" --category Politics -./scripts/rag/rag.ps1 query "upload continuity scandal" --json -``` - -## Inspect Stats - -```powershell -./scripts/rag/rag.ps1 stats -``` - -## Notes - -- The query path uses hybrid scoring: vector similarity plus exact/keyword matching. -- Short notes stay whole; larger notes are split by headings and then by paragraph groups when needed. -- The default embedding model is `BAAI/bge-small-en-v1.5`, which is a strong CPU-friendly baseline for a corpus this size. -- Local Qdrant locks the on-disk store while a command is running, so run `build`/`query` commands one at a time. diff --git a/scripts/rag/__pycache__/rag.cpython-312.pyc b/scripts/rag/__pycache__/rag.cpython-312.pyc deleted file mode 100644 index bd6d58f91b8c2bc95ead0934848988006f42d530..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 33499 zcmcJ&3v?UTnI>2T6o@xKfbTc>6dwXe>MiOml9H$gDeGxbl4*(sQKa~C0hB}-Oc*;m zrV=Hh+UXW`;*9A|I-yTAYx=BrTG`H-mNV0x%tF6{onWEf45pq9Ik)i{AA$2oaMN`qX*^E$q{~KNXv0| zIgu0jA#RxO=6Q;1hBVz8cGq@m*)GAVZNObSWE?hin^+sd zka^hBZ5cLqo7uB&$U1E6wy}7|kbO9_JCo-$oY&@c{G69ZGgz$*FM0oYB05-;OjBZ_ zz5vngY;U$`2$&JeLCnZvx!zpS#A12gyuoIrM)Iz-E9GMh8G{>=IZ6!ah2A`6Y|5SI zj5I^$p`zj9?qc>fmZ6g2((Y0gw<2EFUB+|h-^ThcN6gM*6^LcBSS4Z(7ON7S-PK}N zcMW>e4R$JhC+}jm(sOt16U*eNW$K<-CRZ&}e}xnCe$00_cugAaASdR3n-dFuu95px z=B0bRxKk|rw$SYscZo%~d&J%19t;_lqvvH;S!d zIqsWqufTn?*d|uuzC~;ot8m{c9uTW>-zFXuYjEE#9uhmmb!fRoJS^5CWruh~tiye$ zcvP&%eHX^kfctLon7AJIJz}Tm#(l5YC3t-i<@viAf6F7<9-nNTW~*w`&QgLaNmaeVezcE9rq*R^I{9`N4>|y9an2P zPO@TtJMni^nI~mdU*RV-m7LdA#YqJJn(ku!mEy1L95=(?GF<0wXwPxi`MPtT(nQzs zadYp;$f&>9KQKDt>%_BB?DhBd4fXnbb(*;L^r_ak0V!Vpz_1s&!tq}J72LJ^MkYR` zZxuJ}^ZKuNrGegn;E{U!;~8TUqIb-PGS*Yx8~*m;OI}eN80klX~gc*aXCJz!?f?!Ml>E8c*m@5=bdRi9_jH##yDunzZ*3|#j5{Omby?M-yr zGc+oB1G)S6?dxc~)bDlshI@yG+};uQwPw%efb+un3+HRkzku7tiw$)bE?&48$a$f* z^5y3BTPEu+_!??G4SR5}uL@+J?{x>7+}kfE?ibwaE;j5gqBrDwAS`9T5>o-njNMlC)PwugA3meqUURxr%GAUm1-H zm!;9+xG*q+;)4GQ8b*lbgzGGReZYSuu0!bosgJ&tasf}$#jpG+!n>T8>*htSTXPzi z)2kIVJCVv_S~VuHm=3uaYMNe+8Pr^(nr2dCW;JF}V^$Wkp*_1A%T#k6ES4=g0k6VD zR$We9*GeBUuJ!q)xPg7to&hl~i~*4Ff&=~`?^nj50iXXo3S5j^u6TP1TzXi(zHfBI z4`9LE0G8s$>jPH@h6Vr@aU+1%+vA@Y!*qJ3UbGA#?H`pU;(FisCBNiFI_5TRx;`q2 zJ$<9&h?^6=kBHtIaZ94NRWocy#ocH|VM&B;VgaoQY*zf4I-nKbX5oUfBy zowHMFL~*H!0;0>(Mg(!I0)0chm%Kw?XxT_N2V!e$(}+u{&JsUN%ySiL(~Z zo_@3YUiZ8Zt=%53-5x1vSvFIag|p_)`0obq1edM!Y~!q1iN@{poXJ`9XS3fdy;r*I zpl4_DIg6gNIjeJK)7@=%wk_w-b1rAiOAIxSp7WFK7tnJdXDyhmdb9Cf<8l!_7jxFa z*-g=+hHz2CatWoBrVhW1Qe0@`2f?|vpLM*|5y|r`ms29=Fq&2{OEec-o{`>RZ%Bjw$Zqfk?+)6+>zUL79Pu#~g$E(L;ChMymF|+N}!!c)3L@0_`GhaO_C))EPLVnEd ze6=%Xw7hyK#)`A-%&b?tVvg*HkR5a8M1-6#1gFj<_eLG9A~Jd;FbwEN%WXf%vtCzyI{GB+)6F1)Pp1;LbYv>|0hP%gR6irCCoYk{uTk^UkSB!) zO!qk{#bJp#hH+R0FjhKt0=Blu{YQYgQ zS4GSlqvl;<^R9?_Pt<%cY(DrKL+Hfmi1|!NI3v%f3gl=lVwRs=xK3%PVNs?Sxqap46=3baoxtnCh^=bs;F8`nz4g`L%=jaHam=1OZHt+m zGdpHZ&pvnOT*zD&HP?jAH4$^|{N}K^DI_$>bDKJQG|%XoU#URkE{9o6%|#kr${U~L z-sVX{G4%R+#zuVuH{RwM{gz1}8+zO}LJ$IW(#se~iP*MR@aLlijkhq{Z|}Of>$`hG zLgC}1Q5ioO?HN;6Px|ls`1(=4^hvIg^Q%lAq%mk5l4~)@DdqY!=kN^`sZ8p}DDm}Z z)N!-q9fDYP4P5T1bed93XL^Umz3q}TDoF!KNPTgOIDoI+J0ZXJ2y*`kl^||~$hlTm zwPY`u6{Dq|aH;41M5J_k#NHAzwETX_obx%yC$b~<%14IEWi6h+@)0(CrNqU}WpCB~ zbAGEfJ(GMHrM?c&VD?nFNAbiZ^2#hI_ou--AYIe>m2TXB*}bh!i0g@M9~!s>F6c^c zb36m20m5Y5c*T1|9O#Fj2;d;5Ep?8JREKd-(lum#kFqNf5Txr5PuGF6(kJ;K zpB7>;Ol9&WfiFcZY2GHaN#-MoRYm@48_<{zrFbee0>Nui2nsiNUk=X&wQ1YeCEHK5 zt$XH@89_m^28Ce8RV+Hm5EQ`ZWCSx1>VG5vv+FQrhEC~4j3v$o86AuxRvFN^YU6wz zU|QENjgF52Z4;)JdQp%;L!CaZ1J&!h63_7WO8s7c+%Dfi^Ll{mpqDU*V82!ndgb=XlIajcuph!witTz_xb@Ppp znwAI8J*wF~W1r(6WmY`gKdp;p<~+*WJX;hks1Fy^&u?7dBLy24n!^Q~kylg`Em|Kg zT0ehsp*&KwY2mqW(bkY5|08qe%;`|6=TVL)mRmG)c6NWLtT9yDw7`e5HiRr2V)pE5 z3j=r{(pNFufK~E(j?w%v4TD4snB+kK7T$ce<4DKRj?N=JC)&Z{jbB2VVYpYiDvn+s zk-mdMJ|a=dL^&RtBI*0c{$nZ!5{{W!(HM0(U*vL@yxV(j?wQ>*mme|LhlKjaSD=bN zZn5Q(jyPaFehn^x{(=8_qRiKphI;4^C@uU!rJ=b9@=4hsoqprK{^x`uaDDAa4gbHk zW)w}S?+w)Q6ebpn)Z7^`z6h=1i%=Q97|>j34%jc$U#Le)Jt+|p4VY6N1Ev>bg(CEb z(l_vWU{Q3JhkE;c0o@C`kr>FhF!J)o^^;#|OeXNR1ToSL3T`2Y8;_kndWvAMT_Q%M zP9qUrl4(~e?AiK;yuBkmDsaAzQa{0;kFY->{^Hr^Bj%c^wk5Om_LiGlX4cQ?=KK-! zdNn2FteW%ApNg0_s7W)Px$61sh}n}&anCvDPQB;+^Wt}l7tOnq>9w=_=Y;$F-#PmB z(M9w2WP07~nK`ssKh;LeRm;tm@9qob=s8I-ojL6fL0*tRI^%Te;;^E_O?FRv5}@lc7<7^92r_{l9E4Z$LQ^gEUYEX2bxaBVIp7t2grSH? zsMrr`Cketrl0Zm5LXmgy=Ods75a#5+ej0#KPz2$?UKq7kgzXh`+dj0fpVmV6aJS%2 z!R*F4KH{jDYX;SbIvT=`hWXY7N5rvlVPDv>CFrf|_v95dK&+iu!su6uha#Of+)54j>N__gRh`B2ybUpt01k}L{^DAEhmj4b>_I?_$WmGfj;Q0PI>#TjEoMA_D{qOt)nA-UTA~k`jcZ`(xt@>2e6{a_7^vt zf{72t6>mJF{TeC8K$=Fc$vU!l#&NDC|TR&4_# z=(T6q>+dB~rSuml`YZe;a1hYL?arH>Q|&QR)>P|nb2d#IW9IBx-6LQQXVG-)M;6Dj zp3B_~{d8u*)S(rPB{yT*2oK!PPatV}INJ%S{ipf3f{hZQG3D zuKkWZnpyQIv+6fBQyoh-(A_tC?)5}$?x}-N%FY;O>*mge%F4xa}$A~`z=pT=}UKFPF(}ro|Wi2q11%l2~)gl1oy9DZALpil7P=Ss~!x~aR zVbiLQ(DQsk9P2Bh+0wMcu{FWIBqOq4bB5 z^64n1@+RNW-+Jw)AuZKaM+6jmPk_ru>igd`G78(5d;?kU$K+XJ)BafdB+cJVl|-Z? zAj-fdL13hcn`x!an|wOvM5!6%E~`E8pbSusYoPHnC6s)OX_RI$#JJEmIyQmiP4r;z z8<%MP^&~7>ajTL*OB|~ms#mt|eS}wCQ&Us2EC|kZ+;=5V!c>PnN}(hTVcCC=Dn2rL zfqKN8xp(XC)Xg4?IP0U%rm(Xq;@mK;|H$m5mDqK&Yqlp^(i|>nj+AU%I2$V7^?_wK zl-HIlwm4*Vh9Kx|4U^2`Lg(m+7i^7*jX)l(=}`N=wvNt&kkXh)$HZdkTd0+MeTj%# zok`|zR68C_0ex4ce~GNPz;K(IBTLk}A@4QsP}1s5rZwpyHT*sW46lHYaKr`S+Pime zLeIE*ex!egGJlId->(sTg;`nenK~3JE1xR(b$ftz90k$L%5Y}ohndyW z!cu-kG`}{SUkf%o-!pCf$ea)4bL(KNqM`Oix^9}Ru3;W*g2o-OcJ{GeSL@lnc#Wh#FXz|2K z8=|F6;nJo^>4xbp#zy5t4aH$Y@$9J&3>C|HoTKKmLQZG;Ht<;1RDuMj5<(Swbt5&E zgG%z{31zkeIE&b!HJ1X^{rpKTNlw!d%2YjB8+TjNr|IYVG#4v@>7e*|THB17dTNO~ znp@QZFeX~k`krB=z{>*upb7%6h2%yIMmnqqLapiNNh7D=tEY_<#9b%tO_BRmEQdJ+ ziz;nMFKIlf?`P>#DMOM+KR>CvbtR}vhoO^dG+@gBBoLkC`5!tOus3~R&-yfzn(L%l z4CpV6KyE9HrD1C@h{}29r7M`)Q zdS@uBGNP}VH$Kug$1+{f%$jg!O(e56WUmVu>OR@L_0OMw_xT5dp~Kyao6koaFNHUs zpErhju0(rA!aXA&^o-5a-EF+n_<^%BwE6th;gI8{kIXrbH}r9S;naZ`1;5QH1ACKI zaCgU@9kbWw3+Bz?vW=0fP0_5Da8^qsYv;7#BeUZNSLbRYxefCJk8-!hEY91w0oeLkp}G$&Rm)Z`*Y#N@XRyT#w%gX5 z)|tz9uim*jR};yti{>_la~mVM&5MQ&ADbOh$G-f$l5`O%j zsv>}7M&jcEh~UyC7esl?GNuyuYm!zbc{0_A(m06lJ17}l{XAJg2ww9Kex#vS5iS=CNW2uSbX|UX$+at zc?6_~XC^&3fWWv%CQsj|OUMXlT3YIC>HMnxaF43G%(zj4yP>=sU!lH-C^(G(Dl)i! zFyWFJ3z@2mv3W8nBHS;t-A3=oID}ggU1>!#DVO=~)aAyFq_>e}-9Mwe?@{mz3K$aE zMzQS_sPb;Yv5_p5WXy(-qiT*}+-TecU7d_Yd@>UH8`K9oKYod!7AZoHI+zZh=m^;>BK8AO z`_ZudXvE$*b?75=R?P00HhxT{s>1fFpPY->8^I1+a%9PTsdmf4jY#d@uuwhKI-NhW zeYSrovtYJ1l359XI;UptMkJ?cs*?=XS75`OI`pxpY3lHj-Z`@^qA#A)KGIjl@(O0R z&5VR{s&AV9&a)+Ksh>U?Dy&~BZVWZ;{jhl7j3Jg;7|kpXXO_?HT*wbqw0w}c<1;N+ zvY&_W(YW<9182!iHZpDcxUhKYz*7Cjg~K1#?+F`frgdR^&5Q*jvzN{3=7I}nBlcZj z*9&Xs^CN{DLIoQaHY|7-&OX@x;QE7sP~M?w%TiYH?1e~H{j?!LDVHpcSV8$xPUYO8 zNRAuUs#r-~EWeCg>V9aCm;>IlZSPA1J1c3gAQ1(8H5w~N&f-CDwmSDt#5Q_2ok3Px&Ij@ItidW zhybA2?B~?Z7k!ws861twY5ZVtu3-L1BzM~fhVA%jxmy^IAC2+DN*Ga^1fXXxt&+WGX`4B{og()Z?+tiE_l#Zz4uq#B3GqJZ z|3I(gSO{M@=5q#%@Ktd@id&9$weCCGbNpOe`|*?U%wt_`?MHiB_q85s?>WBj)S+I=uyskWM}?fQkTWaJp9%>%9|#*BhgzB& z2DQtGq>7RXtAW`DDVDEvrXXTkMD2Y+A&Q$x%MLaDMc`8g<+Lu=TuHraI>$tal`6}T ztf-7f)PZ; zqaV?>xPHWYowO7O00x$X7h$ijZ(snviS%LO=GH@}JCB@{zpXF|lTKWB9r{>rT%dgt zah*6mJmzDrq*#!Q6l7e1W)=R5I)=_yugXkcFWie{JkNGrNIX9M1^WDF(H(vMyhWj4 z$(S8A7KDujvz7Du2iYNG!J={Z!?k>*yWK!T*IqldnocyV?qUlV%uRId?LU{69w(oC0< zrb-=9TJmq`Co}w{9AZ~G*hw<0=^W~V8L4cyDBRbj!cIaQ79b91bduJbG5g1GhAzRr#V?I&d%g8dW(ZZgM2vaj@C5db`s$R9Qo*b&*dm>0AJzz0V{gM->p z*~jy^h;3!SsN_oy)FS^P-zYi&BwCzrABbdaeEY!M=9%`phhINDe;}N-F=E*?)&5C# z(d!+tTvyCdw3JslcO;V6w3J&gTmAarQeNTgme*f~zshHJLuSU*p)Z`A&KVVo!$R>J zr+o9; zV&GZ~M`?*2X1dP6?#@vc!!9m{U0j#34M$||+pZVWJ8v@~`9;@dkRrb)&Py_mG-JN& z4ANJ41!6it9(~dYJV^f+1+P#*Y9NUKlFZ({qOVR%p2ILs!bk4^rM!QQU^VtLQ$d9x z$wL<}5x50Vj!cueQF95b&5PzTIA@0n&P8n9Awzepss^U*RD8&=Vbj8a-)0w1x5cvZ z?zY@%d41<}Til)%%gnoLyJLI9|7PG`V7@9+*7!kY(`R}vr-;yDGgQH3sQyQDGX9m1 z1leEY@7HSIch(^Ml~%Xks_#p{0P;9ag~B)-F3qGnBUFUps_zsj2~ ztxA)yt~Q~3MbDGuM5KjyseuyJ$ndNxfXeupscErEh~ein(5H+N^|NI>t*ZXC*A*oo zVLfdpOagG5g^LV>9^y{|(B%1iU{+dO&~Z8(Id9PFPym?PyD zbfnk!NjkJ9`#)z~12YPlk_ZEneq@=)&T99FqkRzRi0{G{TJRMBiyjnI>7k`yuKSl8 z7x#2Ub{t={JO^82ed9X~Z#RU`oR4g|fbcisgMx>f9@aeC+7UW)KD_0^V*SO4`Gs)( z#kq{>1EgE~URS98V%YpbNOA>ey->2oje=SFZ1@T z6IRhA0${9me&xb5z=g8*fFV?lC~5RfZre%04|q+wpq|lmGN*G)99wt9kIH*x*y1Cn zjI{!WRo>Uz)5kXaN>yl!oE;VK51F@=Lv!96Cx22DDRD2FJt1r)K7{60c^4B5vG=@! zk*C5xXk6+#K{2|NH%KQrD6_!K&1vYalJB!6IX{@A*osiveT7d^U#hEw(cRQqiFK4x zOI-^}FOypAGe{~m({8;CYEp2k!uw(!uDyU$72Y>7C*XZ^lv3bXwY1`t(ZB$uO{WUP zjOHtKl6R$+RD@cpVltWk^jhR%FbMs_+ADpnq%v~QwYCP_<1+lEO6p*R5=-6(tCDdg z201Y?BdJ`rf1Q%=uTx@>4B*TPXH`HOU83Q>QK^fspua8LHuS+92q^@B&s4Q~779>V zm&tWFZm?d7CGSvQoyil@URj~-h(F5cHgdiXJr)7Fo0KMW>j!cEC#3~7`hHl9Z>XCSWtE$ zC0nS=>SKbG*9rwn8>Cf@}{-z&LfdaQvmeIVe) zK@woedf_ex&6YhW#4;%{Zc9*J?A0H+JkU>?ungJ=(dU<5#CyLDr)pG|{9x=g1ENIK zTGn%Dq*}z`Z-!mf@H^8$ab~2dz5a2Zq@`jck;*jw&(M-YngvNub<7C@#m*YYltosV z*Q69CFiS=htgB&eY_gaQvBEbxbj>SeQ#(uyA#EljZ;W3|vYE03?MQjmgz&;-WCtX7 zFJ$RtcHGeid&H_Fc_*sn3}aW1!;C!)A-XLwkMWGwu9M_B{PTW?}_6H3QNCh2QO&I)L5py1XT4`CQe{>ffq=|60V; z`Tn&B-udx&0zVHtycYI!Mx0&KdeSkhn=8Lp|N7{(8LH{1xiW07#E$Wv^kZ(Xtt8?Wn;OHn?KNt~a~xbuAT@%YHlyNB(Wi-!%T~#^~V}!-rp7 zEb3j>amMm5IIYq2g^g?2^j_qbIs6qSZ94ajv5R=`0=1;xT^LEd} zGa*CqqT%SrP`X=gT5j1s5wd5_EDFU-f@7v(_S&5PXWxA5n-7G)H2uP~D6}qTa9Mdv zvc{$G?&zJ-m?P(I;hn-1&C9!6B84sQZkey1-TY?D4_g+tgkhl0-H}knEag@IjcG zJNMk8Y2Bhw8#7zJ{2VaR*u{VOMK)gQ_c&6(Xs6+S&-QO@DZwezc%}DA@}~y ztiv|buk8Cb;Nh?Mjfb-gf3>L`@!#mO4mUdgjm?7izbSMcUjKwGj<2ApuT%Gcw5cpL ziYG3G@F@ry3EjVDia9Y5Q3qV1%*!Ph@}mA3t|n{Tz^!@v@;M^@^Ojca9g{siw4Cic z&%DKdYTYTkC+vKJ;QAXZ-Pf#7+8C(JR60|MlDBI6bXuoLG779EicDC=JFdOdR#V+V z(zgnw$78KxpGP`(6}8ojq=*HRn|^KYU@TL1pD6J0)E4zmXb0W}%IE-Jh6H?pMF&Hw z!er3!EPVj>N*LQ|NQyp56XK*v%$PI>HIk9s(y_-y7c`6d070^ns!yaER;3yfsm4{Q z8HrR=BGm-tCRm;Cz{f~!-7M!Ng{^7pWnQ#=+YmH9z9WN;$C_v@Bb`G2rd64RNelck z_*>h9<`m(}BHEOh#9?WeJGre}M`8Rlt7{R5Xn`@3E%@KSca?PrjCgLrBT+ea^~<;X zy@^Ps63JE~oD%u{o>jswi5_6g`IQ<({+L&{lE64J$=`%fd5ueK7BQ{bEW!kEV1Tz! zEpAX82c(0v!l_M{OZM`W{uB=Z*ATc7?C@ zo}V6{H7wd)?-qQ?T!&w^Ir#pK>{{9LDZyLfbwhigvL);!D`-he5Z7Y&DnmMSLe>Oq zQtVtN?eM@zoh`1DwJXFtfHNZn5Ki~Pt^+pHaWgwADv?6+N!&zAJvo8g`utZCaRCO3 zVfnD0L)4azy2?=&$13$L8Tz_&9>?B+Y>h@W^?D95j)PH zsK!1inR32f4Ce+%_RQ(K=kA;Xy_F(4^$|xy)PX%x&Da?=bvV|z>79wUCms|%ytLTZ z5y?CpZtR%z{~&AT)awOz%kGqY;HU^Sc1(4IG7m3Vb7s86#C&YAPqlvppNW^}u7>lP zV+QL?)=kT_PEHyO=QXWK8VKh#u1Q)|>hBzR*p-Xjw@1R}vY5p&bMofV>2|zWgx~_f zqIrj!3~Nj@wj$ zex6n%WezQAM%acH)1*JG#jk}6m}CEtce=*YH=fYbrv7a-YeWkUq_}etMK}f1D9a@D zCv^dV$S9GN8_ek|Ze^H@Tt|CxauGZMnDDfR@D0Em_(>LrtgxTL5z7bvX6$5_UO@hT zq%xnN3{aBZ{MwCIZv3EiR`;gyo)MdRopgppwmX`=F`T_IlD+vs_Fv}zBL6Q-e^DCQ z-tj2A1Ky@oY4VlHH?nX_*SUM=<^l^FA|+cQ1zV#9yTS##A_aRMZu<4E_jg5G&xBjg zL|UJJRPcPF+BaVL#v7Hh-ZuyC4a}PtPDDz#MGCh^3-^Qz_e2W!Jv{a67v6s%+I}wF zelF5}{!!t1xmr|T@JL@khmg=NglOrOaOoCQ{a4zD?Z57LzvDNTLua0kw4ICW?T+q! zF}(N1$lgog(o0m=Y=2xMOnOa*4)(&ZovIO5mc=A*BqTwEZ_Nc}U09QZM+K051xcMa zZ9_;`7H*cBq~|cS!5D*lDZPUg!={A+(i0NdR!^%4YF^je0@1K&*}03brdhK?|H_j%YU`^Mpy{rPCBcugs^@5Gymo z<4DCxeJ}&;-Y_w#K|{JD{cuhPG+8KVOlnnwIJ8h1H<&kGCUwK8RjZUu-j!OSUKx?77Y%P{q2*=gXWkkG)2JP6NY+qd z$~@rxVM$@SV-sLO)g6<@RcRA%ndMhC#_8b!np?sYxWI;+MI&===5I9y^=Y-nCY*?( zm&dBsvn1a;Xc0|hB?zioWJ)3V@YeHqBfZb#SG9;q?eX9y>VNf>ezg5KwfGHS;Ryx)WyM6VOnL!&%muHoo#>~y*^QNS~0$Mw`roZ(O?drMi_rAzwWa`U&~DNzwHQ zCDfCzpOBxgeETQl7ko{A;n$9!NNEAk1rw%!pE9p0*eCt$6pJOwyen-e_a}_DRLK|1 z-cYv&@N{zXMyx&iV5`!1@{Y5v;rM^QT zlr7c{9#m>1?@GQh7OcFw$(-c72D8v_Jxb+JD~LBBzMkTu`-g()`H{LQ4dix`Fsr}R%||X8#@D*R@p}zeub_V11&GEvGzQ>-xk|n@F{T!r?4HFtTWO{3PhyD zEp)OqwnVTKXr&`ahr)u%{5GE#-hVi>KvqaFxm!Nzgd}iq#?%OTb@|4J;F?1m7qiYh zS#_c+PEeMVts+PBm{6UVF;_gDvw@Z`q2*4dJ%P{BTQp1^xkyf=vf#4Pq*PW+FtbD& zl}QySv5z{DH72>o!#lecHqVTQO57jhdGPS%3eC$ab|_77$F=+v)z34JpC(8izvM;q z4<0s|RebpvjXdm}&Tz}I1t=0`yFSQWkNB5TKbmFJ#J<(191Apdrh{ss75w4RNZd z5jv^kP-wxxYC-g*yj34h4m`C>y^XDj!_E^0a1sc$2B)|x*$lafI%H-~W}hVIM&`97 zn+39JrFzZGJopv>fQ(0-kvGWMZ zHy>NO*uwAii{8t@#UYb(oTI0u3{GvQ3$Fxf03B(q0M}8nH49uHx7kx5#fJ+aV=#@l5G7VG}j=DW{M4 zaC`|FQ012;rwbWaFH=1W?I>0Tj1z3}lo+iUb6c{pcJ8jf3(S7A@)%i}(;m_TiR zPj7+>%B6 zwk2orY;eBqovydLLJd1ZJFkUGu1B0VLY5n!>^c0)$w>aOUra7|XLDu)bOv90xS;vL zk}X&is}QcDHOF)tWHZH`oITIh?Ew!}(YpXmjcY1%Zi?UA7vy*kU6bGhO=+2;CT zQPWNPv|*;2x;Z`9wrH=TZZdI5>YIc22Is3InN703bKiGb=&%IZPgWhytVaCfg3?$; zb*#J!9awEEI=#a*t@|RAtEiqkJ-;Vhz2m{@NO|kDZhG_)4y7vLikm-kVZYgu!Fo%) zJ$Q3ap4JBy(6T)2U9=rqG(aDd`AJr8IQRI2!_l3`!aI*G?(B->9*<-lf0T7Lbn5KX z(Isb2IH&VLeYE9BxaG)V%dtof?9iQ$oTqVk@zjxz$uk)q@KHxa*ijL6R6*6dzjIcH zqY}gUjSK4^Y}~H^)jUm}2T>937&@7xqU=w#JHFZ*IT0eeN8#%RMSO|K83t zI;6Yxva0IOBo+)Ij zoI3b1^kWmROsxHC>%s;5f72W~|Kg&pchLa#ROTmwarz+6uRna}@XXFxDP*cx6e^br zD(1E>lt&9T-LwJ{uEln@J$&KO?o-gvp@X?C^PX_!_Qm{`Q1{hoEBctpnbFTkIJPzK zUSkLgv10zr!ojzD9-IuV-}lfFvb8Q6+NhWjMRM2evadT3Of1U)pR`*dsgX;kQum_=`AX-N%y!uTThgWQ(rBsx?g#m9)z&Kyen_ZWpt zPyZ`o;9eO4#Lrl*;`^Ap6xifxq(4V&OeYG1R+#gn<}z4t7R}|%-}p?#b~a=pr7>E1a{EJ&8Zd!;Y+kr^zI>sdL|%4@MPl*&lBDL*A@YO7CNcM~)xzZkAmE0e& zszutx0zKEZqjajdOa3vs#sj@R`A6#5@67l|M~BFTS#sg0*IddEDY>qZ4C{eQG99$e zeozTYGU*Nwz$qvL1Uh1!jP5KX9U7BpuMA{pY*WL2TWQ=Is5&iEkJW97=;yeIP!0L9xGYF@uhl6s}SD@?y4kH;&@J5YU|Ky@cv!f|%0d#HD$e;k_# zUHHu28<;FNbs_I0V*_EzkWWeP6DaN+$A&_Q-Z-Inbm3Ly-De*7q6_pBbkYy;qHMYY znQpf-EVtYY;h4atL!;M;O`J#$ni=xpFOTLfdtuXeUGf6%yl5Lgl6V!L)Dv)|G$!jq z`Y*HVKkf-Mo@ELX8knrxaQX0)E;yiUz(0XkXJ44999*({7mX~TkZ>p3lD)PfdjeaM zZL#@igm#O5UgGPT)VbXlY+!xY7#$^vHN|SXkcJ(+v>Kdqdco;0f5> zZgRZDYxpbvu4c$AyD=T!&9~ zb+Xk>KOlfMyuHKtz}Sn5&&gQvK+{S57^I6WMwfDit%?s4XLKgkV`4#N;umkEevIDd zjobD~{cLM0wgSL^H*Uk>-aW(}vPYcuOV9E{+(L2GWaZ-)DC6*xR7@M^lbJ$dH{)4t z?FaVZ2>PDZt`o;kpOjA*aV8Uw9&K+u)zQ@%w<>AukP)esKz;{WkVvsAkxGiaU|j}j z#s)@|Q83;$F*+RGi+veGbowtFy+!u%SDq4j$3!0O;J^?4$W@7XgzgiU><&Yx3*-Hz z{~6t0CBC);0r*;HVboa{cGiWqj4e7}!iE4xv3$z-LczmRAxA^Rad_(R$2n!uoceH1 z{rvXOj`0w@?Q*V7b;bl73{Vyp%I3@qM?(jPLqgf2FrudZhO@l#;MeGCTU7Z*fOwWNTOG%w$L?SropJ%o|zUawybwg(VLplLy`_ z3AJBfsTbAMpWr|@)NWkWpP%(Np4wl3X#Gi6@|0S2_7CZ2cHn;BgXU0cFKgtI+6aD9 z<2~QQ{mU>c@NF7oet}=cY}cgZH~&)i(1-jJe5;b*mYU!6-uS~l1F43I1) z$i^Ic#gxiIO0Cqg%rL~-VyV;?42IvM%E?rUJ)XKjPW>|~tM(Q!u^*Bp)5%3t`NitY z0zY;_dFe-B)`Q1M38?JqP6I`(A-MvJqQXU1?c&8ceqb-neunfn)Bzi5h+@gPV52k0 zF5)a7-rP9Qm4ekB$&;JGPRvT~F-yMDS7z3GE1lj(!(9Eo(t9)xfgSY2;_{sT75agO zAz3y;9aCA^Cd5W$qem8FbIdzdIUr8>gz?hYWEHH5X%BfI0Pbb(iAAX+>8&&8H67aFQIW`P*M@B9AeISr?(qXE= zY7u!telnG*%l=m?MEP;y66kYc9DzKw?|rKXtujtFgf9A7&&%UrCh@bL66pn{H>tUo zD8}@3H(0|y*utdmQx2(kAUi0(tPKZrfyy!~s*dt-(3hWn&WwF|?Any-PLKT_v=+Ed8_*GUJF;yX>|I{tc%3za$69^&Su)_=FlQQ7JlKK;u=8oYp z*wfnCZ^{9~%cp3tlU8H}2c*SuSb~8^Jn=l z3P6XHSqZ>xfVWarIhE7^DBm8-o3KBfY1J7TMqAB zY0~jKS28__UE&3VHi6$r!E^kI&CGZ4E5$}s&DZihF!N{dxhqaB-?(B~&s$es;5DdV zD&ZX~dwC dict[str, Any]: - return { - "chunk_id": self.chunk_id, - "path": self.path, - "title": self.title, - "heading_path": self.heading_path, - "heading_label": " / ".join(self.heading_path), - "content": self.content, - "text": self.text, - "wikilinks": self.wikilinks, - "note_type": self.note_type, - "era": self.era, - "category": self.category, - "subtree": self.subtree, - "word_count": self.word_count, - "chunk_index": self.chunk_index, - "content_hash": self.content_hash, - "preview": preview_text(self.text), - } - - -def ensure_dependencies() -> None: - if TextEmbedding is None or QdrantClient is None or models is None: - raise SystemExit( - "Missing RAG dependencies. Run:\n" - " ./scripts/rag/rag.ps1 install\n" - "or\n" - f" {sys.executable} -m pip install -r {REPO_ROOT / 'requirements-rag.txt'}" - ) - - -def iso_now() -> str: - return datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def preview_text(text: str, limit: int = 180) -> str: - compact = " ".join(text.split()) - if len(compact) <= limit: - return compact - return compact[: limit - 1].rstrip() + "…" - - -def normalise_path(path: Path) -> str: - return path.as_posix() - - -def display_path(path: Path) -> str: - try: - return normalise_path(path.relative_to(REPO_ROOT)) - except ValueError: - return normalise_path(path) - - -def hash_text(text: str) -> str: - return hashlib.sha1(text.encode("utf-8")).hexdigest() - - -def count_words(text: str) -> int: - return len(re.findall(r"\S+", text)) - - -def replace_wikilinks(markdown: str) -> str: - def repl(match: re.Match[str]) -> str: - target = match.group(1).strip() - target_no_alias = target.split("|", 1) - destination = target_no_alias[0] - alias = target_no_alias[1] if len(target_no_alias) > 1 else "" - label = alias or destination.split("#", 1)[0] - destination_note = destination.split("#", 1)[0] - if alias and alias != destination_note: - return f"{alias} ({destination_note})" - return label - - return WIKILINK_RE.sub(repl, markdown) - - -def clean_markdown(markdown: str) -> str: - text = replace_wikilinks(markdown) - text = re.sub(r"`([^`]+)`", r"\1", text) - text = re.sub(r"\*\*([^*]+)\*\*", r"\1", text) - text = re.sub(r"\*([^*]+)\*", r"\1", text) - text = re.sub(r"^#{1,6}\s*", "", text, flags=re.MULTILINE) - text = re.sub(r"^>\s*", "", text, flags=re.MULTILINE) - text = re.sub(r"\n{3,}", "\n\n", text) - return text.strip() - - -def extract_wikilinks(markdown: str) -> list[str]: - seen: set[str] = set() - ordered: list[str] = [] - for raw in WIKILINK_RE.findall(markdown): - target = raw.split("|", 1)[0].split("#", 1)[0].strip() - if not target: - continue - normalized = target.lower() - if normalized in seen: - continue - seen.add(normalized) - ordered.append(target) - return ordered - - -def iter_markdown_files(corpus_root: Path) -> list[Path]: - return sorted(path for path in corpus_root.rglob("*.md") if path.is_file()) - - -def infer_metadata(rel_path: Path) -> tuple[str, str, str, str]: - parts = rel_path.parts - era = "General" - for candidate in ("Pre-Elysium", "Post-Elysium"): - if candidate in parts: - era = candidate - break - - category = "General" - for candidate in ( - "Politics", - "Technology", - "Concepts", - "Species", - "Factions", - "Timeline", - "Events", - "Movements", - "Powers", - "Territories", - "Reference", - ): - if candidate in parts: - category = candidate - break - - note_type = rel_path.parent.name if rel_path.parent != Path(".") else "Root" - subtree = normalise_path(rel_path.parent) - return era, category, note_type, subtree - - -def split_sections(markdown: str) -> list[tuple[list[str], str]]: - sections: list[tuple[list[str], str]] = [] - current_lines: list[str] = [] - heading_stack: list[str] = [] - current_heading: list[str] = [] - - def flush() -> None: - content = "\n".join(current_lines).strip() - if content: - sections.append((current_heading.copy(), content)) - - for line in markdown.splitlines(): - heading_match = HEADING_RE.match(line) - if heading_match: - flush() - current_lines = [line] - level = len(heading_match.group(1)) - heading_text = heading_match.group(2).strip() - heading_stack = heading_stack[: level - 1] - heading_stack.append(heading_text) - current_heading = heading_stack.copy() - continue - current_lines.append(line) - - flush() - return sections - - -def split_large_section(section_markdown: str, max_words: int = 320) -> list[str]: - if count_words(section_markdown) <= max_words: - return [section_markdown.strip()] - - paragraphs = [paragraph.strip() for paragraph in re.split(r"\n\s*\n", section_markdown) if paragraph.strip()] - if not paragraphs: - return [section_markdown.strip()] - - chunks: list[str] = [] - buffer: list[str] = [] - for paragraph in paragraphs: - proposal = "\n\n".join(buffer + [paragraph]).strip() - if buffer and count_words(proposal) > max_words: - chunks.append("\n\n".join(buffer).strip()) - buffer = [paragraph] - else: - buffer.append(paragraph) - - if buffer: - chunks.append("\n\n".join(buffer).strip()) - - return chunks - - -def build_chunk_text(title: str, heading_path: list[str], markdown: str) -> str: - context = [title] - if heading_path: - context.append(" / ".join(heading_path)) - context_block = "\n".join(context) - return clean_markdown(f"{context_block}\n\n{markdown}") - - -def chunk_note(file_path: Path, corpus_root: Path) -> list[Chunk]: - markdown = file_path.read_text(encoding="utf-8", errors="replace").strip() - if not markdown: - return [] - - rel_path = file_path.relative_to(REPO_ROOT) - title = file_path.stem - era, category, note_type, subtree = infer_metadata(rel_path) - note_hash = hash_text(markdown) - note_wikilinks = extract_wikilinks(markdown) - sections = split_sections(markdown) - - raw_chunks: list[tuple[list[str], str]] = [] - if count_words(markdown) <= 220: - raw_chunks = [([], markdown)] - else: - for heading_path, section_markdown in sections: - for subchunk in split_large_section(section_markdown): - raw_chunks.append((heading_path, subchunk)) - - if not raw_chunks: - raw_chunks = [([], markdown)] - - chunks: list[Chunk] = [] - for index, (heading_path, content) in enumerate(raw_chunks): - text = build_chunk_text(title, heading_path, content) - chunk_source = f"{normalise_path(rel_path)}::{index}::{content}" - chunk_id = hash_text(chunk_source) - chunks.append( - Chunk( - chunk_id=chunk_id, - path=normalise_path(rel_path), - title=title, - heading_path=heading_path, - content=content.strip(), - text=text, - wikilinks=note_wikilinks, - note_type=note_type, - era=era, - category=category, - subtree=subtree, - word_count=count_words(content), - chunk_index=index, - content_hash=note_hash, - ) - ) - return chunks - - -def collect_chunks(corpus_root: Path) -> list[Chunk]: - chunks: list[Chunk] = [] - for file_path in iter_markdown_files(corpus_root): - chunks.extend(chunk_note(file_path, corpus_root)) - return chunks - - -def ensure_storage_dirs() -> None: - RAG_ROOT.mkdir(exist_ok=True) - LOCAL_PYDEPS.mkdir(parents=True, exist_ok=True) - MODEL_CACHE_PATH.mkdir(parents=True, exist_ok=True) - QDRANT_PATH.mkdir(parents=True, exist_ok=True) - - -def save_chunk_cache(chunks: list[Chunk]) -> None: - ensure_storage_dirs() - with CHUNKS_PATH.open("w", encoding="utf-8", newline="\n") as handle: - for chunk in chunks: - handle.write(json.dumps(chunk.payload(), ensure_ascii=False) + "\n") - - -def save_manifest(corpus_root: Path, chunks: list[Chunk], collection: str, model_name: str) -> None: - files = sorted({chunk.path for chunk in chunks}) - manifest = { - "built_at": iso_now(), - "collection": collection, - "model": model_name, - "corpus_root": display_path(corpus_root), - "file_count": len(files), - "chunk_count": len(chunks), - "paths": files, - } - MANIFEST_PATH.write_text(json.dumps(manifest, indent=2), encoding="utf-8") - - -def load_chunks() -> list[dict[str, Any]]: - if not CHUNKS_PATH.exists(): - raise SystemExit("No chunk cache found. Run `./scripts/rag/rag.ps1 build` first.") - records: list[dict[str, Any]] = [] - with CHUNKS_PATH.open("r", encoding="utf-8") as handle: - for line in handle: - line = line.strip() - if line: - records.append(json.loads(line)) - return records - - -def embed_documents(model_name: str, texts: list[str]) -> list[list[float]]: - ensure_dependencies() - model = TextEmbedding(model_name=model_name, cache_dir=str(MODEL_CACHE_PATH)) - return [vector.tolist() for vector in model.embed([f"passage: {text}" for text in texts])] - - -def embed_query(model_name: str, text: str) -> list[float]: - ensure_dependencies() - model = TextEmbedding(model_name=model_name, cache_dir=str(MODEL_CACHE_PATH)) - vectors = [vector.tolist() for vector in model.embed([f"query: {text}"])] - if not vectors: - raise SystemExit("Embedding model did not return a query vector.") - return vectors[0] - - -def qdrant_client() -> QdrantClient: - ensure_dependencies() - ensure_storage_dirs() - return QdrantClient(path=str(QDRANT_PATH)) - - -def build_index(args: argparse.Namespace) -> int: - corpus_root = Path(args.corpus_root).resolve() - if not corpus_root.exists(): - raise SystemExit(f"Corpus root does not exist: {corpus_root}") - - chunks = collect_chunks(corpus_root) - if not chunks: - raise SystemExit(f"No Markdown notes found under {corpus_root}") - - client = qdrant_client() - try: - vectors = embed_documents(args.model, [chunk.text for chunk in chunks]) - vector_size = len(vectors[0]) - - if client.collection_exists(args.collection): - client.delete_collection(args.collection) - client.create_collection( - collection_name=args.collection, - vectors_config=models.VectorParams(size=vector_size, distance=models.Distance.COSINE), - ) - - points = [ - models.PointStruct(id=index, vector=vector, payload=chunk.payload()) - for index, (chunk, vector) in enumerate(zip(chunks, vectors, strict=True), start=1) - ] - client.upload_points(collection_name=args.collection, points=points) - finally: - client.close() - - save_chunk_cache(chunks) - save_manifest(corpus_root, chunks, args.collection, args.model) - - summary = { - "status": "ok", - "built_at": iso_now(), - "collection": args.collection, - "model": args.model, - "corpus_root": display_path(corpus_root), - "file_count": len({chunk.path for chunk in chunks}), - "chunk_count": len(chunks), - "qdrant_path": normalise_path(QDRANT_PATH.relative_to(REPO_ROOT)), - "model_cache_path": normalise_path(MODEL_CACHE_PATH.relative_to(REPO_ROOT)), - } - print(json.dumps(summary, indent=2)) - return 0 - - -def tokenise(text: str) -> list[str]: - tokens = [token for token in TOKEN_RE.findall(text.lower()) if token not in STOPWORDS] - return tokens - - -def lexical_score(query: str, chunk: dict[str, Any]) -> float: - query_lower = query.lower().strip() - title = str(chunk.get("title", "")) - title_lower = title.lower() - text = str(chunk.get("text", "")) - text_lower = text.lower() - path_lower = str(chunk.get("path", "")).lower() - wikilinks = [str(item).lower() for item in chunk.get("wikilinks", [])] - tokens = tokenise(query) - - score = 0.0 - if query_lower and query_lower in title_lower: - score += 6.0 - if query_lower and query_lower in path_lower: - score += 4.0 - if query_lower and query_lower in wikilinks: - score += 5.0 - if query_lower and query_lower in text_lower: - score += 2.0 - - for token in tokens: - if token in title_lower: - score += 1.5 - if any(token in wikilink for wikilink in wikilinks): - score += 1.2 - occurrences = text_lower.count(token) - if occurrences: - score += min(occurrences, 5) * 0.25 - - return score - - -def filters_match(chunk: dict[str, Any], args: argparse.Namespace) -> bool: - if args.era and str(chunk.get("era", "")).lower() != args.era.lower(): - return False - if args.category and str(chunk.get("category", "")).lower() != args.category.lower(): - return False - if args.note_type and str(chunk.get("note_type", "")).lower() != args.note_type.lower(): - return False - if args.path_contains and args.path_contains.lower() not in str(chunk.get("path", "")).lower(): - return False - return True - - -def query_index(args: argparse.Namespace) -> int: - client = qdrant_client() - try: - if not client.collection_exists(args.collection): - raise SystemExit(f"Collection `{args.collection}` does not exist. Run `./scripts/rag/rag.ps1 build` first.") - - cached_chunks = {chunk["chunk_id"]: chunk for chunk in load_chunks()} - query_vector = embed_query(args.model, args.query) - semantic_limit = max(args.semantic_limit, args.limit * 4) - semantic_points = client.query_points( - collection_name=args.collection, - query=query_vector, - limit=semantic_limit, - with_payload=True, - ).points - finally: - client.close() - - candidates: dict[str, dict[str, Any]] = {} - for point in semantic_points: - payload = point.payload or {} - if not filters_match(payload, args): - continue - chunk_id = str(payload.get("chunk_id") or point.id) - candidates[chunk_id] = { - "chunk": payload, - "semantic_score": max(float(point.score), 0.0), - "lexical_score": 0.0, - } - - for chunk_id, chunk in cached_chunks.items(): - if not filters_match(chunk, args): - continue - score = lexical_score(args.query, chunk) - if score <= 0: - continue - entry = candidates.setdefault( - chunk_id, - { - "chunk": chunk, - "semantic_score": 0.0, - "lexical_score": 0.0, - }, - ) - entry["lexical_score"] = max(entry["lexical_score"], score) - - if not candidates: - print(json.dumps({"query": args.query, "results": []}, indent=2)) - return 0 - - max_semantic = max(entry["semantic_score"] for entry in candidates.values()) or 1.0 - max_lexical = max(entry["lexical_score"] for entry in candidates.values()) or 1.0 - - results: list[dict[str, Any]] = [] - for entry in candidates.values(): - semantic_norm = entry["semantic_score"] / max_semantic if max_semantic else 0.0 - lexical_norm = entry["lexical_score"] / max_lexical if max_lexical else 0.0 - fused_score = (semantic_norm * args.semantic_weight) + (lexical_norm * args.lexical_weight) - chunk = entry["chunk"] - results.append( - { - "score": round(fused_score, 6), - "semantic_score": round(entry["semantic_score"], 6), - "lexical_score": round(entry["lexical_score"], 6), - "path": chunk.get("path"), - "title": chunk.get("title"), - "heading_path": chunk.get("heading_path", []), - "era": chunk.get("era"), - "category": chunk.get("category"), - "note_type": chunk.get("note_type"), - "wikilinks": chunk.get("wikilinks", []), - "preview": chunk.get("preview") or preview_text(str(chunk.get("text", ""))), - "content": chunk.get("content"), - } - ) - - results.sort(key=lambda item: item["score"], reverse=True) - results = results[: args.limit] - - if args.json: - print(json.dumps({"query": args.query, "results": results}, indent=2, ensure_ascii=False)) - return 0 - - print(f"Query: {args.query}\n") - for index, result in enumerate(results, start=1): - heading_label = " / ".join(result["heading_path"]) if result["heading_path"] else "(note overview)" - print(f"{index}. {result['title']} [{result['score']:.3f}]") - print(f" Path: {result['path']}") - print(f" Section: {heading_label}") - print(f" Era/Category: {result['era']} / {result['category']}") - print(f" Preview: {result['preview']}") - print() - return 0 - - -def stats(args: argparse.Namespace) -> int: - if not MANIFEST_PATH.exists(): - raise SystemExit("No manifest found. Run `./scripts/rag/rag.ps1 build` first.") - manifest = json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) - print(json.dumps(manifest, indent=2, ensure_ascii=False)) - return 0 - - -def install(args: argparse.Namespace) -> int: - requirements = REPO_ROOT / "requirements-rag.txt" - ensure_storage_dirs() - command = [ - sys.executable, - "-m", - "pip", - "install", - "--target", - str(LOCAL_PYDEPS), - "-r", - str(requirements), - ] - if args.upgrade: - command.append("--upgrade") - return subprocess.call(command) - - -def parser() -> argparse.ArgumentParser: - main_parser = argparse.ArgumentParser(description="Local Qdrant/FastEmbed RAG tools for the Aetheria vault.") - subparsers = main_parser.add_subparsers(dest="command", required=True) - - install_parser = subparsers.add_parser("install", help="Install Python dependencies for the local RAG toolchain.") - install_parser.add_argument("--upgrade", action="store_true", help="Upgrade dependencies during install.") - install_parser.set_defaults(func=install) - - build_parser = subparsers.add_parser("build", help="Index the vault into local Qdrant storage.") - build_parser.add_argument("--corpus-root", default=str(DEFAULT_CORPUS_ROOT), help="Directory of Markdown notes to index.") - build_parser.add_argument("--collection", default=DEFAULT_COLLECTION, help="Qdrant collection name.") - build_parser.add_argument("--model", default=DEFAULT_MODEL, help="FastEmbed model name.") - build_parser.set_defaults(func=build_index) - - query_parser = subparsers.add_parser("query", help="Run a hybrid semantic + lexical search over indexed chunks.") - query_parser.add_argument("query", help="Natural-language or exact-name query.") - query_parser.add_argument("--collection", default=DEFAULT_COLLECTION, help="Qdrant collection name.") - query_parser.add_argument("--model", default=DEFAULT_MODEL, help="FastEmbed model name.") - query_parser.add_argument("--limit", type=int, default=8, help="Number of results to return.") - query_parser.add_argument("--semantic-limit", type=int, default=24, help="How many semantic matches to fetch before reranking.") - query_parser.add_argument("--semantic-weight", type=float, default=0.7, help="Weight of vector similarity in the fused score.") - query_parser.add_argument("--lexical-weight", type=float, default=0.3, help="Weight of exact/keyword similarity in the fused score.") - query_parser.add_argument("--era", help="Optional exact era filter, e.g. Pre-Elysium.") - query_parser.add_argument("--category", help="Optional exact category filter, e.g. Technology.") - query_parser.add_argument("--note-type", help="Optional exact note_type filter, usually the parent folder.") - query_parser.add_argument("--path-contains", help="Optional substring filter against the note path.") - query_parser.add_argument("--json", action="store_true", help="Emit raw JSON for downstream tooling.") - query_parser.set_defaults(func=query_index) - - stats_parser = subparsers.add_parser("stats", help="Show build metadata for the current local index.") - stats_parser.set_defaults(func=stats) - return main_parser - - -def main() -> int: - args = parser().parse_args() - return int(args.func(args)) - - -if __name__ == "__main__": - raise SystemExit(main())