From 6d1766dd72dca9fafd025869c80b95b30bdebf74 Mon Sep 17 00:00:00 2001 From: Eureka Date: Tue, 10 Nov 2026 00:44:25 +0000 Subject: [PATCH 1/2] Sources: skip .obsidian/.voidbot dirs, licence files and *.provenance; bump chunker version to rebuild Co-Authored-By: Claude Sonnet 5.5 --- crates/huginn-daemon/src/sources/chunk.rs | 5 +++-- crates/huginn-daemon/src/sources/git.rs | 24 +++++++++++++++++++---- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/crates/huginn-daemon/src/sources/chunk.rs b/crates/huginn-daemon/src/sources/chunk.rs index 394d0ad..279541c 100644 --- a/crates/huginn-daemon/src/sources/chunk.rs +++ b/crates/huginn-daemon/src/sources/chunk.rs @@ -2,9 +2,10 @@ //! holds and a search answers, so its line range is a citation. /// Bumped whenever the windows or the embedded text would change for the same -/// blob: it is part of every chunk's identity and of the collection's label, so +/// blob, or the skip list in `git::skipped_path` changes (an incremental run +/// never revisits an unchanged path, so only a rebuild drops its old chunks): it is part of every chunk's identity and of the collection's label, so /// a change rebuilds the collection instead of mixing two chunkings. -pub const CHUNKER_VERSION: u32 = 1; +pub const CHUNKER_VERSION: u32 = 2; /// The most lines a chunk holds. pub const CHUNK_LINES_MAX: usize = 48; diff --git a/crates/huginn-daemon/src/sources/git.rs b/crates/huginn-daemon/src/sources/git.rs index af41e21..53d42aa 100644 --- a/crates/huginn-daemon/src/sources/git.rs +++ b/crates/huginn-daemon/src/sources/git.rs @@ -24,13 +24,26 @@ const SKIPPED_NAMES: &[&str] = &["package-lock.json", "pnpm-lock.yaml", "yarn.lo const LFS_POINTER: &[u8] = b"version https://git-lfs.github.com/spec/v1"; +/// Directories of editor settings and generated packets, not the repo's own +/// writing: a path with one of these as a component is not indexed. +const SKIPPED_DIRECTORIES: &[&str] = &[".obsidian", ".voidbot"]; + +/// Licence texts, whatever their extension: they crowd out a front page. +const SKIPPED_STEMS: &[&str] = &["license", "licence", "copying"]; + /// Whether a path is not worth reading, by name alone. pub fn skipped_path(path: &str) -> bool { - let name = path.rsplit('/').next().unwrap_or(path).to_ascii_lowercase(); + let mut components = path.split('/'); + let name = components.next_back().unwrap_or(path).to_ascii_lowercase(); + if components.any(|directory| SKIPPED_DIRECTORIES.contains(&directory.to_ascii_lowercase().as_str())) { + return true; + } if SKIPPED_NAMES.contains(&name.as_str()) { return true; } - name.rsplit_once('.').is_some_and(|(_, extension)| SKIPPED_EXTENSIONS.contains(&extension)) + let stem = name.split('.').next().unwrap_or(""); + let extension = name.rsplit_once('.').map_or("", |(_, extension)| extension); + SKIPPED_STEMS.contains(&stem) || extension == "provenance" || SKIPPED_EXTENSIONS.contains(&extension) } /// The text of a blob worth indexing, or `None`: a NUL byte, bytes that are @@ -186,10 +199,13 @@ mod tests { #[test] fn binary_unity_and_lock_paths_are_skipped_by_name_and_source_is_not() { - for path in ["a.png", "dir/B.PNG", "x.lock", "Assets/a.prefab", "Assets/a.unity", "a.cs.meta", "p/package-lock.json", "Cargo.lock"] { + for path in ["a.png", "dir/B.PNG", "x.lock", "Assets/a.prefab", "Assets/a.unity", "a.cs.meta", "p/package-lock.json", "Cargo.lock", + ".obsidian/app.json", "vault/.obsidian/appearance.json", ".voidbot/packets/a.json", "LICENSE", "LICENSE.md", "sub/licence.txt", "COPYING", + "bundle.provenance", "x/bundle.provenance"] { assert!(skipped_path(path), "{path}"); } - for path in ["src/main.rs", "README.md", "docs/lock.md", "Assets/Scripts/Player.cs", "Makefile", ".gitignore", "locksmith"] { + for path in ["src/main.rs", "README.md", "docs/lock.md", "Assets/Scripts/Player.cs", "Makefile", ".gitignore", "locksmith", + "docs/x.md", "src/obsidian_notes.rs", "docs/licensing.md", "docs/obsidian/notes.md", "provenance.md"] { assert!(!skipped_path(path), "{path}"); } } From 1ee0b86f30309e14d906413e67c14cf7c20e2bb4 Mon Sep 17 00:00:00 2001 From: Eureka Date: Tue, 10 Nov 2026 14:06:10 +0000 Subject: [PATCH 2/2] Sources: licence rule narrowed to bare or prose-extension names, chunker version stays 1 The skip list now matches `license`, `licence`, `copying` only with no extension or md/txt/rst, so src/license.rs and Copying.cs stay indexed. CHUNKER_VERSION is back to 1: a bump rebuilds the live collection; the already-indexed noise paths go by a one-time delete_by sweep instead (work/sweep/sweep-sources-noise.py, same rule). Co-Authored-By: Claude Sonnet 5.5 --- crates/huginn-daemon/src/sources/chunk.rs | 5 ++--- crates/huginn-daemon/src/sources/git.rs | 16 +++++++++------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/crates/huginn-daemon/src/sources/chunk.rs b/crates/huginn-daemon/src/sources/chunk.rs index 279541c..394d0ad 100644 --- a/crates/huginn-daemon/src/sources/chunk.rs +++ b/crates/huginn-daemon/src/sources/chunk.rs @@ -2,10 +2,9 @@ //! holds and a search answers, so its line range is a citation. /// Bumped whenever the windows or the embedded text would change for the same -/// blob, or the skip list in `git::skipped_path` changes (an incremental run -/// never revisits an unchanged path, so only a rebuild drops its old chunks): it is part of every chunk's identity and of the collection's label, so +/// blob: it is part of every chunk's identity and of the collection's label, so /// a change rebuilds the collection instead of mixing two chunkings. -pub const CHUNKER_VERSION: u32 = 2; +pub const CHUNKER_VERSION: u32 = 1; /// The most lines a chunk holds. pub const CHUNK_LINES_MAX: usize = 48; diff --git a/crates/huginn-daemon/src/sources/git.rs b/crates/huginn-daemon/src/sources/git.rs index 53d42aa..c471cf0 100644 --- a/crates/huginn-daemon/src/sources/git.rs +++ b/crates/huginn-daemon/src/sources/git.rs @@ -28,8 +28,10 @@ const LFS_POINTER: &[u8] = b"version https://git-lfs.github.com/spec/v1"; /// writing: a path with one of these as a component is not indexed. const SKIPPED_DIRECTORIES: &[&str] = &[".obsidian", ".voidbot"]; -/// Licence texts, whatever their extension: they crowd out a front page. +/// Licence texts: a file named exactly one of these stems, bare or with a prose +/// extension. They crowd out a front page; `license.rs` and `Copying.cs` are code. const SKIPPED_STEMS: &[&str] = &["license", "licence", "copying"]; +const LICENSE_EXTENSIONS: &[&str] = &["", "md", "txt", "rst"]; /// Whether a path is not worth reading, by name alone. pub fn skipped_path(path: &str) -> bool { @@ -41,9 +43,8 @@ pub fn skipped_path(path: &str) -> bool { if SKIPPED_NAMES.contains(&name.as_str()) { return true; } - let stem = name.split('.').next().unwrap_or(""); - let extension = name.rsplit_once('.').map_or("", |(_, extension)| extension); - SKIPPED_STEMS.contains(&stem) || extension == "provenance" || SKIPPED_EXTENSIONS.contains(&extension) + let (stem, extension) = name.rsplit_once('.').unwrap_or((&name, "")); + (SKIPPED_STEMS.contains(&stem) && LICENSE_EXTENSIONS.contains(&extension)) || extension == "provenance" || SKIPPED_EXTENSIONS.contains(&extension) } /// The text of a blob worth indexing, or `None`: a NUL byte, bytes that are @@ -200,12 +201,13 @@ mod tests { #[test] fn binary_unity_and_lock_paths_are_skipped_by_name_and_source_is_not() { for path in ["a.png", "dir/B.PNG", "x.lock", "Assets/a.prefab", "Assets/a.unity", "a.cs.meta", "p/package-lock.json", "Cargo.lock", - ".obsidian/app.json", "vault/.obsidian/appearance.json", ".voidbot/packets/a.json", "LICENSE", "LICENSE.md", "sub/licence.txt", "COPYING", - "bundle.provenance", "x/bundle.provenance"] { + ".obsidian/app.json", "vault/.obsidian/appearance.json", ".voidbot/packets/a.json", "LICENSE", "LICENSE.md", "sub/licence.txt", "COPYING", "COPYING.rst", "x/.Obsidian/a.md", "x/.VoidBot/p.json", + "bundle.provenance", "x/bundle.provenance", ".provenance"] { assert!(skipped_path(path), "{path}"); } for path in ["src/main.rs", "README.md", "docs/lock.md", "Assets/Scripts/Player.cs", "Makefile", ".gitignore", "locksmith", - "docs/x.md", "src/obsidian_notes.rs", "docs/licensing.md", "docs/obsidian/notes.md", "provenance.md"] { + "docs/x.md", "src/obsidian_notes.rs", "docs/licensing.md", "docs/obsidian/notes.md", "provenance.md", + "src/license.rs", "crates/x/src/license.rs", "Copying.cs", "LICENSE-MIT", "LICENSE.md.bak"] { assert!(!skipped_path(path), "{path}"); } }