GameCult/Huginn
Watch
9
0
Fork
You've already forked Huginn
 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Sources: licence rule narrowed to bare or prose-extension names, chunker version stays 1

Browse source 
The skip list now matches `license`, `licence`, `copying` only with no extension or md/txt/rst, so src/license.rs and Copying.cs stay indexed. CHUNKER_VERSION is back to 1: a bump rebuilds the live collection; the already-indexed noise paths go by a one-time delete_by sweep instead (work/sweep/sweep-sources-noise.py, same rule).

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Eureka 2026-11-10 14:06:10 +00:00
commit 1ee0b86f30
2 changed files with 11 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

5
crates/huginn-daemon/src/sources/chunk.rs
View file

@ -2,10 +2,9 @@
//! holds and a search answers, so its line range is a citation.
/// Bumped whenever the windows or the embedded text would change for the same
/// blob, or the skip list in `git::skipped_path` changes (an incremental run
/// never revisits an unchanged path, so only a rebuild drops its old chunks): it is part of every chunk's identity and of the collection's label, so
/// blob: it is part of every chunk's identity and of the collection's label, so
/// a change rebuilds the collection instead of mixing two chunkings.
pub const CHUNKER_VERSION: u32 = 2;
pub const CHUNKER_VERSION: u32 = 1;
/// The most lines a chunk holds.
pub const CHUNK_LINES_MAX: usize = 48;

16
crates/huginn-daemon/src/sources/git.rs
View file

@ -28,8 +28,10 @@ const LFS_POINTER: &[u8] = b"version https://git-lfs.github.com/spec/v1";
/// writing: a path with one of these as a component is not indexed.
const SKIPPED_DIRECTORIES: &[&str] = &[".obsidian", ".voidbot"];
/// Licence texts, whatever their extension: they crowd out a front page.
/// Licence texts: a file named exactly one of these stems, bare or with a prose
/// extension. They crowd out a front page; `license.rs` and `Copying.cs` are code.
const SKIPPED_STEMS: &[&str] = &["license", "licence", "copying"];
const LICENSE_EXTENSIONS: &[&str] = &["", "md", "txt", "rst"];
/// Whether a path is not worth reading, by name alone.
pub fn skipped_path(path: &str) -> bool {
@ -41,9 +43,8 @@ pub fn skipped_path(path: &str) -> bool {
if SKIPPED_NAMES.contains(&name.as_str()) {
return true;
}
let stem = name.split('.').next().unwrap_or("");
let extension = name.rsplit_once('.').map_or("", |(_, extension)| extension);
SKIPPED_STEMS.contains(&stem) || extension == "provenance" || SKIPPED_EXTENSIONS.contains(&extension)
let (stem, extension) = name.rsplit_once('.').unwrap_or((&name, ""));
(SKIPPED_STEMS.contains(&stem) && LICENSE_EXTENSIONS.contains(&extension)) || extension == "provenance" || SKIPPED_EXTENSIONS.contains(&extension)
}
/// The text of a blob worth indexing, or `None`: a NUL byte, bytes that are
@ -200,12 +201,13 @@ mod tests {
#[test]
fn binary_unity_and_lock_paths_are_skipped_by_name_and_source_is_not() {
for path in ["a.png", "dir/B.PNG", "x.lock", "Assets/a.prefab", "Assets/a.unity", "a.cs.meta", "p/package-lock.json", "Cargo.lock",
".obsidian/app.json", "vault/.obsidian/appearance.json", ".voidbot/packets/a.json", "LICENSE", "LICENSE.md", "sub/licence.txt", "COPYING",
"bundle.provenance", "x/bundle.provenance"] {
".obsidian/app.json", "vault/.obsidian/appearance.json", ".voidbot/packets/a.json", "LICENSE", "LICENSE.md", "sub/licence.txt", "COPYING", "COPYING.rst", "x/.Obsidian/a.md", "x/.VoidBot/p.json",
"bundle.provenance", "x/bundle.provenance", ".provenance"] {
assert!(skipped_path(path), "{path}");
}
for path in ["src/main.rs", "README.md", "docs/lock.md", "Assets/Scripts/Player.cs", "Makefile", ".gitignore", "locksmith",
"docs/x.md", "src/obsidian_notes.rs", "docs/licensing.md", "docs/obsidian/notes.md", "provenance.md"] {
"docs/x.md", "src/obsidian_notes.rs", "docs/licensing.md", "docs/obsidian/notes.md", "provenance.md",
"src/license.rs", "crates/x/src/license.rs", "Copying.cs", "LICENSE-MIT", "LICENSE.md.bak"] {
assert!(!skipped_path(path), "{path}");
}
}