Huginn/crates/huginn-daemon/src/sources/git.rs
2026-11-10 00:44:25 +00:00

220 lines
9.5 KiB
Rust

//! Git, by shelling out: bare mirrors of the forge's repos, what changed
//! between two commits, and the blobs at one. Nothing here knows an index or an
//! embedder; the mirrors are cache and can be deleted whenever.
use std::io::{BufRead, BufReader, Read, Write};
use std::path::{Path, PathBuf};
use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio};
use anyhow::{Context, Result, bail, ensure};
/// A blob over this many bytes is not indexed.
pub const BLOB_BYTES_MAX: u64 = 256 * 1024;
/// Extensions of files that are not text worth finding, VoidBot's list, plus
/// Unity's serialised YAML (huge, machine written) and lock files.
const SKIPPED_EXTENSIONS: &[&str] = &[
"7z", "avi", "bmp", "class", "dll", "dylib", "eot", "exe", "gif", "gz", "ico", "jar", "jpeg", "jpg", "lock", "map", "mp3",
"mp4", "mov", "ogg", "otf", "pdf", "png", "rar", "so", "svg", "tar", "ttf", "wav", "webm", "webp", "woff", "woff2", "zip",
"meta", "prefab", "unity", "asset", "mat", "anim", "controller", "physicmaterial",
];
/// Lock files whose extension is not `.lock`.
const SKIPPED_NAMES: &[&str] = &["package-lock.json", "pnpm-lock.yaml", "yarn.lock", "bun.lockb", "composer.lock"];
const LFS_POINTER: &[u8] = b"version https://git-lfs.github.com/spec/v1";
/// Directories of editor settings and generated packets, not the repo's own
/// writing: a path with one of these as a component is not indexed.
const SKIPPED_DIRECTORIES: &[&str] = &[".obsidian", ".voidbot"];
/// Licence texts, whatever their extension: they crowd out a front page.
const SKIPPED_STEMS: &[&str] = &["license", "licence", "copying"];
/// Whether a path is not worth reading, by name alone.
pub fn skipped_path(path: &str) -> bool {
let mut components = path.split('/');
let name = components.next_back().unwrap_or(path).to_ascii_lowercase();
if components.any(|directory| SKIPPED_DIRECTORIES.contains(&directory.to_ascii_lowercase().as_str())) {
return true;
}
if SKIPPED_NAMES.contains(&name.as_str()) {
return true;
}
let stem = name.split('.').next().unwrap_or("");
let extension = name.rsplit_once('.').map_or("", |(_, extension)| extension);
SKIPPED_STEMS.contains(&stem) || extension == "provenance" || SKIPPED_EXTENSIONS.contains(&extension)
}
/// The text of a blob worth indexing, or `None`: a NUL byte, bytes that are
/// not UTF-8, or a Git LFS pointer make it not text.
pub fn indexable_text(bytes: &[u8]) -> Option<String> {
if bytes.contains(&0) || bytes.starts_with(LFS_POINTER) {
return None;
}
String::from_utf8(bytes.to_vec()).ok()
}
/// One file of a tree: its path, the blob it holds and that blob's size.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TreeFile {
pub path: String,
pub blob: String,
pub size: u64,
}
fn git(dir: &Path, args: &[&str]) -> Command {
let mut command = Command::new("git");
command.arg("-C").arg(dir).args(args).env("GIT_TERMINAL_PROMPT", "0");
command
}
fn run(mut command: Command, what: &str) -> Result<Vec<u8>> {
let output = command.output().with_context(|| format!("starting git to {what}"))?;
ensure!(
output.status.success(),
"git could not {what}: {}",
String::from_utf8_lossy(&output.stderr).trim().chars().take(400).collect::<String>()
);
Ok(output.stdout)
}
/// A bare mirror of one forge repo's default branch.
pub struct Mirror {
dir: PathBuf,
branch: String,
}
impl Mirror {
pub fn path_for(root: &Path, name: &str) -> PathBuf {
root.join(format!("{name}.git"))
}
pub fn new(root: &Path, name: &str, branch: &str) -> Self {
Self { dir: Self::path_for(root, name), branch: branch.to_owned() }
}
/// Fetches the default branch, creating the mirror if it is absent, and
/// returns the commit it now points at. The branch is forced: a rewritten
/// history is followed, not refused.
pub fn fetch(&self, remote: &str) -> Result<String> {
if !self.dir.join("HEAD").exists() {
std::fs::create_dir_all(&self.dir).with_context(|| format!("creating {}", self.dir.display()))?;
let mut init = Command::new("git");
init.arg("init").arg("--bare").arg("--quiet").arg(&self.dir);
run(init, "initialise a mirror")?;
}
let refspec = format!("+refs/heads/{0}:refs/heads/{0}", self.branch);
run(git(&self.dir, &["fetch", "--quiet", "--no-tags", remote, &refspec]), "fetch the default branch")?;
self.head()
}
pub fn head(&self) -> Result<String> {
let out = run(git(&self.dir, &["rev-parse", "--verify", &format!("refs/heads/{}", self.branch)]), "read the branch head")?;
Ok(String::from_utf8_lossy(&out).trim().to_owned())
}
/// Whether `old` is an ancestor of `new`. A commit the mirror no longer
/// holds is not.
pub fn is_ancestor(&self, old: &str, new: &str) -> bool {
git(&self.dir, &["merge-base", "--is-ancestor", old, new]).status().is_ok_and(|status| status.success())
}
/// The paths that differ between two commits, a rename counted as a delete
/// and an add.
pub fn changed_paths(&self, old: &str, new: &str) -> Result<Vec<String>> {
let out = run(git(&self.dir, &["diff", "--no-renames", "--name-only", "-z", old, new]), "diff two commits")?;
Ok(out.split(|byte| *byte == 0).filter(|path| !path.is_empty()).map(|path| String::from_utf8_lossy(path).into_owned()).collect())
}
/// Every regular file at `commit`; symlinks and submodules are not files.
pub fn files(&self, commit: &str) -> Result<Vec<TreeFile>> {
let out = run(git(&self.dir, &["ls-tree", "-r", "-z", "-l", commit]), "list a tree")?;
let mut files = Vec::new();
for entry in out.split(|byte| *byte == 0).filter(|entry| !entry.is_empty()) {
let text = String::from_utf8_lossy(entry);
let (meta, path) = text.split_once('\t').with_context(|| format!("an ls-tree entry has no tab: {text:?}"))?;
let mut fields = meta.split_whitespace();
let (mode, kind, blob, size) = (fields.next(), fields.next(), fields.next(), fields.next());
let (Some(mode), Some("blob"), Some(blob), Some(size)) = (mode, kind, blob, size) else { continue };
if mode == "120000" {
continue;
}
let Ok(size) = size.parse::<u64>() else { continue };
files.push(TreeFile { path: path.to_owned(), blob: blob.to_owned(), size });
}
Ok(files)
}
/// A reader of blobs by id, one `cat-file --batch` process.
pub fn blobs(&self) -> Result<Blobs> {
let mut child = git(&self.dir, &["cat-file", "--batch"])
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::null())
.spawn()
.context("starting git cat-file")?;
let stdin = child.stdin.take().context("git cat-file has no stdin")?;
let stdout = BufReader::new(child.stdout.take().context("git cat-file has no stdout")?);
Ok(Blobs { child, stdin, stdout })
}
}
/// `git cat-file --batch`, asked one blob at a time.
pub struct Blobs {
child: Child,
stdin: ChildStdin,
stdout: BufReader<ChildStdout>,
}
impl Blobs {
pub fn read(&mut self, blob: &str) -> Result<Vec<u8>> {
writeln!(self.stdin, "{blob}").and_then(|()| self.stdin.flush()).context("asking git cat-file for a blob")?;
let mut header = String::new();
self.stdout.read_line(&mut header).context("reading git cat-file's header")?;
let mut fields = header.split_whitespace();
let (Some(_), Some(kind), Some(size)) = (fields.next(), fields.next(), fields.next()) else {
bail!("git cat-file does not hold blob {blob}: {}", header.trim());
};
ensure!(kind == "blob", "git cat-file says {blob} is a {kind}");
let size: usize = size.parse().with_context(|| format!("a blob size of {size:?}"))?;
let mut bytes = vec![0; size];
self.stdout.read_exact(&mut bytes).context("reading a blob")?;
let mut newline = [0_u8; 1];
self.stdout.read_exact(&mut newline).context("reading a blob's end")?;
Ok(bytes)
}
}
impl Drop for Blobs {
fn drop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn binary_unity_and_lock_paths_are_skipped_by_name_and_source_is_not() {
for path in ["a.png", "dir/B.PNG", "x.lock", "Assets/a.prefab", "Assets/a.unity", "a.cs.meta", "p/package-lock.json", "Cargo.lock",
".obsidian/app.json", "vault/.obsidian/appearance.json", ".voidbot/packets/a.json", "LICENSE", "LICENSE.md", "sub/licence.txt", "COPYING",
"bundle.provenance", "x/bundle.provenance"] {
assert!(skipped_path(path), "{path}");
}
for path in ["src/main.rs", "README.md", "docs/lock.md", "Assets/Scripts/Player.cs", "Makefile", ".gitignore", "locksmith",
"docs/x.md", "src/obsidian_notes.rs", "docs/licensing.md", "docs/obsidian/notes.md", "provenance.md"] {
assert!(!skipped_path(path), "{path}");
}
}
#[test]
fn a_nul_byte_an_lfs_pointer_and_invalid_utf8_are_not_text() {
assert_eq!(indexable_text(b"fn main() {}\n").as_deref(), Some("fn main() {}\n"));
assert_eq!(indexable_text(b"a\0b"), None);
assert_eq!(indexable_text(b"version https://git-lfs.github.com/spec/v1\noid sha256:ab\nsize 12\n"), None);
assert_eq!(indexable_text(&[0xff, 0xfe, b'a']), None);
}
}