diff --git a/.gitignore b/.gitignore index 0808c4ad..4a48d6a9 100644 --- a/.gitignore +++ b/.gitignore @@ -293,6 +293,25 @@ FakesAssemblies/ .ntvs_analysis.dat node_modules/ +# CultLib language packages are source workspaces, not NuGet restore output. +!packages/ +!packages/cultcache-ts/ +!packages/cultcache-ts/** +!packages/cultnet-ts/ +!packages/cultnet-ts/** +!packages/cultmesh-ts/ +!packages/cultmesh-ts/** +!packages/cultcache-rs/ +!packages/cultcache-rs/** +!packages/cultnet-rs/ +!packages/cultnet-rs/** +packages/**/node_modules/ +packages/**/dist/ +packages/**/dist-test/ +packages/**/dist-inspector/ +packages/**/release/ +packages/**/release-inspector/ + # Visual Studio 6 build log *.plg diff --git a/package.json b/package.json new file mode 100644 index 00000000..f1b995a2 --- /dev/null +++ b/package.json @@ -0,0 +1,13 @@ +{ + "name": "cultlib", + "private": true, + "workspaces": [ + "packages/cultcache-ts", + "packages/cultnet-ts", + "packages/cultmesh-ts" + ], + "scripts": { + "build:ts": "npm --workspaces run build", + "test:ts": "npm --workspaces run test" + } +} diff --git a/packages/cultcache-rs/.gitignore b/packages/cultcache-rs/.gitignore new file mode 100644 index 00000000..96ef6c0b --- /dev/null +++ b/packages/cultcache-rs/.gitignore @@ -0,0 +1,2 @@ +/target +Cargo.lock diff --git a/packages/cultcache-rs/Cargo.toml b/packages/cultcache-rs/Cargo.toml new file mode 100644 index 00000000..b11a909e --- /dev/null +++ b/packages/cultcache-rs/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "cultcache-rs" +version = "0.1.0" +edition = "2024" +publish = false + +[workspace] +members = ["cultcache-rs-derive"] + +[lib] +name = "cultcache_rs" +path = "src/lib.rs" + +[dependencies] +anyhow = "1" +chrono = { version = "0.4", features = ["serde"] } +cultcache-rs-derive = { path = "cultcache-rs-derive" } +fs2 = "0.4" +rmp-serde = "1.3" +serde = { version = "1", features = ["derive"] } +serde_bytes = "0.11" +serde_json = "1" +uuid = { version = "1", features = ["v4"] } + +[dev-dependencies] +pretty_assertions = "1.4" +tempfile = "3" diff --git a/packages/cultcache-rs/README.md b/packages/cultcache-rs/README.md new file mode 100644 index 00000000..2d75d94e --- /dev/null +++ b/packages/cultcache-rs/README.md @@ -0,0 +1,332 @@ +# CultCache Rust + +`cultcache-rs` is the Rust port of the useful part of GameCult's CultCache: +a polymorphism-aware in-memory cache that remembers through pluggable backing +stores. + +Consumer code should think in domain types: + +```rust +let player = cache.get_required::("player:meta")?; +cache.put("player:meta", &player)?; +``` + +The persistence layer should deal with envelopes, routing, MessagePack payload +bytes, and schema identity. Application code should not paw at loose JSON files +like a sad little bureaucrat with a clipboard. + +## What It Is + +CultCache is a domain cache with persistence adapters. + +- `CultCache` is the query and mutation surface. +- Domain structs implement `DatabaseEntry`. +- Entries are stored behind a `type::key` identity, so multiple entry types + can share a logical key without colliding. +- Each payload is encoded directly from the known `DatabaseEntry` type into + MessagePack bytes. +- Backing stores are adapters, not the public data model. +- Writes persist to the resolved backing store before the in-memory cache is + updated. +- Type-specific backing stores beat generic backing stores. +- `SingleFileMessagePackBackingStore` is the first concrete store and guards + file access with a sidecar lock file. + +This is not an ORM, not a database, and not distributed consensus in a novelty +hat. If multiple processes write the same backing file, use an external lock or +a coordinator. + +## Current API + +```rust +use cultcache_rs::{CultCache, DatabaseEntry, SingleFileMessagePackBackingStore}; + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "settings")] +struct Settings { + #[cultcache(key = 0)] + theme: String, + + #[cultcache(key = 1, default)] + retries: u32, +} + +let mut cache = CultCache::new(); +cache.register_entry_type::()?; +cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new("cache.msgpack")); +cache.pull_all_backing_stores()?; + +cache.put("app", &Settings { + theme: "ash".to_string(), + retries: 3, +})?; + +let settings = cache.get_required::("app")?; +# Ok::<(), anyhow::Error>(()) +``` + +For a closed set of entries, generate a registry instead of repeating yourself: + +```rust +use cultcache_rs::cultcache_registry; + +cultcache_registry!(GameCultEntries { + Settings, + PlayerData, +}); + +let mut cache = CultCache::new(); +cache.register_registry(GameCultEntries)?; +``` + +The important part is that callers retrieve domain values directly: + +```rust +let settings: Settings = cache.get_required("app")?; +let all_settings: Vec = cache.get_all()?; +``` + +The cache is the ergonomic surface. The backing store is the memory prosthetic. + +## How This Maps From Original CultCache + +The original C# CultCache relies on runtime reflection: + +- cacheable models inherit from `DatabaseEntry` +- the cache scans child classes at startup +- `Get(id)` returns a `T` +- optional `INamedEntry` enables name lookup +- optional field/property indexes are registered by member name +- backing stores pull/push/delete `DatabaseEntry` values + +Rust does not have C#-style assembly scanning or runtime subclass discovery. +The closest honest Rust translation is: + +- domain structs implement a marker trait, `DatabaseEntry` +- the derive macro generates a positional array formatter from explicit integer + field slots +- the envelope carries the polymorphic type discriminator +- unknown persisted type ids fail closed instead of constructing arbitrary + runtime types +- a generated registry should eventually replace repetitive manual + registration + +Manual `register_entry_type::()` is still accepted, but entry identity no +longer needs a hand-written impl: `#[derive(DatabaseEntry)]` declares it beside +the domain type. `cultcache_registry!` provides the current generated resolver +surface for a closed entry set. + +## Intended Rust Ergonomics + +The target user-facing shape should be: + +```rust +#[derive(Clone, DatabaseEntry)] +#[cultcache(type = "player")] +pub struct PlayerData { + #[cultcache(key = 0)] + pub id: String, + + #[cultcache(key = 1)] + pub name: String, + + #[cultcache(key = 2)] + pub faction: String, +} + +let mut cache = CultCache::builder() + .with_generated_entries(game_entries()) + .with_generic_store(SingleFileMessagePackBackingStore::new("cache.msgpack")) + .build()?; + +cache.pull_all_backing_stores()?; +let player = cache.get_required::("player:ari")?; +``` + +The current `cultcache_registry!` macro is deliberately simple. The remaining +ergonomic endpoint is automatic inventory or build-script generation so users +do not have to maintain the registry list by hand. + +The registry is the Rust equivalent of C# reflection and MessagePack resolver +setup. It should know: + +- entry type id +- generated MessagePack/serde formatter path +- Rust type name / schema name +- optional store domain route +- optional name key extractor +- optional secondary indexes +- optional global singleton marker + +In other words: code generation should restore the original CultCache feeling +without pretending Rust has runtime reflection hiding under the floorboards. + +## Current Surface + +- `CultCache::new` +- `register_entry_type::` +- `add_generic_backing_store` +- `add_backing_store` +- `pull_all_backing_stores` +- `get::` +- `get_required::` +- `get_envelope::` +- `get_required_envelope::` +- `get_all::` +- `put::` +- `put_envelope::` +- `update::` +- `delete::` +- `snapshot` +- `SingleFileMessagePackBackingStore` + +## Backing Store Routing + +Generic store: + +```rust +cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new("cache.msgpack")); +``` + +Type-specific store: + +```rust +cache.add_backing_store( + SingleFileMessagePackBackingStore::new("players.msgpack"), + ["player"], +); +``` + +When writing a `PlayerData`, the cache checks type-specific stores first. If none +match, it writes to the first generic store. Later matching stores are mirrors. + +This mirrors the C# behavior: + +- specific domain stores own their domain +- the first generic store is the primary generic write target +- later generic stores mirror writes +- this is not multi-master + +## Persistence Semantics + +`put` persists before mutating the in-memory cache. If persistence fails, the +cache does not pretend the write succeeded. + +The envelope is MessagePack, and the payload inside each envelope is also raw +MessagePack bytes encoded from the registered concrete `DatabaseEntry` type. +That avoids the old bootstrap path where payloads were normalized through +`serde_json::Value`. + +For bit-compatible neighbors, that also enables a real fast lane: + +- `get_envelope::()` exports the canonical persisted bytes for a typed entry +- `put_envelope::()` ingests the same envelope into another cache instance + without re-encoding the payload first + +It still decodes once for validation and typed reads. We are not pretending one +`Vec` became metaphysically zero-copy because we believed in it harder. But +the stupid decode/re-encode loop is gone. + +The `DatabaseEntry` derive does not trust Rust source field order as the durable +schema. Every persisted member must declare a stable integer slot: + +```rust +#[derive(Clone, DatabaseEntry)] +#[cultcache(type = "player")] +pub struct PlayerData { + #[cultcache(key = 0)] + pub id: String, + + #[cultcache(key = 1)] + pub name: String, + + #[cultcache(key = 2, default)] + pub level: u32, +} +``` + +The derive emits a tuple/array formatter. Gaps serialize as nil. Missing fields +marked `default` use `Default::default()` when older payloads are read. Deleted +field slots should stay reserved until an explicit store migration rewrites the +data. + +The single-file MessagePack store rewrites an atomic snapshot. That is a sane +starting point for small typed state surfaces, settings, Epiphany agent memory, +heartbeat state, and other compact control-plane data. Large corpora should use +a sharded store or a real database instead of asking one file to become a +warehouse and then acting wounded when physics invoices us. + +The single-file store uses a sidecar lock file for shared reads and exclusive +writes. That protects ordinary multi-process access to the same file. It is +still not a multi-master replication protocol, and it is not a substitute for a +coordinator when higher-level write ordering matters. + +## Near-Term Ergonomic Improvements + +1. **Derive macro** + - `#[derive(DatabaseEntry)]` + - `#[cultcache(type = "...")]` + - `#[cultcache(key = N)]` on every persisted field + - tuple/array formatter generation landed + - optional `#[cultcache(name)]`, `#[cultcache(index)]`, and + `#[cultcache(global)]` are still future work + +2. **Generated registry** + - a `CultCacheRegistry` trait + - `cache.register_registry(GameCultEntries)` + - basic macro registry landed + - automatic inventory/build-script generation and store routes are future work + +3. **Name and index lookups** + - `cache.get_by_name::("Potion")` + - `cache.get_by_index::("faction", "Lucent")` + - generated extractors instead of reflection strings + +4. **Schema/version metadata** + - schema version in the envelope or payload metadata + - explicit migration hooks + - refusal on unknown persisted entry types unless a migration/resolver is + installed + +5. **Projection helpers** + - JSON projection for review and git diffs + - vector stripping / large-field elision for agent memory surfaces + +## Why Not Skip Registration Immediately? + +Rust can only deserialize polymorphic data into concrete types if something +maps the persisted `type` discriminator to the Rust type. In C#, reflection and +MessagePack resolvers can discover a closed `DatabaseEntry` inheritance tree at +runtime and route it through known formatters. In Rust, that map has to come from +somewhere: + +- explicit registration +- a generated registry +- a macro inventory crate +- a hand-written resolver + +Explicit registration is the simplest honest implementation. Generated registry +is the ergonomic destination. Hand-written resolver tables are the punishment +we give ourselves if we get lazy. + +## Security Model + +CultCache should never parse arbitrary structured data into arbitrary runtime +types. It only accepts persisted envelopes whose `type` discriminator resolves +to a known `DatabaseEntry` type registered in the cache or generated registry. + +That is the Rust version of the original trick: + +- C# refuses to store objects outside the `DatabaseEntry` root type. +- Runtime-visible `DatabaseEntry` subclasses get dedicated formatters. +- Unknown or unregistered types are rejected. +- MessagePack payload bytes deserialize through known concrete formatters, not + an open-world type loader or generic structured-value bridge. + +The derive/registry work should preserve that shape. Its job is to remove +manual registration ceremony, not to loosen the closed-world boundary. + +## License + +Private GameCult infrastructure for now. Public packaging can wait until the +API grows enough teeth to deserve strangers. diff --git a/packages/cultcache-rs/cultcache-rs-derive/Cargo.toml b/packages/cultcache-rs/cultcache-rs-derive/Cargo.toml new file mode 100644 index 00000000..74d52a4f --- /dev/null +++ b/packages/cultcache-rs/cultcache-rs-derive/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "cultcache-rs-derive" +version = "0.1.0" +edition = "2024" +publish = false + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1" +quote = "1" +syn = { version = "2", features = ["derive", "parsing"] } diff --git a/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs b/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs new file mode 100644 index 00000000..f05a23f3 --- /dev/null +++ b/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs @@ -0,0 +1,243 @@ +use proc_macro::TokenStream; +use quote::format_ident; +use quote::quote; +use syn::Data; +use syn::DeriveInput; +use syn::Fields; +use syn::LitInt; +use syn::LitStr; +use syn::Type; +use syn::parse_macro_input; + +#[proc_macro_derive(DatabaseEntry, attributes(cultcache))] +pub fn derive_database_entry(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + let ident = input.ident.clone(); + let mut entry_type: Option = None; + let mut schema_name: Option = None; + + for attribute in input + .attrs + .iter() + .filter(|attr| attr.path().is_ident("cultcache")) + { + let parse_result = attribute.parse_nested_meta(|meta| { + if meta.path.is_ident("type") { + entry_type = Some(meta.value()?.parse()?); + Ok(()) + } else if meta.path.is_ident("schema") { + schema_name = Some(meta.value()?.parse()?); + Ok(()) + } else { + Err(meta.error("unsupported cultcache attribute")) + } + }); + if let Err(error) = parse_result { + return error.to_compile_error().into(); + } + } + + let Some(entry_type) = entry_type else { + return syn::Error::new_spanned( + ident, + "DatabaseEntry derive requires #[cultcache(type = \"...\")]", + ) + .to_compile_error() + .into(); + }; + let schema_name = schema_name.unwrap_or_else(|| LitStr::new(&ident.to_string(), ident.span())); + + let fields = match input.data { + Data::Struct(data) => match data.fields { + Fields::Named(fields) => fields.named, + _ => { + return syn::Error::new_spanned( + ident, + "DatabaseEntry derive requires a named struct", + ) + .to_compile_error() + .into(); + } + }, + _ => { + return syn::Error::new_spanned(ident, "DatabaseEntry derive requires a struct") + .to_compile_error() + .into(); + } + }; + + let mut slots: Vec = Vec::new(); + for field in fields { + let Some(field_ident) = field.ident else { + return syn::Error::new_spanned(field.ty, "DatabaseEntry derive requires named fields") + .to_compile_error() + .into(); + }; + let mut key: Option = None; + let mut default = false; + for attribute in field + .attrs + .iter() + .filter(|attr| attr.path().is_ident("cultcache")) + { + let parse_result = attribute.parse_nested_meta(|meta| { + if meta.path.is_ident("key") { + let value: LitInt = meta.value()?.parse()?; + key = Some(value.base10_parse()?); + Ok(()) + } else if meta.path.is_ident("default") { + default = true; + Ok(()) + } else { + Err(meta.error("unsupported field cultcache attribute")) + } + }); + if let Err(error) = parse_result { + return error.to_compile_error().into(); + } + } + let Some(key) = key else { + return syn::Error::new_spanned( + field_ident, + "DatabaseEntry fields require #[cultcache(key = N)]", + ) + .to_compile_error() + .into(); + }; + if slots.iter().any(|slot| slot.key == key) { + return syn::Error::new_spanned( + field_ident, + format!("duplicate DatabaseEntry slot {key}"), + ) + .to_compile_error() + .into(); + } + slots.push(CultCacheSlot { + key, + ident: field_ident, + ty: field.ty, + default, + }); + } + slots.sort_by_key(|slot| slot.key); + let max_slot = slots.iter().map(|slot| slot.key).max().unwrap_or(0); + + let serialize_elements = (0..=max_slot).map(|slot| { + if let Some(field) = slots.iter().find(|field| field.key == slot) { + let field_ident = &field.ident; + quote! { + tuple.serialize_element(&self.#field_ident)?; + } + } else { + quote! { + tuple.serialize_element(&())?; + } + } + }); + + let option_declarations = slots.iter().map(|field| { + let variable = format_ident!("slot_{}", field.key); + let ty = &field.ty; + quote! { + let mut #variable: ::std::option::Option<#ty> = ::std::option::Option::None; + } + }); + + let deserialize_match_arms = (0..=max_slot).map(|slot| { + if let Some(field) = slots.iter().find(|field| field.key == slot) { + let variable = format_ident!("slot_{}", field.key); + let ty = &field.ty; + quote! { + #slot => { + #variable = seq.next_element::<#ty>()?; + } + } + } else { + quote! { + #slot => { + let _ = seq.next_element::<::serde::de::IgnoredAny>()?; + } + } + } + }); + + let build_fields = slots.iter().map(|field| { + let field_ident = &field.ident; + let variable = format_ident!("slot_{}", field.key); + if field.default { + quote! { + #field_ident: #variable.unwrap_or_default() + } + } else { + let field_name = field_ident.to_string(); + quote! { + #field_ident: #variable.ok_or_else(|| ::serde::de::Error::missing_field(#field_name))? + } + } + }); + + quote! { + impl ::cultcache_rs::DatabaseEntry for #ident { + const TYPE: &'static str = #entry_type; + const SCHEMA_NAME: &'static str = #schema_name; + } + + impl ::serde::Serialize for #ident { + fn serialize(&self, serializer: S) -> ::std::result::Result + where + S: ::serde::Serializer, + { + use ::serde::ser::SerializeTuple; + let mut tuple = serializer.serialize_tuple(#max_slot + 1)?; + #(#serialize_elements)* + tuple.end() + } + } + + impl<'de> ::serde::Deserialize<'de> for #ident { + fn deserialize(deserializer: D) -> ::std::result::Result + where + D: ::serde::Deserializer<'de>, + { + struct CultCacheVisitor; + + impl<'de> ::serde::de::Visitor<'de> for CultCacheVisitor { + type Value = #ident; + + fn expecting(&self, formatter: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { + formatter.write_str(concat!("a CultCache array for ", stringify!(#ident))) + } + + fn visit_seq(self, mut seq: A) -> ::std::result::Result + where + A: ::serde::de::SeqAccess<'de>, + { + #(#option_declarations)* + let mut index = 0usize; + while index <= #max_slot { + match index { + #(#deserialize_match_arms)* + _ => unreachable!(), + } + index += 1; + } + while let ::std::option::Option::Some(_) = seq.next_element::<::serde::de::IgnoredAny>()? {} + Ok(#ident { + #(#build_fields),* + }) + } + } + + deserializer.deserialize_tuple(#max_slot + 1, CultCacheVisitor) + } + } + } + .into() +} + +struct CultCacheSlot { + key: usize, + ident: syn::Ident, + ty: Type, + default: bool, +} diff --git a/packages/cultcache-rs/examples/cultcache_interop.rs b/packages/cultcache-rs/examples/cultcache_interop.rs new file mode 100644 index 00000000..4049acf8 --- /dev/null +++ b/packages/cultcache-rs/examples/cultcache_interop.rs @@ -0,0 +1,121 @@ +use anyhow::Result; +use anyhow::anyhow; +use cultcache_rs::CultCache; +use cultcache_rs::DatabaseEntry; +use cultcache_rs::SingleFileMessagePackBackingStore; +use serde_json::json; +use std::collections::BTreeMap; + +const INTEROP_TYPE: &str = "cultcache.interop-note"; +const INTEROP_SCHEMA_VERSION: &str = "cultcache.interop_note.v1"; + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "cultcache.interop-note", schema = "CultCacheInteropNote")] +struct CultCacheInteropNote { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + document_id: String, + #[cultcache(key = 2)] + author_runtime_id: String, + #[cultcache(key = 3)] + title: String, + #[cultcache(key = 4)] + body: String, + #[cultcache(key = 5, default)] + tags: Vec, +} + +fn main() -> Result<()> { + let mut args = std::env::args().skip(1); + let mode = args + .next() + .ok_or_else(|| anyhow!("expected mode: write | read"))?; + let options = parse_args(args.collect()); + let file = require_arg(&options, "file")?; + + match mode.as_str() { + "write" => write_note(file, require_arg(&options, "runtime-id")?)?, + "read" => read_note(file)?, + _ => return Err(anyhow!("unknown mode {mode}")), + } + Ok(()) +} + +fn write_note(file: &str, runtime_id: &str) -> Result<()> { + let mut cache = build_cache(file)?; + cache.pull_all_backing_stores()?; + let note = CultCacheInteropNote { + schema_version: INTEROP_SCHEMA_VERSION.to_string(), + document_id: format!("note:{runtime_id}"), + author_runtime_id: runtime_id.to_string(), + title: format!("{runtime_id} wrote a CultCache note"), + body: "The v1 store format is the contract.".to_string(), + tags: vec![ + runtime_id.to_string(), + "rust".to_string(), + "interop".to_string(), + ], + }; + cache.put(¬e.document_id, ¬e)?; + print_note(¬e) +} + +fn read_note(file: &str) -> Result<()> { + let mut cache = build_cache(file)?; + cache.pull_all_backing_stores()?; + let notes = cache.get_all::()?; + let note = notes + .first() + .ok_or_else(|| anyhow!("no {INTEROP_TYPE} records found"))?; + print_note(note) +} + +fn build_cache(file: &str) -> Result { + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(file)); + Ok(cache) +} + +fn print_note(note: &CultCacheInteropNote) -> Result<()> { + println!( + "{}", + serde_json::to_string(&json!({ + "schemaVersion": note.schema_version, + "documentId": note.document_id, + "authorRuntimeId": note.author_runtime_id, + "title": note.title, + "body": note.body, + "tags": note.tags, + }))? + ); + Ok(()) +} + +fn parse_args(args: Vec) -> BTreeMap { + let mut parsed = BTreeMap::new(); + let mut index = 0; + while index < args.len() { + let token = &args[index]; + if !token.starts_with("--") { + index += 1; + continue; + } + let name = token.trim_start_matches("--").to_string(); + let value = args + .get(index + 1) + .cloned() + .unwrap_or_else(|| panic!("missing value for --{name}")); + parsed.insert(name, value); + index += 2; + } + parsed +} + +fn require_arg<'a>(options: &'a BTreeMap, name: &str) -> Result<&'a str> { + options + .get(name) + .map(String::as_str) + .ok_or_else(|| anyhow!("missing required argument --{name}")) +} diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs new file mode 100644 index 00000000..cb56b8a8 --- /dev/null +++ b/packages/cultcache-rs/src/lib.rs @@ -0,0 +1,885 @@ +use anyhow::Context; +use anyhow::Result; +use anyhow::anyhow; +use serde::Serialize; +use serde::de::DeserializeOwned; +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::fs; +use std::fs::File; +use std::fs::OpenOptions; +use std::path::Path; +use std::path::PathBuf; + +extern crate self as cultcache_rs; + +pub use cultcache_rs_derive::DatabaseEntry; + +pub trait DatabaseEntry: Serialize + DeserializeOwned + Clone + Send + 'static { + const TYPE: &'static str; + const SCHEMA_NAME: &'static str = "DatabaseEntry"; +} + +pub trait CultCacheRegistry { + fn register_entries(&self, cache: &mut CultCache) -> Result<()>; +} + +#[macro_export] +macro_rules! cultcache_registry { + ($name:ident { $($entry:ty),* $(,)? }) => { + #[derive(Clone, Copy, Debug, Default)] + pub struct $name; + + impl $crate::CultCacheRegistry for $name { + fn register_entries(&self, cache: &mut $crate::CultCache) -> ::anyhow::Result<()> { + $( + cache.register_entry_type::<$entry>()?; + )* + Ok(()) + } + } + }; +} + +#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CultCacheEnvelope { + pub key: String, + #[serde(rename = "type")] + pub r#type: String, + #[serde(with = "serde_bytes")] + pub payload: Vec, + pub stored_at: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema_id: Option, +} + +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +struct PersistedStoreSnapshot( + String, + Vec, + Vec, +); + +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +struct PersistedSchemaCatalogEntry( + String, + String, + String, + String, + String, + Vec, + Vec, +); + +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +struct PersistedSchemaCatalogMember( + u32, + String, + String, + bool, + bool, + Option, + bool, + Option, +); + +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +struct PersistedRecord( + String, + String, + String, + #[serde(with = "serde_bytes")] Vec, +); + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct PushAllOptions { + pub soft: bool, +} + +pub trait CacheBackingStore: Send { + fn pull_all(&self) -> Result>; + fn push(&mut self, entry: &CultCacheEnvelope) -> Result<()>; + fn delete(&mut self, entry: &CultCacheEnvelope) -> Result<()>; + + fn push_all(&mut self, entries: &[CultCacheEnvelope], _options: PushAllOptions) -> Result<()> { + let existing = self.pull_all()?; + for entry in existing { + self.delete(&entry)?; + } + for entry in entries { + self.push(entry)?; + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SingleFileMessagePackBackingStore { + path: PathBuf, +} + +impl SingleFileMessagePackBackingStore { + pub fn new(path: impl Into) -> Self { + Self { path: path.into() } + } + + pub fn path(&self) -> &Path { + &self.path + } + + fn read_all_unlocked(&self) -> Result> { + if !self.path.exists() { + return Ok(Vec::new()); + } + let bytes = fs::read(&self.path) + .with_context(|| format!("failed to read {}", self.path.display()))?; + if bytes.is_empty() { + return Ok(Vec::new()); + } + decode_store_snapshot(&bytes).or_else(|_| { + rmp_serde::from_slice(&bytes) + .with_context(|| format!("failed to decode MessagePack {}", self.path.display())) + }) + } + + fn write_all_unlocked(&self, entries: &[CultCacheEnvelope]) -> Result<()> { + if let Some(parent) = self.path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("failed to create {}", parent.display()))?; + } + let bytes = rmp_serde::to_vec(&encode_store_snapshot(entries)) + .context("failed to encode MessagePack")?; + let tmp_path = temporary_path_for(&self.path); + fs::write(&tmp_path, bytes) + .with_context(|| format!("failed to write {}", tmp_path.display()))?; + if self.path.exists() { + fs::remove_file(&self.path) + .with_context(|| format!("failed to replace {}", self.path.display()))?; + } + fs::rename(&tmp_path, &self.path).with_context(|| { + format!( + "failed to move {} to {}", + tmp_path.display(), + self.path.display() + ) + })?; + Ok(()) + } + + fn with_shared_lock(&self, action: impl FnOnce() -> Result) -> Result { + let lock = self.open_lock_file()?; + fs2::FileExt::lock_shared(&lock) + .with_context(|| format!("failed to lock {}", self.lock_path().display()))?; + let result = action(); + fs2::FileExt::unlock(&lock) + .with_context(|| format!("failed to unlock {}", self.lock_path().display()))?; + result + } + + fn with_exclusive_lock(&self, action: impl FnOnce() -> Result) -> Result { + let lock = self.open_lock_file()?; + fs2::FileExt::lock_exclusive(&lock) + .with_context(|| format!("failed to lock {}", self.lock_path().display()))?; + let result = action(); + fs2::FileExt::unlock(&lock) + .with_context(|| format!("failed to unlock {}", self.lock_path().display()))?; + result + } + + fn open_lock_file(&self) -> Result { + let lock_path = self.lock_path(); + if let Some(parent) = lock_path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("failed to create {}", parent.display()))?; + } + OpenOptions::new() + .create(true) + .read(true) + .truncate(false) + .write(true) + .open(&lock_path) + .with_context(|| format!("failed to open {}", lock_path.display())) + } + + fn lock_path(&self) -> PathBuf { + let mut lock_name = self + .path + .file_name() + .map(|value| value.to_os_string()) + .unwrap_or_else(|| "cultcache.msgpack".into()); + lock_name.push(".lock"); + self.path.with_file_name(lock_name) + } +} + +impl CacheBackingStore for SingleFileMessagePackBackingStore { + fn pull_all(&self) -> Result> { + self.with_shared_lock(|| self.read_all_unlocked()) + } + + fn push(&mut self, entry: &CultCacheEnvelope) -> Result<()> { + self.with_exclusive_lock(|| { + let mut entries = self.read_all_unlocked()?; + entries.retain(|candidate| entry_id(candidate) != entry_id(entry)); + entries.push(entry.clone()); + entries.sort_by_key(entry_id); + self.write_all_unlocked(&entries) + }) + } + + fn delete(&mut self, entry: &CultCacheEnvelope) -> Result<()> { + self.with_exclusive_lock(|| { + let mut entries = self.read_all_unlocked()?; + entries.retain(|candidate| entry_id(candidate) != entry_id(entry)); + self.write_all_unlocked(&entries) + }) + } + + fn push_all(&mut self, entries: &[CultCacheEnvelope], _options: PushAllOptions) -> Result<()> { + self.with_exclusive_lock(|| { + let mut entries = entries.to_vec(); + entries.sort_by_key(entry_id); + self.write_all_unlocked(&entries) + }) + } +} + +struct CultCacheStoreRegistration { + store: Box, + types: BTreeSet, +} + +pub struct CultCache { + definitions: BTreeMap, + entries: BTreeMap, + stores: Vec, +} + +impl CultCache { + pub fn new() -> Self { + Self { + definitions: BTreeMap::new(), + entries: BTreeMap::new(), + stores: Vec::new(), + } + } + + pub fn register_entry_type(&mut self) -> Result<()> { + if T::TYPE.trim().is_empty() { + return Err(anyhow!( + "CultCache entry types must declare a non-empty type" + )); + } + if let Some(existing_schema) = self.definitions.get(T::TYPE) + && *existing_schema != T::SCHEMA_NAME + { + return Err(anyhow!( + "CultCache already has a different definition registered for type {:?}", + T::TYPE + )); + } + self.definitions.insert(T::TYPE.to_string(), T::SCHEMA_NAME); + Ok(()) + } + + pub fn register_document_type(&mut self) -> Result<()> { + self.register_entry_type::() + } + + pub fn register_registry(&mut self, registry: R) -> Result<&mut Self> { + registry.register_entries(self)?; + Ok(self) + } + + pub fn add_backing_store( + &mut self, + store: impl CacheBackingStore + 'static, + types: impl IntoIterator>, + ) { + self.stores.push(CultCacheStoreRegistration { + store: Box::new(store), + types: types.into_iter().map(Into::into).collect(), + }); + } + + pub fn add_generic_backing_store(&mut self, store: impl CacheBackingStore + 'static) { + self.add_backing_store(store, Vec::::new()); + } + + pub fn pull_all_backing_stores(&mut self) -> Result<()> { + self.entries.clear(); + let known_types: BTreeSet = self.definitions.keys().cloned().collect(); + for registration in &mut self.stores { + for entry in registration.store.pull_all()? { + if !known_types.contains(&entry.r#type) { + return Err(anyhow!( + "No schema is registered for persisted entry type {:?}", + entry.r#type + )); + } + self.entries.insert(entry_id(&entry), entry); + } + } + Ok(()) + } + + pub fn get(&self, key: &str) -> Result> { + self.require_entry_type::()?; + let Some(entry) = self.entries.get(&entry_id_parts(T::TYPE, key)) else { + return Ok(None); + }; + let payload = rmp_serde::from_slice(&entry.payload).with_context(|| { + format!( + "failed to decode CultCache entry {:?} at key {:?} as {}", + T::TYPE, + key, + T::SCHEMA_NAME + ) + })?; + Ok(Some(payload)) + } + + pub fn get_required(&self, key: &str) -> Result { + self.get::(key)? + .ok_or_else(|| anyhow!("CultCache has no {:?} entry at key {:?}", T::TYPE, key)) + } + + pub fn get_envelope(&self, key: &str) -> Result> { + self.require_entry_type::()?; + Ok(self.entries.get(&entry_id_parts(T::TYPE, key)).cloned()) + } + + pub fn get_required_envelope(&self, key: &str) -> Result { + self.get_envelope::(key)? + .ok_or_else(|| anyhow!("CultCache has no {:?} envelope at key {:?}", T::TYPE, key)) + } + + pub fn get_all(&self) -> Result> { + self.require_entry_type::()?; + let mut values = Vec::new(); + for entry in self.entries.values() { + if entry.r#type != T::TYPE { + continue; + } + values.push(rmp_serde::from_slice(&entry.payload).with_context(|| { + format!( + "failed to decode CultCache entry {:?} at key {:?} as {}", + T::TYPE, + entry.key, + T::SCHEMA_NAME + ) + })?); + } + Ok(values) + } + + pub fn put(&mut self, key: impl Into, value: &T) -> Result { + self.require_entry_type::()?; + let key = key.into(); + let payload = rmp_serde::to_vec(value).with_context(|| { + format!( + "failed to encode CultCache entry {:?} at key {:?} as {}", + T::TYPE, + key, + T::SCHEMA_NAME + ) + })?; + let parsed: T = rmp_serde::from_slice(&payload).with_context(|| { + format!( + "failed to validate CultCache entry {:?} at key {:?} as {}", + T::TYPE, + key, + T::SCHEMA_NAME + ) + })?; + let entry = CultCacheEnvelope { + key: key.clone(), + r#type: T::TYPE.to_string(), + payload, + stored_at: now_utc_second(), + schema_id: Some(T::TYPE.to_string()), + }; + let route = self.resolve_route_indices(T::TYPE); + let Some(primary_index) = route.first().copied() else { + return Err(anyhow!( + "No backing store is registered for entry type {:?}", + T::TYPE + )); + }; + self.stores[primary_index].store.push(&entry)?; + for mirror_index in route.iter().skip(1).copied() { + self.stores[mirror_index].store.push(&entry)?; + } + self.entries.insert(entry_id(&entry), entry); + Ok(parsed) + } + + pub fn put_envelope(&mut self, entry: CultCacheEnvelope) -> Result { + self.require_entry_type::()?; + if entry.r#type != T::TYPE { + return Err(anyhow!( + "CultCache envelope type {:?} does not match registered Rust type {:?}", + entry.r#type, + T::TYPE + )); + } + if entry.key.trim().is_empty() { + return Err(anyhow!( + "CultCache envelope keys for type {:?} must be non-empty", + T::TYPE + )); + } + if entry.stored_at.trim().is_empty() { + return Err(anyhow!( + "CultCache envelope stored_at for type {:?} must be non-empty", + T::TYPE + )); + } + + let parsed: T = rmp_serde::from_slice(&entry.payload).with_context(|| { + format!( + "failed to validate CultCache envelope {:?} at key {:?} as {}", + T::TYPE, + entry.key, + T::SCHEMA_NAME + ) + })?; + let route = self.resolve_route_indices(T::TYPE); + let Some(primary_index) = route.first().copied() else { + return Err(anyhow!( + "No backing store is registered for entry type {:?}", + T::TYPE + )); + }; + self.stores[primary_index].store.push(&entry)?; + for mirror_index in route.iter().skip(1).copied() { + self.stores[mirror_index].store.push(&entry)?; + } + self.entries.insert(entry_id(&entry), entry); + Ok(parsed) + } + + pub fn update(&mut self, key: &str, updater: F) -> Result + where + T: DatabaseEntry, + F: FnOnce(Option) -> T, + { + let current = self.get::(key)?; + self.put::(key.to_string(), &updater(current)) + } + + pub fn delete(&mut self, key: &str) -> Result { + self.require_entry_type::()?; + let id = entry_id_parts(T::TYPE, key); + let Some(entry) = self.entries.get(&id).cloned() else { + return Ok(false); + }; + let route = self.resolve_route_indices(T::TYPE); + let Some(primary_index) = route.first().copied() else { + return Err(anyhow!( + "No backing store is registered for entry type {:?}", + T::TYPE + )); + }; + self.stores[primary_index].store.delete(&entry)?; + for mirror_index in route.iter().skip(1).copied() { + self.stores[mirror_index].store.delete(&entry)?; + } + self.entries.remove(&id); + Ok(true) + } + + pub fn snapshot(&self) -> Vec { + self.entries.values().cloned().collect() + } + + fn require_entry_type(&self) -> Result<()> { + match self.definitions.get(T::TYPE) { + Some(schema_name) if *schema_name == T::SCHEMA_NAME => Ok(()), + _ => Err(anyhow!( + "CultCache entry type {:?} is not registered on this cache instance", + T::TYPE + )), + } + } + + fn resolve_route_indices(&self, type_id: &str) -> Vec { + let type_specific: Vec = self + .stores + .iter() + .enumerate() + .filter_map(|(index, registration)| { + registration.types.contains(type_id).then_some(index) + }) + .collect(); + if !type_specific.is_empty() { + return type_specific; + } + self.stores + .iter() + .enumerate() + .filter_map(|(index, registration)| registration.types.is_empty().then_some(index)) + .collect() + } +} + +impl Default for CultCache { + fn default() -> Self { + Self::new() + } +} + +fn entry_id(entry: &CultCacheEnvelope) -> String { + entry_id_parts(&entry.r#type, &entry.key) +} + +fn entry_id_parts(r#type: &str, key: &str) -> String { + format!("{type}::{key}", type = r#type) +} + +fn now_utc_second() -> String { + chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true) +} + +fn encode_store_snapshot(entries: &[CultCacheEnvelope]) -> PersistedStoreSnapshot { + let mut schema_names = BTreeSet::::new(); + for entry in entries { + schema_names.insert( + entry + .schema_id + .clone() + .unwrap_or_else(|| entry.r#type.clone()), + ); + } + + let catalog = schema_names + .into_iter() + .map(|schema_id| { + PersistedSchemaCatalogEntry( + schema_id.clone(), + schema_id.clone(), + format!("{schema_id}.v1"), + schema_id.clone(), + format!( + "{{\"schemaName\":\"{}\",\"schemaVersion\":\"{}.v1\",\"members\":[]}}", + escape_json_string(&schema_id), + escape_json_string(&schema_id) + ), + vec![schema_id], + Vec::new(), + ) + }) + .collect(); + let records = entries + .iter() + .map(|entry| { + PersistedRecord( + entry.key.clone(), + entry + .schema_id + .clone() + .unwrap_or_else(|| entry.r#type.clone()), + entry.stored_at.clone(), + entry.payload.clone(), + ) + }) + .collect(); + + PersistedStoreSnapshot("cultcache.store.v1".to_string(), catalog, records) +} + +fn decode_store_snapshot(bytes: &[u8]) -> Result> { + let snapshot: PersistedStoreSnapshot = + rmp_serde::from_slice(bytes).context("failed to decode CultCache v1 snapshot")?; + if snapshot.0 != "cultcache.store.v1" { + return Err(anyhow!("unsupported CultCache snapshot {}", snapshot.0)); + } + + let catalog = snapshot + .1 + .into_iter() + .map(|entry| (entry.0, entry.1)) + .collect::>(); + snapshot + .2 + .into_iter() + .map(|record| { + let r#type = catalog.get(&record.1).cloned().ok_or_else(|| { + anyhow!( + "CultCache record {:?} references missing schema {:?}", + record.0, + record.1 + ) + })?; + Ok(CultCacheEnvelope { + key: record.0, + r#type, + stored_at: record.2, + payload: record.3, + schema_id: Some(record.1), + }) + }) + .collect() +} + +fn escape_json_string(value: &str) -> String { + value.replace('\\', "\\\\").replace('"', "\\\"") +} + +fn temporary_path_for(path: &Path) -> PathBuf { + let mut file_name = path + .file_name() + .map(|value| value.to_os_string()) + .unwrap_or_else(|| "cultcache.msgpack".into()); + file_name.push(format!(".{}.tmp", uuid::Uuid::new_v4())); + path.with_file_name(file_name) +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + + #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] + #[cultcache(type = "settings")] + struct Settings { + #[cultcache(key = 0)] + theme: String, + #[cultcache(key = 1, default)] + retries: u32, + } + + #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] + #[cultcache(type = "note")] + struct Note { + #[cultcache(key = 0)] + title: String, + #[cultcache(key = 1)] + body: String, + } + + cultcache_registry!(TestEntries { Settings, Note }); + + #[test] + fn familiar_cultcache_flow_persists_and_reloads_typed_documents() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("cache.msgpack"); + let settings = Settings { + theme: "ash".to_string(), + retries: 3, + }; + + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path)); + cache.pull_all_backing_stores()?; + cache.put("app", &settings)?; + assert_eq!(cache.get_required::("app")?, settings); + + let mut reloaded = CultCache::new(); + reloaded.register_entry_type::()?; + reloaded.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path)); + reloaded.pull_all_backing_stores()?; + assert_eq!(reloaded.get_required::("app")?, settings); + Ok(()) + } + + #[test] + fn entry_identity_is_polymorphic_by_type_and_key() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("cache.msgpack"); + let mut cache = CultCache::new(); + cache.register_registry(TestEntries)?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path)); + + cache.put( + "shared", + &Settings { + theme: "green".to_string(), + retries: 1, + }, + )?; + cache.put( + "shared", + &Note { + title: "same key".to_string(), + body: "different type".to_string(), + }, + )?; + + assert_eq!(cache.snapshot().len(), 2); + assert_eq!(cache.get_required::("shared")?.title, "same key"); + assert_eq!(cache.get_required::("shared")?.theme, "green"); + Ok(()) + } + + #[test] + fn type_specific_store_routes_before_generic_store() -> Result<()> { + let temp = tempfile::tempdir()?; + let generic_path = temp.path().join("generic.msgpack"); + let settings_path = temp.path().join("settings.msgpack"); + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.register_entry_type::()?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic_path)); + cache.add_backing_store( + SingleFileMessagePackBackingStore::new(&settings_path), + ["settings"], + ); + + cache.put( + "app", + &Settings { + theme: "ash".to_string(), + retries: 3, + }, + )?; + cache.put( + "memo", + &Note { + title: "hello".to_string(), + body: "world".to_string(), + }, + )?; + + let generic_entries = SingleFileMessagePackBackingStore::new(&generic_path).pull_all()?; + let settings_entries = SingleFileMessagePackBackingStore::new(&settings_path).pull_all()?; + assert_eq!(generic_entries[0].r#type, "note"); + assert_eq!(settings_entries[0].r#type, "settings"); + Ok(()) + } + + #[test] + fn update_and_delete_follow_the_cache_api() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("cache.msgpack"); + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path)); + + let updated = cache.update::("app", |current| { + let mut current = current.unwrap_or(Settings { + theme: "ash".to_string(), + retries: 0, + }); + current.retries += 1; + current + })?; + assert_eq!(updated.retries, 1); + assert!(cache.delete::("app")?); + assert!(cache.get::("app")?.is_none()); + Ok(()) + } + + #[test] + fn pull_rejects_unregistered_persisted_entry_type() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("cache.msgpack"); + let mut store = SingleFileMessagePackBackingStore::new(&store_path); + store.push(&CultCacheEnvelope { + key: "unknown".to_string(), + r#type: "unregistered".to_string(), + payload: rmp_serde::to_vec(&1_u8)?, + stored_at: now_utc_second(), + schema_id: Some("unregistered".to_string()), + })?; + + let mut cache = CultCache::new(); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path)); + let error = cache.pull_all_backing_stores().unwrap_err(); + assert!( + error + .to_string() + .contains("No schema is registered for persisted entry type") + ); + Ok(()) + } + + #[test] + fn payload_is_binary_messagepack_not_json_value() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("cache.msgpack"); + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path)); + cache.put( + "app", + &Settings { + theme: "ash".to_string(), + retries: 3, + }, + )?; + + let entry = cache.snapshot().remove(0); + let decoded: Settings = rmp_serde::from_slice(&entry.payload)?; + assert_eq!(decoded.theme, "ash"); + assert!(!entry.payload.is_empty()); + Ok(()) + } + + #[test] + fn corrupted_payload_fails_during_typed_retrieval() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("cache.msgpack"); + let mut store = SingleFileMessagePackBackingStore::new(&store_path); + store.push(&CultCacheEnvelope { + key: "app".to_string(), + r#type: "settings".to_string(), + payload: vec![0xc1], + stored_at: now_utc_second(), + schema_id: Some("settings".to_string()), + })?; + + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path)); + cache.pull_all_backing_stores()?; + let error = cache.get_required::("app").unwrap_err(); + assert!( + error + .to_string() + .contains("failed to decode CultCache entry") + ); + Ok(()) + } + + #[test] + fn put_envelope_reuses_existing_messagepack_payload() -> Result<()> { + let temp = tempfile::tempdir()?; + let origin_store = temp.path().join("origin.msgpack"); + let target_store = temp.path().join("target.msgpack"); + + let mut origin = CultCache::new(); + origin.register_entry_type::()?; + origin.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&origin_store)); + origin.put( + "app", + &Settings { + theme: "ash".to_string(), + retries: 3, + }, + )?; + + let envelope = origin.get_required_envelope::("app")?; + + let mut target = CultCache::new(); + target.register_entry_type::()?; + target.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&target_store)); + let applied = target.put_envelope::(envelope.clone())?; + + assert_eq!( + applied, + Settings { + theme: "ash".to_string(), + retries: 3, + } + ); + assert_eq!(target.get_required::("app")?, applied); + assert_eq!( + target.get_required_envelope::("app")?.payload, + envelope.payload + ); + Ok(()) + } +} diff --git a/packages/cultcache-ts/.gitignore b/packages/cultcache-ts/.gitignore new file mode 100644 index 00000000..28b29219 --- /dev/null +++ b/packages/cultcache-ts/.gitignore @@ -0,0 +1,5 @@ +node_modules/ +dist-test/ +dist-inspector/ +release/ +release-inspector/ diff --git a/packages/cultcache-ts/README.md b/packages/cultcache-ts/README.md new file mode 100644 index 00000000..898b97db --- /dev/null +++ b/packages/cultcache-ts/README.md @@ -0,0 +1,290 @@ +# CultCacheTS + +`CultCacheTS` is the TypeScript port of the useful part of GameCult's `CultCache` idea: consumer code talks to typed domain documents, while a closed-world persistence layer deals with schemas, routing, name lookups, indexes, globals, and raw MessagePack bytes. + +This is not an ORM in a fake mustache. It is for cases where you want: + +- one cache API for heterogeneous domain documents +- typed reads and writes instead of every script massaging anonymous blobs +- lookup by type, logical key, name, and indexed property +- singleton-style global documents +- backing-store routing instead of every caller pawing raw files directly +- a persistence boundary that refuses unregistered polymorphic junk + +## Current Shape + +- document types are registered explicitly +- a document schema only needs a `parse(...)` function, so generated JSON-schema + contracts can plug in directly without growing a second hand-written Zod body +- each persisted record stores: + - `key` + - `schemaId` + - raw MessagePack `payload` bytes + - `storedAt` +- the single-file store writes the CultCache v1 snapshot shape: + - `formatVersion` + - embedded schema catalog + - record array +- payload bytes only decode through the registered document definition for that schema +- name and index maps live in the cache, not in the backing store +- global documents are treated as a singleton per type +- backing stores are adapters, not the public data model + +That keeps the polymorphic boundary tight without making callers write boilerplate by candlelight. + +## Huginn Inspector + +CultCache files may use the `.cc` extension. The bytes are still the canonical +`cultcache.store.v1` MessagePack snapshot; the extension is the human handle, not +a second format. + +Run the local inspector during development: + +```sh +npm run dev:inspector +``` + +Build the Vite inspector bundle: + +```sh +npm run build:inspector +``` + +Build the desktop package: + +```sh +npm run dist:inspector +``` + +Release builds must carry a new semantic version before any deployable artifact +is produced. While the package is pre-1.0, breaking public behavior increments +the minor version, compatible fixes increment the patch version, and the +generated Huginn executable must use that package version in its filename. + +Huginn is read-only. Drop a `.cc`, `.msgpack`, or `.mpack` file onto the window to +inspect the snapshot header, schema catalog, records, decoded MessagePack +payload previews, and an Norn graph cloud of the file's structured payload data +without registering application schemas. + +Huginn's background field uses the graph node AABB as its single world-space +authority. The WebGPU particle pass keeps a million particle slots available, +intersects the current screen bounds with the artwork bounds, and assigns only +visible quadtree cells to those slots. Each visible cell is seeded with the +Aetheria Stardust `RandomFirst` hash and xorshift follow-up sequence, then each +particle pair trades opacity with sine/cosine phase while tracing a short +segment through the packed flow texture. The packed map uses `RG` for flow, `B` +for structure/curvature/detail pressure, and `A` for emission/field strength; +particle color comes from the source Huginn albedo at the starting UV so the +flow carries brushstrokes instead of repainting them at the final sample. +Zooming raises the live quadtree level; coarse cells fade down while finer cells +seeded from the same spatial lattice fade in around them, so detail density +rises without changing the field's identity. + +## Example + +```ts +import { z } from "zod"; +import { + CultCache, + SingleFileMessagePackBackingStore, + defineDocumentRegistry, + defineDocumentType, +} from "cultcache-ts"; + +const itemDocument = defineDocumentType({ + type: "item", + schemaId: "gamecult.item.v1", + schemaName: "item", + schemaVersion: "item.v1", + schema: z.object({ + name: z.string(), + category: z.string(), + value: z.number().int(), + }), + name: "name", + indexes: { + category: "category", + }, +}); + +const settingsDocument = defineDocumentType({ + type: "settings", + schema: z.object({ + theme: z.string(), + retries: z.number().int().nonnegative(), + }), + global: true, +}); + +const registry = defineDocumentRegistry(itemDocument, settingsDocument); + +const cache = CultCache.builder() + .withRegistry(registry) + .withGenericStore(new SingleFileMessagePackBackingStore("cache.msgpack")) + .build(); + +await cache.pullAllBackingStores(); + +await cache.put(itemDocument, "item:potion", { + name: "Potion", + category: "Consumable", + value: 50, +}); + +await cache.putGlobal(settingsDocument, { + theme: "ash", + retries: 3, +}); + +const potion = cache.getByName(itemDocument, "Potion"); +const consumable = cache.getByIndex(itemDocument, "category", "Consumable"); +const settings = cache.getRequiredGlobal(settingsDocument); +``` + +## Public Surface + +- `defineDocumentType(...)` +- `defineDocumentRegistry(...)` +- `CultCache.builder()` +- `registerDocumentType(...)` +- `registerRegistry(...)` +- `registerNameLookup(...)` +- `registerIndex(...)` +- `addBackingStore(...)` +- `addGenericBackingStore(...)` +- `pullAllBackingStores()` +- `get(...)` +- `getRequired(...)` +- `getAll(...)` +- `getKeyByName(...)` +- `getByName(...)` +- `getKeyByIndex(...)` +- `getByIndex(...)` +- `getGlobal(...)` +- `getRequiredGlobal(...)` +- `put(...)` +- `putGlobal(...)` +- `update(...)` +- `updateGlobal(...)` +- `delete(...)` +- `deleteGlobal(...)` +- `snapshot()` +- `getEnvelope(...)` +- `getRequiredEnvelope(...)` +- `getGlobalEnvelope(...)` +- `putEnvelope(...)` + +Document definitions may also carry persistence metadata for cross-language +stores: `schemaId`, `schemaName`, `schemaVersion`, `contentHash`, +`canonicalSchemaJson`, `compatibleSchemaIds`, and slot `members`. + +## Name, Index, and Global Semantics + +`name` and `indexes` can be declared on the document definition: + +```ts +const playerDocument = defineDocumentType({ + type: "player", + schema: playerSchema, + name: "displayName", + indexes: { + faction: "faction", + email: "email", + }, +}); +``` + +You can also register them later if you feel like changing the machine after boot: + +```ts +cache.registerNameLookup(playerDocument, "displayName"); +cache.registerIndex(playerDocument, "faction", "faction"); +``` + +Global documents are singleton-style per type: + +```ts +await cache.putGlobal(settingsDocument, { theme: "ash", retries: 3 }); +const settings = cache.getRequiredGlobal(settingsDocument); +``` + +If persisted state contains two globals for the same type, `pullAllBackingStores()` fails closed instead of pretending the contradiction is charming. + +## Typed Formatters + +Each document definition can provide a custom formatter: + +```ts +const combatLogDocument = defineDocumentType({ + type: "combat-log", + schema: combatLogSchema, + formatter: { + encode(value) { + return myFastBinaryEncoder(value); + }, + decode(payload) { + return myFastBinaryDecoder(payload); + }, + }, +}); +``` + +If you do not supply one, `CultCacheTS` uses MessagePack plus Zod validation by default. + +Zod is no longer mandatory. Any schema object with a `parse(...)` function will +do, which means generated Ajv-backed JSON-schema contracts can feed the cache +directly as long as they expose the same parse surface. + +The important boundary is this: + +- payload bytes are persisted as bytes +- decoding goes through the registered formatter for that document type +- the decoded value is still validated against the declared schema + +So callers get typed values, and the persistence layer does not become an open polymorphic sewer. + +## Raw Envelope Fast Path + +For bit-compatible neighbors such as CultNet peers that already share the same +formatter contract, `CultCacheTS` can now move the persisted envelope directly: + +- `getEnvelope(...)` / `getRequiredEnvelope(...)` export the canonical + MessagePack payload bytes for a typed document +- `putEnvelope(...)` ingests that envelope into another cache instance without + re-encoding the payload first + +Raw envelopes may carry `schemaId` and catalog metadata. Domain callers should +not need that sludge. Replication, migration tooling, and wire-compatibility +tests do. + +That still is not magic shared memory. The cache decodes once so lookups and +typed reads stay honest. But it stops doing the stupid part where identical +MessagePack payload bytes get decoded into a generic value, then re-encoded +into the same bytes again just to cross the room. + +## Security Model + +`CultCacheTS` should never deserialize arbitrary persisted payloads into arbitrary runtime shapes. + +It only accepts persisted records whose `schemaId` or legacy `type` discriminator resolves to a registered document definition on the cache instance. Unknown schemas fail immediately. Known schemas decode through the registered formatter and schema only. + +That is the TypeScript version of the original CultCache trick: + +- closed discriminator set +- explicit registration +- typed formatter path +- no runtime type-name roulette + +## Persistence Model + +`SingleFileMessagePackBackingStore` is the first concrete store. + +- it writes atomic CultCache v1 MessagePack snapshots +- the top-level MessagePack value is `[formatVersion, schemaCatalog, records]` +- each record is `[key, schemaId, storedAt, payload]` +- each catalog entry is `[schemaId, schemaName, schemaVersion, contentHash, canonicalSchemaJson, compatibleSchemaIds, members]` +- it still reads the older TS/Rust-style envelope array and rewrites repaired legacy payloads as v1 snapshots +- it queues local writes so one process does not step on its own shoelaces +- it persists raw MessagePack payload bytes inside schema-addressed records + +It is a control-plane store, not a tiny god. If multiple processes hammer the same file, you still need a coordinator or an external lock instead of prayer. diff --git a/packages/cultcache-ts/electron-builder.json b/packages/cultcache-ts/electron-builder.json new file mode 100644 index 00000000..28df4bf3 --- /dev/null +++ b/packages/cultcache-ts/electron-builder.json @@ -0,0 +1,28 @@ +{ + "appId": "com.gamecult.cultcache.inspector", + "productName": "Huginn", + "executableName": "Huginn", + "asar": false, + "icon": "inspector/public/hugin.ico", + "extraMetadata": { + "main": "electron/main.cjs" + }, + "files": [ + "electron/main.cjs", + "dist-inspector/**/*" + ], + "directories": { + "output": "release-inspector" + }, + "win": { + "signAndEditExecutable": false, + "target": ["portable"], + "artifactName": "Huginn-${version}.${ext}" + }, + "mac": { + "target": ["dir"] + }, + "linux": { + "target": ["dir"] + } +} diff --git a/packages/cultcache-ts/electron/main.cjs b/packages/cultcache-ts/electron/main.cjs new file mode 100644 index 00000000..2cf76b67 --- /dev/null +++ b/packages/cultcache-ts/electron/main.cjs @@ -0,0 +1,72 @@ +const { app, BrowserWindow } = require("electron"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); + +const devServer = process.env.CULTCACHE_INSPECTOR_DEV_SERVER; +app.setName("Huginn"); +const logPath = path.join(app.getPath("userData"), "huginn-renderer.log"); + +function writeLog(message) { + const line = `[${new Date().toISOString()}] ${message}${os.EOL}`; + try { + fs.mkdirSync(path.dirname(logPath), { recursive: true }); + fs.appendFileSync(logPath, line); + } catch { + // Logging must never be the reason the inspector refuses to open. + } + console.log(line.trimEnd()); +} + +function createWindow() { + const window = new BrowserWindow({ + width: 1180, + height: 780, + minWidth: 820, + minHeight: 560, + title: "Huginn", + icon: path.join(__dirname, "..", "dist-inspector", "hugin-64.png"), + backgroundColor: "#05090d", + autoHideMenuBar: true, + webPreferences: { + contextIsolation: true, + nodeIntegration: false, + sandbox: true, + }, + }); + + window.webContents.on("console-message", (_event, level, message, line, sourceId) => { + writeLog(`renderer console[${level}] ${sourceId}:${line} ${message}`); + }); + window.webContents.on("did-fail-load", (_event, code, description, validatedUrl) => { + writeLog(`renderer failed load ${code} ${description} ${validatedUrl}`); + }); + window.webContents.on("render-process-gone", (_event, details) => { + writeLog(`renderer process gone ${details.reason} exitCode=${details.exitCode}`); + }); + window.webContents.on("did-finish-load", () => { + writeLog(`renderer loaded ${window.webContents.getURL()}`); + }); + + if (devServer) { + void window.loadURL(devServer); + } else { + void window.loadFile(path.join(__dirname, "..", "dist-inspector", "index.html")); + } +} + +app.whenReady().then(() => { + createWindow(); + + app.on("activate", () => { + if (BrowserWindow.getAllWindows().length === 0) { + createWindow(); + } + }); +}); + +app.on("window-all-closed", () => { + if (process.platform !== "darwin") { + app.quit(); + } +}); diff --git a/packages/cultcache-ts/inspector/index.html b/packages/cultcache-ts/inspector/index.html new file mode 100644 index 00000000..73efecc2 --- /dev/null +++ b/packages/cultcache-ts/inspector/index.html @@ -0,0 +1,12 @@ + + + + + + Huginn + + +
+ + + diff --git a/packages/cultcache-ts/inspector/public/hugin-256.png b/packages/cultcache-ts/inspector/public/hugin-256.png new file mode 100644 index 00000000..4ff5743b Binary files /dev/null and b/packages/cultcache-ts/inspector/public/hugin-256.png differ diff --git a/packages/cultcache-ts/inspector/public/hugin-64.png b/packages/cultcache-ts/inspector/public/hugin-64.png new file mode 100644 index 00000000..9437fb05 Binary files /dev/null and b/packages/cultcache-ts/inspector/public/hugin-64.png differ diff --git a/packages/cultcache-ts/inspector/public/hugin.ico b/packages/cultcache-ts/inspector/public/hugin.ico new file mode 100644 index 00000000..95bd8868 Binary files /dev/null and b/packages/cultcache-ts/inspector/public/hugin.ico differ diff --git a/packages/cultcache-ts/inspector/public/hugin.png b/packages/cultcache-ts/inspector/public/hugin.png new file mode 100644 index 00000000..bb817ad8 Binary files /dev/null and b/packages/cultcache-ts/inspector/public/hugin.png differ diff --git a/packages/cultcache-ts/inspector/public/huginn-fieldpack.png b/packages/cultcache-ts/inspector/public/huginn-fieldpack.png new file mode 100644 index 00000000..6a78b8bc Binary files /dev/null and b/packages/cultcache-ts/inspector/public/huginn-fieldpack.png differ diff --git a/packages/cultcache-ts/inspector/public/huginn-surface.png b/packages/cultcache-ts/inspector/public/huginn-surface.png new file mode 100644 index 00000000..d7b62bd3 Binary files /dev/null and b/packages/cultcache-ts/inspector/public/huginn-surface.png differ diff --git a/packages/cultcache-ts/inspector/src/HuginnFieldCanvas.tsx b/packages/cultcache-ts/inspector/src/HuginnFieldCanvas.tsx new file mode 100644 index 00000000..2fd1247f --- /dev/null +++ b/packages/cultcache-ts/inspector/src/HuginnFieldCanvas.tsx @@ -0,0 +1,1380 @@ +import { useEffect, useRef } from "react"; + +type Brush = { + x: number; + y: number; + size: number; + r: number; + g: number; + b: number; + alpha: number; + energy: number; + curve: number; + gx: number; + gy: number; +}; + +type FieldSample = { + width: number; + height: number; + rgba: Uint8ClampedArray; + fieldRgba: Uint8ClampedArray; + luma: Float32Array; + alpha: Float32Array; + analysis: Float32Array; + flowX: Float32Array; + flowY: Float32Array; + fieldStrength: Float32Array; +}; + +type PyramidLevel = { + width: number; + height: number; + values: Float32Array; + energy: Float32Array; + gx: Float32Array; + gy: Float32Array; + curve: Float32Array; +}; + +const FIELD_SIZE = 256; +const MAX_DEPTH = 6; +const MIN_BRUSH_SIZE = 4; +const ENERGY_SPLIT_THRESHOLD = 0.038; +const PARTICLE_COUNT = new URLSearchParams(globalThis.location?.search ?? "").has("smoke") ? 8_192 : 1_048_576; +const PARTICLE_STRIDE_FLOATS = 12; +const MAX_NODE_ENVELOPES = 96; +const STARDUST_MAX_LEVEL = 13; +const STARDUST_COARSE_PARTICLES_PER_CELL = 2; +const STARDUST_FINE_PARTICLES_PER_CELL = 6; + +type NodeEnvelope = { + id: string; + x: number; + y: number; + radius: number; + strength: number; +}; + +type GraphViewportTransform = { + x: number; + y: number; + scale: number; + bounds: { + x: number; + y: number; + width: number; + height: number; + }; +}; + +const gpuParticleComputeShader = /* wgsl */ ` +struct Particle { + position: vec2f, + velocity: vec2f, + color: vec4f, + seed: f32, + size: f32, + life: f32, + pad: f32, +}; + +struct SimUniforms { + time: f32, + dt: f32, + width: f32, + height: f32, + particleCount: f32, + flowGain: f32, + alpha: f32, + detail: f32, + envelopeCount: f32, + nodeGain: f32, + worldX: f32, + worldY: f32, + worldWidth: f32, + worldHeight: f32, + viewX: f32, + viewY: f32, + viewScale: f32, + pad1: f32, +}; + +struct VisibleBand { + level: u32, + minX: u32, + minY: u32, + cellsX: u32, + cellsY: u32, + pairCount: u32, + weight: f32, + particlesPerCell: u32, +}; + +@group(0) @binding(0) var particles: array; +@group(0) @binding(1) var uniforms: SimUniforms; +@group(0) @binding(2) var fieldSampler: sampler; +@group(0) @binding(3) var fieldTexture: texture_2d; +@group(0) @binding(4) var nodeEnvelopes: array; +@group(0) @binding(5) var albedoTexture: texture_2d; + +fn hash32(value: u32) -> u32 { + var x = value; + x = (x ^ 61u) ^ (x >> 16u); + x = x * 9u; + x = x ^ (x >> 4u); + x = x * 0x27d4eb2du; + x = x ^ (x >> 15u); + return x; +} + +fn hashPair(a: u32, b: u32) -> u32 { + var x = hash32(a); + var y = b; + y = y - (y << 6u); + y = y ^ (x >> 17u); + y = y - (y << 9u); + y = y ^ (x << 4u); + y = y - (y << 3u); + y = y ^ (y << 10u); + y = y ^ (y >> 15u); + return x ^ y; +} + +fn randomFirst(a: u32, b: u32) -> f32 { + return f32(hashPair(a, b) & 0x00ffffffu) / 16777216.0; +} + +fn xorshift(value: u32) -> u32 { + var state = value; + state = state ^ (state << 13u); + state = state ^ (state >> 17u); + state = state ^ (state << 5u); + return state; +} + +fn randomValue(state: u32) -> f32 { + return f32(state & 0x00ffffffu) / 16777216.0; +} + +fn fadeCurve(t: f32) -> f32 { + return t * t * t * (t * (6.0 * t - 15.0) + 10.0); +} + +fn grad(seed: u32, p: f32) -> f32 { + if ((hash32(seed) & 1u) == 0u) { + return p; + } + return -p; +} + +fn perlinNoise1D(p: f32, seed: u32) -> f32 { + let pi = floor(p); + let pf = p - pi; + let cell = u32(max(pi, 0.0)); + let w = fadeCurve(pf); + return mix(grad(cell ^ seed, pf), grad((cell + 1u) ^ seed, pf - 1.0), w) * 2.0; +} + +fn tri(value: f32) -> f32 { + return abs(fract(value) - 0.5); +} + +fn tri3(point: vec3f) -> vec3f { + return vec3f( + tri(point.z + tri(point.y)), + tri(point.z + tri(point.x)), + tri(point.y + tri(point.x)) + ); +} + +fn triFlow(point: vec2f, seed: f32) -> vec2f { + let p1 = normalize(tri3(vec3f(point.x, seed, point.y)) * 2.0 - vec3f(1.0)); + let p2 = normalize(tri3(vec3f(point.x, seed, point.y) * 1.6180339) * 2.0 - vec3f(1.0)); + let c = cross(p1, p2); + return normalize(c.xz + vec2f(0.0001, 0.0)); +} + +fn parabola(x: f32, k: f32) -> f32 { + return pow(max(0.0, 4.0 * x * (1.0 - x)), k); +} + +fn samplePacked(uv: vec2f) -> vec4f { + return textureSampleLevel(fieldTexture, fieldSampler, clamp(uv, vec2f(0.001), vec2f(0.999)), 0.0); +} + +fn sampleAlbedo(uv: vec2f) -> vec4f { + return textureSampleLevel(albedoTexture, fieldSampler, clamp(uv, vec2f(0.001), vec2f(0.999)), 0.0); +} + +fn channelGradient(uv: vec2f, channel: u32) -> vec2f { + let step = vec2f(1.0 / 256.0, 1.0 / 256.0) / max(uniforms.detail, 1.0); + let left = samplePacked(uv - vec2f(step.x, 0.0)); + let right = samplePacked(uv + vec2f(step.x, 0.0)); + let up = samplePacked(uv - vec2f(0.0, step.y)); + let down = samplePacked(uv + vec2f(0.0, step.y)); + if (channel == 2u) { + return vec2f(right.b - left.b, down.b - up.b); + } + return vec2f(right.a - left.a, down.a - up.a); +} + +fn visibleUvBounds() -> vec4f { + let viewScale = max(uniforms.viewScale, 0.0001); + let worldMin = (vec2f(0.0, 0.0) - vec2f(uniforms.viewX, uniforms.viewY)) / viewScale; + let worldMax = (vec2f(uniforms.width, uniforms.height) - vec2f(uniforms.viewX, uniforms.viewY)) / viewScale; + let uvMin = clamp((worldMin - vec2f(uniforms.worldX, uniforms.worldY)) / vec2f(uniforms.worldWidth, uniforms.worldHeight), vec2f(0.0), vec2f(1.0)); + let uvMax = clamp((worldMax - vec2f(uniforms.worldX, uniforms.worldY)) / vec2f(uniforms.worldWidth, uniforms.worldHeight), vec2f(0.0), vec2f(1.0)); + return vec4f(min(uvMin, uvMax), max(uvMin, uvMax)); +} + +fn stardustLevelValue() -> f32 { + let screenSide = max(uniforms.worldWidth, uniforms.worldHeight) * max(uniforms.viewScale, 0.0001); + return clamp(log2(max(screenSide / 7.0, 1.0)) + uniforms.detail * 0.72, 4.0, f32(${STARDUST_MAX_LEVEL})); +} + +fn visibleBand(level: u32, particlesPerCell: u32, weight: f32) -> VisibleBand { + let span = 1u << level; + let uv = visibleUvBounds(); + let margin = 1.0 / f32(span); + let minUv = clamp(uv.xy - vec2f(margin), vec2f(0.0), vec2f(1.0)); + let maxUv = clamp(uv.zw + vec2f(margin), vec2f(0.0), vec2f(1.0)); + let minX = min(u32(floor(minUv.x * f32(span))), span - 1u); + let minY = min(u32(floor(minUv.y * f32(span))), span - 1u); + let maxX = min(u32(ceil(maxUv.x * f32(span))), span); + let maxY = min(u32(ceil(maxUv.y * f32(span))), span); + let cellsX = max(1u, maxX - minX); + let cellsY = max(1u, maxY - minY); + let visibleCells = cellsX * cellsY; + return VisibleBand(level, minX, minY, cellsX, cellsY, visibleCells * particlesPerCell, weight, particlesPerCell); +} + +@compute @workgroup_size(128) +fn updateParticles(@builtin(global_invocation_id) id: vec3u) { + let index = id.x; + if (index >= u32(uniforms.particleCount)) { + return; + } + + var particle = particles[index]; + let pairIndex = index / 2u; + let companion = index - pairIndex * 2u; + let pairCount = max(1u, u32(uniforms.particleCount * 0.5)); + let levelValue = stardustLevelValue(); + let coarseLevel = u32(floor(levelValue)); + let fineLevel = min(coarseLevel + 1u, ${STARDUST_MAX_LEVEL}u); + let fineBlend = smoothstep(0.18, 0.86, fract(levelValue)); + let coarseBand = visibleBand(coarseLevel, ${STARDUST_COARSE_PARTICLES_PER_CELL}u, 1.0 - fineBlend * 0.72); + let fineBand = visibleBand(fineLevel, ${STARDUST_FINE_PARTICLES_PER_CELL}u, fineBlend); + let coarsePairs = min(coarseBand.pairCount, max(1u, pairCount / 3u)); + let finePairs = min(fineBand.pairCount, pairCount - coarsePairs); + let activePairs = coarsePairs + finePairs; + if (pairIndex >= activePairs) { + particle.color = vec4f(0.0); + particle.size = 0.0; + particles[index] = particle; + return; + } + + var band = coarseBand; + var localPair = pairIndex; + if (pairIndex >= coarsePairs) { + band = fineBand; + localPair = pairIndex - coarsePairs; + } + + let level = band.level; + let span = 1u << level; + let particleInCell = localPair % band.particlesPerCell; + let cellOrdinal = localPair / band.particlesPerCell; + let cellX = band.minX + (cellOrdinal % band.cellsX); + let cellY = band.minY + (cellOrdinal / band.cellsX); + let cell = vec2f(f32(cellX), f32(cellY)); + let offset = 65535u; + let seedU = hashPair(level * 73856093u + cellX * 19349663u + particleInCell * 119954089u, cellY * 83492791u + companion * 2654435761u); + var rngState = hashPair(cellX + offset + particleInCell * 251u, cellY + offset + level * 509u); + let firstRandom = randomFirst(cellX + offset + particleInCell * 251u, cellY + offset + level * 509u); + rngState = xorshift(rngState); + let period = 5.2 + randomValue(rngState) * 1.8; + rngState = xorshift(rngState); + let jitterX = perlinNoise1D(randomValue(rngState) * 1024.0, seedU ^ 17u); + rngState = xorshift(rngState); + let jitterY = perlinNoise1D(randomValue(rngState) * 1024.0, seedU ^ 29u); + rngState = xorshift(rngState); + let triSeed = randomValue(rngState); + rngState = xorshift(rngState); + let sizeRandom = randomValue(rngState); + let lifetime = fract(uniforms.time / period + firstRandom); + let spacing = 1.0 / max(f32(span), 1.0); + let base = clamp( + (cell + vec2f(0.5)) * spacing + + vec2f(jitterX, jitterY) * spacing * 0.58, + vec2f(0.001), + vec2f(0.999) + ); + let sourceAlbedo = sampleAlbedo(base); + + let startPacked = samplePacked(base); + var packed = startPacked; + let levelDetail = f32(level) / f32(${STARDUST_MAX_LEVEL}u); + let detail = clamp(uniforms.detail, 0.75, 2.0); + let fieldVector = startPacked.rg * 2.0 - vec2f(1.0); + let fieldDirection = normalize(fieldVector + vec2f(0.0001, 0.0)); + let curvatureGradient = channelGradient(base, 2u); + let tangent = normalize(vec2f(-curvatureGradient.y, curvatureGradient.x) + fieldDirection * 0.28 + vec2f(0.0001, 0.0)); + let sourceStructure = startPacked.b; + let sourceEmission = startPacked.a; + let domainNoise = triFlow(base * (14.0 + sourceStructure * 18.0) - vec2f(0.0, uniforms.time * (0.026 + sourceEmission * 0.018)), triSeed); + let fieldBlend = normalize(fieldDirection * (0.56 + sourceEmission * 0.18) + tangent * (0.26 + sourceStructure * 0.36) + domainNoise * (0.13 + sourceStructure * 0.27)); + var nodePush = vec2f(0.0); + var nodeHeat = 0.0; + for (var envelopeIndex = 0u; envelopeIndex < ${MAX_NODE_ENVELOPES}u; envelopeIndex = envelopeIndex + 1u) { + if (f32(envelopeIndex) >= uniforms.envelopeCount) { + break; + } + let envelope = nodeEnvelopes[envelopeIndex]; + let delta = base - envelope.xy; + let radius = max(envelope.z, 0.008); + let normalized = dot(delta, delta) / (radius * radius); + let edge = exp(-4.0); + let gaussian = exp(-4.0 * normalized); + let influence = pow(clamp((gaussian - edge) / max(1.0 - edge, 0.000001), 0.0, 1.0), 0.92) * envelope.w * uniforms.nodeGain; + let away = normalize(delta + vec2f(0.0001, 0.0)); + let swirl = vec2f(-away.y, away.x); + nodePush += (swirl * 0.72 + away * 0.18) * influence; + nodeHeat += influence; + } + + let flowLine = normalize(fieldBlend + nodePush); + var uv = clamp(base - flowLine * lifetime * (uniforms.flowGain / 24.0) * (0.009 + sourceEmission * 0.030 + sourceStructure * 0.016) * spacing, vec2f(0.001), vec2f(0.999)); + packed = samplePacked(uv); + + particle.position = uv; + let advectedFlow = packed.rg * 2.0 - vec2f(1.0); + let advectedShear = length(advectedFlow - fieldVector); + let emission = clamp(sourceEmission * 0.42 + packed.a * 0.78, 0.0, 1.0); + let structure = clamp(sourceStructure * 0.52 + packed.b * 0.62, 0.0, 1.0); + particle.velocity = flowLine * (3.2 + emission * 10.5 + structure * 5.4 + advectedShear * 2.2); + particle.life = lifetime; + + let flowAxis = abs(fieldVector.x - fieldVector.y); + let lifeFade = parabola(lifetime, 2.0); + let phase = lifetime * 6.28318530718 + f32(companion) * 1.57079632679; + let pairFade = pow(clamp(sin(phase) * 0.5 + 0.5, 0.0, 1.0), 1.35); + let heat = clamp(emission * 0.82 + structure * 0.48 + advectedShear * 0.22 + nodeHeat * 0.38 + lifeFade * 0.16, 0.0, 1.0); + let brass = clamp((structure - emission * 0.22) * 1.22 + flowAxis * 0.22, 0.0, 1.0); + let cyan = vec3f(0.07 + emission * 0.24, 0.40 + heat * 0.50, 0.52 + emission * 0.72); + let gold = vec3f(0.96, 0.65 + structure * 0.14, 0.24); + let violet = vec3f(0.40 + sourceAlbedo.r * 0.18, 0.14 + sourceAlbedo.g * 0.08, 0.54 + sourceAlbedo.b * 0.24); + let deep = vec3f(0.012, 0.052, 0.065); + let albedoLift = mix(deep, sourceAlbedo.rgb, clamp(sourceAlbedo.a * (0.64 + emission * 0.30 + structure * 0.16), 0.0, 1.0)); + let fieldTint = mix(mix(cyan, gold, brass * 0.34), violet, clamp(sourceStructure * (1.0 - sourceEmission) * 0.28, 0.0, 0.28)); + let emissiveInk = sourceAlbedo.rgb * (0.25 + emission * 0.72) + fieldTint * (0.20 + structure * 0.22); + let rgb = mix(albedoLift, emissiveInk, clamp(0.16 + heat * 0.28 + sourceAlbedo.a * 0.18, 0.0, 0.74)); + particle.color = vec4f( + rgb, + uniforms.alpha * (0.0022 + emission * 0.013 + structure * 0.006 + advectedShear * 0.003 + nodeHeat * 0.0025) * lifeFade * pairFade * band.weight * (0.72 + levelDetail * 0.28) + ); + particle.size = mix(0.20, 0.98, sizeRandom) * (0.62 + emission * 0.82 + structure * 0.36 + advectedShear * 0.18 + nodeHeat * 0.14) * (0.92 + detail * 0.035) * (0.82 + levelDetail * 0.22) * lifeFade * (0.35 + pairFade * 0.65) * (0.82 + band.weight * 0.18); + particles[index] = particle; +} +`; + +const gpuParticleRenderShader = /* wgsl */ ` +struct Particle { + position: vec2f, + velocity: vec2f, + color: vec4f, + seed: f32, + size: f32, + life: f32, + pad: f32, +}; + +struct SimUniforms { + time: f32, + dt: f32, + width: f32, + height: f32, + particleCount: f32, + flowGain: f32, + alpha: f32, + detail: f32, + envelopeCount: f32, + nodeGain: f32, + worldX: f32, + worldY: f32, + worldWidth: f32, + worldHeight: f32, + viewX: f32, + viewY: f32, + viewScale: f32, + pad1: f32, +}; + +@group(0) @binding(0) var renderParticles: array; +@group(0) @binding(1) var renderUniforms: SimUniforms; + +struct VertexOut { + @builtin(position) position: vec4f, + @location(0) color: vec4f, + @location(1) local: vec2f, +}; + +@vertex +fn vertexMain(@builtin(vertex_index) vertexIndex: u32, @builtin(instance_index) instanceIndex: u32) -> VertexOut { + let corners = array( + vec2f(-1.0, -1.0), + vec2f(1.0, -1.0), + vec2f(-1.0, 1.0), + vec2f(-1.0, 1.0), + vec2f(1.0, -1.0), + vec2f(1.0, 1.0) + ); + let particle = renderParticles[instanceIndex]; + let local = corners[vertexIndex]; + let direction = normalize(particle.velocity + vec2f(0.001, 0.0)); + let tangent = vec2f(-direction.y, direction.x); + let stretch = clamp(length(particle.velocity) / 16.0, 0.0, 1.4); + let ellipse = direction * local.x * particle.size * (1.0 + stretch * 0.62) + tangent * local.y * particle.size * (0.64 + renderUniforms.detail * 0.025); + let world = vec2f(renderUniforms.worldX, renderUniforms.worldY) + particle.position * vec2f(renderUniforms.worldWidth, renderUniforms.worldHeight); + let viewScale = max(renderUniforms.viewScale, 0.05); + let basePosition = vec2f(renderUniforms.viewX, renderUniforms.viewY) + world * viewScale; + let pixelPosition = basePosition + ellipse * sqrt(viewScale); + var out: VertexOut; + out.position = vec4f((pixelPosition.x / renderUniforms.width) * 2.0 - 1.0, 1.0 - (pixelPosition.y / renderUniforms.height) * 2.0, 0.0, 1.0); + out.color = particle.color; + out.local = local; + return out; +} + +@fragment +fn fragmentMain(input: VertexOut) -> @location(0) vec4f { + let radius2 = dot(input.local, input.local); + let falloff = 5.1 + renderUniforms.detail * 0.52; + let edge = exp(-falloff); + let gaussian = exp(-falloff * radius2); + let compact = pow(clamp((gaussian - edge) / max(1.0 - edge, 0.000001), 0.0, 1.0), 0.72); + let core = exp(-radius2 * 18.0) * 0.42; + return vec4f(input.color.rgb * (compact * 1.52 + core), input.color.a * compact); +} +`; + +type HuginnFieldCanvasProps = { + imageUrl: string; + fieldUrl: string; +}; + +export function HuginnFieldCanvas({ imageUrl, fieldUrl }: HuginnFieldCanvasProps) { + const canvasRef = useRef(null); + + useEffect(() => { + const canvas = canvasRef.current; + if (!canvas) { + return; + } + + let cancelled = false; + const render = async () => { + try { + const [image, fieldImage] = await Promise.all([ + loadImage(imageUrl), + loadImage(fieldUrl), + ]); + if (cancelled) { + return; + } + + const sample = sampleImage(image, fieldImage); + const gpuCleanup = await startGpuParticleField(canvas, sample); + if (gpuCleanup) { + console.info("Huginn WebGPU particle field active."); + return gpuCleanup; + } + + console.info("Huginn WebGPU unavailable; using canvas field fallback."); + const pyramid = buildCurvaturePyramid(sample); + const brushes = buildBrushes(sample, pyramid); + const observer = new ResizeObserver(() => drawFieldFallback(canvas, brushes, sample)); + observer.observe(canvas); + drawFieldFallback(canvas, brushes, sample); + + return () => observer.disconnect(); + } catch { + return; + } + }; + + let cleanup: (() => void) | undefined; + void render().then((nextCleanup) => { + cleanup = nextCleanup; + }); + + return () => { + cancelled = true; + cleanup?.(); + }; + }, [fieldUrl, imageUrl]); + + return