Restore cross-runtime schema recovery and prove staging cleanup
Taking the standalone body as the CultCache base droppeda0b31b8, and it was a real regression rather than a missing convenience. A record whose catalog entry is absent -- written by another runtime, or older than the catalog it was stamped against -- made decode_store_snapshot fail, and one such record failed the entire pull. Recovering the type from the payload is what "across runtimes" meant in that commit's title. Restored in full: the payload is a MessagePack array whose first field is the schema version, so the schema name is recoverable from the record itself; the registry now also indexes schema name to entry type and refuses a name claimed by a second type; and pull resolves a persisted type that is either a registered type or a recovered schema name. The stale schema id is preserved on the envelope rather than rewritten. 743eaa5's durability half needed no code. The standalone lineage reached the same place independently through crash-atomic single-file snapshots and abandoned staging cleanup, and its cleanup carries the same protections: only a candidate named for this exact store plus a canonical UUID is removed, and anything that is not a regular file is skipped. Untested, though, which is how a careful deletion rule quietly becomes a careless one. Now asserted: an abandoned canonical candidate is removed while a non-UUID lookalike, a bare .tmp suffix, and another store's candidate all survive, and the committed snapshot still reloads. 48 tests pass. This clears the cultcache half of the retained debt fromb672fb7. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
fd676ce74d
commit
eb74ead837
1 changed files with 225 additions and 10 deletions
|
|
@ -1750,6 +1750,7 @@ struct CultCacheStoreRegistration {
|
|||
|
||||
pub struct CultCache {
|
||||
definitions: BTreeMap<String, &'static str>,
|
||||
schema_name_definitions: BTreeMap<String, String>,
|
||||
entries: BTreeMap<(String, String), CultCacheEnvelope>,
|
||||
stores: Vec<CultCacheStoreRegistration>,
|
||||
}
|
||||
|
|
@ -1758,6 +1759,7 @@ impl CultCache {
|
|||
pub fn new() -> Self {
|
||||
Self {
|
||||
definitions: BTreeMap::new(),
|
||||
schema_name_definitions: BTreeMap::new(),
|
||||
entries: BTreeMap::new(),
|
||||
stores: Vec::new(),
|
||||
}
|
||||
|
|
@ -1777,7 +1779,18 @@ impl CultCache {
|
|||
T::TYPE
|
||||
));
|
||||
}
|
||||
if let Some(existing_type) = self.schema_name_definitions.get(T::SCHEMA_NAME)
|
||||
&& existing_type != T::TYPE
|
||||
{
|
||||
return Err(anyhow!(
|
||||
"CultCache schema name {:?} is already registered for type {:?}",
|
||||
T::SCHEMA_NAME,
|
||||
existing_type
|
||||
));
|
||||
}
|
||||
self.definitions.insert(T::TYPE.to_string(), T::SCHEMA_NAME);
|
||||
self.schema_name_definitions
|
||||
.insert(T::SCHEMA_NAME.to_string(), T::TYPE.to_string());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -1812,14 +1825,20 @@ impl CultCache {
|
|||
pub fn pull_all_backing_stores(&mut self) -> Result<()> {
|
||||
self.entries.clear();
|
||||
let known_types: BTreeSet<String> = self.definitions.keys().cloned().collect();
|
||||
let schema_name_definitions = self.schema_name_definitions.clone();
|
||||
for registration in &mut self.stores {
|
||||
for entry in registration.store.pull_all()? {
|
||||
if !known_types.contains(&entry.r#type) {
|
||||
for mut entry in registration.store.pull_all()? {
|
||||
let Some(canonical_type) = resolve_registered_type(
|
||||
&known_types,
|
||||
&schema_name_definitions,
|
||||
&entry.r#type,
|
||||
) else {
|
||||
return Err(anyhow!(
|
||||
"No schema is registered for persisted entry type {:?}",
|
||||
entry.r#type
|
||||
));
|
||||
}
|
||||
};
|
||||
entry.r#type = canonical_type;
|
||||
self.entries.insert(entry_id(&entry), entry);
|
||||
}
|
||||
}
|
||||
|
|
@ -2311,13 +2330,32 @@ fn decode_store_snapshot(bytes: &[u8]) -> Result<Vec<CultCacheEnvelope>> {
|
|||
.2
|
||||
.into_iter()
|
||||
.map(|record| {
|
||||
let r#type = catalog.get(&record.1).cloned().ok_or_else(|| {
|
||||
anyhow!(
|
||||
"CultCache record {:?} references missing schema {:?}",
|
||||
record.0,
|
||||
record.1
|
||||
)
|
||||
})?;
|
||||
let r#type = match catalog.get(&record.1) {
|
||||
Some(schema_name) => schema_name.clone(),
|
||||
None => {
|
||||
// A record can outlive its catalog entry when another runtime
|
||||
// wrote it, or when the record is older than the catalog it
|
||||
// was stamped against. The payload is a MessagePack array
|
||||
// whose first field is the schema version, so the type can be
|
||||
// recovered from the record itself rather than failing the
|
||||
// whole snapshot.
|
||||
let schema_version =
|
||||
infer_schema_version_from_payload(&record.3).ok_or_else(|| {
|
||||
anyhow!(
|
||||
"CultCache record {:?} references missing schema {:?}",
|
||||
record.0,
|
||||
record.1
|
||||
)
|
||||
})?;
|
||||
infer_schema_name(&schema_version).ok_or_else(|| {
|
||||
anyhow!(
|
||||
"CultCache record {:?} references missing schema {:?}",
|
||||
record.0,
|
||||
record.1
|
||||
)
|
||||
})?
|
||||
}
|
||||
};
|
||||
Ok(CultCacheEnvelope {
|
||||
key: record.0,
|
||||
r#type,
|
||||
|
|
@ -2329,6 +2367,83 @@ fn decode_store_snapshot(bytes: &[u8]) -> Result<Vec<CultCacheEnvelope>> {
|
|||
.collect()
|
||||
}
|
||||
|
||||
/// A record recovered from its payload carries a schema name, while the
|
||||
/// registry is keyed by entry type. Accept either, so a snapshot whose catalog
|
||||
/// entry is gone still resolves to the registered definition.
|
||||
fn resolve_registered_type(
|
||||
known_types: &BTreeSet<String>,
|
||||
schema_name_definitions: &BTreeMap<String, String>,
|
||||
persisted_type: &str,
|
||||
) -> Option<String> {
|
||||
if known_types.contains(persisted_type) {
|
||||
return Some(persisted_type.to_string());
|
||||
}
|
||||
|
||||
schema_name_definitions.get(persisted_type).cloned()
|
||||
}
|
||||
|
||||
fn infer_schema_version_from_payload(payload: &[u8]) -> Option<String> {
|
||||
let mut offset = 0usize;
|
||||
read_array_header(payload, &mut offset)?;
|
||||
read_string(payload, &mut offset)
|
||||
}
|
||||
|
||||
fn read_array_header(payload: &[u8], offset: &mut usize) -> Option<u32> {
|
||||
let marker = *payload.get(*offset)?;
|
||||
*offset += 1;
|
||||
match marker {
|
||||
0x90..=0x9f => Some((marker & 0x0f) as u32),
|
||||
0xdc => {
|
||||
let bytes = payload.get(*offset..(*offset + 2))?;
|
||||
*offset += 2;
|
||||
Some(u16::from_be_bytes([bytes[0], bytes[1]]) as u32)
|
||||
}
|
||||
0xdd => {
|
||||
let bytes = payload.get(*offset..(*offset + 4))?;
|
||||
*offset += 4;
|
||||
Some(u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn read_string(payload: &[u8], offset: &mut usize) -> Option<String> {
|
||||
let marker = *payload.get(*offset)?;
|
||||
*offset += 1;
|
||||
let length = match marker {
|
||||
0xa0..=0xbf => (marker & 0x1f) as usize,
|
||||
0xd9 => {
|
||||
let length = *payload.get(*offset)? as usize;
|
||||
*offset += 1;
|
||||
length
|
||||
}
|
||||
0xda => {
|
||||
let bytes = payload.get(*offset..(*offset + 2))?;
|
||||
*offset += 2;
|
||||
u16::from_be_bytes([bytes[0], bytes[1]]) as usize
|
||||
}
|
||||
0xdb => {
|
||||
let bytes = payload.get(*offset..(*offset + 4))?;
|
||||
*offset += 4;
|
||||
u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) as usize
|
||||
}
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let bytes = payload.get(*offset..(*offset + length))?;
|
||||
std::str::from_utf8(bytes).ok().map(str::to_string)
|
||||
}
|
||||
|
||||
fn infer_schema_name(schema_version: &str) -> Option<String> {
|
||||
let marker = schema_version.rfind(".v")?;
|
||||
let version = schema_version.get((marker + 2)..)?;
|
||||
if marker == 0 || version.is_empty() || !version.bytes().all(|value| value.is_ascii_digit()) {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(schema_version[..marker].to_string())
|
||||
}
|
||||
|
||||
fn escape_json_string(value: &str) -> String {
|
||||
value.replace('\\', "\\\\").replace('"', "\\\"")
|
||||
}
|
||||
|
|
@ -2514,6 +2629,17 @@ mod tests {
|
|||
symbol: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||
#[cultcache(type = "runtime-policy", schema = "tests.schema_stamped_entry")]
|
||||
struct SchemaStamped {
|
||||
#[cultcache(key = 0)]
|
||||
schema_version: String,
|
||||
#[cultcache(key = 1)]
|
||||
name: String,
|
||||
#[cultcache(key = 2)]
|
||||
value: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||
#[cultcache(type = "nested-note")]
|
||||
struct NestedNote {
|
||||
|
|
@ -3711,4 +3837,93 @@ mod tests {
|
|||
let error = table.validate().unwrap_err();
|
||||
assert!(error.to_string().contains("column types has length 0"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn messagepack_store_recovers_schema_stamped_records_missing_catalog_entries() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let store_path = temp.path().join("missing-catalog.msgpack");
|
||||
let expected = SchemaStamped {
|
||||
schema_version: "tests.schema_stamped_entry.v1".to_string(),
|
||||
name: "schema-stamped".to_string(),
|
||||
value: "still readable".to_string(),
|
||||
};
|
||||
// An empty catalog with a stale schema id is what a record written by
|
||||
// another runtime, or one older than its catalog, looks like on disk.
|
||||
let snapshot = PersistedStoreSnapshot(
|
||||
"cultcache.store.v1".to_string(),
|
||||
Vec::new(),
|
||||
vec![PersistedRecord(
|
||||
"record-1".to_string(),
|
||||
"sha256:stale-schema-id-from-cold-record".to_string(),
|
||||
"2026-06-25T12:00:00Z".to_string(),
|
||||
rmp_serde::to_vec(&expected)?,
|
||||
)],
|
||||
);
|
||||
std::fs::write(&store_path, rmp_serde::to_vec(&snapshot)?)?;
|
||||
|
||||
let mut cache = CultCache::new();
|
||||
cache.register_entry_type::<SchemaStamped>()?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path));
|
||||
cache.pull_all_backing_stores()?;
|
||||
|
||||
assert_eq!(cache.get_required::<SchemaStamped>("record-1")?, expected);
|
||||
assert_eq!(
|
||||
cache
|
||||
.get_required_envelope::<SchemaStamped>("record-1")?
|
||||
.schema_id,
|
||||
Some("sha256:stale-schema-id-from-cold-record".to_string())
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn abandoned_staging_cleanup_removes_only_canonical_candidates() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let store_path = temp.path().join("store.cc");
|
||||
let mut cache = CultCache::new();
|
||||
cache.register_entry_type::<Settings>()?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path));
|
||||
cache.put(
|
||||
"primary",
|
||||
&Settings {
|
||||
theme: "dark".to_string(),
|
||||
retries: 2,
|
||||
},
|
||||
)?;
|
||||
|
||||
// A crashed writer leaves a canonically named candidate behind.
|
||||
let abandoned = temp
|
||||
.path()
|
||||
.join(format!("store.cc.{}.tmp", uuid::Uuid::new_v4()));
|
||||
std::fs::write(&abandoned, b"partial snapshot")?;
|
||||
// These only look like candidates. Cleanup must not touch them: the
|
||||
// staging name is the store's own file name plus a canonical UUID, and
|
||||
// anything else is somebody's file.
|
||||
let lookalike = temp.path().join("store.cc.not-a-uuid.tmp");
|
||||
std::fs::write(&lookalike, b"keep me")?;
|
||||
let bare_suffix = temp.path().join("store.cc.tmp");
|
||||
std::fs::write(&bare_suffix, b"keep me too")?;
|
||||
let neighbour = temp.path().join("other.cc.00000000-0000-0000-0000-000000000000.tmp");
|
||||
std::fs::write(&neighbour, b"not this store")?;
|
||||
|
||||
cache.put(
|
||||
"primary",
|
||||
&Settings {
|
||||
theme: "light".to_string(),
|
||||
retries: 3,
|
||||
},
|
||||
)?;
|
||||
|
||||
assert!(!abandoned.exists(), "canonical abandoned candidate survived");
|
||||
assert!(lookalike.exists(), "non-UUID lookalike was deleted");
|
||||
assert!(bare_suffix.exists(), "bare .tmp suffix was deleted");
|
||||
assert!(neighbour.exists(), "another store's candidate was deleted");
|
||||
|
||||
let mut reloaded = CultCache::new();
|
||||
reloaded.register_entry_type::<Settings>()?;
|
||||
reloaded.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path));
|
||||
reloaded.pull_all_backing_stores()?;
|
||||
assert_eq!(reloaded.get_required::<Settings>("primary")?.theme, "light");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue