Republish Warming while awaiting a write lease

This commit is contained in:
Meta Vens 2026-09-03 10:39:04 +02:00
commit 6a79cb093a

View file

@ -22,6 +22,7 @@ use cultnet_rs::{
ServiceIdentitySigner, encode_cultnet_message_to_vec, open_service_identity_credential_reader, ServiceIdentitySigner, encode_cultnet_message_to_vec, open_service_identity_credential_reader,
}; };
use std::{ use std::{
collections::VecDeque,
fs::{self, File}, fs::{self, File},
net::{SocketAddr, UdpSocket}, net::{SocketAddr, UdpSocket},
path::{Path, PathBuf}, path::{Path, PathBuf},
@ -48,6 +49,8 @@ const STATE_CONTRACT_SHA256: &str =
const CULTNET_RUDP_PROTOCOL_ID: &str = "cultnet.transport.rudp.v0"; const CULTNET_RUDP_PROTOCOL_ID: &str = "cultnet.transport.rudp.v0";
const ODIN_CULTMESH_CATALOG_CONNECTION_ID: u32 = 0x0d1d_0002; const ODIN_CULTMESH_CATALOG_CONNECTION_ID: u32 = 0x0d1d_0002;
const RUNTIME_PRESENCE_IDENTITY_FD_NAME: &str = "gamecult-runtime-presence-identity"; const RUNTIME_PRESENCE_IDENTITY_FD_NAME: &str = "gamecult-runtime-presence-identity";
const WARMING_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(5);
const MAX_RECENT_WARMING_PROOFS: usize = 64;
const ODIN_CAPABILITY: (&str, &str, &str) = const ODIN_CAPABILITY: (&str, &str, &str) =
("odin.verse-rendezvous", "odin.verse-topology.v1", "v1"); ("odin.verse-rendezvous", "odin.verse-topology.v1", "v1");
const CONNECTOR_CAPABILITY: (&str, &str, &str) = ( const CONNECTOR_CAPABILITY: (&str, &str, &str) = (
@ -256,27 +259,38 @@ impl RuntimePresencePublisher {
} }
pub(crate) async fn wait_for_write_lease( pub(crate) async fn wait_for_write_lease(
&self, &mut self,
warming: &PublishedRuntimePresence, warming: &PublishedRuntimePresence,
timeout: Duration, timeout: Duration,
) -> Result<ProcessWriteLeaseGuard> { ) -> Result<ProcessWriteLeaseGuard> {
let deadline = Instant::now() + timeout; let deadline = Instant::now() + timeout;
let mut next_heartbeat = Instant::now() + WARMING_HEARTBEAT_INTERVAL;
let mut recent_warming_proofs = VecDeque::new();
remember_warming_proof(&mut recent_warming_proofs, warming.canonical_sha256.clone());
loop { loop {
if self.write_lease_path.exists() && sibling_lock_path(&self.write_lease_path).exists() if self.write_lease_path.exists() && sibling_lock_path(&self.write_lease_path).exists()
{ {
return ProcessWriteLeaseGuard::acquire( return ProcessWriteLeaseGuard::acquire_recent(
self.write_lease_path.clone(), self.write_lease_path.clone(),
self.expected.clone(), self.expected.clone(),
self.activation.clone(), self.activation.clone(),
warming.canonical_sha256.clone(), recent_warming_proofs.iter().cloned().collect(),
); );
} }
if Instant::now() >= deadline { let now = Instant::now();
if now >= deadline {
bail!( bail!(
"timed out waiting for process write lease {}", "timed out waiting for process write lease {}",
self.write_lease_path.display() self.write_lease_path.display()
); );
} }
if now >= next_heartbeat {
let heartbeat = self
.publish_warming()
.context("publishing Warming heartbeat while awaiting process lease")?;
remember_warming_proof(&mut recent_warming_proofs, heartbeat.canonical_sha256);
next_heartbeat = now + WARMING_HEARTBEAT_INTERVAL;
}
tokio::time::sleep(Duration::from_millis(100)).await; tokio::time::sleep(Duration::from_millis(100)).await;
} }
} }
@ -769,13 +783,13 @@ pub(crate) struct ProcessWriteLeaseGuard {
} }
impl ProcessWriteLeaseGuard { impl ProcessWriteLeaseGuard {
fn acquire( fn acquire_recent(
path: PathBuf, path: PathBuf,
expected: IdunnExpectedIncarnationRecord, expected: IdunnExpectedIncarnationRecord,
activation: IdunnRuntimeActivationRecord, activation: IdunnRuntimeActivationRecord,
warming_presence_sha256: String, warming_presence_sha256es: Vec<String>,
) -> Result<Self> { ) -> Result<Self> {
Self::acquire_inner(path, expected, activation, warming_presence_sha256, true) Self::acquire_inner(path, expected, activation, warming_presence_sha256es, true)
} }
#[cfg(test)] #[cfg(test)]
@ -785,16 +799,36 @@ impl ProcessWriteLeaseGuard {
activation: IdunnRuntimeActivationRecord, activation: IdunnRuntimeActivationRecord,
warming_presence_sha256: String, warming_presence_sha256: String,
) -> Result<Self> { ) -> Result<Self> {
Self::acquire_inner(path, expected, activation, warming_presence_sha256, false) Self::acquire_inner(
path,
expected,
activation,
vec![warming_presence_sha256],
false,
)
}
#[cfg(test)]
fn acquire_recent_fixture(
path: PathBuf,
expected: IdunnExpectedIncarnationRecord,
activation: IdunnRuntimeActivationRecord,
warming_presence_sha256es: Vec<String>,
) -> Result<Self> {
Self::acquire_inner(path, expected, activation, warming_presence_sha256es, false)
} }
fn acquire_inner( fn acquire_inner(
path: PathBuf, path: PathBuf,
expected: IdunnExpectedIncarnationRecord, expected: IdunnExpectedIncarnationRecord,
activation: IdunnRuntimeActivationRecord, activation: IdunnRuntimeActivationRecord,
warming_presence_sha256: String, warming_presence_sha256es: Vec<String>,
enforce_custody: bool, enforce_custody: bool,
) -> Result<Self> { ) -> Result<Self> {
ensure!(
!warming_presence_sha256es.is_empty(),
"process write lease has no provider-owned Warming proof"
);
if enforce_custody { if enforce_custody {
require_process_write_lease_custody(&path)?; require_process_write_lease_custody(&path)?;
} }
@ -813,7 +847,7 @@ impl ProcessWriteLeaseGuard {
&entries, &entries,
&expected, &expected,
&activation, &activation,
&warming_presence_sha256, &warming_presence_sha256es,
)?; )?;
ready_tx ready_tx
.send(Ok(record)) .send(Ok(record))
@ -937,7 +971,7 @@ fn decode_write_lease(
entries: &[CultCacheEnvelope], entries: &[CultCacheEnvelope],
expected: &IdunnExpectedIncarnationRecord, expected: &IdunnExpectedIncarnationRecord,
activation: &IdunnRuntimeActivationRecord, activation: &IdunnRuntimeActivationRecord,
warming_presence_sha256: &str, warming_presence_sha256es: &[String],
) -> Result<IdunnProcessWriteLeaseRecord> { ) -> Result<IdunnProcessWriteLeaseRecord> {
let [envelope] = entries else { let [envelope] = entries else {
bail!("process write-lease store is absent or ambiguous"); bail!("process write-lease store is absent or ambiguous");
@ -968,12 +1002,21 @@ fn decode_write_lease(
.context("Expected has no state contract")? .context("Expected has no state contract")?
&& lease.runtime_id == expected.runtime_id && lease.runtime_id == expected.runtime_id
&& lease.runtime_instance_id == activation.runtime_instance_id && lease.runtime_instance_id == activation.runtime_instance_id
&& lease.warming_presence_sha256 == warming_presence_sha256, && warming_presence_sha256es
"process write lease does not bind this exact warming incarnation" .iter()
.any(|sha256| sha256 == &lease.warming_presence_sha256),
"process write lease does not bind a provider-owned Warming proof for this incarnation"
); );
Ok(lease) Ok(lease)
} }
fn remember_warming_proof(proofs: &mut VecDeque<String>, sha256: String) {
proofs.push_back(sha256);
while proofs.len() > MAX_RECENT_WARMING_PROOFS {
proofs.pop_front();
}
}
fn publish_runtime_presence( fn publish_runtime_presence(
endpoint: SocketAddr, endpoint: SocketAddr,
record: &GameCultRuntimePresenceHealthRecord, record: &GameCultRuntimePresenceHealthRecord,
@ -1417,11 +1460,18 @@ pub(crate) mod tests {
fn process_write_lease_guard_holds_and_rechecks_the_exact_incarnation() -> Result<()> { fn process_write_lease_guard_holds_and_rechecks_the_exact_incarnation() -> Result<()> {
let root = tempfile::tempdir()?; let root = tempfile::tempdir()?;
let mut fixture = fixture(root.path())?; let mut fixture = fixture(root.path())?;
let warming = let first_warming =
fixture fixture
.publisher .publisher
.signed_record("warming", "process-write-lease-pending", None)?; .signed_record("warming", "process-write-lease-pending", None)?;
let warming_sha256 = warming.canonical_sha256()?; let first_warming_sha256 = first_warming.canonical_sha256()?;
let latest_warming =
fixture
.publisher
.signed_record("warming", "process-write-lease-pending", None)?;
let latest_warming_sha256 = latest_warming.canonical_sha256()?;
assert!(latest_warming.publisher_sequence > first_warming.publisher_sequence);
assert_ne!(latest_warming_sha256, first_warming_sha256);
let lease = IdunnProcessWriteLeaseRecord { let lease = IdunnProcessWriteLeaseRecord {
schema_version: IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), schema_version: IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(),
target: fixture.expected.target.clone(), target: fixture.expected.target.clone(),
@ -1434,7 +1484,7 @@ pub(crate) mod tests {
state_contract_sha256: fixture.expected.state_contract_sha256.clone().unwrap(), state_contract_sha256: fixture.expected.state_contract_sha256.clone().unwrap(),
runtime_id: fixture.expected.runtime_id.clone(), runtime_id: fixture.expected.runtime_id.clone(),
runtime_instance_id: fixture.activation.runtime_instance_id.clone(), runtime_instance_id: fixture.activation.runtime_instance_id.clone(),
warming_presence_sha256: warming_sha256.clone(), warming_presence_sha256: first_warming_sha256.clone(),
lease_epoch: 1, lease_epoch: 1,
issued_at_unix_millis: 200, issued_at_unix_millis: 200,
}; };
@ -1447,11 +1497,11 @@ pub(crate) mod tests {
schema_id: Some(IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into()), schema_id: Some(IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into()),
})?; })?;
let guard = ProcessWriteLeaseGuard::acquire_fixture( let guard = ProcessWriteLeaseGuard::acquire_recent_fixture(
path.clone(), path.clone(),
fixture.expected.clone(), fixture.expected.clone(),
fixture.activation.clone(), fixture.activation.clone(),
warming_sha256, vec![first_warming_sha256, latest_warming_sha256],
)?; )?;
guard.require_current()?; guard.require_current()?;
assert_eq!(guard.canonical_sha256(), lease.canonical_sha256()?); assert_eq!(guard.canonical_sha256(), lease.canonical_sha256()?);