Move coverage recovery into projector authority
This commit is contained in:
Meta Vens
parent
d66e35ef67
commit
0601e28048
9 changed files with 1118 additions and 128 deletions
|
|
@ -57,17 +57,18 @@ use epiphany_core::{EpiphanyPackagedReleaseEntry, authenticate_epiphany_packaged
|
|||
use epiphany_core::{
|
||||
ProcessInstanceIdentity, ProcessInstanceObservation,
|
||||
WORKSPACE_COVERAGE_PROCESS_LAUNCH_SCHEMA_VERSION, WorkspaceCoverageManagedProcessLaunchEntry,
|
||||
WorkspaceCoverageProcessBootstrap, authenticate_workspace_coverage_managed_process_launch,
|
||||
WorkspaceCoverageProcessBootstrap, authenticate_current_workspace_coverage_claim_sight,
|
||||
authenticate_workspace_coverage_managed_process_launch,
|
||||
authenticate_workspace_coverage_provider_heartbeat,
|
||||
authenticate_workspace_coverage_termination_with_envelope_digest, capture_process_instance,
|
||||
current_workspace_coverage_recovery_target,
|
||||
load_latest_workspace_coverage_managed_process_launch,
|
||||
load_latest_workspace_coverage_provider_heartbeat,
|
||||
load_workspace_coverage_process_termination_observation, native_boot_identity,
|
||||
observe_process_instance, open_default_host_identity, recover_workspace_coverage_projection,
|
||||
sign_workspace_coverage_launch, workspace_coverage_host_identity_record_digest,
|
||||
observe_process_instance, open_default_host_identity, sign_workspace_coverage_launch,
|
||||
workspace_coverage_host_identity_record_digest,
|
||||
write_workspace_coverage_managed_process_launch, write_workspace_coverage_process_bootstrap,
|
||||
write_workspace_coverage_process_termination_observation,
|
||||
write_workspace_coverage_recovery_directive,
|
||||
};
|
||||
use epiphany_core::{
|
||||
authenticate_current_workspace_coverage_advancement_sight,
|
||||
|
|
@ -2226,11 +2227,18 @@ fn reconcile_workspace_coverage_projector(
|
|||
);
|
||||
return Ok(());
|
||||
}
|
||||
let runtime_store = args
|
||||
.runtime_store
|
||||
.clone()
|
||||
.context("workspace coverage reconciliation requires --runtime-store")?;
|
||||
let Some(target) = current_workspace_coverage_recovery_target(&runtime_store)? else {
|
||||
let runtime_store = args.runtime_store.clone().context(
|
||||
"workspace coverage reconciliation requires --runtime-store for Body/route sight",
|
||||
)?;
|
||||
let host = open_default_host_identity()
|
||||
.context("workspace coverage reconciliation requires an enrolled host identity")?;
|
||||
let Some(target) = authenticate_current_workspace_coverage_claim_sight(
|
||||
&args.store,
|
||||
&runtime_store,
|
||||
&args.runtime_id,
|
||||
host.entry(),
|
||||
)?
|
||||
else {
|
||||
let latest = load_latest_workspace_coverage_managed_process_launch(
|
||||
&args.store,
|
||||
args.runtime_id.clone(),
|
||||
|
|
@ -2315,19 +2323,17 @@ fn reconcile_workspace_coverage_projector(
|
|||
}
|
||||
};
|
||||
|
||||
let host = open_default_host_identity()
|
||||
.context("workspace coverage reconciliation requires an enrolled host identity")?;
|
||||
if load_workspace_coverage_process_termination_observation(
|
||||
&args.store,
|
||||
args.runtime_id.clone(),
|
||||
&target.managed_process_launch_id,
|
||||
&target.launch_id,
|
||||
)?
|
||||
.is_none()
|
||||
{
|
||||
if let Err(error) = write_workspace_coverage_process_termination_observation(
|
||||
&args.store,
|
||||
args.runtime_id.clone(),
|
||||
&target.managed_process_launch_id,
|
||||
&target.launch_id,
|
||||
&host,
|
||||
) {
|
||||
println!(
|
||||
|
|
@ -2349,11 +2355,11 @@ fn reconcile_workspace_coverage_projector(
|
|||
authenticate_workspace_coverage_termination_with_envelope_digest(
|
||||
&args.store,
|
||||
args.runtime_id.clone(),
|
||||
&target.managed_process_launch_id,
|
||||
&target.launch_id,
|
||||
host.entry(),
|
||||
)?;
|
||||
let replacement_evidence = CoverageReplacementEvidence {
|
||||
old_launch_id: target.managed_process_launch_id.clone(),
|
||||
old_launch_id: target.launch_id.clone(),
|
||||
termination_id: termination.termination_id.clone(),
|
||||
termination_envelope_digest: termination_digest,
|
||||
};
|
||||
|
|
@ -2362,7 +2368,7 @@ fn reconcile_workspace_coverage_projector(
|
|||
args.runtime_id.clone(),
|
||||
)?
|
||||
.filter(|launch| {
|
||||
launch.replaces_launch_id.as_deref() == Some(target.managed_process_launch_id.as_str())
|
||||
launch.replaces_launch_id.as_deref() == Some(target.launch_id.as_str())
|
||||
&& launch.replaces_termination_id.as_deref()
|
||||
== Some(termination.termination_id.as_str())
|
||||
});
|
||||
|
|
@ -2405,28 +2411,24 @@ fn reconcile_workspace_coverage_projector(
|
|||
let Some(ready) = ready else {
|
||||
anyhow::bail!("workspace coverage replacement did not publish signed readiness in time");
|
||||
};
|
||||
let recovered = recover_workspace_coverage_projection(
|
||||
&runtime_store,
|
||||
let directive = write_workspace_coverage_recovery_directive(
|
||||
&args.store,
|
||||
&runtime_store,
|
||||
&args.runtime_id,
|
||||
host.entry(),
|
||||
&target.managed_process_launch_id,
|
||||
&target,
|
||||
&replacement.launch_id,
|
||||
&ready.heartbeat_id,
|
||||
&target.claim_id,
|
||||
&host,
|
||||
)?;
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&json!({
|
||||
"schemaVersion": "epiphany.workspace_coverage_reconcile.v0",
|
||||
"status": "recovered",
|
||||
"status": "recovery-directed",
|
||||
"serviceId": policy.service_id,
|
||||
"oldClaimId": target.claim_id,
|
||||
"newClaimId": recovered.claim_id,
|
||||
"claimEpoch": recovered.claim_epoch,
|
||||
"replacementLaunchId": replacement.launch_id,
|
||||
"recoveryReceiptId": recovered.recovery_receipt_id,
|
||||
"recoveryReceiptDigest": recovered.recovery_receipt_digest,
|
||||
"directiveId": directive.directive_id,
|
||||
"restarted": true,
|
||||
"privateStateExposed": false,
|
||||
}))?
|
||||
|
|
@ -3935,10 +3937,14 @@ mod semantic_projector_authority_tests {
|
|||
let readiness = body
|
||||
.rfind("load_latest_workspace_coverage_provider_heartbeat")
|
||||
.unwrap();
|
||||
let recovery = body.rfind("recover_workspace_coverage_projection").unwrap();
|
||||
let recovery = body
|
||||
.rfind("write_workspace_coverage_recovery_directive")
|
||||
.unwrap();
|
||||
assert!(termination < launch && launch < readiness && readiness < recovery);
|
||||
assert!(body.contains("write_workspace_coverage_process_termination_observation"));
|
||||
assert!(body.contains("status == \"ready\""));
|
||||
assert!(!body.contains("open_workspace_coverage_authority"));
|
||||
assert!(!body.contains("recover_workspace_coverage_projection"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -8,13 +8,16 @@ use crate::default_substrate_gate_cultnet_contracts;
|
|||
use crate::packaged_release::{EpiphanyPackagedReleaseEntry, EpiphanyPackagedReleaseHead};
|
||||
use crate::workspace_coverage_process_documents::{
|
||||
WORKSPACE_COVERAGE_ADVANCEMENT_SIGHT_SCHEMA_VERSION,
|
||||
WORKSPACE_COVERAGE_CLAIM_SIGHT_SCHEMA_VERSION,
|
||||
WORKSPACE_COVERAGE_PROCESS_EVIDENCE_HEAD_SCHEMA_VERSION,
|
||||
WORKSPACE_COVERAGE_PROCESS_LAUNCH_SCHEMA_VERSION,
|
||||
WORKSPACE_COVERAGE_PROCESS_TERMINATION_SCHEMA_VERSION,
|
||||
WORKSPACE_COVERAGE_PROVIDER_HEARTBEAT_SCHEMA_VERSION,
|
||||
WORKSPACE_COVERAGE_RECOVERY_DIRECTIVE_SCHEMA_VERSION,
|
||||
WORKSPACE_COVERAGE_TERMINAL_SIGHT_SCHEMA_VERSION, WorkspaceCoverageAdvancementSightEntry,
|
||||
WorkspaceCoverageManagedProcessLaunchEntry, WorkspaceCoverageProcessEvidenceHead,
|
||||
WorkspaceCoverageProcessTerminationObservationEntry, WorkspaceCoverageProviderHeartbeatEntry,
|
||||
WorkspaceCoverageClaimSightEntry, WorkspaceCoverageManagedProcessLaunchEntry,
|
||||
WorkspaceCoverageProcessEvidenceHead, WorkspaceCoverageProcessTerminationObservationEntry,
|
||||
WorkspaceCoverageProviderHeartbeatEntry, WorkspaceCoverageRecoveryDirectiveEntry,
|
||||
WorkspaceCoverageTerminalSightEntry,
|
||||
};
|
||||
use crate::workspace_coverage_projection_progress::{
|
||||
|
|
@ -2665,6 +2668,8 @@ cultmesh_documents!(EpiphanyCultMeshDocuments {
|
|||
WorkspaceCoverageProcessTerminationObservationEntry => WORKSPACE_COVERAGE_PROCESS_TERMINATION_SCHEMA_VERSION,
|
||||
WorkspaceCoverageProjectionProgressEntry => WORKSPACE_COVERAGE_PROJECTION_PROGRESS_SCHEMA_VERSION,
|
||||
WorkspaceCoverageAdvancementSightEntry => WORKSPACE_COVERAGE_ADVANCEMENT_SIGHT_SCHEMA_VERSION,
|
||||
WorkspaceCoverageClaimSightEntry => WORKSPACE_COVERAGE_CLAIM_SIGHT_SCHEMA_VERSION,
|
||||
WorkspaceCoverageRecoveryDirectiveEntry => WORKSPACE_COVERAGE_RECOVERY_DIRECTIVE_SCHEMA_VERSION,
|
||||
WorkspaceCoverageTerminalSightEntry => WORKSPACE_COVERAGE_TERMINAL_SIGHT_SCHEMA_VERSION,
|
||||
EpiphanyPackagedReleaseEntry => crate::packaged_release::EPIPHANY_PACKAGED_RELEASE_SCHEMA_VERSION,
|
||||
EpiphanyPackagedReleaseHead => crate::packaged_release::EPIPHANY_PACKAGED_RELEASE_HEAD_SCHEMA_VERSION,
|
||||
|
|
|
|||
|
|
@ -1197,15 +1197,20 @@ pub use workspace_coverage_process_bootstrap::{
|
|||
};
|
||||
pub use workspace_coverage_process_documents::{
|
||||
WORKSPACE_COVERAGE_ADVANCEMENT_SIGHT_SCHEMA_VERSION, WORKSPACE_COVERAGE_ADVANCEMENT_SIGHT_TYPE,
|
||||
WORKSPACE_COVERAGE_CLAIM_SIGHT_SCHEMA_VERSION, WORKSPACE_COVERAGE_CLAIM_SIGHT_TYPE,
|
||||
WORKSPACE_COVERAGE_PROCESS_LAUNCH_LATEST_KEY, WORKSPACE_COVERAGE_PROCESS_LAUNCH_SCHEMA_VERSION,
|
||||
WORKSPACE_COVERAGE_PROCESS_LAUNCH_TYPE, WORKSPACE_COVERAGE_PROCESS_TERMINATION_SCHEMA_VERSION,
|
||||
WORKSPACE_COVERAGE_PROCESS_TERMINATION_TYPE, WORKSPACE_COVERAGE_PROVIDER_HEARTBEAT_LATEST_KEY,
|
||||
WORKSPACE_COVERAGE_PROVIDER_HEARTBEAT_SCHEMA_VERSION,
|
||||
WORKSPACE_COVERAGE_PROVIDER_HEARTBEAT_TYPE, WORKSPACE_COVERAGE_TERMINAL_SIGHT_SCHEMA_VERSION,
|
||||
WORKSPACE_COVERAGE_PROVIDER_HEARTBEAT_TYPE,
|
||||
WORKSPACE_COVERAGE_RECOVERY_DIRECTIVE_SCHEMA_VERSION,
|
||||
WORKSPACE_COVERAGE_RECOVERY_DIRECTIVE_TYPE, WORKSPACE_COVERAGE_TERMINAL_SIGHT_SCHEMA_VERSION,
|
||||
WORKSPACE_COVERAGE_TERMINAL_SIGHT_TYPE, WorkspaceCoverageAdvancementSightEntry,
|
||||
WorkspaceCoverageManagedProcessLaunchEntry,
|
||||
WorkspaceCoverageClaimSightEntry, WorkspaceCoverageManagedProcessLaunchEntry,
|
||||
WorkspaceCoverageProcessTerminationObservationEntry, WorkspaceCoverageProviderHeartbeatEntry,
|
||||
WorkspaceCoverageTerminalSightEntry, authenticate_current_workspace_coverage_advancement_sight,
|
||||
WorkspaceCoverageRecoveryDirectiveEntry, WorkspaceCoverageTerminalSightEntry,
|
||||
authenticate_current_workspace_coverage_advancement_sight,
|
||||
authenticate_current_workspace_coverage_claim_sight,
|
||||
authenticate_current_workspace_coverage_terminal_sight,
|
||||
authenticate_workspace_coverage_managed_process_launch,
|
||||
authenticate_workspace_coverage_process_termination_observation,
|
||||
|
|
@ -1222,7 +1227,7 @@ pub use workspace_coverage_process_documents::{
|
|||
workspace_coverage_host_identity_record_digest, workspace_coverage_launch_statement,
|
||||
workspace_coverage_termination_statement, write_workspace_coverage_managed_process_launch,
|
||||
write_workspace_coverage_process_termination_observation,
|
||||
write_workspace_coverage_provider_heartbeat,
|
||||
write_workspace_coverage_provider_heartbeat, write_workspace_coverage_recovery_directive,
|
||||
};
|
||||
pub use workspace_coverage_projection_progress::{
|
||||
WORKSPACE_COVERAGE_PROJECTION_PROGRESS_SCHEMA_VERSION,
|
||||
|
|
@ -1234,10 +1239,8 @@ pub use workspace_coverage_projection_progress::{
|
|||
};
|
||||
pub use workspace_coverage_projector::{
|
||||
WORKSPACE_COVERAGE_MAXIMUM_FILE_BYTES, WorkspaceCoverageRecoveryOutcome,
|
||||
WorkspaceCoverageRecoveryTarget, WorkspaceCoverageTerminalAuthority,
|
||||
authenticate_current_workspace_coverage_terminal_authority,
|
||||
authenticate_workspace_coverage_recovery_receipt, current_workspace_coverage_recovery_target,
|
||||
recover_workspace_coverage_projection,
|
||||
WorkspaceCoverageTerminalAuthority, authenticate_current_workspace_coverage_terminal_authority,
|
||||
authenticate_workspace_coverage_recovery_receipt,
|
||||
};
|
||||
pub use workspace_coverage_projector_service::WorkspaceCoverageProjectorConfig;
|
||||
pub use workspace_coverage_projector_service::WorkspaceCoverageProjectorPulseStatus;
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -107,14 +107,6 @@ pub(crate) struct WorkspaceCoverageObservedBinding {
|
|||
vector_binding_set_sha256: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct WorkspaceCoverageRecoveryTarget {
|
||||
pub claim_id: String,
|
||||
pub claim_epoch: u64,
|
||||
pub plan_id: String,
|
||||
pub managed_process_launch_id: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct WorkspaceCoverageRecoveryOutcome {
|
||||
pub claim_id: String,
|
||||
|
|
@ -2107,34 +2099,8 @@ pub(crate) fn acquire_workspace_coverage_projection(
|
|||
/// Atomically transfers a projection lease only after CultMesh proves the old
|
||||
/// process dead and the replacement process ready. This does not launch either
|
||||
/// process; the supervisor must establish the evidence ordering first.
|
||||
pub fn current_workspace_coverage_recovery_target(
|
||||
runtime_store: impl AsRef<Path>,
|
||||
) -> Result<Option<WorkspaceCoverageRecoveryTarget>> {
|
||||
let runtime_store = runtime_store.as_ref();
|
||||
let body_basis = observe_runtime_repository_body_basis(runtime_store)?;
|
||||
let coverage = crate::open_workspace_coverage_authority(runtime_store)?;
|
||||
let opening = coverage.store.pull_all()?;
|
||||
let claim_key = workspace_coverage_claim_key(&body_basis.observation_id, body_basis.generation);
|
||||
let Some(claim_env) = find(&opening, CLAIM_TYPE, &claim_key) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let claim: WorkspaceCoverageProjectionClaim = decode(claim_env)?;
|
||||
let attempt_env = find(&opening, ATTEMPT_TYPE, &claim.attempt_id)
|
||||
.ok_or_else(|| anyhow!("workspace coverage current attempt is absent"))?;
|
||||
let attempt: WorkspaceCoverageProjectionAttempt = decode(attempt_env)?;
|
||||
validate_claim_attempt_link(&claim, &attempt)?;
|
||||
if claim.status != "running" {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(WorkspaceCoverageRecoveryTarget {
|
||||
claim_id: claim.claim_id,
|
||||
claim_epoch: claim.claim_epoch,
|
||||
plan_id: claim.plan_id,
|
||||
managed_process_launch_id: claim.managed_process_launch_id,
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn recover_workspace_coverage_projection(
|
||||
pub(crate) fn recover_workspace_coverage_projection_with_authority(
|
||||
coverage: &crate::WorkspaceCoverageAuthority,
|
||||
runtime_store: impl AsRef<Path>,
|
||||
cultmesh_store: impl AsRef<Path>,
|
||||
runtime_id: &str,
|
||||
|
|
@ -2185,6 +2151,14 @@ pub fn recover_workspace_coverage_projection(
|
|||
&replacement_launch.launch_id,
|
||||
)?
|
||||
.ok_or_else(|| anyhow!("workspace coverage recovery replacement has no current heartbeat"))?;
|
||||
let (latest, _) = authenticate_workspace_coverage_provider_heartbeat_with_envelope_digest(
|
||||
cultmesh_store,
|
||||
runtime_id,
|
||||
&latest.heartbeat_id,
|
||||
host,
|
||||
)?;
|
||||
let ready_at = chrono::DateTime::parse_from_rfc3339(&ready.observed_at_utc)?;
|
||||
let latest_at = chrono::DateTime::parse_from_rfc3339(&latest.observed_at_utc)?;
|
||||
if old_launch.launch_id == replacement_launch.launch_id
|
||||
|| replacement_launch.replaces_launch_id.as_deref() != Some(old_launch.launch_id.as_str())
|
||||
|| replacement_launch.replaces_termination_id.as_deref()
|
||||
|
|
@ -2194,9 +2168,14 @@ pub fn recover_workspace_coverage_projection(
|
|||
.as_deref()
|
||||
!= Some(termination_digest.as_str())
|
||||
|| ready.launch_id != replacement_launch.launch_id
|
||||
|| latest.heartbeat_id != ready.heartbeat_id
|
||||
|| ready.status != "ready"
|
||||
|| ready.sequence == 0
|
||||
|| latest.launch_id != ready.launch_id
|
||||
|| latest.provider_incarnation_id != ready.provider_incarnation_id
|
||||
|| latest.provider_public_key != ready.provider_public_key
|
||||
|| latest.status != "ready"
|
||||
|| latest.sequence < ready.sequence
|
||||
|| latest_at < ready_at
|
||||
{
|
||||
bail!("workspace coverage recovery replacement is not exact and ready");
|
||||
}
|
||||
|
|
@ -2212,7 +2191,6 @@ pub fn recover_workspace_coverage_projection(
|
|||
RepositoryBodyReadSession::open(runtime_store, &body_basis)?;
|
||||
let body_opening = SingleFileMessagePackBackingStore::new(&body_store).pull_all()?;
|
||||
let body_authority = exact_body_authority(&body_opening, &body_basis)?;
|
||||
let coverage = crate::open_workspace_coverage_authority(runtime_store)?;
|
||||
let opening = coverage.store.pull_all()?;
|
||||
let claim_key = workspace_coverage_claim_key(&body_basis.observation_id, body_basis.generation);
|
||||
let claim_env = find(&opening, CLAIM_TYPE, &claim_key)
|
||||
|
|
@ -2222,6 +2200,47 @@ pub fn recover_workspace_coverage_projection(
|
|||
.ok_or_else(|| anyhow!("workspace coverage recovery running attempt is absent"))?;
|
||||
let old_attempt: WorkspaceCoverageProjectionAttempt = decode(attempt_env)?;
|
||||
validate_claim_attempt_link(&old_claim, &old_attempt)?;
|
||||
// The coverage CAS may have committed before the successor projector could
|
||||
// publish its claim sight. Replaying the exact host directive must observe
|
||||
// that committed transfer, not attempt a second transfer or strand recovery.
|
||||
if old_claim.claim_id != expected_old_claim_id
|
||||
&& let (Some(recovery_id), Some(expected_digest)) = (
|
||||
old_claim.recovery_receipt_id.as_deref(),
|
||||
old_claim.recovery_receipt_digest.as_deref(),
|
||||
)
|
||||
{
|
||||
let receipt_env = find(&opening, RECOVERY_TYPE, recovery_id)
|
||||
.ok_or_else(|| anyhow!("recovered successor receipt is absent"))?;
|
||||
let receipt: WorkspaceCoverageRecoveryReceipt = decode(receipt_env)?;
|
||||
validate_recovery_receipt(&receipt)?;
|
||||
let receipt_digest = cultcache_envelope_digest(receipt_env);
|
||||
if receipt_digest != expected_digest
|
||||
|| receipt.old_claim_id != expected_old_claim_id
|
||||
|| receipt.termination_id != termination.termination_id
|
||||
|| receipt.termination_envelope_digest != termination_digest.as_str()
|
||||
|| receipt.replacement_launch_id != replacement_launch.launch_id
|
||||
|| receipt.replacement_launch_envelope_digest != replacement_launch_digest.as_str()
|
||||
|| receipt.ready_heartbeat_id != ready.heartbeat_id
|
||||
|| receipt.ready_heartbeat_envelope_digest != ready_digest.as_str()
|
||||
|| receipt.new_claim_id != old_claim.claim_id
|
||||
|| receipt.new_attempt_id != old_attempt.attempt_id
|
||||
|| receipt.new_claim_epoch != old_claim.claim_epoch
|
||||
|| old_claim.managed_process_launch_id != replacement_launch.launch_id
|
||||
|| old_claim.executor_incarnation != replacement_launch.provider_incarnation_id
|
||||
|| old_claim.status != "running"
|
||||
|| old_attempt.status != "running"
|
||||
{
|
||||
bail!("workspace coverage recovered successor disagrees with exact directive evidence");
|
||||
}
|
||||
return Ok(WorkspaceCoverageRecoveryOutcome {
|
||||
claim_id: old_claim.claim_id,
|
||||
claim_epoch: old_claim.claim_epoch,
|
||||
managed_process_launch_id: old_claim.managed_process_launch_id,
|
||||
executor_incarnation: old_claim.executor_incarnation,
|
||||
recovery_receipt_id: recovery_id.to_string(),
|
||||
recovery_receipt_digest: receipt_digest,
|
||||
});
|
||||
}
|
||||
if old_claim.schema_version != CLAIM_SCHEMA
|
||||
|| old_attempt.schema_version != ATTEMPT_SCHEMA
|
||||
|| old_claim.status != "running"
|
||||
|
|
@ -5157,7 +5176,6 @@ mod tests {
|
|||
.is_none()
|
||||
);
|
||||
drop(authority);
|
||||
assert!(current_workspace_coverage_recovery_target(&runtime)?.is_none());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,10 @@
|
|||
use crate::repository_body_observer::RepositoryBodyReadSession;
|
||||
use crate::semantic_backend::{OllamaConfig, OllamaEmbedder, QdrantBackend, QdrantConfig};
|
||||
use crate::workspace_coverage_process_documents::publish_workspace_coverage_terminal_sight;
|
||||
use crate::workspace_coverage_process_documents::{
|
||||
authenticate_current_workspace_coverage_claim_sight,
|
||||
authenticate_workspace_coverage_recovery_directive, publish_workspace_coverage_claim_sight,
|
||||
publish_workspace_coverage_terminal_sight,
|
||||
};
|
||||
use crate::workspace_coverage_projector::{
|
||||
WorkspaceCoverageAcquireResult, WorkspaceCoverageCurrentState,
|
||||
WorkspaceCoverageProjectionExecutionAuthority, acquire_workspace_coverage_projection,
|
||||
|
|
@ -214,6 +218,53 @@ impl WorkspaceCoverageProjectorServiceBody {
|
|||
}
|
||||
|
||||
fn pulse_inner(&mut self) -> Result<WorkspaceCoverageProjectorServicePulse> {
|
||||
let authority = self.execution_authority.as_ref().ok_or_else(|| {
|
||||
anyhow!("workspace coverage projector has no installed execution authority")
|
||||
})?;
|
||||
let has_successor_sight = authenticate_current_workspace_coverage_claim_sight(
|
||||
&authority.local_verse_store,
|
||||
&self.runtime_store,
|
||||
&authority.runtime_id,
|
||||
&authority.trusted_host,
|
||||
)?
|
||||
.is_some_and(|sight| {
|
||||
sight.launch_id == self.managed_process_launch_id && sight.recovery_receipt_id.is_some()
|
||||
});
|
||||
if !has_successor_sight
|
||||
&& let Some(directive) = authenticate_workspace_coverage_recovery_directive(
|
||||
&authority.local_verse_store,
|
||||
&self.runtime_store,
|
||||
&authority.runtime_id,
|
||||
&self.managed_process_launch_id,
|
||||
&authority.trusted_host,
|
||||
)?
|
||||
{
|
||||
let basis = load_current_runtime_repository_body_basis(&self.runtime_store)?;
|
||||
if directive.workspace_id != basis.workspace_id
|
||||
|| directive.body_binding_sha256 != basis.body_binding_sha256
|
||||
|| directive.body_observation_id != basis.observation_id
|
||||
|| directive.body_generation != basis.generation
|
||||
|| directive.manifest_root_sha256 != basis.manifest_root_sha256
|
||||
|| directive.coverage_store_binding_id
|
||||
!= self.coverage_authority.store_binding.binding_id
|
||||
|| directive.coverage_store_binding_envelope_digest
|
||||
!= self.coverage_authority.store_binding_envelope_sha256
|
||||
|| directive.coverage_store_file_identity
|
||||
!= self.coverage_authority.store_binding.store_file_identity
|
||||
|| directive.runtime_coverage_route_envelope_digest
|
||||
!= self
|
||||
.coverage_authority
|
||||
.runtime_coverage_route_envelope_sha256
|
||||
{
|
||||
bail!("workspace coverage recovery directive disagrees with held authority");
|
||||
}
|
||||
crate::workspace_coverage_projector::recover_workspace_coverage_projection_with_authority(
|
||||
&self.coverage_authority, &self.runtime_store, &authority.local_verse_store,
|
||||
&authority.runtime_id, &authority.trusted_host, &directive.old_launch_id,
|
||||
&directive.replacement_launch_id, &directive.replacement_ready_heartbeat_id,
|
||||
&directive.old_claim_id,
|
||||
)?;
|
||||
}
|
||||
// Retirement is derived from typed Body history and is deliberately
|
||||
// performed before the idle fast path. Qdrant never nominates its own
|
||||
// garbage and incompatible same-name collections stop the pulse.
|
||||
|
|
@ -308,6 +359,20 @@ impl WorkspaceCoverageProjectorServiceBody {
|
|||
let authority = self.execution_authority.as_ref().ok_or_else(|| {
|
||||
anyhow!("workspace coverage projector has no installed execution authority")
|
||||
})?;
|
||||
// Publish the exact provider-signed lease before the first backend
|
||||
// operation. Recovery sight must exist even when the projector dies
|
||||
// before it can mint a progress checkpoint.
|
||||
publish_workspace_coverage_claim_sight(
|
||||
&authority.local_verse_store,
|
||||
&self.runtime_store,
|
||||
&self.coverage_authority,
|
||||
&acquisition,
|
||||
&authority.runtime_id,
|
||||
&self.managed_process_launch_id,
|
||||
&authority.trusted_host,
|
||||
&authority.provider_signing_key,
|
||||
chrono::Utc::now(),
|
||||
)?;
|
||||
let receipt = execute_workspace_coverage_projection(
|
||||
&acquisition,
|
||||
&prepared,
|
||||
|
|
@ -441,4 +506,18 @@ mod tests {
|
|||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn acquired_claim_is_visible_before_any_projection_backend_work() {
|
||||
let source = include_str!("workspace_coverage_projector_service.rs");
|
||||
let acquired = source
|
||||
.find("WorkspaceCoverageAcquireResult::Acquired(acquisition)")
|
||||
.unwrap();
|
||||
let body = &source[acquired..source.find("#[cfg(test)]").unwrap()];
|
||||
let sight = body
|
||||
.find("publish_workspace_coverage_claim_sight(")
|
||||
.unwrap();
|
||||
let execution = body.find("execute_workspace_coverage_projection(").unwrap();
|
||||
assert!(sight < execution, "claim sight must precede backend work");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2413,3 +2413,80 @@ supervisor binaries pass. The old all-at-once executor survives only under
|
|||
deployed body. The next authority cut is Idunn and aggregate health: consume the
|
||||
exact provider progress document with a no-advance lease, never a generic job
|
||||
progress noun or total projection wall clock.
|
||||
|
||||
## Organizational Epiphany is the product, not a remote-control experiment (2026-07-18)
|
||||
|
||||
The Yggdrasil deployment is the first ordinary organizational Epiphany body.
|
||||
It must remain awake without continual instruction: Modeling persistently maps
|
||||
the bounded domain into typed Mind state, Self extrapolates pressure and
|
||||
direction from that map, and Imagination authors inspectable improvement and
|
||||
feature proposals. This initiative is advisory until the appropriate adoption
|
||||
and consequence authorities act; operator inactivity permits observation,
|
||||
modeling, and proposals, not merge, publication, deployment, or scope growth.
|
||||
|
||||
The repository Persona is the organization's shared social crossing. Members
|
||||
may talk to it, correct it, and provide feedback through ordinary Discord
|
||||
conversation. Persona owns discussion and legibility. Conversation produces
|
||||
typed feedback pressure for Modeling, Imagination, and Mind review; it does not
|
||||
directly mutate canonical Mind, invoke Hands, authorize a release, or command
|
||||
Idunn. Authenticated operational capabilities such as brake, objective intake,
|
||||
adoption approval, release authorization, and deployment remain distinct typed
|
||||
Bifrost routes with explicit receipts. VoidBot observes the room and Bifrost
|
||||
routes the typed crossing; do not invent a second Discord bot or grant ordinary
|
||||
chat ambient actuator authority.
|
||||
|
||||
The July 22 continuity criterion is stronger than remote command. Yggdrasil
|
||||
must host the resident scheduler/coordinator, model runtime, tool spine, Persona
|
||||
crossing, writable domain worktree, persistent Mind/Modeling state, and local
|
||||
credentials. Starfire may be absent through August 5. The offline rehearsal
|
||||
must prove conversation/feedback -> persistent domain map -> Imagination
|
||||
proposal -> explicit adoption/review -> Hands consequence -> Bifrost exact
|
||||
release authority -> Idunn deployment receipt, plus the negative that
|
||||
conversation alone cannot cross any later boundary.
|
||||
|
||||
The immediate live wound precedes that work. The workspace projector is the
|
||||
lifetime owner of `workspace-coverage.cc`, but the supervisor currently opens
|
||||
that private store to discover and perform recovery. Replace that transitive
|
||||
authority leak with a provider-signed claim sight published immediately after
|
||||
acquisition and a host-signed recovery directive consumed by the replacement
|
||||
projector. Heartbeat remains liveness only. The supervisor must never open or
|
||||
mutate the coverage store; it declares recovery only after authenticating the
|
||||
replacement claim sight. Keep Epiphany and Idunn stopped on Yggdrasil until the
|
||||
corrected exact release is authorized through Bifrost and deployed through
|
||||
Idunn. No host reboot is authorized.
|
||||
|
||||
## Workspace-coverage recovery ownership cut complete locally (2026-07-18)
|
||||
|
||||
The supervisor no longer opens or mutates `workspace-coverage.cc`, directly or
|
||||
through an imported recovery helper. The lifetime projector owner publishes a
|
||||
provider-signed claim sight before its first backend operation. Claim sight is
|
||||
keyed to the exact current Body observation and generation and binds the exact
|
||||
launch, provider key, coverage-store binding/file/route, Body basis, claim,
|
||||
attempt envelope, plan, and recovery lineage. A signed sight from an older Body
|
||||
is not current recovery authority.
|
||||
|
||||
After exact termination and replacement readiness, the supervisor publishes
|
||||
one immutable host-signed recovery directive. It binds the authenticated old
|
||||
claim sight, termination, replacement launch, readiness heartbeat, Body, and
|
||||
coverage route by exact envelope digests. Reconciliation reauthenticates and
|
||||
returns the same signed directive byte-for-byte; a later heartbeat cannot
|
||||
rewrite it, and conflicting lineage is refused. Provider heartbeats now retain
|
||||
immutable per-heartbeat evidence beside the latest pointer because durable
|
||||
termination and recovery documents may cite an earlier heartbeat. Causally safe
|
||||
bounded retirement remains future work; naive deletion is forbidden.
|
||||
|
||||
The replacement projector authenticates the directive and performs the
|
||||
recovery CAS through its already-held `WorkspaceCoverageAuthority`. Recovery is
|
||||
idempotent across the CAS-to-successor-sight crash window: an exact committed
|
||||
successor and receipt are returned on replay, then ordinary acquisition
|
||||
republishes successor claim sight. A directive naming readiness heartbeat H
|
||||
remains valid when the same launch/provider has advanced normally to ready
|
||||
H+1. Heartbeat remains liveness evidence and does not own the lease.
|
||||
|
||||
Focused proof passes: eight process-document tests, the exclusive lifetime
|
||||
owner test, supervisor authority-order/prohibition test, pre-checkpoint sight,
|
||||
stale-Body refusal, H/H+1 consumption, immutable directive reissue, conflicting
|
||||
lineage refusal, exact CAS replay, successor sight publication, `cargo check
|
||||
--bins`, and `git diff --check`. Soul found no remaining P0/P1/P2. This code is
|
||||
ready to commit and push, then cross Bifrost/Idunn as an exact release while the
|
||||
host remains braked.
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -739,4 +739,4 @@ current_status:
|
|||
- Release `sha256-c94109f94ee42c1257089830f399cb87cd5ad672772c274194371995ce4df923` is the first deployed immutable sibling generation. Native packaging builds from detached exact source commit `dd80f4a51c18425a6665710698a357be61154abf`, initializes pinned submodules, uses the tracked lockfile under `--locked`, and publishes four exact binaries plus a typed CultCache witness/head only after atomic directory publication and byte verification. Task Scheduler pins release id/digest and the witnessed supervisor path; Idunn reauthenticates the complete set and reserved policy role paths. Current-boot proof: reconciler PID 21228 launched direct child 22736; witnessed Mind/Modeling queries ranked semantically; killing that exact-path child caused Idunn to launch PID 19748 with receipt `86a42d51-9930-4b5d-9e61-e74c82a0d808`, ready heartbeat `4fc91977-b635-474b-812b-c364847e2b4b`, and witnessed executable digest `sha256-1060dd64e8c2b520ec8744b67e8306db04a6618d463fc2117c352adf9e30e6b0`. This closes current-boot artifact generation and recurrence, not reboot/logon survival.
|
||||
- Release `sha256-6f9ab4848207a110e7e3836f41e6a74c0f463369ad4b04ce2d0d3831a6b262ae` packages repository-readiness source commit `360dcb433dda9599fca14282592608d26b8b6cc8`; witnessed Idunn PID 29476 directly owns witnessed projector PID 22840 with launch receipt `50879209-b65f-4411-ae3f-6f647d37bf42` and ready heartbeat `134df620-9e20-4220-ad4e-117ea4ca8188`. Deployment exposed receipt-v2 migration pressure: v1 receipts became correctly ineligible, but the old Succeeded pulse shortcut kept both partitions idle. The shortcut is now cut in source. Valid v2 terminal evidence derives Ready; an exact same-obligation succeeded claim opens Repair only after its consumed grant/recovery authority and exact succeeded attempt are authenticated. Failed repair retries through execute at the next epoch. Forged predecessor authority cannot be laundered by a repair grant.
|
||||
- Yggdrasil exact-SHA pressure release `586147a751c6eec2d59cd8dc10bd17ce0d02a4d1` proved useful GPU work but failed deployment honestly. GTX 1080 stayed near saturation at 2,419 MiB; bounded four-text requests completed continuously. Mind published 215 vectors, while workspace coverage completed 3,389 Ollama HTTP calls total (about 13.3k workspace chunks after Mind) before Idunn's 50-minute candidate-health deadline at 2026-07-17T01:23:50+02:00. Idunn then stopped the service, published no deployment witness, and reported no exact-candidate daemon-health proof; the Modeling collection remained at the inherited 22 points because the monolithic executor had not yet published its accumulated vectors. This is failure evidence for the execution/deployment contract, not for GPU acceleration. Total wall-time extension is forbidden as the repair.
|
||||
next_action: Migrate workspace projection obligation/plan/claim/attempt/recovery/checkpoint/progress/receipt/head ownership into the pinned transactional workspace-coverage store, bind/bootstrap it on every deployment path, and cut all Body/Verse projection writers and fallback reads. Preserve Body as observations only. Do not redeploy c94fa580 or any descendant as a live candidate until that cut is complete and verified against body.cc rewrite, store-substitution, stale-Body claim, atomic checkpoint/progress, restart, and terminal-proof negatives. No reboot is authorized.
|
||||
next_action: "Commit and push the verified workspace-coverage recovery ownership cut, authorize its exact SHA through Bifrost, deploy only through Idunn, and prove Ygg-local exclusive ownership, recovery replay, bounded Body/coverage I/O, Qdrant progress, GPU work, terminal health, and service restart while the host remains braked until admission. Then package the resident organizational swarm (scheduler/coordinator, model runtime, tool spine, Persona), provision its writable domain worktree and local credentials, wire VoidBot/Bifrost Discord feedback and separately authenticated operations, and complete a Starfire-offline rehearsal before 2026-07-22. Ordinary Persona conversation may create typed feedback pressure but cannot adopt Mind state, invoke Hands, authorize release, or deploy. No host reboot is authorized."
|
||||
|
|
|
|||
Reference in a new issue