diff --git a/tools/eureka-cut5-mutations.ps1 b/tools/eureka-cut5-mutations.ps1 deleted file mode 100644 index 125aa5fa..00000000 --- a/tools/eureka-cut5-mutations.ps1 +++ /dev/null @@ -1,238 +0,0 @@ -# Eureka Cut 5 mutation suite. -# -# Cut 5 collapsed `TypedCommitStore` into `commit_authorized_mind_mutation`. -# The collapse is only safe if the rules the profile's tests used to pin are -# still pinned against the concrete owner. Each entry below names one such rule -# and the exact source change that removes it. A mutation counts only if the -# named test fails while the mutation is applied. The script applies one -# mutation at a time, runs that test alone, and restores every file it touched -# before the next one, so a failed run never leaves the tree mutated. -# -# powershell -File tools/eureka-cut5-mutations.ps1 -# -# Windows PowerShell 5.1 is the interpreter on this host; there is no `pwsh`. -# The guard below refuses anything else rather than failing later on a path or -# cmdlet difference. -# -# Requires CARGO_TARGET_DIR to be set the way the cut ran it, or it will build -# into the repo-local target/. -# -# M1-M5 are the collapse's own rules. MS1 and MS2 close Soul's findings S7 and -# S8: each is a mutation the suite used to survive, kept here so the gap cannot -# reopen. -# -# This suite differs from Cut 4's in one way: an entry carries a list of edits -# rather than a single Old/New pair, because two of the seven rules are about -# the *order* of two statements. Removing a statement and reinserting it -# elsewhere is two edits and cannot be expressed as one replacement. Every edit -# is still asserted to match exactly once. - -$ErrorActionPreference = 'Stop' - -if ($PSVersionTable.PSVersion.Major -lt 5 -or $env:OS -ne 'Windows_NT') { - throw "This script runs under Windows PowerShell 5.1 on Windows (powershell -File tools/eureka-cut5-mutations.ps1). Found PowerShell $($PSVersionTable.PSVersion) on $(if ($env:OS) { $env:OS } else { 'a non-Windows host' })." -} - -$repo = Split-Path -Parent $PSScriptRoot -$context = 'epiphany-core/src/reasoning_context.rs' -$spine = 'epiphany-core/src/runtime_spine.rs' - -# The validation loop, as it stands in the collapsed owner. Two mutations move -# it; one deletes it. -$validationLoop = @' - for write in &writes { - crate::mind_documents::validate_mind_write_envelope(write)?; - } -'@ - -$mutations = @( - @{ - Id = 'M1' - Rule = 'The store opener refuses a store at a foreign schema epoch, before the commit can write to it.' - Test = 'reasoning_context::tests::mind_commit_refuses_a_foreign_epoch_store' - Edits = @( - @{ - File = $spine - Old = ' validate_runtime_store_epoch(&backing_store.pull_all()?)?;' - New = ' let _ = &backing_store;' - } - ) - }, - @{ - Id = 'M2' - Rule = 'The batch is validated against the Mind document rules at all. MS2 pins that every write is.' - Test = 'reasoning_context::tests::mind_commit_keeps_validation_receipts_replay_and_conflicts' - Edits = @( - @{ - File = $context - Old = $validationLoop - New = @' - for write in &writes { - let _ = write; - } -'@ - } - ) - }, - @{ - Id = 'M3' - Rule = 'The owner resolves one backing store and both reads and writes that one.' - Test = 'reasoning_context::tests::mind_commit_reads_and_writes_one_store' - Edits = @( - @{ - File = $context - Old = ' if backing_store.compare_and_swap_batch(&expected, replacements)? {' - New = ' if runtime_spine_backing_store(&store_path.with_file_name("other.cc"))?.compare_and_swap_batch(&expected, replacements)? {' - } - ) - }, - # Ruling 12 order, first half. Hoisting validation above the uniqueness - # checks changes which refusal a caller is told about for a batch that is - # both duplicated and invalid. - @{ - Id = 'M4' - Rule = 'Identity uniqueness is checked before the writes are validated.' - Test = 'reasoning_context::tests::mind_commit_refuses_repeated_write_identities_before_validating' - Edits = @( - @{ - File = $context - Old = $validationLoop - New = '' - }, - @{ - File = $context - Old = ' validate_unique_envelope_identities(&strong_reads, "strong read")?;' - New = @' - for write in &writes { - crate::mind_documents::validate_mind_write_envelope(write)?; - } - validate_unique_envelope_identities(&strong_reads, "strong read")?; -'@ - } - ) - }, - # Ruling 12 order, second half, and Soul's finding S6. Moving validation - # below the replay answer lets a stored receipt answer a batch the current - # validator would refuse. - @{ - Id = 'M5' - Rule = 'Writes are validated before a stored receipt may answer a replay.' - Test = 'reasoning_context::tests::mind_commit_validates_before_answering_a_replay' - Edits = @( - @{ - File = $context - Old = $validationLoop - New = '' - }, - @{ - File = $context - Old = @' - return Ok(EpiphanyMindCommitOutcome::Committed(existing)); - } -'@ - New = @' - return Ok(EpiphanyMindCommitOutcome::Committed(existing)); - } - for write in &writes { - crate::mind_documents::validate_mind_write_envelope(write)?; - } -'@ - } - ) - }, - # Soul's finding S7. M3 sends the CAS to a *different* path, which leaves a - # second file the one-store test can see. This sends it to the same path, - # which leaves no trace in the directory at all and survived M3's test. It is - # still a second handle, and the map's redb plan makes that fatal: redb - # permits one writable handle per path. - @{ - Id = 'MS1' - Rule = 'The owner resolves the backing store once and reuses that handle, rather than re-deriving it.' - Test = 'reasoning_context::tests::mind_commit_reads_and_writes_one_store' - Edits = @( - @{ - File = $context - Old = ' if backing_store.compare_and_swap_batch(&expected, replacements)? {' - New = ' if runtime_spine_backing_store(store_path)?.compare_and_swap_batch(&expected, replacements)? {' - } - ) - }, - # Soul's finding S8. M2 neuters validation for every write, which any - # single-write batch catches. This keeps validation and narrows it to the - # first write, which nothing caught until a batch of two carried an invalid - # second write. - @{ - Id = 'MS2' - Rule = 'Every write in the batch is validated, not just the first.' - Test = 'reasoning_context::tests::mind_commit_validates_every_write_in_the_batch' - Edits = @( - @{ - File = $context - Old = $validationLoop - New = @' - for write in writes.iter().take(1) { - crate::mind_documents::validate_mind_write_envelope(write)?; - } -'@ - } - ) - } -) - -$results = @() -foreach ($mutation in $mutations) { - $originals = @{} - try { - foreach ($edit in $mutation.Edits) { - $path = Join-Path $repo $edit.File - if (-not $originals.ContainsKey($path)) { - $originals[$path] = [IO.File]::ReadAllText($path) - } - $current = [IO.File]::ReadAllText($path) - # The anchors here are written with plain newlines and the working - # tree is CRLF, so a multi-line anchor matches nothing unless it is - # converted to the line endings the file actually has. Convert - # towards the file. - $eol = if ($current.Contains("`r`n")) { "`r`n" } else { "`n" } - $old = ($edit.Old -replace "`r`n", "`n") -replace "`n", $eol - $new = ($edit.New -replace "`r`n", "`n") -replace "`n", $eol - # A multi-line here-string drops its final newline; the anchors - # above are whole statements, so put it back on both sides. - if ($edit.Old.Contains("`n")) { $old += $eol } - if ($edit.New -and $edit.New.Contains("`n")) { $new += $eol } - # Exactly one site, not merely at least one: `.Replace` below - # mutates every match, so a second site would change more than the - # entry describes and the test could then fail for a reason the - # entry does not name. - $sites = [regex]::Matches($current, [regex]::Escape($old)).Count - if ($sites -ne 1) { - throw "$($mutation.Id): anchor matches $sites times in $($edit.File), expected exactly 1. The mutation is stale." - } - $mutated = $current.Replace($old, $new) - if ($mutated -eq $current) { - throw "$($mutation.Id): replacement changed nothing in $($edit.File)." - } - [IO.File]::WriteAllText($path, $mutated) - } - & cargo test -p epiphany-core --lib -- --exact $mutation.Test | Out-Host - $survived = ($LASTEXITCODE -eq 0) - } - finally { - foreach ($path in $originals.Keys) { - [IO.File]::WriteAllText($path, $originals[$path]) - } - } - $results += [pscustomobject]@{ - Id = $mutation.Id - Test = $mutation.Test.Split(':')[-1] - Killed = -not $survived - Rule = $mutation.Rule - } - Write-Host "$($mutation.Id): $(if ($survived) { 'SURVIVED - the rule is not pinned' } else { 'killed' })" -} - -$results | Format-Table -AutoSize -if ($results | Where-Object { -not $_.Killed }) { - throw 'At least one mutation survived.' -} -Write-Host 'Every mutation was killed by its own test.' diff --git a/tools/eureka-cut5-mutations.psd1 b/tools/eureka-cut5-mutations.psd1 new file mode 100644 index 00000000..40ab37b9 --- /dev/null +++ b/tools/eureka-cut5-mutations.psd1 @@ -0,0 +1,174 @@ +# Eureka Cut 5 mutation suite. Entries only; the harness is +# tools/eureka-mutations.ps1. +# +# powershell -File tools/eureka-mutations.ps1 ` +# -Entries tools/eureka-cut5-mutations.psd1 ` +# -Target epiphany-core/src/reasoning_context.rs,epiphany-core/src/runtime_spine.rs ` +# -Test 'cargo test -p epiphany-core --lib' +# +# Cut 5 collapsed `TypedCommitStore` into `commit_authorized_mind_mutation`. +# The collapse is only safe if the rules the profile's tests used to pin are +# still pinned against the concrete owner. M1-M5 are the collapse's own rules. +# MS1 and MS2 close Soul's findings S7 and S8: each is a mutation the suite +# used to survive, kept here so the gap cannot reopen. +# +# Two of the rules are about the *order* of two statements, and removing a +# statement and reinserting it elsewhere is two edits, so those entries carry +# an `Edits` list. The validation loop they move is +# +# for write in &writes { +# crate::mind_documents::validate_mind_write_envelope(write)?; +# } +# +# written out in each entry, since a data file cannot share it. +@{ + Mutations = @( + @{ + Id = 'M1' + Rule = 'The store opener refuses a store at a foreign schema epoch, before the commit can write to it.' + Test = 'reasoning_context::tests::mind_commit_refuses_a_foreign_epoch_store' + Edits = @( + @{ + File = 'epiphany-core/src/runtime_spine.rs' + Old = ' validate_runtime_store_epoch(&backing_store.pull_all()?)?;' + New = ' let _ = &backing_store;' + } + ) + }, + @{ + Id = 'M2' + Rule = 'The batch is validated against the Mind document rules at all. MS2 pins that every write is.' + Test = 'reasoning_context::tests::mind_commit_keeps_validation_receipts_replay_and_conflicts' + Edits = @( + @{ + File = 'epiphany-core/src/reasoning_context.rs' + Old = @' + for write in &writes { + crate::mind_documents::validate_mind_write_envelope(write)?; + } +'@ + New = @' + for write in &writes { + let _ = write; + } +'@ + } + ) + }, + @{ + Id = 'M3' + Rule = 'The owner resolves one backing store and both reads and writes that one.' + Test = 'reasoning_context::tests::mind_commit_reads_and_writes_one_store' + Edits = @( + @{ + File = 'epiphany-core/src/reasoning_context.rs' + Old = ' if backing_store.compare_and_swap_batch(&expected, replacements)? {' + New = ' if runtime_spine_backing_store(&store_path.with_file_name("other.cc"))?.compare_and_swap_batch(&expected, replacements)? {' + } + ) + }, + # Ruling 12 order, first half. Hoisting validation above the uniqueness + # checks changes which refusal a caller is told about for a batch that + # is both duplicated and invalid. + @{ + Id = 'M4' + Rule = 'Identity uniqueness is checked before the writes are validated.' + Test = 'reasoning_context::tests::mind_commit_refuses_repeated_write_identities_before_validating' + Edits = @( + @{ + File = 'epiphany-core/src/reasoning_context.rs' + Old = @' + for write in &writes { + crate::mind_documents::validate_mind_write_envelope(write)?; + } +'@ + New = '' + }, + @{ + File = 'epiphany-core/src/reasoning_context.rs' + Old = ' validate_unique_envelope_identities(&strong_reads, "strong read")?;' + New = @' + for write in &writes { + crate::mind_documents::validate_mind_write_envelope(write)?; + } + validate_unique_envelope_identities(&strong_reads, "strong read")?; +'@ + } + ) + }, + # Ruling 12 order, second half, and Soul's finding S6. Moving validation + # below the replay answer lets a stored receipt answer a batch the + # current validator would refuse. + @{ + Id = 'M5' + Rule = 'Writes are validated before a stored receipt may answer a replay.' + Test = 'reasoning_context::tests::mind_commit_validates_before_answering_a_replay' + Edits = @( + @{ + File = 'epiphany-core/src/reasoning_context.rs' + Old = @' + for write in &writes { + crate::mind_documents::validate_mind_write_envelope(write)?; + } +'@ + New = '' + }, + @{ + File = 'epiphany-core/src/reasoning_context.rs' + Old = @' + return Ok(EpiphanyMindCommitOutcome::Committed(existing)); + } +'@ + New = @' + return Ok(EpiphanyMindCommitOutcome::Committed(existing)); + } + for write in &writes { + crate::mind_documents::validate_mind_write_envelope(write)?; + } +'@ + } + ) + }, + # Soul's finding S7. M3 sends the CAS to a *different* path, which + # leaves a second file the one-store test can see. This sends it to the + # same path, which leaves no trace in the directory at all and survived + # M3's test. It is still a second handle, and the map's redb plan makes + # that fatal: redb permits one writable handle per path. + @{ + Id = 'MS1' + Rule = 'The owner resolves the backing store once and reuses that handle, rather than re-deriving it.' + Test = 'reasoning_context::tests::mind_commit_reads_and_writes_one_store' + Edits = @( + @{ + File = 'epiphany-core/src/reasoning_context.rs' + Old = ' if backing_store.compare_and_swap_batch(&expected, replacements)? {' + New = ' if runtime_spine_backing_store(store_path)?.compare_and_swap_batch(&expected, replacements)? {' + } + ) + }, + # Soul's finding S8. M2 neuters validation for every write, which any + # single-write batch catches. This keeps validation and narrows it to + # the first write, which nothing caught until a batch of two carried an + # invalid second write. + @{ + Id = 'MS2' + Rule = 'Every write in the batch is validated, not just the first.' + Test = 'reasoning_context::tests::mind_commit_validates_every_write_in_the_batch' + Edits = @( + @{ + File = 'epiphany-core/src/reasoning_context.rs' + Old = @' + for write in &writes { + crate::mind_documents::validate_mind_write_envelope(write)?; + } +'@ + New = @' + for write in writes.iter().take(1) { + crate::mind_documents::validate_mind_write_envelope(write)?; + } +'@ + } + ) + } + ) +} diff --git a/tools/eureka-cut6-mutations.ps1 b/tools/eureka-cut6-mutations.ps1 deleted file mode 100644 index e61c7e7a..00000000 --- a/tools/eureka-cut6-mutations.ps1 +++ /dev/null @@ -1,262 +0,0 @@ -# Eureka Cut 6 mutation suite. -# -# Each entry below is one rule that a surviving test claims to pin, together -# with the exact source change that removes the rule. A mutation counts only if -# the named test fails while the mutation is applied. The script applies one -# mutation at a time, runs that test alone, and restores the file from disk -# before the next one, so a failed run never leaves the tree mutated. -# -# powershell -File tools/eureka-cut6-mutations.ps1 -# -# Windows PowerShell 5.1 is the interpreter on this host; there is no `pwsh`. -# The guard below refuses anything else rather than failing later on a path or -# cmdlet difference. -# -# Requires CARGO_TARGET_DIR to be set the way the cut ran it, or it will build -# into the repo-local target/. -# -# M1-M9 are Cut 4's suite re-anchored: the code they mutate moved from -# `epiphany-core/src/pipeline_documents.rs` to `epiphany-pipeline/src/lib.rs`, -# so every entry names its own package as well as its own file. M10 still -# mutates the spine, which kept its rule and its test. M11 and M12 are new with -# the three kinds this cut added, and M13-M15 came out of Soul's pass on them. -# -# A stale anchor is a hard failure, not a skipped entry: the check below asserts -# each anchor matches exactly once, since `.Replace` hits every match and a -# second site would change more than the entry describes. - -$ErrorActionPreference = 'Stop' - -if ($PSVersionTable.PSVersion.Major -lt 5 -or $env:OS -ne 'Windows_NT') { - throw "This script runs under Windows PowerShell 5.1 on Windows (powershell -File tools/eureka-cut6-mutations.ps1). Found PowerShell $($PSVersionTable.PSVersion) on $(if ($env:OS) { $env:OS } else { 'a non-Windows host' })." -} - -$repo = Split-Path -Parent $PSScriptRoot -$documents = 'epiphany-pipeline/src/lib.rs' -$spine = 'epiphany-core/src/runtime_spine.rs' -$pipeline = 'epiphany-pipeline' -$core = 'epiphany-core' - -$mutations = @( - @{ - Id = 'M1' - Package = $pipeline - File = $documents - Rule = 'A document decodes back only from an envelope of its own type id.' - Test = 'tests::every_pipeline_kind_round_trips_through_named_slot_zero' - Old = '$(if envelope.r#type == <$document as DatabaseEntry>::TYPE {' - New = '$(if envelope.r#type != <$document as DatabaseEntry>::TYPE {' - }, - @{ - Id = 'M2' - Package = $pipeline - File = $documents - Rule = 'Text bounds are measured in UTF-8 bytes, not characters.' - Test = 'tests::bounds_refuse_in_utf8_bytes' - Old = ' move |field: &str, value: &str| bound(field, limit, value.len())' - New = ' move |field: &str, value: &str| bound(field, limit, value.chars().count())' - }, - @{ - Id = 'M3' - Package = $pipeline - File = $documents - Rule = 'An OrgRepo carries exactly one slash.' - Test = 'tests::repo_fields_must_be_org_slash_repo' - Old = " if !org.is_empty() && !repo.is_empty() && !repo.contains('/') && value.len() <= 200 =>" - New = ' if !org.is_empty() && !repo.is_empty() && value.len() <= 200 =>' - }, - @{ - Id = 'M4' - Package = $pipeline - File = $documents - Rule = 'A write envelope''s key is recomputed from the value and a mismatch refuses.' - Test = 'tests::keys_are_derived_and_mismatch_refuses' - Old = ' if envelope.key != expected {' - New = ' if false {' - }, - @{ - Id = 'M5' - Package = $pipeline - File = $documents - Rule = 'A finding label carries no dot, so a composed key segments one way.' - Test = 'tests::composed_keys_cannot_collide' - Old = ' label_text("finding.label", &value.label.0)?;' - New = ' let _ = &value.label.0;' - }, - @{ - Id = 'M6' - Package = $pipeline - File = $documents - Rule = 'A parent local must carry its own kind''s marker character.' - Test = 'tests::parent_ids_are_parsed_strictly' - Old = ' let digits = suffix.strip_prefix(marker).ok_or_else(invalid)?;' - New = ' let digits = suffix.strip_prefix(marker).unwrap_or(suffix);' - }, - @{ - Id = 'M7' - Package = $pipeline - File = $documents - Rule = 'A PipelineRef''s id is parsed as a full id of its declared kind.' - Test = 'tests::resolution_subject_is_a_full_id_of_its_kind' - Old = ' pipeline_id(&format!("{at}.id"), &self.id.0, self.kind).map(|_| ())' - New = ' Ok(())' - }, - @{ - Id = 'M8' - Package = $pipeline - File = $documents - Rule = 'Published schemas are the Rust derivation, byte for byte.' - Test = 'tests::pipeline_published_schemas_match_derivation' - Old = ' Short = 200, |field, value| within(200)(field, value);' - New = ' Short = 201, |field, value| within(200)(field, value);' - }, - # Soul F2. M1 removes the positive match; this removes the refusal that - # catches everything the positive match misses, leaving a decode that says - # "malformed payload" about an envelope that was never a document at all. - @{ - Id = 'M9' - Package = $pipeline - File = $documents - Rule = 'An envelope whose type id belongs to another kind is refused as ForeignStore.' - Test = 'tests::decode_refuses_an_envelope_of_a_foreign_type' - Old = 'Err(PipelineRefusal::ForeignStore { r#type: envelope.r#type.clone() })' - New = 'Err(format_error("payload", &envelope.r#type))' - }, - # Soul F4. The spine cache refuses a pipeline store by registering its own - # types and nothing else, so the mutation that removes the rule is the one - # that registers a pipeline type into it. Cut 4 registered the documents - # themselves; epiphany-core no longer has them, so the mutation declares its - # own type carrying the same type id, exactly as the test now does. - @{ - Id = 'M10' - Package = $core - File = $spine - Rule = 'The runtime spine cache refuses an epiphany.pipeline.* envelope.' - Test = 'runtime_spine::tests::runtime_spine_cache_refuses_a_pipeline_store' - Old = @' -fn runtime_spine_schema_cache() -> Result { - let mut cache = CultCache::new(); -'@ - New = @' -fn runtime_spine_schema_cache() -> Result { - let mut cache = CultCache::new(); - #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] - #[cultcache( - type = "epiphany.pipeline.campaign.v1", - schema = "EpiphanyPipelineCampaignDocument" - )] - struct RegisteredPipelineDocument { - #[cultcache(key = 0)] - slug: String, - } - cache.register_entry_type::()?; -'@ - }, - # The cut's first named mutation: change a value type's shape and leave the - # published schema alone. A field cannot be added without breaking every - # struct literal that builds the value, which would fail the whole target - # and prove nothing about this test in particular, so the shape change is a - # new enum variant: it compiles, and it changes exactly one derived schema. - @{ - Id = 'M11' - Package = $pipeline - File = $documents - Rule = 'A change to a value type''s shape is caught unless its schema is regenerated.' - Test = 'tests::pipeline_published_schemas_match_derivation' - Old = ' FindingSeverity { Blocker, High, Medium, Low }' - New = ' FindingSeverity { Blocker, High, Medium, Low, Cosmetic }' - }, - # The cut's second named mutation. `/` is not a Label byte, so an unescaped - # repo both adds a segment the reader cannot tell from a real one and puts a - # byte in the key that no key segment may carry. - @{ - Id = 'M12' - Package = $pipeline - File = $documents - Rule = 'A repo inside a key is escaped.' - Test = 'tests::stewardship_key_escapes_the_repo_slash' - Old = ' Ok(repo.0.replace(''_'', "__").replace(''/'', "_-"))' - New = ' Ok(repo.0.clone())' - }, - # Soul F1. Escaping is not enough on its own: the escape has to be - # injective, or two repos claim one key and a mind stewarding both silently - # loses a document. This is the escape the cut shipped, which is not. - @{ - Id = 'M13' - Package = $pipeline - File = $documents - Rule = 'The repo escape is injective, so two repos cannot claim one key.' - Test = 'tests::stewardship_key_escapes_the_repo_slash' - Old = ' Ok(repo.0.replace(''_'', "__").replace(''/'', "_-"))' - New = ' Ok(repo.0.replace(''/'', "_"))' - }, - # Soul F2, which is Soul's own survivor. Stewardship and hand-off validate - # every slug they key by; the instance root did not, so a `Slug` reached the - # key unchecked and `thought cage` composed a store key carrying a space. - @{ - Id = 'M14' - Package = $pipeline - File = $documents - Rule = 'An instance''s slug is validated where its key is composed.' - Test = 'tests::keys_read_back_as_ids_of_their_kind' - Old = ' dotted_text("instance.instance", &value.instance.0)?;' - New = ' let _ = &value.instance.0;' - }, - # Soul F3. The key writer made the instance a root and the id reader was - # left behind, so an instance key would not read back as an instance id and - # no `PipelineRef` could name one. This is the reader as the cut shipped it. - @{ - Id = 'M15' - Package = $pipeline - File = $documents - Rule = 'The id reader knows the same roots the key writer does.' - Test = 'tests::keys_read_back_as_ids_of_their_kind' - Old = ' if matches!(kind, PipelineKind::Campaign | PipelineKind::Instance) {' - New = ' if kind == PipelineKind::Campaign {' - } -) - -$results = @() -foreach ($mutation in $mutations) { - $path = Join-Path $repo $mutation.File - $original = [IO.File]::ReadAllText($path) - # The anchors here are written with plain newlines and the working tree is - # CRLF, so a multi-line anchor matches nothing unless it is converted to the - # line endings the file actually has. Convert towards the file. - $eol = if ($original.Contains("`r`n")) { "`r`n" } else { "`n" } - $old = ($mutation.Old -replace "`r`n", "`n") -replace "`n", $eol - $new = ($mutation.New -replace "`r`n", "`n") -replace "`n", $eol - # Exactly one site, not merely at least one: `.Replace` below mutates every - # match, so a second site would change more than the entry describes and the - # test could then fail for a reason the entry does not name. - $sites = [regex]::Matches($original, [regex]::Escape($old)).Count - if ($sites -ne 1) { - throw "$($mutation.Id): anchor matches $sites times in $($mutation.File), expected exactly 1. The mutation is stale." - } - $mutated = $original.Replace($old, $new) - if ($mutated -eq $original) { - throw "$($mutation.Id): replacement changed nothing." - } - [IO.File]::WriteAllText($path, $mutated) - try { - & cargo test -p $mutation.Package --lib -- --exact $mutation.Test | Out-Host - $survived = ($LASTEXITCODE -eq 0) - } - finally { - [IO.File]::WriteAllText($path, $original) - } - $results += [pscustomobject]@{ - Id = $mutation.Id - Package = $mutation.Package - Test = $mutation.Test.Split(':')[-1] - Killed = -not $survived - Rule = $mutation.Rule - } - Write-Host "$($mutation.Id): $(if ($survived) { 'SURVIVED - the rule is not pinned' } else { 'killed' })" -} - -$results | Format-Table -AutoSize -if ($results | Where-Object { -not $_.Killed }) { - throw 'At least one mutation survived.' -} -Write-Host 'Every mutation was killed by its own test.' diff --git a/tools/eureka-cut6-mutations.psd1 b/tools/eureka-cut6-mutations.psd1 new file mode 100644 index 00000000..aa5e90e5 --- /dev/null +++ b/tools/eureka-cut6-mutations.psd1 @@ -0,0 +1,143 @@ +# Eureka Cut 6 mutation suite. Entries only; the harness is +# tools/eureka-mutations.ps1. +# +# powershell -File tools/eureka-mutations.ps1 ` +# -Entries tools/eureka-cut6-mutations.psd1 ` +# -Target epiphany-pipeline/src/lib.rs,epiphany-core/src/runtime_spine.rs ` +# -Test 'cargo test -p epiphany-pipeline --lib' +# +# M1-M9 are Cut 4's suite re-anchored: the code they mutate moved from +# `epiphany-core/src/pipeline_documents.rs` to `epiphany-pipeline/src/lib.rs`. +# M10 still mutates the spine in epiphany-core, so it carries its own command. +# M11 is the first of the two the cut named for the kinds it added. +# +# Five entries Cut 6b's grammar made stale are gone rather than re-anchored; +# each note below names where the rule is pinned now. +# +# - M5, "a finding label carries no dot": the finding arm no longer validates +# the label itself; the label goes through `local`, which refuses a dotted +# part as `finding.key`. Cut 6b's F5 accepted that refusal; the rule is +# pinned by `composed_keys_cannot_collide` and Cut 6b's M6. +# - M12, "a repo inside a key is escaped", and M13, "the repo escape is +# injective": `key_segment` replaced `repo_segment`; the escape and its +# injectivity are pinned by Cut 6b's M6 and the S2 receiver pair. +# - M14, "an instance's slug is validated where its key is composed": the +# instance arm composes through the shared root check; pinned by Cut 6b's +# M5 and M1. +# - M15, "the id reader knows the same roots the key writer does": every kind +# reads back with no root excused; pinned by Cut 6b's M1. +@{ + Mutations = @( + @{ + Id = 'M1' + File = 'epiphany-pipeline/src/lib.rs' + Rule = 'A document decodes back only from an envelope of its own type id.' + Test = 'tests::every_pipeline_kind_round_trips_through_named_slot_zero' + Old = '$(if envelope.r#type == <$document as DatabaseEntry>::TYPE {' + New = '$(if envelope.r#type != <$document as DatabaseEntry>::TYPE {' + }, + @{ + Id = 'M2' + File = 'epiphany-pipeline/src/lib.rs' + Rule = 'Text bounds are measured in UTF-8 bytes, not characters.' + Test = 'tests::bounds_refuse_in_utf8_bytes' + Old = ' move |field: &str, value: &str| bound(field, limit, value.len())' + New = ' move |field: &str, value: &str| bound(field, limit, value.chars().count())' + }, + @{ + Id = 'M3' + File = 'epiphany-pipeline/src/lib.rs' + Rule = 'An OrgRepo carries exactly one slash.' + Test = 'tests::repo_fields_must_be_org_slash_repo' + Old = " if !org.is_empty() && !repo.is_empty() && !repo.contains('/') && value.len() <= 200 =>" + New = ' if !org.is_empty() && !repo.is_empty() && value.len() <= 200 =>' + }, + @{ + Id = 'M4' + File = 'epiphany-pipeline/src/lib.rs' + Rule = 'A write envelope''s key is recomputed from the value and a mismatch refuses.' + Test = 'tests::keys_are_derived_and_mismatch_refuses' + Old = ' if envelope.key != expected {' + New = ' if false {' + }, + @{ + Id = 'M6' + File = 'epiphany-pipeline/src/lib.rs' + Rule = 'A parent local must carry its own kind''s marker character.' + Test = 'tests::parent_ids_are_parsed_strictly' + Old = ' let digits = suffix.strip_prefix(marker).ok_or_else(invalid)?;' + New = ' let digits = suffix.strip_prefix(marker).unwrap_or(suffix);' + }, + @{ + Id = 'M7' + File = 'epiphany-pipeline/src/lib.rs' + Rule = 'A PipelineRef''s id is parsed as a full id of its declared kind.' + Test = 'tests::resolution_subject_is_a_full_id_of_its_kind' + Old = ' pipeline_id(&format!("{at}.id"), &self.id.0, self.kind).map(|_| ())' + New = ' Ok(())' + }, + @{ + Id = 'M8' + File = 'epiphany-pipeline/src/lib.rs' + Rule = 'Published schemas are the Rust derivation, byte for byte.' + Test = 'tests::pipeline_published_schemas_match_derivation' + Old = ' Short = 200, |field, value| within(200)(field, value);' + New = ' Short = 201, |field, value| within(200)(field, value);' + }, + # Soul F2. M1 removes the positive match; this removes the refusal that + # catches everything the positive match misses, leaving a decode that + # says "malformed payload" about an envelope that was never a document. + @{ + Id = 'M9' + File = 'epiphany-pipeline/src/lib.rs' + Rule = 'An envelope whose type id belongs to another kind is refused as ForeignStore.' + Test = 'tests::decode_refuses_an_envelope_of_a_foreign_type' + Old = 'Err(PipelineRefusal::ForeignStore { r#type: envelope.r#type.clone() })' + New = 'Err(format_error("payload", &envelope.r#type))' + }, + # Soul F4. The spine cache refuses a pipeline store by registering its + # own types and nothing else, so the mutation that removes the rule is + # the one that registers a pipeline type into it. epiphany-core no + # longer has the documents, so the mutation declares its own type + # carrying the same type id, exactly as the test does. + @{ + Id = 'M10' + File = 'epiphany-core/src/runtime_spine.rs' + Command = 'cargo test -p epiphany-core --lib' + Rule = 'The runtime spine cache refuses an epiphany.pipeline.* envelope.' + Test = 'runtime_spine::tests::runtime_spine_cache_refuses_a_pipeline_store' + Old = @' +fn runtime_spine_schema_cache() -> Result { + let mut cache = CultCache::new(); +'@ + New = @' +fn runtime_spine_schema_cache() -> Result { + let mut cache = CultCache::new(); + #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] + #[cultcache( + type = "epiphany.pipeline.campaign.v1", + schema = "EpiphanyPipelineCampaignDocument" + )] + struct RegisteredPipelineDocument { + #[cultcache(key = 0)] + slug: String, + } + cache.register_entry_type::()?; +'@ + }, + # The cut's first named mutation: change a value type's shape and leave + # the published schema alone. A field cannot be added without breaking + # every struct literal that builds the value, which would fail the whole + # target and prove nothing about this test in particular, so the shape + # change is a new enum variant: it compiles, and it changes exactly one + # derived schema. + @{ + Id = 'M11' + File = 'epiphany-pipeline/src/lib.rs' + Rule = 'A change to a value type''s shape is caught unless its schema is regenerated.' + Test = 'tests::pipeline_published_schemas_match_derivation' + Old = ' FindingSeverity { Blocker, High, Medium, Low }' + New = ' FindingSeverity { Blocker, High, Medium, Low, Cosmetic }' + } + ) +} diff --git a/tools/eureka-cut6b-mutations.ps1 b/tools/eureka-cut6b-mutations.ps1 deleted file mode 100644 index 7ff824bb..00000000 --- a/tools/eureka-cut6b-mutations.ps1 +++ /dev/null @@ -1,185 +0,0 @@ -# Eureka Cut 6b mutation suite: the key grammar. -# -# Each entry below is one rule the cut claims a test pins, together with the -# exact source change that restores the old permissiveness. A mutation counts -# only if the named test fails while the mutation is applied. The script applies -# one mutation at a time, runs the whole `epiphany-pipeline` suite so collateral -# kills are visible, and restores the file by the reverse edit before the next -# one, then asserts the restored bytes equal the original. -# -# powershell -File tools/eureka-cut6b-mutations.ps1 -# -# Windows PowerShell 5.1 is the interpreter on this host; there is no `pwsh`. -# Requires CARGO_TARGET_DIR to be set the way the cut ran it, or it will build -# into the repo-local target/. -# -# Harness rules: -# - Every read and write is byte-exact UTF-8 with no BOM. The file holds an `é` -# literal in `bounds_refuse_in_utf8_bytes`; the 5.1 default round-trip -# corrupts it, which fails that test on its own and fakes a kill. -# - M0 is a no-op control: the file is rewritten through the same I/O path with -# no change and the suite must stay green. If M0 fails anything, the harness -# is broken and no verdict below it can be trusted. -# - Every anchor must match exactly once. Zero (a stale anchor, or a multi-line -# anchor written against the wrong line endings) and more than one (`.Replace` -# would change more than the entry describes) both throw. - -$ErrorActionPreference = 'Stop' - -if ($PSVersionTable.PSVersion.Major -lt 5 -or $env:OS -ne 'Windows_NT') { - throw "This script runs under Windows PowerShell 5.1 on Windows. Found PowerShell $($PSVersionTable.PSVersion) on $(if ($env:OS) { $env:OS } else { 'a non-Windows host' })." -} - -$repo = Split-Path -Parent $PSScriptRoot -$package = 'epiphany-pipeline' -$path = Join-Path $repo 'epiphany-pipeline/src/lib.rs' -$utf8 = [System.Text.UTF8Encoding]::new($false) - -function Read-Source { [System.IO.File]::ReadAllText($path, $utf8) } -function Write-Source([string] $text) { [System.IO.File]::WriteAllText($path, $text, $utf8) } -function Get-SourceHash { (Get-FileHash -Algorithm SHA256 -Path $path).Hash } - -# Runs the suite; returns the list of failed test names and the exit code. -function Invoke-Suite { - # cargo writes its build chatter to stderr; under `Stop` a redirected stderr - # line is a terminating error, so the redirect runs under `Continue`. - $ErrorActionPreference = 'Continue' - $output = & cargo test -p $package --lib 2>&1 | ForEach-Object { "$_" } - $ErrorActionPreference = 'Stop' - $output | Out-Host - $failed = @($output | ForEach-Object { - if ($_ -match '^test (\S+) \.\.\. FAILED') { $Matches[1] } - }) - [pscustomobject]@{ Failed = $failed; ExitCode = $LASTEXITCODE } -} - -$mutations = @( - @{ - Id = 'M1' - Rule = 'Campaign and instance roots do not share one namespace (defect 1).' - Test = 'tests::roots_of_different_kinds_do_not_share_a_key' - Old = ' D::Campaign(value) => ("campaign.slug", value.slug.0.as_str(), local(&key_field, &[ROOT_LOCAL])?),' - New = ' D::Campaign(value) => return Ok(value.slug.0.clone()),' - }, - @{ - Id = 'M2' - Rule = 'A resolution''s key carries its subject''s kind (defect 2).' - Test = 'tests::a_resolution_names_its_subjects_kind' - Old = ' let parts = std::iter::once(value.subject.kind.name()).chain(subject_local.split(''.'')).collect::>();' - New = ' let parts = subject_local.split(''.'').collect::>();' - }, - @{ - Id = 'M3' - Rule = 'A resolution of a resolution composes a key the reader accepts (defect 3).' - Test = 'tests::a_resolution_of_a_resolution_reads_back' - Old = ' (field, subject_root, local(&key_field, &parts)?)' - New = ' return Ok(format!("resolution:{}", value.subject.id.0))' - }, - @{ - Id = 'M4' - Rule = 'A composed local is bounded whole, not only per part.' - Test = 'tests::a_composed_local_is_bounded_whole' - Old = @' - if joined.len() > 64 { - return Err(format_error(field, &joined)); - } -'@ - New = '' - }, - @{ - Id = 'M5' - Rule = 'The root segment is validated where the key is composed.' - Test = 'tests::every_key_has_exactly_three_segments' - Old = ' dotted_text(root_field, root)?;' - New = ' let _ = root_field;' - }, - @{ - Id = 'M6' - Rule = 'No local part carries the separator, the head included (R2).' - Test = 'tests::no_local_part_carries_the_separator' - Old = @' - for part in parts { - label_text(field, part)?; - } -'@ - New = @' - for (index, part) in parts.iter().enumerate() { - if index == 0 { dotted_text(field, part)?; } else { label_text(field, part)?; } - } -'@ - } -) - -$original = Read-Source -$originalHash = Get-SourceHash -$eol = if ($original.Contains("`r`n")) { "`r`n" } else { "`n" } -Write-Host "Line endings in $path`: $(if ($eol -eq "`r`n") { 'CRLF' } else { 'LF' })" - -# M0: the no-op control. -Write-Host '--- M0: no-op control' -Write-Source $original -if ((Get-SourceHash) -ne $originalHash) { - throw 'M0: rewriting the file through the harness I/O path changed its bytes. The harness is broken.' -} -$control = Invoke-Suite -if ($control.ExitCode -ne 0 -or $control.Failed.Count -ne 0) { - throw "M0: the unmutated suite failed ($($control.Failed -join ', ')). The harness is broken; no verdict below can be trusted." -} -Write-Host 'M0: green, bytes unchanged' - -$results = @([pscustomobject]@{ Id = 'M0'; Test = '(whole suite)'; Killed = $false; Collateral = ''; Rule = 'No-op control: rewrite through the I/O path, suite stays green.' }) - -foreach ($mutation in $mutations) { - Write-Host "--- $($mutation.Id): $($mutation.Rule)" - $old = ($mutation.Old -replace "`r`n", "`n") -replace "`n", $eol - $new = ($mutation.New -replace "`r`n", "`n") -replace "`n", $eol - $sites = [regex]::Matches($original, [regex]::Escape($old)).Count - if ($sites -ne 1) { - throw "$($mutation.Id): anchor matches $sites times, expected exactly 1. The mutation is stale or the line endings differ." - } - $mutated = $original.Replace($old, $new) - if ($mutated -eq $original) { - throw "$($mutation.Id): replacement changed nothing." - } - Write-Source $mutated - try { - $run = Invoke-Suite - } - finally { - # Reverse edit, never a checkout: the restored text must be the original. - if ($new -eq '') { - $restored = $original - } else { - $restoreSites = [regex]::Matches($mutated, [regex]::Escape($new)).Count - if ($restoreSites -ne 1) { - Write-Source $original - throw "$($mutation.Id): the mutated text matches $restoreSites times on restore; wrote the original instead." - } - $restored = $mutated.Replace($new, $old) - } - Write-Source $restored - if ((Get-SourceHash) -ne $originalHash) { - throw "$($mutation.Id): the reverse edit did not restore the original bytes." - } - } - if ($run.ExitCode -ne 0 -and $run.Failed.Count -eq 0) { - throw "$($mutation.Id): the mutated tree did not build, so no test ran. That is not a kill; fix the mutation." - } - $named = $mutation.Test.Split(':')[-1] - $killed = $run.Failed -contains $mutation.Test - $collateral = @($run.Failed | Where-Object { $_ -ne $mutation.Test } | ForEach-Object { $_.Split(':')[-1] }) - $results += [pscustomobject]@{ - Id = $mutation.Id - Test = $named - Killed = $killed - Collateral = ($collateral -join ', ') - Rule = $mutation.Rule - } - Write-Host "$($mutation.Id): $(if ($killed) { "killed by $named" } else { 'SURVIVED - the rule is not pinned' })$(if ($collateral.Count) { "; collateral: $($collateral -join ', ')" })" -} - -$results | Format-Table -AutoSize -Wrap -if ($results | Where-Object { $_.Id -ne 'M0' -and -not $_.Killed }) { - throw 'At least one mutation survived.' -} -Write-Host 'M0 green; every mutation was killed by its own test.' diff --git a/tools/eureka-cut6b-mutations.psd1 b/tools/eureka-cut6b-mutations.psd1 new file mode 100644 index 00000000..d2647ae6 --- /dev/null +++ b/tools/eureka-cut6b-mutations.psd1 @@ -0,0 +1,104 @@ +# Eureka Cut 6b mutation suite: the key grammar. Entries only; the harness is +# tools/eureka-mutations.ps1. +# +# powershell -File tools/eureka-mutations.ps1 ` +# -Entries tools/eureka-cut6b-mutations.psd1 ` +# -Target epiphany-pipeline/src/lib.rs ` +# -Test 'cargo test -p epiphany-pipeline --lib' +# +# M1-M6 are the cut's own rules. S2, S4 and S5 are the mutations Soul's pass +# found surviving; each is kept exactly as Soul wrote it, against the test the +# fix batch extended to kill it. The file holds an `é` literal in +# `bounds_refuse_in_utf8_bytes`, which is why the harness I/O is byte-exact. +@{ + Mutations = @( + @{ + Id = 'M1' + Rule = 'Campaign and instance roots do not share one namespace (defect 1).' + Test = 'tests::roots_of_different_kinds_do_not_share_a_key' + Old = ' D::Campaign(value) => ("campaign.slug", value.slug.0.as_str(), local(&key_field, &[ROOT_LOCAL])?),' + New = ' D::Campaign(value) => return Ok(value.slug.0.clone()),' + }, + @{ + Id = 'M2' + Rule = 'A resolution''s key carries its subject''s kind (defect 2).' + Test = 'tests::a_resolution_names_its_subjects_kind' + Old = ' let parts = std::iter::once(value.subject.kind.name()).chain(subject_local.split(''.'')).collect::>();' + New = ' let parts = subject_local.split(''.'').collect::>();' + }, + @{ + Id = 'M3' + Rule = 'A resolution of a resolution composes a key the reader accepts (defect 3).' + Test = 'tests::a_resolution_of_a_resolution_reads_back' + Old = ' (field, subject_root, local(&key_field, &parts)?)' + New = ' return Ok(format!("resolution:{}", value.subject.id.0))' + }, + @{ + Id = 'M4' + Rule = 'A composed local is bounded whole, not only per part.' + Test = 'tests::a_composed_local_is_bounded_whole' + Old = @' + if joined.len() > 64 { + return Err(format_error(field, &joined)); + } +'@ + New = '' + }, + @{ + Id = 'M5' + Rule = 'The root segment is validated where the key is composed.' + Test = 'tests::every_key_has_exactly_three_segments' + Old = ' dotted_text(root_field, root)?;' + New = ' let _ = root_field;' + }, + @{ + Id = 'M6' + Rule = 'No local part carries the separator, the head included (R2).' + Test = 'tests::no_local_part_carries_the_separator' + Old = @' + for part in parts { + label_text(field, part)?; + } +'@ + New = @' + for (index, part) in parts.iter().enumerate() { + if index == 0 { dotted_text(field, part)?; } else { label_text(field, part)?; } + } +'@ + }, + # Soul S2. The escape's `_` half was pinned only through an OrgRepo, so + # an escape that skipped it for a value with no slash survived: `a_db` + # and `a.b` both keyed to `a_db`. The receiver pair kills it. + @{ + Id = 'S2' + Rule = 'key_segment escapes `_` for a slug as well as a repo, so `a_db` and `a.b` are two receivers.' + Test = 'tests::hand_off_names_both_instances' + Old = ' value.replace(''_'', "__").replace(''/'', "_-").replace(''.'', "_d")' + New = @' + if !value.contains('/') { + return value.replace('.', "_d"); + } + value.replace('_', "__").replace('/', "_-").replace('.', "_d") +'@ + }, + # Soul S4. The composer's root check was pinned by `a:b` alone, so a + # check that trimmed a trailing dot first survived. The root fixtures + # kill it. + @{ + Id = 'S4' + Rule = 'The composer''s root check refuses a trailing dot, not only a colon.' + Test = 'tests::every_key_has_exactly_three_segments' + Old = ' dotted_text(root_field, root)?;' + New = ' dotted_text(root_field, root.trim_end_matches(''.''))?;' + }, + # Soul S5. The whole-local bound was unpinned on both sides, so an + # off-by-one survived. The 64- and 65-byte fixtures kill it. + @{ + Id = 'S5' + Rule = 'The whole-local bound is 64 exactly: 65 refuses.' + Test = 'tests::a_composed_local_is_bounded_whole' + Old = ' if joined.len() > 64 {' + New = ' if joined.len() > 65 {' + } + ) +} diff --git a/tools/eureka-mutations.ps1 b/tools/eureka-mutations.ps1 new file mode 100644 index 00000000..2b0c4dd6 --- /dev/null +++ b/tools/eureka-mutations.ps1 @@ -0,0 +1,240 @@ +# Eureka mutation harness: one script for every cut's mutation suite. +# +# A suite is a data file of entries; each entry is one rule a test claims to +# pin, together with the exact source change that removes the rule and the +# test that must fail while the change is applied. The harness applies one +# entry at a time, runs that test alone, restores the file by the reverse edit, +# and reports a verdict per entry. Nothing here knows which cut it is serving. +# +# powershell -File tools/eureka-mutations.ps1 ` +# -Entries tools/eureka-cut6b-mutations.psd1 ` +# -Target epiphany-pipeline/src/lib.rs ` +# -Test 'cargo test -p epiphany-pipeline --lib' +# +# -Entries a `.psd1` whose root is `@{ Mutations = @(...) }`, or a `.ps1` that +# returns the array. Paths are relative to the repo root. +# -Target the file (or files) the suite mutates, relative to the repo root. +# With one target every edit is against it; with more than one, each +# edit names its `File`, which must be one of the targets. +# -Test the cargo command that runs the suite's tests, as one string; the +# harness appends `-- --exact ` per entry. An entry may +# carry its own `Command` when its test lives in another package. +# +# Entry shape: `Id`, `Rule`, `Test`, and either `Old`/`New` (with an optional +# `File`) or `Edits = @(@{ File; Old; New }, ...)`. `New = ''` deletes the +# anchor. `MustNotCompile = $true` marks an entry whose proof is a compile +# error: it is killed when the mutated tree does not build and survives when +# it does; for every other entry a tree that does not build is no verdict. +# +# Harness rules: +# - Every read and write is byte-exact UTF-8 with no BOM. Windows PowerShell +# 5.1 is the interpreter on this host and its default round-trip corrupts a +# non-ASCII literal, which fails a test on its own and fakes a kill. +# - Anchors are written with plain newlines and converted to the target's own +# line endings before matching. +# - Every anchor must match exactly once. Zero is a stale entry; more than one +# means `.Replace` would change more than the entry describes. Both throw, +# naming the entry. +# - Restore is the reverse edit, never a checkout, and the restored bytes must +# hash to the original. If the reverse edit cannot be applied the original +# text is written back and the run throws. +# - M0 is built in and cannot be omitted: before any entry, every target is +# rewritten through the harness I/O path with no change and every command +# the suite uses is run bare. If a byte moves or a test fails, the harness is +# broken and no entry runs. + +param( + [Parameter(Mandatory = $true)] [string] $Entries, + [Parameter(Mandatory = $true)] [string[]] $Target, + [Parameter(Mandatory = $true)] [string] $Test +) + +$ErrorActionPreference = 'Stop' + +if ($PSVersionTable.PSVersion.Major -lt 5 -or $env:OS -ne 'Windows_NT') { + throw "This script runs under Windows PowerShell 5.1 on Windows. Found PowerShell $($PSVersionTable.PSVersion) on $(if ($env:OS) { $env:OS } else { 'a non-Windows host' })." +} +if (-not $env:CARGO_TARGET_DIR) { + throw 'CARGO_TARGET_DIR is not set; set it the way the cut ran it rather than building into the repo-local target/.' +} + +$repo = Split-Path -Parent $PSScriptRoot +$utf8 = [System.Text.UTF8Encoding]::new($false) +# `powershell -File` hands a comma-separated argument over as one string, so +# split it here rather than asking the caller to know that. +$Target = @($Target | ForEach-Object { $_ -split ',' } | ForEach-Object { $_.Trim() } | Where-Object { $_ }) + +function Read-Text([string] $path) { [System.IO.File]::ReadAllText($path, $utf8) } +function Write-Text([string] $path, [string] $text) { [System.IO.File]::WriteAllText($path, $text, $utf8) } +function Get-Hash([string] $path) { (Get-FileHash -Algorithm SHA256 -LiteralPath $path).Hash } + +# Runs one cargo command from the repo root; returns the test lines it printed +# and its exit code. cargo writes build chatter to stderr, and under `Stop` a +# redirected stderr line is a terminating error, so the redirect runs under +# `Continue`. +function Invoke-Cargo([string] $command, [string[]] $filter) { + $words = @($command -split '\s+' | Where-Object { $_ }) + if ($words.Count -lt 1) { throw "Empty test command." } + $arguments = @($words | Select-Object -Skip 1) + $filter + $ErrorActionPreference = 'Continue' + Push-Location $repo + try { + $output = & $words[0] @arguments 2>&1 | ForEach-Object { "$_" } + $code = $LASTEXITCODE + } + finally { + Pop-Location + $ErrorActionPreference = 'Stop' + } + $output | Out-Host + $passed = @($output | ForEach-Object { if ($_ -match '^test (\S+) \.\.\. ok$') { $Matches[1] } }) + $failed = @($output | ForEach-Object { if ($_ -match '^test (\S+) \.\.\. FAILED$') { $Matches[1] } }) + # A test failure also prints an `error:` line, so only a compiler error + # code or cargo's own "could not compile" counts as a build failure. + $compiled = -not ($output | Where-Object { $_ -match '^error\[E\d+\]' -or $_ -match 'could not compile' }) + [pscustomobject]@{ Passed = $passed; Failed = $failed; ExitCode = $code; Compiled = $compiled } +} + +# --- Load the suite --------------------------------------------------------- + +$entriesPath = Join-Path $repo $Entries +$mutations = switch ([System.IO.Path]::GetExtension($entriesPath)) { + '.psd1' { (Import-PowerShellDataFile -LiteralPath $entriesPath).Mutations } + '.ps1' { & $entriesPath } + default { throw "$Entries`: an entries file is a .psd1 or a .ps1." } +} +if (-not $mutations -or @($mutations).Count -lt 1) { throw "$Entries`: no mutations." } + +$targets = @{} +foreach ($file in $Target) { + $path = Join-Path $repo $file + if (-not (Test-Path -LiteralPath $path)) { throw "Target $file does not exist." } + $targets[$file] = $path +} + +# Normalise every entry to a list of edits against a named target. +$suite = foreach ($mutation in $mutations) { + if (-not $mutation.Id) { throw 'An entry has no Id.' } + if (-not $mutation.Test) { throw "$($mutation.Id): no Test." } + $edits = if ($mutation.ContainsKey('Edits')) { @($mutation.Edits) } else { @(@{ File = $mutation.File; Old = $mutation.Old; New = $mutation.New }) } + if ($edits.Count -lt 1) { throw "$($mutation.Id): no edits." } + $resolved = foreach ($edit in $edits) { + $file = $edit.File + if (-not $file) { + if ($Target.Count -ne 1) { throw "$($mutation.Id): an edit names no File and the suite has $($Target.Count) targets." } + $file = $Target[0] + } + if (-not $targets.ContainsKey($file)) { throw "$($mutation.Id): edits $file, which is not a target of this run." } + if ([string]::IsNullOrEmpty($edit.Old)) { throw "$($mutation.Id): an edit against $file has an empty Old." } + [pscustomobject]@{ File = $file; Old = [string] $edit.Old; New = [string] $edit.New } + } + [pscustomobject]@{ + Id = [string] $mutation.Id + Rule = [string] $mutation.Rule + Test = [string] $mutation.Test + Command = if ($mutation.Command) { [string] $mutation.Command } else { $Test } + MustNotCompile = [bool] $mutation.MustNotCompile + Edits = @($resolved) + } +} + +# --- M0: the no-op control ---------------------------------------------------- + +$originals = @{} +$hashes = @{} +$eols = @{} +Write-Host '--- M0: no-op control' +foreach ($file in $Target) { + $path = $targets[$file] + $text = Read-Text $path + $hash = Get-Hash $path + Write-Text $path $text + if ((Get-Hash $path) -ne $hash) { + throw "harness broken: rewriting $file through the harness I/O path changed its bytes. No entry ran." + } + $originals[$file] = $text + $hashes[$file] = $hash + $eols[$file] = if ($text.Contains("`r`n")) { "`r`n" } else { "`n" } + Write-Host "M0: $file rewritten, bytes unchanged ($(if ($eols[$file] -eq "`r`n") { 'CRLF' } else { 'LF' }), SHA-256 $hash)" +} +foreach ($command in @($suite | ForEach-Object { $_.Command } | Select-Object -Unique)) { + $control = Invoke-Cargo $command @() + if ($control.ExitCode -ne 0 -or $control.Failed.Count -ne 0 -or $control.Passed.Count -lt 1) { + throw "harness broken: '$command' is not green unmutated (exit $($control.ExitCode), $($control.Passed.Count) passed, failed: $($control.Failed -join ', ')). No entry ran." + } + Write-Host "M0: '$command' green, $($control.Passed.Count) passed" +} +$results = @([pscustomobject]@{ Id = 'M0'; Test = '(suite)'; Verdict = 'green'; Rule = 'No-op control: targets rewritten through the I/O path, every command green.' }) + +# --- Entries -------------------------------------------------------------------- + +foreach ($mutation in $suite) { + Write-Host "--- $($mutation.Id): $($mutation.Rule)" + # Prepare every edit against the original text before touching a file, so + # a stale anchor throws with the tree untouched. + $texts = @{} + foreach ($file in @($mutation.Edits | ForEach-Object { $_.File } | Select-Object -Unique)) { $texts[$file] = $originals[$file] } + $applied = @() + foreach ($edit in $mutation.Edits) { + $eol = $eols[$edit.File] + $old = ($edit.Old -replace "`r`n", "`n") -replace "`n", $eol + $new = ($edit.New -replace "`r`n", "`n") -replace "`n", $eol + $sites = [regex]::Matches($texts[$edit.File], [regex]::Escape($old)).Count + if ($sites -ne 1) { + throw "$($mutation.Id): anchor matches $sites times in $($edit.File), expected exactly 1." + } + if ($old -eq $new) { throw "$($mutation.Id): replacement changes nothing in $($edit.File)." } + # The edit is applied by position, so the reverse edit below is the + # same splice backwards and does not depend on the replacement being + # unique (`Ok(())` is not) or non-empty. + $at = $texts[$edit.File].IndexOf($old, [System.StringComparison]::Ordinal) + $texts[$edit.File] = $texts[$edit.File].Substring(0, $at) + $new + $texts[$edit.File].Substring($at + $old.Length) + $applied += [pscustomobject]@{ File = $edit.File; Old = $old; New = $new; At = $at } + } + foreach ($file in $texts.Keys) { Write-Text $targets[$file] $texts[$file] } + try { + $run = Invoke-Cargo $mutation.Command @('--', '--exact', $mutation.Test) + } + finally { + # Reverse every edit, last first, each at the offset it was applied. + $restored = @{} + foreach ($file in $texts.Keys) { $restored[$file] = $texts[$file] } + for ($index = $applied.Count - 1; $index -ge 0; $index--) { + $edit = $applied[$index] + $text = $restored[$edit.File] + if ($text.Length -lt $edit.At + $edit.New.Length -or $text.Substring($edit.At, $edit.New.Length) -ne $edit.New) { + foreach ($file in $texts.Keys) { Write-Text $targets[$file] $originals[$file] } + throw "$($mutation.Id): the mutated text is not at its offset on restore in $($edit.File); wrote the originals instead." + } + $restored[$edit.File] = $text.Substring(0, $edit.At) + $edit.Old + $text.Substring($edit.At + $edit.New.Length) + } + foreach ($file in $texts.Keys) { + Write-Text $targets[$file] $restored[$file] + if ((Get-Hash $targets[$file]) -ne $hashes[$file]) { + Write-Text $targets[$file] $originals[$file] + throw "$($mutation.Id): the reverse edit did not restore $file to the original bytes; wrote the original instead." + } + } + } + $named = $mutation.Test.Split(':')[-1] + $verdict = if ($mutation.MustNotCompile) { + if (-not $run.Compiled) { 'killed (did not compile)' } else { 'SURVIVED (compiled)' } + } elseif (-not $run.Compiled) { + 'DID NOT BUILD (no verdict)' + } elseif ($run.Failed -contains $mutation.Test) { + 'killed' + } elseif ($run.Passed -contains $mutation.Test) { + 'SURVIVED' + } else { + 'TEST NOT RUN (stale test name?)' + } + $results += [pscustomobject]@{ Id = $mutation.Id; Test = $named; Verdict = $verdict; Rule = $mutation.Rule } + Write-Host "$($mutation.Id): $verdict" +} + +$results | Format-Table -AutoSize -Wrap +$notKilled = @($results | Where-Object { $_.Id -ne 'M0' -and $_.Verdict -notlike 'killed*' }) +if ($notKilled.Count) { + throw "Not every mutation was killed: $(($notKilled | ForEach-Object { "$($_.Id) $($_.Verdict)" }) -join '; ')." +} +Write-Host 'M0 green; every mutation was killed by its own test.'