#!/usr/bin/env bash
# Hand-mutant kill table: run a list of mutants against a revision on
# Yggdrasil, in ONE verify job, and get one table back.
#
#   tools/mutation-table.sh <local-repo> <rev> <image> <mutants-file> [workdir]
#     workdir   directory inside the repo where the commands run (e.g.
#               packages/cultcache-rs; CultLib has no Cargo.toml at its root).
#     env       TEST       default test command for mutants with no `test:` line
#               BUILD      optional compile-only command, run on the UNMUTATED
#                          tree first (a red or missing build aborts like
#                          BASELINE RED); when it fails on a mutant the mutant is
#                          UNVIABLE. Without it a mutant that does not compile
#                          has no failing test and reads ERROR, never CAUGHT.
#               MUT_TIMEOUT  seconds per build or test run (default 600)
#               TIMEOUT    seconds for the WHOLE job (ygg-verify's own cap;
#                          default 3600 here, stated in the START line). Size it
#                          to mutants x (build + test) plus the baselines and one
#                          snapshot commit (about 20 s per 400 MB of build output).
#               everything else ygg-verify.sh reads (SUMMARY=<log>, MEM, ...)
#     Typical, from an agent: launch as ONE background shell with
#       SUMMARY=<fresh scratch>/mut1.log TEST='cargo test' tools/mutation-table.sh ...
#     then `tools/stopgap/ygg-verify.sh wait <scratch>/mut1.log` in the foreground,
#     with a Bash tool timeout of at least 280000 ms.
#
# Mutants file (plain text; the text before the first `===` is ignored):
#     === label-unique-in-the-file      (any text; `|` is allowed)
#     file: src/lib.rs              (relative to workdir)
#     test: cargo test add_         (optional; else $TEST)
#     --- before
#     a + b                         (exact text, may span lines; must occur ONCE;
#                                    LF text also matches a CRLF file, a file
#                                    with mixed endings may read NOAPPLY)
#     --- after
#     a - b
#
# ISOLATION. Docker owns the per-mutant snapshot; no script restores a tree. The
# job (tools/stopgap/ygg-verify.sh RUN_ON_HOST=1) copies the revision into ONE
# baseline container's own filesystem, runs the baselines there (so target/,
# node_modules and every other build or test cache exist), and `docker commit`s
# that container to a throwaway image. Each mutant then runs in a fresh
# `docker run --rm` container from that image, sequentially, under the same
# CPUS/MEM caps and nice. Everything a mutant's build or test writes, tracked or
# ignored, lives in that container's copy-on-write layer and is gone when it
# exits, so no run can decide the next one. The image is removed when the job
# ends, including by timeout or kill. No container gains privilege. What is NOT
# isolated: the shared registry, nuget and sccache volumes (compiler caches keyed
# by input hash) and anything a test sends to the network.
# The outcome of each mutant container travels as its exit status (10 CAUGHT,
# 11 SURVIVED, 12 UNVIABLE, 13 TIMEOUT, 14 NOAPPLY, 15 ERROR); any other status
# (OOM kill, watchdog kill, docker failure) is an ERROR row.
#
# One row prints as each mutant finishes, so a job timeout keeps the finished rows.
# Row (a `YGG-SUMMARY/$YGG_NONCE: ` line, plain `YGG-SUMMARY: ` when no nonce is
# set): [i/N] label | outcome | killing test or detail. The outcome is a field of
# its own (the exit status), tallied as such, never read back from the rendered
# row. Outcomes:
#   CAUGHT    the test command failed AND a failing test name was parsed from it
#   SURVIVED  the test command passed with the mutation applied
#   UNVIABLE  BUILD failed, or only a test FILE failed (vitest `FAIL <file> [ <file> ]`,
#             jest `FAIL <file>` + `Test suite failed to run`, see tools/testparse.sh
#             LOADFAIL) with no test name: nothing ran, not a kill
#   TIMEOUT   a run hit MUT_TIMEOUT (not a kill)
#   NOAPPLY   before text absent, ambiguous, or equal to after (no mutation made)
#   ERROR     the test command failed but no failing test was parsed (a build
#             error without BUILD, a crash): not a kill
# Before any mutant, every distinct test command and BUILD run UNMUTATED. If one
# fails or a test command runs zero tests the job aborts with `BASELINE RED` or
# `BASELINE RAN ZERO TESTS`, because a red or empty baseline would make every
# mutant look killed or every one survive. Then the same unmutated run repeats in
# a fresh container from the snapshot (the state every mutant starts from); red
# there aborts with `SNAPSHOT RED` and no row counted: a test that cannot run
# twice (it creates a directory, claims a port) passes the baseline and fails
# every mutant, which would read as every mutant CAUGHT.
# Exit 0 means every mutant was CAUGHT or UNVIABLE and nothing else happened.
#
# DELETION LINE: this runner replaces the per-agent runmut/mutrun/strykrun
# scripts. It dies with tools/stopgap/ygg-verify.sh when `idunn verify` ships a
# typed mutation step. It reads tools/testparse.sh for failure names, the same
# parser the summary uses.
#
# Modes (this one file, three places): no flag = launcher on the workstation;
# --host = the driver on the Yggdrasil host, started by ygg-verify.sh RUN_ON_HOST=1;
# --baseline / --control / --mutant N = the work inside a container.
set -euo pipefail

SUMTAG="YGG-SUMMARY${YGG_NONCE:+/$YGG_NONCE}"   # ygg-verify.sh carries only lines tagged with its per-run nonce
# outcome <-> exit status of a --mutant container
OUTCOMES=(CAUGHT SURVIVED UNVIABLE TIMEOUT NOAPPLY ERROR); CODE0=10

# ---- launcher (workstation) -------------------------------------------------
if [ "${1:-}" != --host ] && [ "${1:-}" != --baseline ] && [ "${1:-}" != --mutant ] && [ "${1:-}" != --control ]; then
  here=$(cd "$(dirname "$0")" && pwd)
  repo=${1:?usage: mutation-table.sh <local-repo> <rev> <image> <mutants-file> [workdir]}
  rev=${2:?revision}; image=${3:?image}; mfile=${4:?mutants file}; workdir=${5:-.}
  [ -r "$mfile" ] || { echo "mutation-table: cannot read $mfile" >&2; exit 2; }
  stage=$(mktemp -d)
  trap 'rm -rf "$stage"' EXIT
  cp "$here/testparse.sh" "$here/mutation-table.sh" "$stage/"
  tr -d '\r' <"$mfile" >"$stage/mutants.txt"
  bundle=$(tar -C "$stage" -cz testparse.sh mutation-table.sh mutants.txt | base64 | tr -d '\n')
  # The driver runs on the host: it unpacks the bundle into its own directory (mounted read-only into every container) and removes it when done.
  cmd=$(printf 'd=$(mktemp -d) && echo %s | base64 -d | tar -xz -C "$d" && WORKDIR=%q BUILD=%q TEST=%q MUT_TIMEOUT=%q JOB_TIMEOUT=%q MT_BUNDLE="$d" bash "$d/mutation-table.sh" --host' \
    "$bundle" "$workdir" "${BUILD:-}" "${TEST:-}" "${MUT_TIMEOUT:-600}" "${TIMEOUT:-3600}")
  RUN_ON_HOST=1 TIMEOUT=${TIMEOUT:-3600} "$here/stopgap/ygg-verify.sh" "$repo" "$rev" "$image" "$cmd"
  exit $?
fi

# ---- driver (Yggdrasil host, inside the job's slot) --------------------------
if [ "$1" = --host ]; then
  : "${YGG_WORK:?not started by ygg-verify.sh RUN_ON_HOST=1}" "${YGG_IMAGE:?}" "${YGG_CNAME:?}" "${YGG_LABEL:?}"
  bundle=${MT_BUNDLE:?}
  set +e   # every docker step is checked by hand: a mutant container exits non-zero by design
  base="$YGG_CNAME-base"; snap="eureka-mt-$YGG_CNAME:snap"
  # A TERM waits for the foreground container (bounded by MUT_TIMEOUT); the stopgap's watchdog kills every
  # job-labelled container before it signals this driver, so cleanup has no running mutant to stop.
  cleanup() {
    sudo docker rm -f "$base" >/dev/null 2>&1 || true
    sudo docker rmi -f "$snap" >/dev/null 2>&1 || true
    [ -z "$bundle" ] || [ "$bundle" = / ] || rm -rf -- "$bundle"
  }
  trap cleanup EXIT; trap 'exit 143' TERM INT HUP
  # Every container carries the job label (the stopgap's watchdog kills by it) and the same caps.
  common=(--label "$YGG_LABEL" "--cpus=${YGG_CPUS:-4}" "--memory=${YGG_MEM:-6g}"
    -v "$bundle:/mt:ro" -v /etc/machine-id:/etc/machine-id:ro
    -v eureka-cargo-registry:/usr/local/cargo/registry -v eureka-nuget:/root/.nuget/packages -v eureka-sccache:/sccache
    -e BASH_ENV=/tmp/.ygg-bash-env
    -e "WORKDIR=${WORKDIR:-.}" -e "BUILD=${BUILD:-}" -e "TEST=${TEST:-}" -e "MUT_TIMEOUT=${MUT_TIMEOUT:-600}" -e "JOB_TIMEOUT=${JOB_TIMEOUT:-}"
    -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e 'GIT_CONFIG_VALUE_0=*')
  [ -z "${YGG_NONCE:-}" ] || common+=(-e "YGG_NONCE=$YGG_NONCE")
  # DOCKER_ARGS is word-split on purpose: it is a flag string.
  # shellcheck disable=SC2086
  drun() { sudo nice -n 10 docker run "$@"; }
  pre="printf 'set -o pipefail\n' >/tmp/.ygg-bash-env"

  sudo docker rm -f "$base" >/dev/null 2>&1 || true
  # shellcheck disable=SC2086
  drun --name "$base" "${common[@]}" -v "$YGG_WORK:/seed:ro" -w /src ${DOCKER_ARGS:-} "$YGG_IMAGE" \
    bash -o pipefail -c "$pre; bash /mt/mutation-table.sh --baseline"
  rc=$?
  [ "$rc" = 0 ] || exit "$rc"          # the baseline container printed why
  count=$(sudo docker cp "$base:/tmp/mtm/count" - | tar -xO)
  case $count in ''|*[!0-9]*) echo "$SUMTAG: cannot read the mutant count from the baseline container"; exit 2 ;; esac
  t0=$(date +%s)
  sudo docker commit "$base" "$snap" >/dev/null || { echo "$SUMTAG: docker commit failed"; exit 2; }
  sudo docker rm -f "$base" >/dev/null
  echo "$SUMTAG: SNAPSHOT committed in $(( $(date +%s) - t0 ))s; a control run of the unmutated snapshot comes first, then each mutant in a fresh container from it"
  # A test that passed once and cannot pass again (it creates a directory, claims a port, appends to a
  # fixture) is green in the baseline and red in every mutant's container, which would read as a fully
  # killed suite. The control is the baseline's tests run once more from the snapshot, unmutated.
  # shellcheck disable=SC2086
  drun --rm --name "$YGG_CNAME-ctl" "${common[@]}" -w /src ${DOCKER_ARGS:-} "$snap" \
    bash -o pipefail -c "$pre; bash /mt/mutation-table.sh --control"
  rc=$?
  [ "$rc" = 0 ] || { echo "$SUMTAG: the unmutated snapshot is red though the baseline was green, so no mutant result would mean anything; no rows counted"; exit "$rc"; }

  declare -A tally; for o in "${OUTCOMES[@]}"; do tally[$o]=0; done
  for n in $(seq 1 "$count"); do
    if [ -n "${YGG_KILLED:-}" ] && [ -e "$YGG_KILLED" ]; then
      echo "$SUMTAG: JOB KILLED by the watchdog before mutant $n of $count; the finished rows above stand"; exit 124
    fi
    # shellcheck disable=SC2086
    drun --rm --name "$YGG_CNAME-m$n" "${common[@]}" -w /src ${DOCKER_ARGS:-} "$snap" \
      bash -o pipefail -c "$pre; bash /mt/mutation-table.sh --mutant $n"
    rc=$?
    if [ "$rc" -ge "$CODE0" ] && [ "$rc" -lt $((CODE0 + ${#OUTCOMES[@]})) ]; then
      o=${OUTCOMES[$((rc - CODE0))]}
    else
      o=ERROR; echo "$SUMTAG: [$n/$count] mutant $n | ERROR | the mutant container exited $rc without a row (137 is a kill or OOM)"
    fi
    tally[$o]=$(( ${tally[$o]} + 1 ))
  done
  echo "$SUMTAG: MUTATION $count mutants: caught=${tally[CAUGHT]} survived=${tally[SURVIVED]} unviable=${tally[UNVIABLE]} timeout=${tally[TIMEOUT]} noapply=${tally[NOAPPLY]} error=${tally[ERROR]}  (baseline green, tests ran)"
  [ $(( ${tally[CAUGHT]} + ${tally[UNVIABLE]} )) -eq "$count" ]
  exit $?
fi

# ---- inside a container -----------------------------------------------------
set +e
# overridable only so tools/test-tools.sh can run this half against a mock
SRC=${MT_SRC:-/src}; MT_DIR=${MT_DIR:-/mt}; M=${MT_M:-/tmp/mtm}; SEED=${MT_SEED:-/seed}
# shellcheck source=testparse.sh
. "$MT_DIR/testparse.sh"
tmo=${MUT_TIMEOUT:-600}
# run <command> <logfile>: prints the exit status; 124 means timeout
run() { timeout "$tmo" bash -o pipefail -c "$1" >"$2" 2>&1; echo $?; }

# unmutated <BASELINE|SNAPSHOT> <count>: run every distinct test command, then BUILD, on the tree as it
# stands. A red or empty run aborts the job (exit 2) under that name: a red baseline would make every
# mutant look killed, an empty one every mutant survive, and a SNAPSHOT that is red where the BASELINE
# was green (a test that cannot run twice) would do the first again.
unmutated() {
  local what=$1 count=$2 n t rc scan tests; declare -A based
  for n in $(seq 1 "$count"); do
    t=$(cat "$M/$n/test"); t=${t:-$TEST}
    [ -n "$t" ] || { echo "$SUMTAG: mutant $(cat "$M/$n/label") has no test: line and TEST is unset"; exit 2; }
    [ -z "${based[$t]:-}" ] || continue
    based[$t]=1
    rc=$(run "$t" "$M/base.log")
    scan=$(tp_scan "$M/base.log")
    tests=$(printf '%s\n' "$scan" | awk '/^FRAMEWORK/{for(i=1;i<=NF;i++){split($i,a,"=");if(a[1]=="passed"||a[1]=="failed")s+=a[2]}}END{print s+0}')
    if [ "$rc" != 0 ]; then
      echo "$SUMTAG: $what RED (exit $rc) for: $t"; tail -n 15 "$M/base.log" | sed "s|^|$SUMTAG:   |"; exit 2
    fi
    if [ "$tests" -eq 0 ]; then
      echo "$SUMTAG: $what RAN ZERO TESTS (or printed counts no parser reads) for: $t"; exit 2
    fi
  done
  # BUILD runs unmutated too. A mistyped or missing build command fails every
  # mutant, and a table of all-UNVIABLE rows reads as a fully killed suite.
  if [ -n "${BUILD:-}" ]; then
    rc=$(run "$BUILD" "$M/base.log")
    if [ "$rc" != 0 ]; then
      echo "$SUMTAG: $what RED: BUILD failed on the unmutated tree (exit $rc; 124 is a timeout): $BUILD"
      tail -n 15 "$M/base.log" | sed "s|^|$SUMTAG:   |"; exit 2
    fi
  fi
}

if [ "$1" = --baseline ]; then
  # The revision lives in this container's own filesystem, so `docker commit` keeps it.
  if [ -d "$SEED" ]; then mkdir -p "$SRC" && cp -a "$SEED/." "$SRC/" || { echo "$SUMTAG: cannot copy $SEED into $SRC"; exit 2; }; fi
  cd "$SRC/${WORKDIR:-.}" || { echo "$SUMTAG: BAD WORKDIR ${WORKDIR:-.}"; exit 2; }
  rm -rf "$M"; mkdir -p "$M"
  if ! perl -e '
    my ($mf, $out) = @ARGV; my ($n, $sec, %c, %seen) = (0, "");
    open my $f, "<", $mf or die "cannot open $mf\n";
    sub flush {
      return unless exists $c{label};
      for my $k (qw(file before after)) { die "mutant $c{label}: no $k\n" unless defined $c{$k} }
      my $d = "$out/$n"; mkdir $d;
      for my $k (qw(label file test before after)) {
        my $v = join("\n", ref $c{$k} ? @{$c{$k}} : ($c{$k} // ""));
        open my $o, ">:raw", "$d/$k" or die; print $o $v; close $o;
      }
    }
    while (<$f>) {
      chomp;
      if (/^=== (.+?)\s*$/) { flush(); die "duplicate label $1\n" if $seen{$1}++; %c = (label => $1); $n++; $sec = ""; next }
      next unless exists $c{label};
      if ($sec eq "" && /^file: (.+?)\s*$/) { $c{file} = $1; next }
      if ($sec eq "" && /^test: (.+?)\s*$/) { $c{test} = $1; next }
      if (/^--- (before|after)\s*$/) { $sec = $1; $c{$sec} = []; next }
      push @{$c{$sec}}, $_ if $sec;
    }
    flush(); die "no mutants in $mf\n" unless $n;
    print "$n\n";' "$MT_DIR/mutants.txt" "$M" >"$M/count" 2>"$M/err"; then
    echo "$SUMTAG: BAD MUTANT FILE: $(cat "$M/err")"; exit 2
  fi
  count=$(cat "$M/count")
  echo "$SUMTAG: START $count mutants; per-run timeout ${tmo}s; whole-job timeout ${JOB_TIMEOUT:-unknown}s; one row per mutant follows as it finishes, so a timeout keeps the finished rows"
  unmutated BASELINE "$count"
  # A baseline that rewrote a tracked file would hand every mutant a changed tree
  # and make `git diff` blind to the mutation; untracked and ignored output stays.
  git -C "$SRC" reset -q --hard 2>/dev/null
  exit 0
fi

if [ "$1" = --control ]; then
  # One unmutated run from the committed snapshot, the state every mutant starts from.
  cd "$SRC/${WORKDIR:-.}" || { echo "$SUMTAG: BAD WORKDIR ${WORKDIR:-.}"; exit 2; }
  unmutated SNAPSHOT "$(cat "$M/count")"
  exit 0
fi

# --mutant <n>
n=${2:?mutant number}
cd "$SRC/${WORKDIR:-.}" || { echo "$SUMTAG: BAD WORKDIR ${WORKDIR:-.}"; exit 2; }
count=$(cat "$M/count")
label=$(cat "$M/$n/label"); file=$(cat "$M/$n/file"); t=$(cat "$M/$n/test"); t=${t:-$TEST}
out=''; detail=''
if ! detail=$(perl -e '
    local $/; my ($file, $bf, $af) = @ARGV;
    open my $h, "<:raw", $file or do { print "cannot read $file\n"; exit 3 }; my $c = <$h>; close $h;
    open my $b, "<:raw", $bf; my $before = <$b> // ""; open my $a, "<:raw", $af; my $after = <$a> // "";
    if (length($before) == 0) { print "empty before text\n"; exit 3 }
    if ($before eq $after) { print "after equals before\n"; exit 3 }
    # a CRLF file matches multi-line before text written with LF
    if ($c =~ /\r\n/ && $before !~ /\r/) { s/\n/\r\n/g for $before, $after }
    my ($cnt, $p) = (0, 0); while (($p = index($c, $before, $p)) >= 0) { $cnt++; $p += length($before) }
    if ($cnt == 0) { print "before text not found\n"; exit 3 }
    if ($cnt > 1) { print "before text occurs $cnt times; make it unique\n"; exit 3 }
    substr($c, index($c, $before), length($before)) = $after;
    open my $o, ">:raw", $file or exit 3; print $o $c; close $o;' "$file" "$M/$n/before" "$M/$n/after" 2>&1); then
  out=NOAPPLY
elif git diff --quiet -- "$file"; then
  out=NOAPPLY; detail="applied, but git sees no change in $file"
else
  out=''; detail=''
  if [ -n "${BUILD:-}" ]; then
    rc=$(run "$BUILD" "$M/run.log")
    if [ "$rc" = 124 ]; then out=TIMEOUT; detail="build exceeded ${tmo}s"
    elif [ "$rc" != 0 ]; then out=UNVIABLE; detail="build exit $rc"; fi
  fi
  if [ -z "$out" ]; then
    rc=$(run "$t" "$M/run.log")
    if [ "$rc" = 0 ]; then out=SURVIVED; detail="tests passed with the mutation applied: $t"
    elif [ "$rc" = 124 ]; then out=TIMEOUT; detail="test exceeded ${tmo}s: $t"
    else
      scan=$(tp_scan "$M/run.log")
      named=$(printf '%s\n' "$scan" | grep '^FAILED ' | awk '!s[$0]++' | sed 's/^FAILED //')
      loads=$(printf '%s\n' "$scan" | grep '^LOADFAIL ' | awk '!s[$0]++' | sed 's/^LOADFAIL //')
      if [ -n "$named" ]; then
        out=CAUGHT; detail=$(printf '%s\n' "$named" | head -n 1)
        k=$(printf '%s\n' "$named" | wc -l); [ "$k" -le 1 ] || detail="$detail (+$((k - 1)) more)"
      elif [ -n "$loads" ]; then out=UNVIABLE; detail="a test file failed to load, no test ran: $(printf '%s\n' "$loads" | head -n 1)"
      else out=ERROR; detail="exit $rc, no failing test parsed: $(tail -n 1 "$M/run.log" | cut -c1-120)"; fi
    fi
  fi
fi
detail=${detail//$'\n'/ }
printf '%s: [%s/%s] %s | %s | %s\n' "$SUMTAG" "$n" "$count" "${label//|/\\|}" "$out" "$detail"
for i in "${!OUTCOMES[@]}"; do [ "${OUTCOMES[$i]}" = "$out" ] && exit $((CODE0 + i)); done
exit 1