diff --git a/tools/test-tools.sh b/tools/test-tools.sh index bc7d97d..60c6ab7 100644 --- a/tools/test-tools.sh +++ b/tools/test-tools.sh @@ -235,7 +235,8 @@ g() { git -C "$R" -c user.name=t -c user.email=t@t -c core.autocrlf=false "$@" > seqf() { local i; for i in $(seq 1 "$2"); do echo "$1$i"; done; } g init -q seqf a 40 >"$R/a.txt"; seqf b 40 >"$R/b.txt"; seqf c 10 >"$R/c.txt"; seqf d 20 >"$R/d.txt" -seqf e 10 >"$R/e.txt"; seqf f 20 >"$R/f.txt"; seqf g 300 >"$R/big.txt"; seqf h 5 | sed 's/$/ /' >"$R/h.txt"; seqf m 40 >"$R/m.txt" +seqf e 10 >"$R/e.txt"; seqf f 20 >"$R/f.txt"; seqf g 300 >"$R/big.txt"; seqf h 5 | sed 's/$/ +/' >"$R/h.txt"; seqf m 40 >"$R/m.txt" g add -A; g commit -q -m base; BASE=$(git -C "$R" rev-parse HEAD) { sed -n 1,5p "$R/a.txt"; printf 'ins1\nins2\nins3\n'; sed -n '6,$p' "$R/a.txt"; } >"$R/a.new"; mv "$R/a.new" "$R/a.txt" sed -e '2,3d' -e 's/^b10$/b10-EDITED/' "$R/b.txt" >"$R/b.new"; mv "$R/b.new" "$R/b.txt" @@ -276,7 +277,8 @@ cin "hunk ending just before the range shifts it" "[23] b.txt:4-5 MOVED(-2) -> cin "NEW file present in the worktree gets a window" "== [24] n.txt:1-2" cin "NEW window shows the file" " 2 n2" cin "CRLF window text" " 2 h2" -{ tr -d ' ' <"$T/cp.out" | cmp -s - "$T/cp.out"; } && echo "ok context-pack CRLF is stripped from windows" || { echo "FAIL context-pack CR in output"; fail=1; } +{ tr -d ' +' <"$T/cp.out" | cmp -s - "$T/cp.out"; } && echo "ok context-pack CRLF is stripped from windows" || { echo "FAIL context-pack CR in output"; fail=1; } cin "window numbers are worktree lines" " 6 b8" cin "window shows the edit" "b10-EDITED" cin "GONE window is labelled AT BASE" "== [7] c.txt:2-3 AT BASE" @@ -803,6 +805,240 @@ except mind.MindError as e: else: raise AssertionError("no error") PYE +# recall.py admit: the PreToolUse hook that holds an admit once, showing the nearest in-force neighbours. The driver +# feeds it real hook JSON (the shape the harness sends: session_id, scratchpad_dir, agent_id, tool_name, tool_input) +# against the fake binding; R1-R12 are the rules the cut_spec names. +cat >"$T/ad.py" <<'PYE' +import hashlib, json, os, subprocess, sys, tempfile, time +RC, T = sys.argv[1], sys.argv[2] +ARGS, COUNT = os.environ["FAKE_ARGS"], os.environ["FAKE_COUNT"] +TMP = tempfile.mkdtemp(dir=T) +CANARY = "CANARY-9f3e7a" +DEL = object() +fails, outs, n = 0, [], 0 +def check(label, ok, why=""): + global fails + print("%s recall admit %s%s" % ("ok " if ok else "FAIL", label, "" if ok else ": " + str(why)[:300])) + fails += not ok +def doc(kind, label, title, **extra): + return {"kind": kind, "value": dict({"campaign": "t", "label": label, "title": title}, **extra)} +def item(label, title, quote=None, kind="ruling"): + return {"id": {"id": "t:%s:%s" % (kind, label), "kind": kind.title()}, + "document": {"kind": kind, "value": {"title": title, "operator_quote": quote}}, "status": "InForce"} +NB = [item("n1", "Neighbour 1", "q1")] + [item("n%d" % i, "Neighbour %d" % i) for i in (2, 3, 4)] +def hook(docs, **kw): + global n + n += 1 + h = {"hook_event_name": "PreToolUse", "session_id": "sess-%d" % n, "scratchpad_dir": tempfile.mkdtemp(dir=T), + "tool_name": "mcp__eureka-state__admit", "tool_use_id": "tu%d" % n, + "tool_input": {"faculty": "SelfFaculty", "agent": "a", "session": "s", "documents": docs}} + h.update(kw) + return {k: v for k, v in h.items() if v is not DEL} +def run(h, items=NB, stdin=None, **env): + open(ARGS, "w").close(); open(COUNT, "w").close() + json.dump(items, open(os.path.join(T, "ad.items"), "w")) + e = dict(os.environ, FAKE_ITEMS=os.path.join(T, "ad.items"), TMPDIR=TMP, **env) + data = stdin if stdin is not None else json.dumps(h, ensure_ascii=False).encode("utf-8") + t = time.perf_counter() + r = subprocess.run([sys.executable, RC, "admit"], input=data, capture_output=True, env=e) + ms = (time.perf_counter() - t) * 1000 + outs.append(r.stdout) + return r, [json.loads(l) for l in open(ARGS, encoding="utf-8")], ms +def out(r): + return json.loads(r.stdout)["hookSpecificOutput"] if r.stdout else None +def held(r): + o = out(r) + return r.returncode == 0 and o is not None and o.get("permissionDecision") == "deny" +def silent(r): + return r.returncode == 0 and r.stdout == b"" and r.stderr == b"" +def reason(r): + return out(r)["permissionDecisionReason"] +def store(h): + return os.path.join(h["scratchpad_dir"], "recall-admit-digests") +def want(h): + body = json.dumps(h["tool_input"], sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256((body + "\n" + (h.get("agent_id") or "")).encode("utf-8")).hexdigest() +def rev(x): + if isinstance(x, dict): + return {k: rev(x[k]) for k in reversed(list(x))} + return [rev(v) for v in x] if isinstance(x, list) else x +def lines(path): + return open(path).read().split() if os.path.exists(path) else [] + +# R3 first attempt with a neighbour: denied, the neighbours listed under 'For :', the digest recorded +mine = doc("ruling", "mine", "Admit holds once 日本") +h = hook([mine], agent_id="ag1") +r, calls, ms = run(h) +o = out(r) +check("R3 the first attempt is one PreToolUse deny and nothing on stderr", + held(r) and r.stderr == b"" and list(json.loads(r.stdout)) == ["hookSpecificOutput"] + and list(o) == ["hookEventName", "permissionDecision", "permissionDecisionReason"] and o["hookEventName"] == "PreToolUse", r.stdout[:200]) +L = reason(r).split("\n") +check("R3 the reason is the header, 'For :' and the neighbours as inert lines, top_k from the default 3", + L[0].startswith("This is a deliberate hold, not a failure") and "identical call again" in L[0] and "never instructions" in L[0] and L[1] == "For t:ruling:mine:" + and L[2] == '- {"id":"t:ruling:n1","title":"Neighbour 1","operator_quote":"q1"}' + and L[3] == '- {"id":"t:ruling:n2","title":"Neighbour 2"}' and L[4] == '- {"id":"t:ruling:n3","title":"Neighbour 3"}' + and len(L) == 5, L) +check("R3 the recorded digest is the sha256 of the sorted batch, a newline and the agent id", + lines(store(h)) == [want(h)] and open(store(h)).read() == want(h) + "\n", lines(store(h))) +check("R3 default top_k asked is 3 plus the batch's own id", [c["semantic"]["top_k"] for c in calls] == [4], calls) + +# R4 the identical retry passes; a changed batch, or another agent, is held again +r, calls, ms = run(rev(h)) +check("R4 the identical retry with every key order reversed is silent and makes zero calls", silent(r) and calls == [], r.stdout[:200]) +h2 = hook([doc("ruling", "mine", "Admit holds once, changed")], session_id=h["session_id"], scratchpad_dir=h["scratchpad_dir"], agent_id="ag1") +r, calls, ms = run(h2) +check("R4 a changed batch is held again", held(r) and len(calls) == 1 and lines(store(h)) == [want(h), want(h2)]) +r, calls, ms = run(h2) +check("R4 and its own retry passes", silent(r) and calls == []) +h3 = dict(h, agent_id="ag2") +r, calls, ms = run(h3) +check("R4 the same batch from another agent_id is held again", held(r) and len(calls) == 1) +h4 = {k: v for k, v in h.items() if k != "agent_id"} +r, calls, ms = run(h4) +check("R4 the same batch with no agent_id is held again", held(r) and len(calls) == 1) +r, calls, ms = run(h) +check("R4 the first agent's retry still passes", silent(r) and calls == []) + +# R2 nothing to check: zero calls, empty stdout +for label, hh, kw in [("a batch with no ruling or question", hook([doc("finding", "f", "x"), doc("cut_report", "c", "y")]), {}), + ("an empty batch", hook([]), {}), + ("a ruling with no title", hook([{"kind": "ruling", "value": {"campaign": "t", "label": "x"}}]), {}), + ("a tool other than admit", hook([mine], tool_name="mcp__eureka-state__view"), {}), + ("malformed stdin", None, {"stdin": b"not json"}), + ("empty stdin", None, {"stdin": b""})]: + r, calls, ms = run(hh, **kw) + check("R2 %s: zero calls, empty stdout, exit 0" % label, silent(r) and calls == [], (r.stdout[:100], calls)) + +# R5 exclusions: own id, the batch's ids, the answered question, superseded subjects never appear +other = doc("question", "other", "Another question") +batch = [doc("ruling", "mine", "Admit holds once", answers="t:question:q1"), other, + {"kind": "resolution", "value": {"subject": {"kind": "Ruling", "id": "t:ruling:old"}, "sequence": 1, + "outcome": {"Superseded": {"by": [{"kind": "Ruling", "id": "t:ruling:mine"}]}}}}, + {"kind": "resolution", "value": {"subject": {"kind": "Ruling", "id": "t:ruling:kept"}, "sequence": 1, + "outcome": {"Recorded": {"reason": "x"}}}}] +pool = [item("mine", "Own"), item("other", "Batch", kind="question"), item("q1", "Answered", kind="question"), + item("old", "Superseded"), item("kept", "Kept")] + NB +ids = lambda r: [json.loads(l[2:])["id"] for l in reason(r).split("\n") if l.startswith("- ")] +h = hook(batch) +r, calls, ms = run(h, items=pool) +check("R5 own, batch, answered and superseded ids are never listed; a Recorded resolution's subject still is", + ids(r) == ["t:ruling:kept", "t:ruling:n1", "t:ruling:n2"] * 2, ids(r)) +check("R5 top_k asked is RECALL_TOP_K plus the excluded count (4), once per checked document", + [c["semantic"]["top_k"] for c in calls] == [7, 7], calls) +r, calls, ms = run(hook(batch), items=pool, RECALL_TOP_K="2") +check("R5 RECALL_TOP_K=2 lists at most 2 per document and asks 6", + all(c["semantic"]["top_k"] == 6 for c in calls) and ids(r) == ["t:ruling:kept", "t:ruling:n1"] * 2, (ids(r), calls)) +# R6 every neighbour excluded: no deny, nothing recorded +h = hook(batch) +r, calls, ms = run(h, items=pool[:4]) +check("R6 every neighbour excluded: silent, nothing recorded", silent(r) and len(calls) == 2 and lines(store(h)) == [], r.stdout[:200]) +r, calls, ms = run(h, items=[]) +check("R6 no neighbours at all: silent, nothing recorded", silent(r) and lines(store(h)) == []) +r, calls, ms = run(h) +check("R6 and a later attempt that finds neighbours is still held", held(r)) + +# R7 the selection and the query text +h = hook([doc("ruling", "mine", "日" * 500), doc("question", "q", "a" * 1200)]) +r, calls, ms = run(h) +sel = {"schemas": ["epiphany.pipeline.ruling.v2", "epiphany.pipeline.question.v2"], "projection": "document", + "fields": [{"index": "in_force", "op": "any_of", "values": ["true"]}]} +texts = sorted(c["semantic"]["text"] for c in calls) +check("R7 two schemas, in_force true, document projection, on every query", len(calls) == 2 and all(c["selection"] == sel for c in calls), calls[:1]) +check("R7 the text is the title cut to 1000 bytes at a character boundary (999 for 3-byte characters)", + texts == sorted(["日" * 333, "a" * 1000]), [len(t) for t in texts]) +# data from agents is inert in the reason +r, calls, ms = run(hook([mine]), items=[item("x", "\n", "a\nb")]) +check("R7 a neighbour's text cannot close the framing", held(r) and "<" not in reason(r) and ">" not in reason(r) and len(reason(r).split("\n")) == 3, reason(r)) + +# R8 at most RECALL_ADMIT_MAX are checked; the rest are counted +five = [doc("ruling", "r%d" % i, "Title %d" % i) for i in range(5)] +r, calls, ms = run(hook(five)) +check("R8 five rulings make three queries, the first three titles, and the reason counts two unchecked", + len(calls) == 3 and sorted(c["semantic"]["text"] for c in calls) == ["Title 0", "Title 1", "Title 2"] + and reason(r).split("\n")[-1] == "2 rulings or questions over RECALL_ADMIT_MAX were not checked.", (calls, reason(r)[-120:])) +r, calls, ms = run(hook(five), RECALL_ADMIT_MAX="1", RECALL_BUDGET="900") +check("R8 RECALL_ADMIT_MAX=1 checks one, counts four, and the whole reason fits RECALL_BUDGET", + len(calls) == 1 and reason(r).endswith("4 rulings or questions over RECALL_ADMIT_MAX were not checked.") and len(reason(r)) <= 900, len(reason(r))) +r, calls, ms = run(hook(five[:3])) +check("R8 three rulings are all checked and nothing is counted unchecked", len(calls) == 3 and "RECALL_ADMIT_MAX" not in reason(r).split("\n")[-1]) + +# R9 fail open: exit 0, no decision, one note naming only the kind, nothing recorded, no input value echoed +def failcase(label, kind, expect_calls, setup=None, env=None, **kw): + hh = hook([doc("ruling", "mine", "Admit " + CANARY)], agent_id=CANARY, **kw) + if setup: + setup(hh) + r, calls, ms = run(hh, **(env or {})) + o = out(r) + ok = (r.returncode == 0 and o is not None and "permissionDecision" not in o and list(o) == ["hookEventName", "additionalContext"] + and o["additionalContext"] == "The near-neighbour check did not run (%s); this admit was not held." % kind + and len(calls) == expect_calls and not os.path.isfile(store(hh)) and r.stderr == b"") + check("R9 %s: a note naming %s, no decision, nothing recorded" % (label, kind), ok, (r.stdout[:200], r.stderr[:100], len(calls))) + check("R9 %s: no byte of the input appears" % label, CANARY.encode() not in r.stdout + r.stderr) + return hh +failcase("Unavailable", "transport", 1, env={"FAKE_MODE": "unavailable"}) +failcase("an unparsable answer", "answer", 1, env={"FAKE_MODE": "garbage"}) +failcase("a binding that dies", "transport", 1, env={"FAKE_MODE": "die"}) +failcase("no binding", "binding", 0, env={"HOME": T + "/nohome", "USERPROFILE": T + "/nohome"}) +failcase("an unspawnable binding", "transport", 0, env={"EUREKA_STATE_EXE": T + "/" + CANARY + "-exe"}) +failcase("no session_id", "session", 0, session_id=DEL) +failcase("an unreadable store", "store", 0, setup=lambda hh: os.mkdir(store(hh))) +def dangling(hh): + os.symlink(os.path.join(T, "no-such-dir", "x"), store(hh)) +try: + failcase("an unwritable store (the deny is not printed before the record)", "store", 1, setup=dangling) +except OSError: + check("R9 an unwritable store (symlinks unavailable here)", True) +hh = failcase("Unavailable", "transport", 1, env={"FAKE_MODE": "unavailable"}) +r, calls, ms = run(hh) +check("R9 after a failed check the same batch is still held on its first real check", held(r)) + +# R10 the deadline is RECALL_DEADLINE_MS for the whole run: empty stdout, nothing recorded +for dl, sl in (("1000", "3"), ("2000", "4")): + h = hook([mine]) + r, calls, ms = run(h, RECALL_DEADLINE_MS=dl, FAKE_SLEEP=sl) + d = int(dl) + check("R10 deadline %s ms with a %s s fake ends in %.0f ms, silent, nothing recorded" % (dl, sl, ms), + silent(r) and d - 100 <= ms <= d + 600 and not os.path.exists(store(h)), (r.stdout[:100], ms)) +h = hook([mine]) +r, calls, ms = run(h, FAKE_SLEEP="4.5") +check("R10 the default deadline in admit mode is 10 s, not the prompt mode's 4 s", held(r) and ms > 4400, (r.stdout[:100], ms)) + +# R11 the checked queries run concurrently +h = hook([doc("ruling", "a", "Alpha"), doc("question", "b", "Beta"), doc("ruling", "c", "Gamma")]) +r, calls, ms = run(h, FAKE_SLEEP="1.5") +check("R11 three checked documents with a 1.5 s fake end in %.0f ms, under 3500, all three listed" % ms, + held(r) and ms < 3500 and ms > 1400 and reason(r).count("\nFor t:") == 3, (ms, reason(r)[:300] if r.stdout else "")) + +# R12 the store: scratchpad_dir when a directory, else the session-keyed temp file +h = hook([mine], agent_id="x") +r, calls, ms = run(h) +tmpfile = lambda sid: os.path.join(TMP, "eureka-recall-admit-" + hashlib.sha256(sid.encode()).hexdigest()[:16]) +check("R12 scratchpad_dir is used when it is a directory, and no temp file is made", lines(store(h)) == [want(h)] and not os.path.exists(tmpfile(h["session_id"]))) +for label, extra in (("absent", {"scratchpad_dir": DEL}), ("not a directory", {"scratchpad_dir": os.path.join(T, "no-such-dir")})): + h = hook([mine], **extra) + r, calls, ms = run(h) + check("R12 a scratchpad_dir that is %s: held, recorded in the session-keyed temp file" % label, + held(r) and lines(tmpfile(h["session_id"])) == [want(h)] and not os.path.exists(os.path.join(T, "no-such-dir"))) + hr = {k: v for k, v in h.items() if k != "scratchpad_dir"} + r, calls, ms = run(hr) + check("R12 a retry with the same session_id and no scratchpad_dir is recognised (%s)" % label, silent(r) and calls == []) + r, calls, ms = run(dict(hr, session_id=h["session_id"] + "-other")) + check("R12 the same batch in another session is held again (%s)" % label, held(r)) + +# R1 across every case above: never allow, never ask +bad = [] +for s in outs: + if s: + d = json.loads(s)["hookSpecificOutput"].get("permissionDecision") + if d not in (None, "deny"): + bad.append(d) +check("R1 across %d runs no permissionDecision other than deny was ever printed (%d denies)" % (len(outs), sum(b"permissionDecision" in s for s in outs)), not bad and sum(b"permissionDecision" in s for s in outs) > 10, bad) +sys.exit(1 if fails else 0) +PYE +rcl $PY "$(wp "$T/ad.py")" "$(wp "$RC")" "$(wp "$T")" >"$T/ad.out" 2>&1; ar=$? +cat "$T/ad.out" +[ $ar = 0 ] && [ "$(grep -c '^ok recall admit' "$T/ad.out")" -ge 45 ] && ! grep -q '^FAIL' "$T/ad.out" || { echo "FAIL recall admit driver: exit $ar, $(grep -c '^ok recall admit' "$T/ad.out") ok lines"; fail=1; } # the mind client and the recall hook: standard library only for f in mind.py recall.py; do if grep -qE '^import (requests|yaml)|^from (requests|yaml)' "$here/$f"; then echo "FAIL $f: a dependency beyond the standard library"; fail=1; else echo "ok $f: standard library only"; fi