"""Validate Ghostlight responder output captures.""" from __future__ import annotations import argparse import json from pathlib import Path from typing import Any ROOT = Path(__file__).resolve().parents[1] DEFAULT_CAPTURES = sorted((ROOT / "experiments" / "responder-packets").glob("*.capture.json")) DEFAULT_MUTATIONS = sorted((ROOT / "experiments" / "responder-packets").glob("*.mutation.json")) REVIEW_STATUSES = {"draft", "accepted", "accepted_as_draft", "useful_needs_revision", "rejected", "superseded"} FIXTURE_LANES = {"historical_grounded", "transition_grounded", "future_branch"} CANON_STATUSES = {"single_history", "branch_local", "promoted_branch", "template", "draft"} ISOLATION_METHODS = {"subagent_no_fork", "manual_packet_only", "external_model_packet_only"} ACTION_LABELS = { "speak", "silence", "gesture", "movement", "inspect_object", "show_object", "withhold_object", "transfer_object", "set_condition", "refuse", "wait", "withdraw", "attack", } LEAK_MARKERS = [ "current_activation", "plasticity", "canonical_state", "coordinator rationale", "author-only", "future branch", ] class ValidationError(Exception): pass def load_json(path: Path) -> Any: return json.loads(path.read_text(encoding="utf-8-sig")) def require(condition: bool, message: str) -> None: if not condition: raise ValidationError(message) def require_keys(obj: dict[str, Any], keys: list[str], path: str) -> None: missing = [key for key in keys if key not in obj] require(not missing, f"{path} missing required keys: {', '.join(missing)}") def require_string(value: Any, path: str) -> None: require(isinstance(value, str) and value.strip(), f"{path} must be a non-empty string") def require_string_array(value: Any, path: str, *, nonempty: bool = False) -> None: require(isinstance(value, list), f"{path} must be an array") require(not nonempty or bool(value), f"{path} must not be empty") for index, item in enumerate(value): require_string(item, f"{path}[{index}]") def require_update_candidate_array(value: Any, path: str) -> None: require(isinstance(value, list), f"{path} must be an array") for index, item in enumerate(value): item_path = f"{path}[{index}]" if isinstance(item, str): require_string(item, item_path) continue require(isinstance(item, dict), f"{item_path} must be a string or object") require("value" in item, f"{item_path} object must include value") require_string(item["value"], f"{item_path}.value") if "key" in item: require_string(item["key"], f"{item_path}.key") if "target_agent_id" in item: require_string(item["target_agent_id"], f"{item_path}.target_agent_id") def require_research_trace(value: Any, path: str) -> None: require(isinstance(value, list), f"{path} must be an array") require(value, f"{path} must not be empty") seen_ids: set[str] = set() for index, item in enumerate(value): item_path = f"{path}[{index}]" require(isinstance(item, dict), f"{item_path} must be an object") require_keys( item, [ "trace_id", "origin", "query", "source_ref", "source_path", "chunk_index", "line_start", "line_end", "extracted_constraint", "used_for", ], item_path, ) require_string(item["trace_id"], f"{item_path}.trace_id") require(item["trace_id"] not in seen_ids, f"{item_path}.trace_id is duplicated") seen_ids.add(item["trace_id"]) require( item["origin"] in {"responder_reported", "coordinator_reconstructed", "runner_captured"}, f"{item_path}.origin is invalid", ) for key in ["query", "source_ref", "source_path", "extracted_constraint", "used_for"]: require_string(item[key], f"{item_path}.{key}") for key in ["chunk_index", "line_start", "line_end"]: require(isinstance(item[key], int), f"{item_path}.{key} must be an integer") require(item["chunk_index"] >= 0, f"{item_path}.chunk_index must be non-negative") require(item["line_start"] >= 1, f"{item_path}.line_start must be positive") require(item["line_end"] >= item["line_start"], f"{item_path}.line_end must be >= line_start") def validate_parsed_response(value: Any, path: str) -> None: require(isinstance(value, dict), f"{path} must be an object") require_keys( value, [ "responder_agent_id", "action_label", "visible_action", "spoken_text", "private_interpretation", "intended_effect", "state_update_candidates", "relationship_update_candidates", "unresolved_hooks", ], path, ) for key in ["responder_agent_id", "action_label", "visible_action", "private_interpretation", "intended_effect"]: require_string(value[key], f"{path}.{key}") require(isinstance(value["spoken_text"], str), f"{path}.spoken_text must be a string") require(value["action_label"] in ACTION_LABELS, f"{path}.action_label is invalid: {value['action_label']}") for key in ["state_update_candidates", "relationship_update_candidates"]: require_update_candidate_array(value[key], f"{path}.{key}") require_string_array(value["unresolved_hooks"], f"{path}.unresolved_hooks") if "consulted_refs" in value: require_string_array(value["consulted_refs"], f"{path}.consulted_refs") if "followed_refs" in value: require_string_array(value["followed_refs"], f"{path}.followed_refs") if "research_summary" in value: require(isinstance(value["research_summary"], str), f"{path}.research_summary must be a string") if "research_trace" in value: require_research_trace(value["research_trace"], f"{path}.research_trace") def validate_capture(document: dict[str, Any], source: Path) -> None: base = str(source) require(document.get("schema_version") == "ghostlight.responder_output.v0", f"{base}.schema_version is wrong") require_keys( document, [ "capture_id", "review_status", "fixture_lane", "canon_status", "packet_ref", "coordinator_artifact_ref", "responder_agent_id", "generation_lane", "lore_access", "isolation", "raw_output", "parsed_output", "leakage_audit", "coordinator_review", ], base, ) require(document["review_status"] in REVIEW_STATUSES, f"{base}.review_status is invalid") require(document["fixture_lane"] in FIXTURE_LANES, f"{base}.fixture_lane is invalid") require(document["canon_status"] in CANON_STATUSES, f"{base}.canon_status is invalid") for key in ["capture_id", "packet_ref", "coordinator_artifact_ref", "responder_agent_id", "raw_output"]: require_string(document[key], f"{base}.{key}") require(document["generation_lane"] in {"packet_only", "retrieval_augmented"}, f"{base}.generation_lane is invalid") require((ROOT / document["packet_ref"]).exists(), f"{base}.packet_ref does not exist") require((ROOT / document["coordinator_artifact_ref"]).exists(), f"{base}.coordinator_artifact_ref does not exist") lore_access = document["lore_access"] require_keys(lore_access, ["mode", "consulted_refs"], f"{base}.lore_access") require( lore_access["mode"] in { "curated_excerpts_only", "coordinator_scoped_retrieval", "responder_scoped_repository_search", }, f"{base}.lore_access.mode is invalid", ) require_string_array(lore_access["consulted_refs"], f"{base}.lore_access.consulted_refs") if document["generation_lane"] == "packet_only": require(lore_access["mode"] == "curated_excerpts_only", f"{base}.packet_only must use curated_excerpts_only lore access") if document["generation_lane"] == "retrieval_augmented": require( lore_access["mode"] in {"coordinator_scoped_retrieval", "responder_scoped_repository_search"}, f"{base}.retrieval_augmented must use scoped retrieval lore access", ) if "research_trace" in lore_access: require_research_trace(lore_access["research_trace"], f"{base}.lore_access.research_trace") require( lore_access.get("research_trace_status") in {"responder_reported", "coordinator_reconstructed", "runner_captured"}, f"{base}.lore_access.research_trace_status is invalid", ) elif document["review_status"] == "accepted": raise ValidationError(f"{base}.lore_access.research_trace is required for accepted retrieval-augmented captures") if document["review_status"] == "accepted": require( lore_access.get("research_trace_status") == "runner_captured", f"{base}.accepted retrieval-augmented captures must use runner_captured research_trace_status", ) if lore_access["mode"] == "responder_scoped_repository_search": require_string_array(lore_access["consulted_refs"], f"{base}.lore_access.consulted_refs", nonempty=True) require_string(lore_access.get("research_summary"), f"{base}.lore_access.research_summary") if "followed_refs" in lore_access: require_string_array(lore_access["followed_refs"], f"{base}.lore_access.followed_refs") isolation = document["isolation"] require_keys(isolation, ["isolation_method", "fork_context", "visible_input_ref", "hidden_context_refs", "worker_model_family"], f"{base}.isolation") require(isolation["isolation_method"] in ISOLATION_METHODS, f"{base}.isolation.isolation_method is invalid") require(isolation["fork_context"] is False, f"{base}.isolation.fork_context must be false for sandbox captures") require_string(isolation["visible_input_ref"], f"{base}.isolation.visible_input_ref") require_string(isolation["worker_model_family"], f"{base}.isolation.worker_model_family") require_string_array(isolation["hidden_context_refs"], f"{base}.isolation.hidden_context_refs") if lore_access["mode"] == "responder_scoped_repository_search": require( "repo" in isolation["visible_input_ref"].lower() or "lore" in isolation["visible_input_ref"].lower(), f"{base}.isolation.visible_input_ref should describe scoped lore access", ) raw_output = document["raw_output"] parsed_raw = json.loads(raw_output) validate_parsed_response(document["parsed_output"], f"{base}.parsed_output") require(parsed_raw == document["parsed_output"], f"{base}.raw_output does not match parsed_output") require(document["parsed_output"]["responder_agent_id"] == document["responder_agent_id"], f"{base}.parsed_output.responder_agent_id mismatch") if lore_access["mode"] == "responder_scoped_repository_search": require( document["parsed_output"].get("consulted_refs") == lore_access["consulted_refs"], f"{base}.parsed_output.consulted_refs must match lore_access.consulted_refs", ) require( document["parsed_output"].get("research_summary") == lore_access["research_summary"], f"{base}.parsed_output.research_summary must match lore_access.research_summary", ) if "followed_refs" in lore_access: require( document["parsed_output"].get("followed_refs") == lore_access["followed_refs"], f"{base}.parsed_output.followed_refs must match lore_access.followed_refs", ) if "research_trace" in document["parsed_output"]: require( document["parsed_output"]["research_trace"] == lore_access.get("research_trace"), f"{base}.parsed_output.research_trace must match lore_access.research_trace when present", ) lowered_raw = raw_output.lower() leaked = [marker for marker in LEAK_MARKERS if marker in lowered_raw] require(not leaked, f"{base}.raw_output leaks marker(s): {', '.join(leaked)}") audit = document["leakage_audit"] require_keys( audit, [ "private_state_leak", "author_only_leak", "future_branch_leak", "raw_state_soup", "prompt_parroting", "body_or_object_omniscience", "notes", ], f"{base}.leakage_audit", ) for key in ["private_state_leak", "author_only_leak", "future_branch_leak", "raw_state_soup", "prompt_parroting", "body_or_object_omniscience"]: require(isinstance(audit[key], bool), f"{base}.leakage_audit.{key} must be boolean") require_string_array(audit["notes"], f"{base}.leakage_audit.notes") require(not any(audit[key] for key in ["private_state_leak", "author_only_leak", "future_branch_leak", "raw_state_soup", "prompt_parroting", "body_or_object_omniscience"]), f"{base}.leakage_audit has leak flags set") review = document["coordinator_review"] require_keys(review, ["reviewer", "accepted_for_training", "review_notes", "coordinator_interventions", "failure_labels"], f"{base}.coordinator_review") require_string(review["reviewer"], f"{base}.coordinator_review.reviewer") require(isinstance(review["accepted_for_training"], bool), f"{base}.coordinator_review.accepted_for_training must be boolean") require(isinstance(review["review_notes"], list), f"{base}.coordinator_review.review_notes must be an array") require(isinstance(review["coordinator_interventions"], list), f"{base}.coordinator_review.coordinator_interventions must be an array") require_string_array(review["failure_labels"], f"{base}.coordinator_review.failure_labels") def validate_mutation_receipt(document: dict[str, Any], source: Path) -> None: base = str(source) require(document.get("schema_version") == "ghostlight.reviewed_state_mutation.v0", f"{base}.schema_version is wrong") require_keys( document, [ "mutation_id", "source_fixture_ref", "source_capture_ref", "mutated_fixture_ref", "event_id", "responder_output_ref", "participant_appraisals", "observed_action", "observed_response", "applied_mutations", "manual_review_notes", "deferred_mutations", ], base, ) for key in ["mutation_id", "source_fixture_ref", "source_capture_ref", "mutated_fixture_ref", "event_id", "responder_output_ref"]: require_string(document[key], f"{base}.{key}") require((ROOT / document["source_fixture_ref"]).exists(), f"{base}.source_fixture_ref does not exist") require((ROOT / document["source_capture_ref"]).exists(), f"{base}.source_capture_ref does not exist") require((ROOT / document["mutated_fixture_ref"]).exists(), f"{base}.mutated_fixture_ref does not exist") require(isinstance(document["participant_appraisals"], list) and document["participant_appraisals"], f"{base}.participant_appraisals must be non-empty") for index, appraisal in enumerate(document["participant_appraisals"]): appraisal_path = f"{base}.participant_appraisals[{index}]" require_keys(appraisal, ["participant_agent_id", "private_interpretation", "state_deltas", "relationship_deltas", "next_action_constraints"], appraisal_path) require(isinstance(document["applied_mutations"], list) and document["applied_mutations"], f"{base}.applied_mutations must be non-empty") for index, mutation in enumerate(document["applied_mutations"]): mutation_path = f"{base}.applied_mutations[{index}]" require_keys(mutation, ["target", "path", "before", "after"], mutation_path) require(isinstance(mutation["before"], (int, float)), f"{mutation_path}.before must be numeric") require(isinstance(mutation["after"], (int, float)), f"{mutation_path}.after must be numeric") require(0 <= mutation["before"] <= 1, f"{mutation_path}.before must be between 0 and 1") require(0 <= mutation["after"] <= 1, f"{mutation_path}.after must be between 0 and 1") def main() -> int: parser = argparse.ArgumentParser(description="Validate Ghostlight responder output captures.") parser.add_argument("paths", nargs="*", type=Path, help="Responder output capture files to validate.") args = parser.parse_args() paths = args.paths or DEFAULT_CAPTURES if not paths: print("ok: no responder output captures found") for path in paths: validate_capture(load_json(path), path) print(f"ok: {path}") for path in DEFAULT_MUTATIONS: validate_mutation_receipt(load_json(path), path) print(f"ok: {path}") return 0 if __name__ == "__main__": raise SystemExit(main())