Ghostlight/tools/ghostlight_state_store.py

from __future__ import annotations

import hashlib
import json
from pathlib import Path
from typing import Any

try:
    from cultcache_py import CultCache, JsonLinesBackingStore, define_document_type
except ModuleNotFoundError as error:
    if error.name != "cultcache_py":
        raise
    raise SystemExit(
        "Ghostlight state tooling needs cultcache-py. Install it with "
        "`npm run state:bootstrap`, or with this interpreter directly: "
        "`python -m pip install -r tools/requirements.txt`."
    ) from error


ROOT = Path(__file__).resolve().parents[1]
STATE_DIR = ROOT / "state"
CACHE_PATH = STATE_DIR / "ghostlight-state.cultcache.jsonl"
BRANCHES_PATH = STATE_DIR / "branches.json"
EVIDENCE_PATH = STATE_DIR / "evidence.jsonl"
ARCHIVE_PATH = STATE_DIR / "evidence.archive.jsonl"


BRANCHES_DOC = define_document_type(
    "ghostlight.branches.v0",
    global_document=True,
)

EVIDENCE_DOC = define_document_type(
    "ghostlight.evidence_record.v0",
    indexes={
        "type": "type",
        "status": "status",
        "branch": "branch",
    },
)


def _read_json(path: Path) -> Any:
    return json.loads(path.read_text(encoding="utf-8-sig"))


def _write_json(path: Path, value: Any) -> None:
    path.write_text(json.dumps(value, indent=2, ensure_ascii=True) + "\n", encoding="utf-8")


def _read_jsonl(path: Path) -> list[dict[str, Any]]:
    if not path.exists():
        return []
    records: list[dict[str, Any]] = []
    for line in path.read_text(encoding="utf-8-sig").splitlines():
        if line.strip():
            records.append(json.loads(line))
    return records


def _write_jsonl(path: Path, records: list[dict[str, Any]]) -> None:
    content = "".join(json.dumps(record, ensure_ascii=True) + "\n" for record in records)
    path.write_text(content, encoding="utf-8")


def _evidence_key(record: dict[str, Any], index: int = 0) -> str:
    payload = json.dumps(record, ensure_ascii=True, sort_keys=True)
    digest = hashlib.sha1(payload.encode("utf-8")).hexdigest()[:12]
    ts = str(record.get("ts", "unstamped")).replace(":", "").replace("+", "Z")
    return f"{ts}-{index:04d}-{digest}"


def open_state_cache() -> CultCache:
    cache = (
        CultCache.builder()
        .register_document_type(BRANCHES_DOC)
        .register_document_type(EVIDENCE_DOC)
        .add_generic_store(JsonLinesBackingStore(CACHE_PATH))
        .build()
    )
    cache.pull_all_backing_stores()
    _bootstrap_if_needed(cache)
    return cache


def load_branches() -> dict[str, Any]:
    cache = open_state_cache()
    branches = cache.get_global(BRANCHES_DOC)
    if branches is None:
        return {"branches": []}
    return branches


def save_branches(data: dict[str, Any]) -> None:
    cache = open_state_cache()
    cache.put_global(BRANCHES_DOC, data)
    _write_json(BRANCHES_PATH, data)


def append_evidence(record: dict[str, Any]) -> None:
    cache = open_state_cache()
    existing = cache.get_all(EVIDENCE_DOC)
    key = _evidence_key(record, len(existing))
    cache.put(EVIDENCE_DOC, key, record)
    _export_evidence(cache)


def replace_evidence(records: list[dict[str, Any]]) -> tuple[int, int]:
    """Replace the whole evidence document with distilled records.

    Every record currently in the store is appended verbatim to
    ``state/evidence.archive.jsonl`` before deletion, so the archive stays
    the complete history and the export carries only what still steers.
    Returns (archived, kept).
    """
    cache = open_state_cache()
    snapshot = cache.snapshot().get(EVIDENCE_DOC.type, {})
    archived = sorted(snapshot.values(), key=lambda record: record.get("ts", ""))
    with ARCHIVE_PATH.open("a", encoding="utf-8") as archive:
        for record in archived:
            archive.write(json.dumps(record, ensure_ascii=True) + "\n")
    for key in list(snapshot):
        cache.delete(EVIDENCE_DOC, key)
    for index, record in enumerate(records):
        cache.put(EVIDENCE_DOC, _evidence_key(record, index), record)
    _export_evidence(cache)
    return len(archived), len(records)


def migrate_legacy_evidence() -> int:
    """Import legacy JSONL records that predate the CultCache authority cut."""
    cache = open_state_cache()
    existing = cache.get_all(EVIDENCE_DOC)
    existing_payloads = {
        json.dumps(record, ensure_ascii=True, sort_keys=True) for record in existing
    }
    imported = 0
    for record in _read_jsonl(EVIDENCE_PATH):
        payload = json.dumps(record, ensure_ascii=True, sort_keys=True)
        if payload in existing_payloads:
            continue
        key = _evidence_key(record, len(existing) + imported)
        cache.put(EVIDENCE_DOC, key, record)
        existing_payloads.add(payload)
        imported += 1
    _export_evidence(cache)
    return imported


def _bootstrap_if_needed(cache: CultCache) -> None:
    changed = False
    if cache.get_global(BRANCHES_DOC) is None and BRANCHES_PATH.exists():
        cache.put_global(BRANCHES_DOC, _read_json(BRANCHES_PATH))
        changed = True
    if not cache.get_all(EVIDENCE_DOC):
        for index, record in enumerate(_read_jsonl(EVIDENCE_PATH)):
            cache.put(EVIDENCE_DOC, _evidence_key(record, index), record)
            changed = True
    if changed:
        _export_evidence(cache)
        branches = cache.get_global(BRANCHES_DOC)
        if branches is not None:
            _write_json(BRANCHES_PATH, branches)


def _export_evidence(cache: CultCache) -> None:
    records = sorted(cache.get_all(EVIDENCE_DOC), key=lambda record: record.get("ts", ""))
    _write_jsonl(EVIDENCE_PATH, records)