diff --git a/README.md b/README.md index 91e59e4..fe849f0 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ Current tile-graph status: - those new diagnostics also settled the design argument about relax: on the pinned badge check under `artifacts/optimizer-relax-purpose-check/`, relax improves source-fidelity a bit (`0.07549` without relax to `0.07451` with it), but it does not act like a strong broad-swatch phase smoother; the displacement field stays roughly as jittery, so the current relax stage is better described as a soft consensus / handoff stabilizer over fixed local candidates - that diagnosis finally forced the obvious architectural verdict: the current optimizer is still a tray-based discrete chooser in a fake moustache, so the repo now also carries `docs/lean-optimizer-algorithm-map.md`, a replacement target for a real displacement-field optimizer with one field, one objective, and one final sample - the first version of that replacement now exists behind `--reconstruction-mode phase-field`; it is a deliberately lean displacement-field solver in `src/repixelizer/phase_field.py`, and the first pinned badge smoke under `artifacts/phase-field-v1-badge-126/` lands at `0.1461`, which is not competitive yet but at least comes from a machine that is no longer lying about what it is +- that first phase-field run also exposed that the old `source_fidelity` metric is too loyal to the lattice accountant's mean/medoid portrait; `run.json` and compare-mode outputs now include a new `source_structure` breakdown that tracks source-size structure instead: edge support F1, stroke wobble, foreground reconstruction, and exact-match ratio - after the first pruning pass, tile-graph no longer participates in pipeline phase-rerank probes and no longer carries hybrid geometry priors through its unary cost; the path is now one lattice-conditioned candidate generator plus one local discrete solver - this fixes the core design mismatch that had allowed repeated distant labels to create big same-color patches and opaque black background blocks - on the current `24x24` emblem smoke case, an end-to-end `tile-graph` run dropped from about `2.57s` on CPU to `0.61s` on CUDA on this machine diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 8e238cf..4df3b44 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -229,6 +229,19 @@ First honest result: That is a respectable first brick: not good enough, but finally the right shape. +Important metric correction: + +- that same first phase-field badge run is also the clearest proof that `source_fidelity` is not enough as the repo's north star +- visually, the phase-field output preserves the important internal badge structure better than the pinned continuous output +- numerically, the old `source_fidelity` score still calls it worse because it rewards agreement with the lattice reference's mean / sharp portraits +- the repo now also reports `source_structure`, a source-size structural metric that combines: + - foreground reconstruction error + - edge-position error + - stroke wobble + - edge support precision / recall / F1 + - exact-match ratio +- on the pinned badge comparison under `artifacts/phase-field-metric-check/`, that new metric finally agrees with human judgment: `phase-field` scores `0.4244` versus continuous `0.4291` where lower is better + ## High-value regression case ### AI badge emblem with baked checkerboard background diff --git a/src/repixelizer/compare.py b/src/repixelizer/compare.py index 589d8ce..36e7e7b 100644 --- a/src/repixelizer/compare.py +++ b/src/repixelizer/compare.py @@ -14,10 +14,12 @@ from .metrics import ( foreground_adjacency_error, foreground_edge_concentration, foreground_edge_position_error, + foreground_edge_support_breakdown, foreground_motif_error, foreground_reconstruction_error, foreground_stroke_wobble_error, reconstruction_error, + source_structure_breakdown, ) from .palette import load_palette from .pipeline import run_pipeline @@ -111,6 +113,8 @@ def run_compare( ): preview = nearest_resize(image, width=source.shape[1], height=source.shape[0]) coherence = coherence_breakdown(image) + edge_support = foreground_edge_support_breakdown(preview, source) + structure = source_structure_breakdown(source, image) rows.append( { "name": name, @@ -125,8 +129,12 @@ def run_compare( "foreground_edge_concentration": foreground_edge_concentration(image), "foreground_edge_position_error": foreground_edge_position_error(preview, source), "foreground_stroke_wobble_error": foreground_stroke_wobble_error(preview, source), + "foreground_edge_precision": edge_support["precision"], + "foreground_edge_recall": edge_support["recall"], + "foreground_edge_f1": edge_support["f1"], "foreground_adjacency_error": foreground_adjacency_error(preview, source), "foreground_motif_error": foreground_motif_error(preview, source), + "source_structure_score": structure["score"], } ) write_compare_csv(diagnostics_path / "compare.csv", rows) diff --git a/src/repixelizer/diagnostics.py b/src/repixelizer/diagnostics.py index e9790e5..8806bd1 100644 --- a/src/repixelizer/diagnostics.py +++ b/src/repixelizer/diagnostics.py @@ -13,6 +13,7 @@ from .metrics import ( coherence_breakdown, foreground_reconstruction_error, reconstruction_error, + source_structure_breakdown, source_lattice_consistency_breakdown, ) from .types import InferenceResult, RunResult @@ -210,6 +211,7 @@ def summarize_run(result: RunResult) -> dict[str, Any]: "source_preview_foreground_error": foreground_reconstruction_error(source_preview, result.source_rgba), "output_colors_from_source_ratio": _source_color_ratio(result.source_rgba, result.output_rgba), "source_fidelity": source_fidelity, + "source_structure": source_structure_breakdown(result.source_rgba, result.output_rgba), "phase_rerank_candidates": rerank_candidates, "loss_history": result.solver.loss_history, "optimizer_displacement": displacement_metrics, diff --git a/src/repixelizer/metrics.py b/src/repixelizer/metrics.py index eb9c988..f21fb2b 100644 --- a/src/repixelizer/metrics.py +++ b/src/repixelizer/metrics.py @@ -193,6 +193,37 @@ def foreground_edge_position_error( return float(np.mean((deficit_a[support_mask] + deficit_b[support_mask]) * 0.5)) +def foreground_edge_support_breakdown( + a: np.ndarray, + b: np.ndarray, + alpha_threshold: float = 0.05, + halo: int = 1, + radius: int = 1, + distance_penalty: float = 0.35, + edge_threshold: float = 0.02, +) -> dict[str, float]: + edge_a = _edge_strength_map(a) + edge_b = _edge_strength_map(b) + mask = sprite_mask(a, b, alpha_threshold=alpha_threshold, halo=halo) + output_edge_mask = mask & (edge_a >= edge_threshold) + source_edge_mask = mask & (edge_b >= edge_threshold) + + support_from_source = _best_local_edge_support(edge_b, radius=radius, distance_penalty=distance_penalty) + support_from_output = _best_local_edge_support(edge_a, radius=radius, distance_penalty=distance_penalty) + + precision = float(np.mean(np.clip(support_from_source[output_edge_mask], 0.0, 1.0))) if np.any(output_edge_mask) else 1.0 + recall = float(np.mean(np.clip(support_from_output[source_edge_mask], 0.0, 1.0))) if np.any(source_edge_mask) else 1.0 + if precision + recall <= 1e-6: + f1 = 0.0 + else: + f1 = float(2.0 * precision * recall / (precision + recall)) + return { + "precision": precision, + "recall": recall, + "f1": f1, + } + + def foreground_edge_concentration( rgba: np.ndarray, alpha_threshold: float = 0.05, @@ -300,6 +331,46 @@ def source_lattice_evidence_breakdown( } +def source_structure_breakdown( + source_rgba: np.ndarray, + output_rgba: np.ndarray, + *, + alpha_threshold: float = 0.05, + halo: int = 1, +) -> dict[str, float]: + if output_rgba.shape[:2] != source_rgba.shape[:2]: + from .io import nearest_resize + + preview = nearest_resize(output_rgba, width=source_rgba.shape[1], height=source_rgba.shape[0]) + else: + preview = output_rgba + + fg_recon = foreground_reconstruction_error(preview, source_rgba, alpha_threshold=alpha_threshold, halo=halo) + edge_position = foreground_edge_position_error(preview, source_rgba, alpha_threshold=alpha_threshold, halo=halo) + stroke_wobble = foreground_stroke_wobble_error(preview, source_rgba, alpha_threshold=alpha_threshold, halo=halo) + edge_support = foreground_edge_support_breakdown(preview, source_rgba, alpha_threshold=alpha_threshold, halo=halo) + exact_ratio = foreground_exact_match_ratio(preview, source_rgba, alpha_threshold=alpha_threshold, halo=halo) + wobble_penalty = float(stroke_wobble / (1.0 + stroke_wobble)) + score = ( + fg_recon * 0.30 + + edge_position * 0.20 + + wobble_penalty * 0.15 + + (1.0 - edge_support["f1"]) * 0.20 + + (1.0 - exact_ratio) * 0.15 + ) + return { + "foreground_reconstruction_error": fg_recon, + "edge_position_error": edge_position, + "stroke_wobble_error": stroke_wobble, + "stroke_wobble_penalty": wobble_penalty, + "edge_precision": edge_support["precision"], + "edge_recall": edge_support["recall"], + "edge_f1": edge_support["f1"], + "exact_match_ratio": exact_ratio, + "score": score, + } + + def exact_match_ratio(a: np.ndarray, b: np.ndarray) -> float: a8 = np.clip(np.rint(premultiply(a) * 255.0), 0, 255).astype(np.uint8) b8 = np.clip(np.rint(premultiply(b) * 255.0), 0, 255).astype(np.uint8) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 905081e..657f4db 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -7,12 +7,14 @@ from repixelizer.metrics import ( foreground_adjacency_error, foreground_edge_concentration, foreground_edge_position_error, + foreground_edge_support_breakdown, foreground_exact_match_ratio, foreground_motif_error, foreground_reconstruction_error, foreground_stroke_wobble_error, - source_lattice_consistency_breakdown, reconstruction_error, + source_lattice_consistency_breakdown, + source_structure_breakdown, ) @@ -112,6 +114,46 @@ def test_stroke_wobble_metric_penalizes_local_line_jitter() -> None: assert foreground_stroke_wobble_error(original, wobbly) > 0.0 +def test_edge_support_breakdown_rewards_nearby_edge_recall() -> None: + source = np.zeros((9, 9, 4), dtype=np.float32) + source[2:7, 4] = np.asarray([1.0, 1.0, 1.0, 1.0], dtype=np.float32) + + stable = source.copy() + + shifted = np.zeros_like(source) + shifted[2:7, 5] = np.asarray([1.0, 1.0, 1.0, 1.0], dtype=np.float32) + + missing = np.zeros_like(source) + + stable_support = foreground_edge_support_breakdown(stable, source) + shifted_support = foreground_edge_support_breakdown(shifted, source) + missing_support = foreground_edge_support_breakdown(missing, source) + + assert stable_support["f1"] >= shifted_support["f1"] > missing_support["f1"] + + +def test_source_structure_breakdown_prefers_exact_and_supported_structure() -> None: + source = np.zeros((9, 9, 4), dtype=np.float32) + source[2:7, 4] = np.asarray([1.0, 1.0, 1.0, 1.0], dtype=np.float32) + + exact = source.copy() + + shifted = np.zeros_like(source) + shifted[2:7, 5] = np.asarray([1.0, 1.0, 1.0, 1.0], dtype=np.float32) + + blurred = np.zeros_like(source) + blurred[2:7, 3] = np.asarray([0.4, 0.4, 0.4, 1.0], dtype=np.float32) + blurred[2:7, 4] = np.asarray([0.7, 0.7, 0.7, 1.0], dtype=np.float32) + blurred[2:7, 5] = np.asarray([0.4, 0.4, 0.4, 1.0], dtype=np.float32) + + exact_score = source_structure_breakdown(source, exact)["score"] + shifted_score = source_structure_breakdown(source, shifted)["score"] + blurred_score = source_structure_breakdown(source, blurred)["score"] + + assert exact_score < shifted_score + assert exact_score < blurred_score + + def test_source_lattice_consistency_prefers_matching_cell_structure() -> None: lowres = np.asarray( [ diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 91f94a5..4d1cfd4 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -36,6 +36,8 @@ def test_pipeline_writes_output_and_diagnostics(tmp_path: Path) -> None: run_json = json.loads((diagnostics_dir / "run.json").read_text(encoding="utf-8")) assert set(run_json["source_fidelity"].keys()) == {"snap_initial", "solver_target", "final_output"} + assert "source_structure" in run_json + assert {"edge_f1", "exact_match_ratio", "score"} <= set(run_json["source_structure"].keys()) assert "phase_rerank_candidates" in run_json assert {"snap", "relax_handoff", "relax_mode", "final_output"} <= set(run_json["optimizer_displacement"].keys()) assert result.output_rgba.shape[0] == result.inference.target_height