From 4a7f1033f7f579b6297da668ee9537c45b170455 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 23 Apr 2026 00:32:05 +0100 Subject: [PATCH] Add hybrid continuous tile-graph mode --- README.md | 4 ++ docs/implementation-plan.md | 21 ++++--- src/repixelizer/cli.py | 2 +- src/repixelizer/params.py | 2 + src/repixelizer/pipeline.py | 39 ++++++++++++- src/repixelizer/tile_graph.py | 60 ++++++++++++++++++- tests/test_pipeline.py | 105 +++++++++++++++++++++++++++++++++- 7 files changed, 219 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 7e66933..9416670 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ Recent adjacency-focused status: - the snap-to-refine handoff is now source-first rather than dominated by the softened representative lattice - low-confidence phase reranking now uses a soft size penalty instead of a hard size-jump reject - an experimental `tile-graph` reconstruction mode now exists behind `--reconstruction-mode tile-graph` +- an experimental `hybrid` reconstruction mode now exists behind `--reconstruction-mode hybrid` Current tile-graph status: @@ -48,6 +49,8 @@ Current tile-graph status: - the remaining large-fixture bottleneck is the per-component one-cell window cutting pass, which still hops back to CPU after labeling - elongated source regions now get a stroke-aware slicing pass that follows their principal axis instead of only marching with cardinal queue steps, and the repo now has a shallow-stroke regression that checks for fan-out on that kind of component - that stroke-aware slicer is a real synthetic improvement but not a real-badge win yet: the latest badge probe under `artifacts/badge-tile-graph-stroke-v2-cuda/` still keeps the initial assignment and slightly regresses source-fidelity (`0.1832` vs `0.1800`) +- the new low-risk hybrid mode keeps tile-graph's source-owned candidates but scores them against a continuous-optimizer geometry prepass; on the cleaned badge it improves tile-graph from `0.1832` to `0.1785` under `artifacts/badge-hybrid-v2-cuda/` +- that hybrid gain is still modest and still far behind the stronger continuous badge result, but it is the first combined path that moves the real badge in the right direction without reintroducing smeared ownership - this fixes the core design mismatch that had allowed repeated distant labels to create big same-color patches and opaque black background blocks - on the current `24x24` emblem smoke case, an end-to-end `tile-graph` run dropped from about `2.57s` on CPU to `0.61s` on CUDA on this machine - the older hard-edge-only candidate widening pass under `artifacts/full-emblem-tile-graph-hard-edge-v2-cuda/` remains a useful negative result: more edge choices alone sharpen some cells locally but still regress full-emblem source-fidelity (`0.0377`) @@ -66,6 +69,7 @@ repixelize input.png --out output.png repixelize input.png --out output.png --diagnostics-dir diagnostics --device auto repixelize input.png --out output.png --reconstruction-mode tile-graph --diagnostics-dir diagnostics --device cpu repixelize input.png --out output.png --reconstruction-mode tile-graph --diagnostics-dir diagnostics --device cuda +repixelize input.png --out output.png --reconstruction-mode hybrid --diagnostics-dir diagnostics --device cuda ``` Run the optimizer plus baselines: diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 393370b..b10b8a7 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -14,12 +14,13 @@ Implemented: - source-first snap/refine scoring with explicit source-vs-representative weights - low-confidence phase reranking with soft size penalties instead of hard size-jump rejection - an experimental `tile-graph` reconstruction path with coord-local literal source-pixel candidates +- an experimental `hybrid` reconstruction path that combines a continuous geometry prepass with tile-graph ownership - stage-aware diagnostics writing, including per-stage source-fidelity and rerank traces - synthetic test fixtures and automated tests Verified: - local editable install in a dedicated venv -- full test suite currently passing (`56 passed`) +- full test suite currently passing (`59 passed`) - compare-mode smoke run against a real emblem image - the cleaned real badge fixture in `tests/fixtures/real/ai-badge-cleaned.png` still beats naive resize on source-lattice consistency on the continuous path - the latest full-emblem tile-graph atomic probe under `artifacts/full-emblem-tile-graph-atomic-v3-cuda/` lands at `0.0224` source-fidelity, beating the older full-emblem tile-graph CUDA baseline at `0.0283` @@ -56,22 +57,25 @@ The most provisional areas are: ### Active pass -This pass tried the next geometry-focused experiment on tile-graph: cutting elongated source regions like strokes instead of treating them like generic blobs. +This pass adds the first low-risk hybrid path between the continuous solver and tile-graph instead of treating them as completely separate experiments. What landed: -- `tile_graph.py` now computes principal-axis stats for source-side atomic regions and gives elongated components a stroke-aware slicing path -- stroke-like components are now seeded and cut along their fitted axis, and then thinned to one best candidate per dominant row or column band before they reach the per-cell candidate buckets -- a new shallow-stroke regression in `tests/test_tile_graph.py` checks that connected slanted strokes do not fan out across too many output cells -- tile-graph still keeps its initial assignment whenever the propagation loop would worsen source-lattice fidelity, because that safeguard is still doing real work on the badge +- `pipeline.py` now supports `reconstruction_mode="hybrid"` and runs a zero-step continuous prepass before tile-graph +- `tile_graph.py` now accepts an optional geometry reference image plus guidance map and adds a geometry-match term to the tile-graph unary cost +- `cli.py` now exposes `--reconstruction-mode hybrid` +- `tests/test_pipeline.py` now covers both the direct hybrid handoff seam and an actual end-to-end hybrid smoke run Current implementation note: - source-region labeling is still device-backed, but the one-cell window cutting pass remains CPU-side and still dominates large-fixture runtime -- the stroke-aware slicer is a meaningful synthetic change, but it is not yet a real-badge quality win: the latest run under `artifacts/badge-tile-graph-stroke-v2-cuda/` lands at `0.1832`, slightly worse than the previous CCL-only tile-graph badge run at `0.1800` -- the latest badge stroke-aware run still keeps its initial assignment, which reinforces that the current badge weakness is mostly in initial candidate ownership/selection rather than in the later propagation loop +- the hybrid is intentionally conservative for this first pass: it only biases tile-graph's unary cost with the continuous prepass layout and does not yet replace the pairwise objective or the source-region builder +- on the cleaned badge, that conservative hybrid still helps: the latest run under `artifacts/badge-hybrid-v2-cuda/` lands at `0.1785`, improving on the current tile-graph badge baseline at `0.1832` +- the hybrid is still far behind the stronger continuous badge result (`0.0832`), so the seam looks promising but it is not yet enough to solve the real contour problem by itself +- the runtime overhead is modest relative to the current stroke-aware tile-graph pass: about `460.7s` for the hybrid badge probe versus about `443.1s` for the current non-hybrid tile-graph badge probe on this machine - the latest full-emblem atomic probe under `artifacts/full-emblem-tile-graph-atomic-v3-cuda/` lands at `0.0224` source-fidelity with `34278` candidates and about `2.16` average choices per cell - that beats the earlier full-CUDA tile-graph baseline (`0.0283`) and materially improves on the first atomic-only attempt (`0.1571`), which chose atomic regions too eagerly and then let the solver blur them out again Next after that: +- decide whether the next hybrid pass should add geometry-aware pairwise terms before changing the source-region builder again - keep pushing the initial candidate builder toward true source-side stroke ownership instead of only better stroke slicing within the same builder - port the per-component window cutting pass back toward a fully device-friendly formulation instead of only accelerating the labeling step - decide whether the next stroke pass should use a real contour/skeleton trace instead of only a principal-axis approximation @@ -103,6 +107,7 @@ Current status: - the remaining weakness on tile-graph is not candidate color purity anymore; it is candidate extraction cost plus deciding how aggressive the propagation step should be once the initial assignment is already strong - on the cleaned badge, the new CCL-backed tile-graph run under `artifacts/badge-tile-graph-ccl-cuda/` still lands at a weak `0.1800`, so the current CCL work should be treated as an infrastructure/performance step rather than a visual-quality fix - the new stroke-aware badge run under `artifacts/badge-tile-graph-stroke-v2-cuda/` still does not solve the guard-contour problem and slightly regresses to `0.1832`, so the next stroke pass should be judged on whether it changes the source-side path model more fundamentally +- the first hybrid badge run under `artifacts/badge-hybrid-v2-cuda/` improves that tile-graph baseline to `0.1785`, which is modest but real progress in the combined direction Important note: - the checkerboard is baked into the source image, not transparency diff --git a/src/repixelizer/cli.py b/src/repixelizer/cli.py index 34409fb..1df03f1 100644 --- a/src/repixelizer/cli.py +++ b/src/repixelizer/cli.py @@ -32,7 +32,7 @@ def build_parser() -> argparse.ArgumentParser: target.add_argument( "--reconstruction-mode", default="continuous", - choices=("continuous", "tile-graph"), + choices=("continuous", "tile-graph", "hybrid"), help="Reconstruction engine to run after lattice estimation", ) target.add_argument( diff --git a/src/repixelizer/params.py b/src/repixelizer/params.py index 4b9774b..17a11e0 100644 --- a/src/repixelizer/params.py +++ b/src/repixelizer/params.py @@ -92,6 +92,8 @@ class SolverHyperParams: tile_graph_coverage_weight: float = 0.05 tile_graph_delta_weight: float = 0.45 tile_graph_iterations: int = 10 + hybrid_geometry_match_weight: float = 0.35 + hybrid_geometry_edge_boost: float = 0.55 def to_dict(self) -> dict[str, float | int]: return asdict(self) diff --git a/src/repixelizer/pipeline.py b/src/repixelizer/pipeline.py index d1e2f06..ef1eb5e 100644 --- a/src/repixelizer/pipeline.py +++ b/src/repixelizer/pipeline.py @@ -53,7 +53,7 @@ def run_pipeline( if strip_background: source = strip_edge_background(source) inference = infer_lattice(source, target_size=target_size, device=device) - analysis = analyze_source(source, seed=seed, device=device if reconstruction_mode == "tile-graph" else None) + analysis = analyze_source(source, seed=seed, device=device if reconstruction_mode in {"tile-graph", "hybrid"} else None) inference = _select_phase_candidate( source, inference, @@ -308,6 +308,43 @@ def _run_reconstruction( device=device, solver_params=solver_params, ) + if reconstruction_mode == "hybrid": + geometry_solver = optimize_uv_field( + source, + inference=inference, + analysis=analysis, + steps=0, + seed=seed, + device=device, + solver_params=solver_params, + ) + hybrid_solver, hybrid_diagnostics = optimize_tile_graph( + source, + inference=inference, + analysis=analysis, + steps=steps, + seed=seed, + device=device, + solver_params=solver_params, + geometry_reference_rgba=geometry_solver.target_rgba, + geometry_guidance_strength=geometry_solver.guidance_strength, + ) + geometry_score = source_lattice_consistency_breakdown( + source, + geometry_solver.target_rgba, + target_width=inference.target_width, + target_height=inference.target_height, + phase_x=inference.phase_x, + phase_y=inference.phase_y, + )["score"] + diagnostics = dict(hybrid_diagnostics) + diagnostics["mode"] = "hybrid" + diagnostics["hybrid_geometry_prepass_mode"] = "continuous" + diagnostics["hybrid_geometry_prepass_steps"] = 0.0 + diagnostics["hybrid_geometry_prepass_source_fidelity"] = float(geometry_score) + diagnostics["hybrid_geometry_match_weight"] = float(solver_params.hybrid_geometry_match_weight) + diagnostics["hybrid_geometry_edge_boost"] = float(solver_params.hybrid_geometry_edge_boost) + return hybrid_solver, diagnostics solver = optimize_uv_field( source, inference=inference, diff --git a/src/repixelizer/tile_graph.py b/src/repixelizer/tile_graph.py index 613ee96..5cb2131 100644 --- a/src/repixelizer/tile_graph.py +++ b/src/repixelizer/tile_graph.py @@ -67,6 +67,8 @@ class TileGraphModel: edge_density: float average_choices: float model_device: str + geometry_reference_rgba: np.ndarray | None = None + geometry_strength: np.ndarray | None = None def _make_regular_uv(height: int, width: int, target_height: int, target_width: int, phase_x: float, phase_y: float) -> np.ndarray: @@ -744,6 +746,8 @@ def build_tile_graph_model( analysis: SourceAnalysis, solver_params: SolverHyperParams | None = None, device: str = "cpu", + geometry_reference_rgba: np.ndarray | None = None, + geometry_guidance_strength: np.ndarray | None = None, ) -> TileGraphModel: torch = _require_torch() resolved_device = _resolve_device(torch, device) @@ -923,6 +927,20 @@ def build_tile_graph_model( edge_density = float( np.mean(source_reference.edge_strength >= solver_params.source_edge_detail_threshold) ) + geometry_reference_np = None + if geometry_reference_rgba is not None: + geometry_reference_np = np.asarray(geometry_reference_rgba, dtype=np.float32) + if geometry_reference_np.shape[:2] != (inference.target_height, inference.target_width): + raise ValueError( + "geometry_reference_rgba must match the inferred output size when provided to tile-graph." + ) + geometry_strength_np = None + if geometry_guidance_strength is not None: + geometry_strength_np = np.asarray(geometry_guidance_strength, dtype=np.float32) + if geometry_strength_np.shape != (inference.target_height, inference.target_width): + raise ValueError( + "geometry_guidance_strength must match the inferred output size when provided to tile-graph." + ) return TileGraphModel( candidate_rgba=candidate_rgba_np, @@ -940,6 +958,8 @@ def build_tile_graph_model( edge_density=edge_density, average_choices=average_choices, model_device=resolved_device, + geometry_reference_rgba=geometry_reference_np, + geometry_strength=geometry_strength_np, ) @@ -993,12 +1013,25 @@ def _tile_graph_unary_cost(model: TileGraphModel, solver_params: SolverHyperPara area_error = np.abs(np.log(np.clip(model.candidate_area_ratio[indices], 1e-4, None) + 1e-4)) alpha_error = np.abs(candidates[:, 3] - reference_mean[3]) coverage_error = 1.0 - np.clip(model.candidate_coverage[indices], 0.0, 1.0) - unary_cost[start:end] = ( + total_cost = ( color_error + area_error * solver_params.tile_graph_area_weight + alpha_error * solver_params.tile_graph_alpha_weight + coverage_error * solver_params.tile_graph_coverage_weight - ).astype(np.float32) + ) + if model.geometry_reference_rgba is not None: + geometry_reference = model.geometry_reference_rgba[y, x] + geometry_error = np.mean(np.abs(geometry_reference[None, :] - candidates), axis=-1) + geometry_strength = 1.0 + if model.geometry_strength is not None: + geometry_strength += ( + float(np.clip(model.geometry_strength[y, x], 0.0, 1.0)) + * solver_params.hybrid_geometry_edge_boost + ) + total_cost = total_cost + geometry_error * ( + solver_params.hybrid_geometry_match_weight * geometry_strength + ) + unary_cost[start:end] = total_cost.astype(np.float32) return unary_cost @@ -1019,6 +1052,16 @@ def _tile_graph_unary_cost_torch( reference_mean_t = torch.from_numpy(model.reference_mean_rgba).to(device=device, dtype=torch.float32) reference_edge_t = torch.from_numpy(model.reference_edge_rgba).to(device=device, dtype=torch.float32) edge_strength_t = torch.from_numpy(model.edge_strength).to(device=device, dtype=torch.float32) + geometry_reference_t = ( + torch.from_numpy(model.geometry_reference_rgba).to(device=device, dtype=torch.float32) + if model.geometry_reference_rgba is not None + else None + ) + geometry_strength_t = ( + torch.from_numpy(model.geometry_strength).to(device=device, dtype=torch.float32) + if model.geometry_strength is not None + else None + ) choice_rgba_t = candidate_rgba_t[choice_indices_t] choice_deltas_t = candidate_deltas_t[choice_indices_t] @@ -1041,6 +1084,14 @@ def _tile_graph_unary_cost_torch( + alpha_error * solver_params.tile_graph_alpha_weight + coverage_error * solver_params.tile_graph_coverage_weight ) + if geometry_reference_t is not None: + geometry_error_t = (geometry_reference_t[..., None, :] - choice_rgba_t).abs().mean(dim=-1) + geometry_scale_t = 1.0 + if geometry_strength_t is not None: + geometry_scale_t = 1.0 + geometry_strength_t[..., None].clamp(0.0, 1.0) * solver_params.hybrid_geometry_edge_boost + unary_cost_t = unary_cost_t + geometry_error_t * ( + solver_params.hybrid_geometry_match_weight * geometry_scale_t + ) unary_cost_t = unary_cost_t.masked_fill(~choice_mask_t, float("inf")) return unary_cost_t, candidate_rgba_t, candidate_deltas_t, choice_rgba_t, choice_deltas_t @@ -1213,6 +1264,8 @@ def optimize_tile_graph( seed: int, device: str, solver_params: SolverHyperParams | None = None, + geometry_reference_rgba: np.ndarray | None = None, + geometry_guidance_strength: np.ndarray | None = None, ) -> tuple[SolverArtifacts, dict[str, Any]]: del steps torch = _require_torch() @@ -1227,6 +1280,8 @@ def optimize_tile_graph( analysis=analysis, solver_params=solver_params, device=resolved_device, + geometry_reference_rgba=geometry_reference_rgba, + geometry_guidance_strength=geometry_guidance_strength, ) choice_indices_np, choice_mask_np = _build_choice_grid(model) choice_indices_t = torch.from_numpy(choice_indices_np).to(device=resolved_device, dtype=torch.long) @@ -1349,6 +1404,7 @@ def optimize_tile_graph( "tile_graph_initial_source_fidelity": float(initial_source_fidelity), "tile_graph_final_source_fidelity": float(final_source_fidelity), "tile_graph_kept_initial_assignment": kept_initial_assignment, + "tile_graph_geometry_prior": model.geometry_reference_rgba is not None, } return ( SolverArtifacts( diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 2284d25..791f657 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -8,10 +8,11 @@ from repixelizer.analysis import analyze_source from repixelizer.baselines import naive_resize_baseline from repixelizer.continuous import optimize_uv_field from repixelizer.metrics import coherence_breakdown, foreground_reconstruction_error, source_lattice_consistency_breakdown -from repixelizer.pipeline import _select_phase_candidate, run_pipeline +from repixelizer.params import SolverHyperParams +from repixelizer.pipeline import _run_reconstruction, _select_phase_candidate, run_pipeline from repixelizer.io import load_rgba, nearest_resize from repixelizer.synthetic import fake_pixelize, make_emblem, make_sprite -from repixelizer.types import InferenceCandidate, InferenceResult +from repixelizer.types import InferenceCandidate, InferenceResult, SolverArtifacts def test_pipeline_writes_output_and_diagnostics(tmp_path: Path) -> None: @@ -221,6 +222,106 @@ def test_phase_rerank_rejects_large_size_jump(monkeypatch) -> None: assert selected.target_height == candidate_a.target_height +def test_hybrid_reconstruction_passes_continuous_geometry_into_tile_graph(monkeypatch) -> None: + source = np.zeros((4, 4, 4), dtype=np.float32) + geometry_rgba = np.zeros((2, 2, 4), dtype=np.float32) + geometry_rgba[0, 0] = np.asarray([1.0, 0.0, 0.0, 1.0], dtype=np.float32) + geometry_guidance = np.asarray([[0.8, 0.2], [0.1, 0.6]], dtype=np.float32) + inference = InferenceResult( + target_width=2, + target_height=2, + phase_x=0.0, + phase_y=0.0, + confidence=1.0, + top_candidates=[], + ) + + geometry_solver = SolverArtifacts( + target_rgba=geometry_rgba, + uv_field=np.zeros((2, 2, 2), dtype=np.float32), + guidance_strength=geometry_guidance, + initial_rgba=geometry_rgba.copy(), + loss_history=[], + ) + + def fake_optimize_uv_field(source_rgba, inference, analysis, steps, seed, device, solver_params=None): + assert steps == 0 + return geometry_solver + + def fake_optimize_tile_graph( + source_rgba, + inference, + analysis, + steps, + seed, + device, + solver_params=None, + geometry_reference_rgba=None, + geometry_guidance_strength=None, + ): + assert np.allclose(geometry_reference_rgba, geometry_rgba) + assert np.allclose(geometry_guidance_strength, geometry_guidance) + return ( + SolverArtifacts( + target_rgba=geometry_rgba.copy(), + uv_field=np.zeros((2, 2, 2), dtype=np.float32), + guidance_strength=np.zeros((2, 2), dtype=np.float32), + initial_rgba=geometry_rgba.copy(), + loss_history=[], + ), + {"mode": "tile-graph", "tile_graph_geometry_prior": True}, + ) + + monkeypatch.setattr("repixelizer.pipeline.optimize_uv_field", fake_optimize_uv_field) + monkeypatch.setattr("repixelizer.pipeline.optimize_tile_graph", fake_optimize_tile_graph) + + solver, diagnostics = _run_reconstruction( + source, + inference=inference, + analysis=object(), + steps=12, + seed=7, + device="cpu", + solver_params=SolverHyperParams(), + reconstruction_mode="hybrid", + ) + + assert np.allclose(solver.target_rgba, geometry_rgba) + assert diagnostics["mode"] == "hybrid" + assert diagnostics["hybrid_geometry_prepass_mode"] == "continuous" + assert diagnostics["tile_graph_geometry_prior"] is True + + +def test_pipeline_hybrid_mode_writes_reconstruction_diagnostics(tmp_path: Path) -> None: + source = make_emblem(16, 16) + fake = fake_pixelize(source, upscale=8, phase_x=0.1, phase_y=-0.05, blur_radius=0.4, warp_strength=0.15, warp_detail=4) + input_path = tmp_path / "input.png" + output_path = tmp_path / "output.png" + diagnostics_dir = tmp_path / "diagnostics" + + from repixelizer.io import save_rgba + + save_rgba(input_path, fake) + result = run_pipeline( + input_path, + output_path, + diagnostics_dir=diagnostics_dir, + reconstruction_mode="hybrid", + steps=0, + device="cpu", + ) + + import json + + run_json = json.loads((diagnostics_dir / "run.json").read_text(encoding="utf-8")) + assert output_path.exists() + assert run_json["settings"]["reconstruction_mode"] == "hybrid" + assert run_json["reconstruction"]["mode"] == "hybrid" + assert run_json["reconstruction"]["tile_graph_geometry_prior"] is True + assert run_json["reconstruction"]["hybrid_geometry_prepass_mode"] == "continuous" + assert result.diagnostics["reconstruction"]["mode"] == "hybrid" + + def test_phase_rerank_can_accept_low_confidence_size_jump_with_strong_support(monkeypatch) -> None: source = np.zeros((16, 16, 4), dtype=np.float32) candidate_a = InferenceCandidate(target_width=16, target_height=16, phase_x=0.0, phase_y=0.0, score=0.91, breakdown={})