diff --git a/README.md b/README.md index 20cff8b..963d744 100644 --- a/README.md +++ b/README.md @@ -21,13 +21,20 @@ That is the two-headed pitch in one image: force non-pixel art onto a coherent g This is already a serious experimental baseline, not a toy prototype: - CUDA-backed inference and solver paths are wired up +- a shared source-lattice reference now drives solver, metrics, and reranking - round-trip corpus benchmarking is built in - soft and crisp corruption profiles are supported -- diagnostics and baseline comparisons are written automatically +- diagnostics and baseline comparisons are written automatically, including per-stage source-fidelity in `run.json` - a black-box tuning harness exists for longer offline parameter sweeps The current pipeline is especially strong on benchmarked sprite and emblem inputs where local fake-pixel texture is meaningful but the global grid is inconsistent. +Recent adjacency-focused status: + +- the cleaned AI badge regression now beats naive resize on the repo's source-lattice consistency metric under the selected lattice +- the snap-to-refine handoff is now source-first rather than dominated by the softened representative lattice +- low-confidence phase reranking now uses a soft size penalty instead of a hard size-jump reject + ## Quickstart ```powershell diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 5f4fc3b..eddb0cd 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -10,16 +10,37 @@ Implemented: - lattice/phase inference with coherence scoring plus source-period prior - source-image structural analysis - PyTorch-based continuous UV-field optimization +- shared source-lattice reference built from actual inferred cell assignments +- source-first snap/refine scoring with explicit source-vs-representative weights +- low-confidence phase reranking with soft size penalties instead of hard size-jump rejection - discrete cleanup on the output grid - optional palette loading and quantization modes - compare mode with naive and diffusion baselines -- diagnostics writing +- stage-aware diagnostics writing, including per-stage source-fidelity and rerank traces - synthetic test fixtures and automated tests Verified: - local editable install in a dedicated venv - full test suite currently passing - compare-mode smoke run against a real emblem image +- focused real-fixture probe on `tests/fixtures/real/ai-badge-cleaned.png` now beats naive resize on source-lattice consistency under the selected lattice (`0.0832` final vs `0.1304` naive with the current `126x126` / `(0.0, -0.2)` pick) + +## Status after adjacency-first pass + +What changed: +- `src/repixelizer/source_reference.py` now owns the lattice-indexed source reference used by metrics, the solver, and phase reranking +- snap and refine now prefer sharp per-cell source evidence and keep the relaxed-mode neighborhood in the final discrete choice +- `run.json` now includes `source_fidelity` for `snap_initial`, `solver_target`, and `final_output`, plus `phase_rerank_candidates` +- `scripts/render_focus_crop.py` now prints stage-level source-fidelity when generating the guard/tip crop sheet + +What that fixed: +- the cleaned badge regression no longer loses badly to naive resize on adjacency and motif preservation once the solver starts from the selected lattice +- the solver no longer regresses relative to its own snap stage on the added thin-feature regression case + +What is still provisional: +- the cleaned badge still lands on the conservative `126x126` lattice candidate, just with a much lower confidence than before; that means candidate selection and weight tuning are improved but not “done” +- tuning has not been rerun yet against the new source-first hyperparameters, so the benchmark/tuning baseline should be treated as stale until the next sweep +- the baked-checkerboard version of the badge remains a separate background-suppression problem ## What is still provisional @@ -51,6 +72,11 @@ Current failure modes: - the right-hand guard wing loses short dark-light-dark adjacency motifs and collapses into a chunky gold slab - some guard closeups look nearly identical across `snap`, `relaxed`, and `final`, which suggests the wrong local motif is often chosen before later refinement even starts +Current status after this pass: +- on the cleaned fixture, `snap`, `relaxed`, and `final` now all preserve source-lattice structure substantially better than the old baseline, with `final` currently landing at `0.0832` source-fidelity score in the reproducible probe under `artifacts/badge-final-probe/` +- that is a meaningful improvement over both the previous documented `0.1494` final score and the naive resize score under the same selected lattice +- the remaining weakness is candidate choice confidence, not a collapse inside the snap/refine handoff + Important note: - the checkerboard is baked into the source image, not transparency - when projected onto a `122x122` output grid it aliases into a visible `2-3-2` cadence @@ -70,10 +96,9 @@ Current optimizer diagnosis: - diagonal structure is also underrepresented in the final structure score, which is a bad fit for curved one-pixel outlines and hooks Most likely next fixes: -- derive a sharper per-cell source reference from actual sampled source patches instead of using the soft representative as the only "source truth" -- point snap, relaxation, and structure-scoring source terms at that sharper source reference -- harden the relaxed solution more honestly by considering the relaxed mode assignment, not just a per-cell handoff argmin -- include diagonal boundary agreement in the final structure score so oblique tiny features matter earlier +- rerun tuning so the new source-first weights are benchmarked against the corpus instead of being judged only by defaults +- keep pressure on low-confidence badge-like cases where larger lattice candidates have better support but still lose on the combined rerank score +- evaluate whether the compare-mode summary should surface per-stage source-fidelity directly, not only the final output metrics Repository fixtures: - `tests/fixtures/real/ai-badge-cleaned.png` is the manually cleaned transparent version of the same emblem diff --git a/scripts/render_focus_crop.py b/scripts/render_focus_crop.py index 61831db..7c6d8ba 100644 --- a/scripts/render_focus_crop.py +++ b/scripts/render_focus_crop.py @@ -8,7 +8,7 @@ from PIL import Image, ImageDraw from repixelizer.continuous import ( _discrete_refine_output, - _exemplar_colors, + _build_source_reliability, _make_patch_offsets, _make_regular_uv, _relax_candidate_selection, @@ -18,12 +18,13 @@ from repixelizer.continuous import ( _sample_cell_patches, _select_colors, _snap_output_to_source_pixels, - _source_boundary_deltas, premultiply, ) from repixelizer.inference import infer_lattice from repixelizer.io import load_rgba +from repixelizer.metrics import source_lattice_consistency_breakdown from repixelizer.params import SolverHyperParams +from repixelizer.source_reference import build_source_lattice_reference def _parse_args() -> argparse.Namespace: @@ -87,7 +88,7 @@ def _build_focus_states( *, steps: int, device: str, -) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, tuple[float, float]]: +) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, tuple[float, float], dict[str, dict[str, float]]]: source = load_rgba(input_path) solver_params = SolverHyperParams() inference = infer_lattice(source, None, device=device) @@ -112,15 +113,45 @@ def _build_focus_states( offsets_t = torch.from_numpy( _make_patch_offsets(height=height, width=width, target_height=inference.target_height, target_width=inference.target_width) ).to(device=resolved_device, dtype=torch.float32) + source_lattice_reference = build_source_lattice_reference( + source, + target_width=inference.target_width, + target_height=inference.target_height, + phase_x=inference.phase_x, + phase_y=inference.phase_y, + alpha_threshold=solver_params.alpha_transparent_threshold, + ) initial_patches = _sample_cell_patches(F, source_t, uv0_t, offsets_t) representative_t, _ = _representative_colors(initial_patches, solver_params) representative_t = representative_t.detach() - source_reference_t = _exemplar_colors(initial_patches).detach() - - source_delta_x_t = _source_boundary_deltas(F, source_t, uv0_t, axis="x") - source_delta_y_t = _source_boundary_deltas(F, source_t, uv0_t, axis="y") - source_delta_diag_t = _source_boundary_deltas(F, source_t, uv0_t, axis="diag") - source_delta_anti_t = _source_boundary_deltas(F, source_t, uv0_t, axis="anti") + source_reference_t = torch.from_numpy(premultiply(source_lattice_reference.sharp_rgba)[None, ...]).to( + device=resolved_device, + dtype=torch.float32, + ) + source_reliability_t = torch.from_numpy(_build_source_reliability(source_lattice_reference)[None, ...]).to( + device=resolved_device, + dtype=torch.float32, + ) + source_delta_x_t = ( + torch.from_numpy(source_lattice_reference.delta_x[None, ...]).to(device=resolved_device, dtype=torch.float32) + if source_lattice_reference.delta_x is not None + else None + ) + source_delta_y_t = ( + torch.from_numpy(source_lattice_reference.delta_y[None, ...]).to(device=resolved_device, dtype=torch.float32) + if source_lattice_reference.delta_y is not None + else None + ) + source_delta_diag_t = ( + torch.from_numpy(source_lattice_reference.delta_diag[None, ...]).to(device=resolved_device, dtype=torch.float32) + if source_lattice_reference.delta_diag is not None + else None + ) + source_delta_anti_t = ( + torch.from_numpy(source_lattice_reference.delta_anti[None, ...]).to(device=resolved_device, dtype=torch.float32) + if source_lattice_reference.delta_anti is not None + else None + ) snap_rgba = _snap_output_to_source_pixels( torch, @@ -128,6 +159,7 @@ def _build_focus_states( uv0_t, representative_t, source_reference_t, + source_reliability_t, solver_params, cell_x=cell_x, cell_y=cell_y, @@ -169,7 +201,12 @@ def _build_focus_states( distance_energy = ((offset_x / max(cell_x, 1e-4)) ** 2 + (offset_y / max(cell_y, 1e-4)) ** 2).reshape(1, 1, -1) relax_base_energy = ( anchor_energy * solver_params.refine_anchor_weight * solver_params.relax_anchor_scale - + (rep_energy * 0.45 + source_reference_energy * 0.55) * solver_params.refine_representative_weight + + ( + source_reference_energy * solver_params.refine_source_match_weight + + rep_energy * solver_params.refine_representative_match_weight + ) + / max(solver_params.refine_source_match_weight + solver_params.refine_representative_match_weight, 1e-4) + * solver_params.refine_representative_weight + alpha_energy * solver_params.refine_alpha_weight + distance_energy * solver_params.refine_distance_weight ) @@ -210,6 +247,7 @@ def _build_focus_states( uv0_t, representative_t, source_reference_t, + source_reliability_t, snap_t, source_delta_x_t, source_delta_y_t, @@ -220,7 +258,33 @@ def _build_focus_states( cell_y=cell_y, iterations=steps, ) - return source, snap_rgba, relaxed_rgba, final_rgba, (cell_x, cell_y) + fidelity = { + "snap": source_lattice_consistency_breakdown( + source, + snap_rgba, + target_width=inference.target_width, + target_height=inference.target_height, + phase_x=inference.phase_x, + phase_y=inference.phase_y, + ), + "relaxed": source_lattice_consistency_breakdown( + source, + relaxed_rgba, + target_width=inference.target_width, + target_height=inference.target_height, + phase_x=inference.phase_x, + phase_y=inference.phase_y, + ), + "final": source_lattice_consistency_breakdown( + source, + final_rgba, + target_width=inference.target_width, + target_height=inference.target_height, + phase_x=inference.phase_x, + phase_y=inference.phase_y, + ), + } + return source, snap_rgba, relaxed_rgba, final_rgba, (cell_x, cell_y), fidelity def main() -> None: @@ -229,7 +293,7 @@ def main() -> None: if x1 <= x0 or y1 <= y0: raise SystemExit("Invalid --cell-bbox: expected X1 > X0 and Y1 > Y0.") - source, snap_rgba, relaxed_rgba, final_rgba, (cell_x, cell_y) = _build_focus_states( + source, snap_rgba, relaxed_rgba, final_rgba, (cell_x, cell_y), fidelity = _build_focus_states( args.input, steps=args.steps, device=args.device, @@ -271,6 +335,7 @@ def main() -> None: "source_bbox": list(source_bbox), "panels": panel_names, "scale": args.scale, + "source_fidelity": fidelity, } ) diff --git a/src/repixelizer/continuous.py b/src/repixelizer/continuous.py index 20ceb4f..e52e17d 100644 --- a/src/repixelizer/continuous.py +++ b/src/repixelizer/continuous.py @@ -4,6 +4,7 @@ import numpy as np from .io import premultiply, unpremultiply from .params import SolverHyperParams +from .source_reference import build_source_lattice_reference from .types import InferenceResult, SolverArtifacts, SourceAnalysis @@ -315,6 +316,64 @@ def _blend_reference_delta(primary_delta, secondary_delta, secondary_weight: flo return primary_delta * primary_weight + secondary_delta * secondary_weight +def _build_source_reliability(reference) -> np.ndarray: + baseline_dispersion = max(reference.dispersion, 1e-4) + dispersion_scale = reference.cell_dispersion / baseline_dispersion + dispersion_confidence = np.exp(-np.maximum(0.0, dispersion_scale - 1.0)) + support_confidence = np.clip(reference.cell_support, 0.0, 1.0) + alpha_confidence = np.clip(np.maximum(reference.cell_alpha_max, reference.sharp_rgba[..., 3]), 0.0, 1.0) + reliability = np.clip(0.2 + support_confidence * 0.5 + alpha_confidence * 0.3, 0.0, 1.0) + return (reliability * dispersion_confidence).astype(np.float32) + + +def _reference_match_energy( + representative_energy, + source_energy, + source_reliability, + *, + representative_weight: float, + source_weight: float, +): + source_mix = source_reliability[..., None] * source_weight + representative_mix = representative_weight + (1.0 - source_reliability[..., None]) * source_weight + normalization = (source_mix + representative_mix).clamp_min(1e-4) + return (representative_energy * representative_mix + source_energy * source_mix) / normalization + + +def _edge_reliability(source_reliability, axis: str): + if axis == "x": + return (source_reliability[:, 1:] + source_reliability[:, :-1]) * 0.5 + if axis == "y": + return (source_reliability[1:, :] + source_reliability[:-1, :]) * 0.5 + if axis == "diag": + return (source_reliability[1:, 1:] + source_reliability[:-1, :-1]) * 0.5 + if axis == "anti": + return (source_reliability[1:, :-1] + source_reliability[:-1, 1:]) * 0.5 + raise ValueError(f"Unsupported edge axis: {axis}") + + +def _blend_reference_delta_map( + representative_delta, + source_delta, + source_reliability, + *, + axis: str, + representative_weight: float, + source_weight: float, +): + if representative_delta is None: + return source_delta + if source_delta is None: + return representative_delta + edge_reliability = _edge_reliability(source_reliability, axis=axis) + source_mix = edge_reliability * source_weight + representative_mix = representative_weight + (1.0 - edge_reliability) * source_weight + normalization = (source_mix + representative_mix).clamp_min(1e-4) + return ( + representative_delta * representative_mix[..., None] + source_delta * source_mix[..., None] + ) / normalization[..., None] + + def _motif_candidate_energy(torch, candidate_colors, context_colors, anchor): if candidate_colors.shape[0] < 2 or candidate_colors.shape[1] < 2: return candidate_colors.new_zeros(candidate_colors.shape[:3]) @@ -686,6 +745,7 @@ def _snap_output_to_source_pixels( uv_t, representative_t, source_reference_t, + source_reliability_t, solver_params: SolverHyperParams, *, cell_x: float, @@ -697,6 +757,7 @@ def _snap_output_to_source_pixels( uv = uv_t[0] representative = representative_t[0] source_reference = source_reference_t[0] + source_reliability = source_reliability_t[0] output_height = representative.shape[0] output_width = representative.shape[1] @@ -713,8 +774,12 @@ def _snap_output_to_source_pixels( representative_match = (candidate_colors - representative[..., None, :]).abs().mean(dim=-1) source_match = (candidate_colors - source_reference[..., None, :]).abs().mean(dim=-1) - base_energy = ( - representative_match * 0.45 + source_match * 0.55 + base_energy = _reference_match_energy( + representative_match, + source_match, + source_reliability, + representative_weight=solver_params.snap_representative_match_weight, + source_weight=solver_params.snap_source_match_weight, ) * solver_params.snap_base_match_weight representative_delta_x = representative[:, 1:, :] - representative[:, :-1, :] if output_width > 1 else None representative_delta_y = representative[1:, :, :] - representative[:-1, :, :] if output_height > 1 else None @@ -725,12 +790,41 @@ def _snap_output_to_source_pixels( representative[1:, :-1, :] - representative[:-1, 1:, :] if output_height > 1 and output_width > 1 else None ) source_delta_x, source_delta_y, source_delta_diag, source_delta_anti = _reference_deltas(source_reference) - desired_delta_x = _blend_reference_delta(representative_delta_x, source_delta_x, 0.65) - desired_delta_y = _blend_reference_delta(representative_delta_y, source_delta_y, 0.65) - desired_delta_diag = _blend_reference_delta(representative_delta_diag, source_delta_diag, 0.65) - desired_delta_anti = _blend_reference_delta(representative_delta_anti, source_delta_anti, 0.65) + desired_delta_x = _blend_reference_delta_map( + representative_delta_x, + source_delta_x, + source_reliability, + axis="x", + representative_weight=solver_params.snap_representative_delta_weight, + source_weight=solver_params.snap_source_delta_weight, + ) + desired_delta_y = _blend_reference_delta_map( + representative_delta_y, + source_delta_y, + source_reliability, + axis="y", + representative_weight=solver_params.snap_representative_delta_weight, + source_weight=solver_params.snap_source_delta_weight, + ) + desired_delta_diag = _blend_reference_delta_map( + representative_delta_diag, + source_delta_diag, + source_reliability, + axis="diag", + representative_weight=solver_params.snap_representative_delta_weight, + source_weight=solver_params.snap_source_delta_weight, + ) + desired_delta_anti = _blend_reference_delta_map( + representative_delta_anti, + source_delta_anti, + source_reliability, + axis="anti", + representative_weight=solver_params.snap_representative_delta_weight, + source_weight=solver_params.snap_source_delta_weight, + ) selected = torch.argmin(base_energy, dim=-1) + ranking_energy = base_energy for _ in range(4): selected_colors = torch.gather( candidate_colors, @@ -792,13 +886,14 @@ def _snap_output_to_source_pixels( anti_desired[1:, :-1, :] = desired_delta_anti anti_error = ((candidate_colors - anti_selected[..., None, :]) - anti_desired[..., None, :]).abs().mean(dim=-1) energy = energy + anti_error * anti_mask * solver_params.snap_diagonal_weight + ranking_energy = energy selected = torch.argmin(energy, dim=-1) hardened_selected, use_opaque, use_transparent = _harden_binary_alpha_selection( torch, candidate_colors, selected, - base_energy, + ranking_energy, representative[..., 3], solver_params=solver_params, ) @@ -909,6 +1004,7 @@ def _discrete_refine_output( uv_t, representative_t, source_reference_t, + source_reliability_t, anchor_t, source_delta_x_t, source_delta_y_t, @@ -926,6 +1022,7 @@ def _discrete_refine_output( uv = uv_t[0] representative = representative_t[0] source_reference = source_reference_t[0] + source_reliability = source_reliability_t[0] anchor = anchor_t[0].permute(1, 2, 0) output_height = representative.shape[0] output_width = representative.shape[1] @@ -950,15 +1047,22 @@ def _discrete_refine_output( source_reference_energy = (candidate_colors - source_reference[..., None, :]).abs().mean(dim=-1) alpha_energy = (candidate_colors[..., 3] - anchor[..., None, 3]).abs() distance_energy = ((offset_x / max(cell_x, 1e-4)) ** 2 + (offset_y / max(cell_y, 1e-4)) ** 2).reshape(1, 1, -1) + match_energy = _reference_match_energy( + rep_energy, + source_reference_energy, + source_reliability, + representative_weight=solver_params.refine_representative_match_weight, + source_weight=solver_params.refine_source_match_weight, + ) base_energy = ( anchor_energy * solver_params.refine_anchor_weight - + (rep_energy * 0.45 + source_reference_energy * 0.55) * solver_params.refine_representative_weight + + match_energy * solver_params.refine_representative_weight + alpha_energy * solver_params.refine_alpha_weight + distance_energy * solver_params.refine_distance_weight ) relax_base_energy = ( anchor_energy * solver_params.refine_anchor_weight * solver_params.relax_anchor_scale - + (rep_energy * 0.45 + source_reference_energy * 0.55) * solver_params.refine_representative_weight + + match_energy * solver_params.refine_representative_weight + alpha_energy * solver_params.refine_alpha_weight + distance_energy * solver_params.refine_distance_weight ) @@ -1002,9 +1106,15 @@ def _discrete_refine_output( solver_params, iterations=relax_iterations, ) + relaxed_mode_colors = _select_colors(candidate_colors, relaxed_mode_selected) refine_base_energy = base_energy + ( (candidate_colors - relaxed_context[..., None, :]).abs().mean(dim=-1) * solver_params.relax_handoff_weight ) + if solver_params.refine_relaxed_mode_weight > 0.0: + refine_base_energy = refine_base_energy + ( + (candidate_colors - relaxed_mode_colors[..., None, :]).abs().mean(dim=-1) + * solver_params.refine_relaxed_mode_weight + ) start_candidates = [selected, relaxed_mode_selected] selected = selected.clone() best_selected = selected.clone() @@ -1012,6 +1122,44 @@ def _discrete_refine_output( loss_history: list[float] = list(relax_history) anchor_hw = anchor_t.permute(0, 2, 3, 1) + def candidate_energy(context_colors): + energy = refine_base_energy.clone() + energy = energy + _pairwise_candidate_energy( + candidate_colors, + context_colors, + desired_delta_x, + desired_delta_y, + desired_delta_diag, + desired_delta_anti, + orthogonal_weight=solver_params.refine_orthogonal_weight, + diagonal_weight=solver_params.refine_diagonal_weight, + ) + if solver_params.refine_motif_weight > 0.0: + motif_energy = _motif_candidate_energy(torch, candidate_colors, context_colors, anchor) + energy = energy + motif_energy * solver_params.refine_motif_weight + if solver_params.structure_source_motif_weight > 0.0: + source_motif_energy = _motif_candidate_energy(torch, candidate_colors, context_colors, source_reference) + energy = energy + source_motif_energy * solver_params.structure_source_motif_weight + if solver_params.refine_line_weight > 0.0: + line_energy = _line_candidate_energy(torch, candidate_colors, context_colors, anchor) + energy = energy + line_energy * solver_params.refine_line_weight + if solver_params.structure_source_line_weight > 0.0: + source_line_energy = _line_candidate_energy(torch, candidate_colors, context_colors, source_reference) + energy = energy + source_line_energy * solver_params.structure_source_line_weight + if solver_params.structure_source_adjacency_weight > 0.0: + source_adjacency_energy = _pairwise_candidate_energy( + candidate_colors, + context_colors, + source_ref_dx, + source_ref_dy, + source_ref_diag, + source_ref_anti, + orthogonal_weight=solver_params.refine_orthogonal_weight, + diagonal_weight=solver_params.refine_diagonal_weight, + ) + energy = energy + source_adjacency_energy * solver_params.structure_source_adjacency_weight + return energy + for start_selected in start_candidates: start_colors = _select_colors(candidate_colors, start_selected) start_score = float( @@ -1034,41 +1182,7 @@ def _discrete_refine_output( for step in range(passes + 1): selected_colors = _select_colors(candidate_colors, selected) - energy = refine_base_energy.clone() - energy = energy + _pairwise_candidate_energy( - candidate_colors, - selected_colors, - desired_delta_x, - desired_delta_y, - desired_delta_diag, - desired_delta_anti, - orthogonal_weight=solver_params.refine_orthogonal_weight, - diagonal_weight=solver_params.refine_diagonal_weight, - ) - if solver_params.refine_motif_weight > 0.0: - motif_energy = _motif_candidate_energy(torch, candidate_colors, selected_colors, anchor) - energy = energy + motif_energy * solver_params.refine_motif_weight - if solver_params.structure_source_motif_weight > 0.0: - source_motif_energy = _motif_candidate_energy(torch, candidate_colors, selected_colors, source_reference) - energy = energy + source_motif_energy * solver_params.structure_source_motif_weight - if solver_params.refine_line_weight > 0.0: - line_energy = _line_candidate_energy(torch, candidate_colors, selected_colors, anchor) - energy = energy + line_energy * solver_params.refine_line_weight - if solver_params.structure_source_line_weight > 0.0: - source_line_energy = _line_candidate_energy(torch, candidate_colors, selected_colors, source_reference) - energy = energy + source_line_energy * solver_params.structure_source_line_weight - if solver_params.structure_source_adjacency_weight > 0.0: - source_adjacency_energy = _pairwise_candidate_energy( - candidate_colors, - selected_colors, - source_ref_dx, - source_ref_dy, - source_ref_diag, - source_ref_anti, - orthogonal_weight=solver_params.refine_orthogonal_weight, - diagonal_weight=solver_params.refine_diagonal_weight, - ) - energy = energy + source_adjacency_energy * solver_params.structure_source_adjacency_weight + energy = candidate_energy(selected_colors) candidate_t = selected_colors[None, ...] score = float( @@ -1097,11 +1211,35 @@ def _discrete_refine_output( break selected = next_selected + final_candidates = [best_selected, relaxed_mode_selected] + final_selected = best_selected + final_score = best_score + for candidate_selected in final_candidates: + candidate_colors_selected = _select_colors(candidate_colors, candidate_selected) + candidate_score = float( + _structure_score( + torch, + F, + source_t, + uv_t, + candidate_colors_selected[None, ...], + representative_t, + source_reference_t, + anchor_hw, + solver_params, + ).detach().cpu().item() + ) + if candidate_score < final_score: + final_score = candidate_score + final_selected = candidate_selected.clone() + + final_selected_colors = _select_colors(candidate_colors, final_selected) + final_ranking_energy = candidate_energy(final_selected_colors) hardened_selected, use_opaque, use_transparent = _harden_binary_alpha_selection( torch, candidate_colors, - best_selected, - refine_base_energy, + final_selected, + final_ranking_energy, representative[..., 3], solver_params=solver_params, ) @@ -1144,20 +1282,52 @@ def optimize_uv_field( _make_patch_offsets(height=height, width=width, target_height=inference.target_height, target_width=inference.target_width) ).to(device=device, dtype=torch.float32) guide_small = F.interpolate(edge_t, size=(inference.target_height, inference.target_width), mode="bilinear", align_corners=True) + source_lattice_reference = build_source_lattice_reference( + rgba, + target_width=inference.target_width, + target_height=inference.target_height, + phase_x=inference.phase_x, + phase_y=inference.phase_y, + alpha_threshold=solver_params.alpha_transparent_threshold, + ) initial_patches = _sample_cell_patches(F, source_t, uv0_t, offsets_t) initial_representative_t, _ = _representative_colors(initial_patches, solver_params) initial_representative_t = initial_representative_t.detach() - initial_source_reference_t = _exemplar_colors(initial_patches).detach() - source_delta_x_t = _source_boundary_deltas(F, source_t, uv0_t, axis="x") - source_delta_y_t = _source_boundary_deltas(F, source_t, uv0_t, axis="y") - source_delta_diag_t = _source_boundary_deltas(F, source_t, uv0_t, axis="diag") - source_delta_anti_t = _source_boundary_deltas(F, source_t, uv0_t, axis="anti") + initial_source_reference_t = torch.from_numpy(premultiply(source_lattice_reference.sharp_rgba)[None, ...]).to( + device=device, + dtype=torch.float32, + ) + source_reliability_t = torch.from_numpy(_build_source_reliability(source_lattice_reference)[None, ...]).to( + device=device, + dtype=torch.float32, + ) + source_delta_x_t = ( + torch.from_numpy(source_lattice_reference.delta_x[None, ...]).to(device=device, dtype=torch.float32) + if source_lattice_reference.delta_x is not None + else None + ) + source_delta_y_t = ( + torch.from_numpy(source_lattice_reference.delta_y[None, ...]).to(device=device, dtype=torch.float32) + if source_lattice_reference.delta_y is not None + else None + ) + source_delta_diag_t = ( + torch.from_numpy(source_lattice_reference.delta_diag[None, ...]).to(device=device, dtype=torch.float32) + if source_lattice_reference.delta_diag is not None + else None + ) + source_delta_anti_t = ( + torch.from_numpy(source_lattice_reference.delta_anti[None, ...]).to(device=device, dtype=torch.float32) + if source_lattice_reference.delta_anti is not None + else None + ) snap_rgba = _snap_output_to_source_pixels( torch, source_t, uv0_t, initial_representative_t, initial_source_reference_t, + source_reliability_t, solver_params, cell_x=cell_x, cell_y=cell_y, @@ -1174,6 +1344,7 @@ def optimize_uv_field( uv0_t, initial_representative_t, initial_source_reference_t, + source_reliability_t, snap_t, source_delta_x_t, source_delta_y_t, diff --git a/src/repixelizer/diagnostics.py b/src/repixelizer/diagnostics.py index aa5cf9d..2ef2cc1 100644 --- a/src/repixelizer/diagnostics.py +++ b/src/repixelizer/diagnostics.py @@ -9,7 +9,12 @@ import numpy as np from PIL import Image, ImageDraw from .io import nearest_resize, save_rgba -from .metrics import coherence_breakdown, foreground_reconstruction_error, reconstruction_error +from .metrics import ( + coherence_breakdown, + foreground_reconstruction_error, + reconstruction_error, + source_lattice_consistency_breakdown, +) from .types import InferenceResult, RunResult @@ -105,6 +110,48 @@ def summarize_run(result: RunResult) -> dict[str, Any]: width=result.source_rgba.shape[1], height=result.source_rgba.shape[0], ) + source_fidelity = { + "snap_initial": source_lattice_consistency_breakdown( + result.source_rgba, + result.solver.initial_rgba, + target_width=result.inference.target_width, + target_height=result.inference.target_height, + phase_x=result.inference.phase_x, + phase_y=result.inference.phase_y, + ), + "solver_target": source_lattice_consistency_breakdown( + result.source_rgba, + result.solver.target_rgba, + target_width=result.inference.target_width, + target_height=result.inference.target_height, + phase_x=result.inference.phase_x, + phase_y=result.inference.phase_y, + ), + "final_output": source_lattice_consistency_breakdown( + result.source_rgba, + result.output_rgba, + target_width=result.inference.target_width, + target_height=result.inference.target_height, + phase_x=result.inference.phase_x, + phase_y=result.inference.phase_y, + ), + } + rerank_candidates = [ + { + "target_width": candidate.target_width, + "target_height": candidate.target_height, + "phase_x": candidate.phase_x, + "phase_y": candidate.phase_y, + "score": candidate.score, + "phase_rerank_score": candidate.breakdown.get("phase_rerank_score"), + "phase_rerank_rank": candidate.breakdown.get("phase_rerank_rank"), + "phase_rerank_support_score": candidate.breakdown.get("phase_rerank_support_score"), + "phase_rerank_size_delta_ratio": candidate.breakdown.get("phase_rerank_size_delta_ratio"), + "phase_rerank_size_penalty": candidate.breakdown.get("phase_rerank_size_penalty"), + } + for candidate in result.inference.top_candidates + if "phase_rerank_score" in candidate.breakdown + ] return { "target_width": result.inference.target_width, "target_height": result.inference.target_height, @@ -115,6 +162,8 @@ def summarize_run(result: RunResult) -> dict[str, Any]: "source_preview_reconstruction_error": reconstruction_error(source_preview, result.source_rgba), "source_preview_foreground_error": foreground_reconstruction_error(source_preview, result.source_rgba), "output_colors_from_source_ratio": _source_color_ratio(result.source_rgba, result.output_rgba), + "source_fidelity": source_fidelity, + "phase_rerank_candidates": rerank_candidates, "loss_history": result.solver.loss_history, } diff --git a/src/repixelizer/metrics.py b/src/repixelizer/metrics.py index c118d01..eb9c988 100644 --- a/src/repixelizer/metrics.py +++ b/src/repixelizer/metrics.py @@ -3,6 +3,7 @@ from __future__ import annotations import numpy as np from .io import premultiply +from .source_reference import build_source_lattice_reference def luminance(rgba: np.ndarray) -> np.ndarray: @@ -247,7 +248,7 @@ def source_lattice_consistency_breakdown( phase_y: float, alpha_threshold: float = 0.05, ) -> dict[str, float]: - lattice_source, dispersion = lattice_source_rgba( + reference = build_source_lattice_reference( source_rgba, target_width=target_width, target_height=target_height, @@ -255,20 +256,20 @@ def source_lattice_consistency_breakdown( phase_y=phase_y, alpha_threshold=alpha_threshold, ) - cell_error = foreground_reconstruction_error(output_rgba, lattice_source, alpha_threshold=alpha_threshold, halo=0) - adjacency_error = foreground_adjacency_error(output_rgba, lattice_source, alpha_threshold=alpha_threshold, halo=0) - motif_error = foreground_motif_error(output_rgba, lattice_source, alpha_threshold=alpha_threshold, halo=0) + cell_error = foreground_reconstruction_error(output_rgba, reference.mean_rgba, alpha_threshold=alpha_threshold, halo=0) + adjacency_error = foreground_adjacency_error(output_rgba, reference.sharp_rgba, alpha_threshold=alpha_threshold, halo=0) + motif_error = foreground_motif_error(output_rgba, reference.sharp_rgba, alpha_threshold=alpha_threshold, halo=0) score = ( cell_error * 0.34 + adjacency_error * 0.24 + motif_error * 0.24 - + dispersion * 0.18 + + reference.dispersion * 0.18 ) return { "cell_error": cell_error, "adjacency_error": adjacency_error, "motif_error": motif_error, - "cell_dispersion": dispersion, + "cell_dispersion": reference.dispersion, "score": score, } @@ -282,7 +283,7 @@ def source_lattice_evidence_breakdown( phase_y: float, alpha_threshold: float = 0.05, ) -> dict[str, float]: - lattice_source, dispersion = lattice_source_rgba( + reference = build_source_lattice_reference( source_rgba, target_width=target_width, target_height=target_height, @@ -290,10 +291,10 @@ def source_lattice_evidence_breakdown( phase_y=phase_y, alpha_threshold=alpha_threshold, ) - adjacency_strength = foreground_adjacency_strength(lattice_source, alpha_threshold=alpha_threshold) - evidence_score = adjacency_strength - dispersion * 0.8 + adjacency_strength = foreground_adjacency_strength(reference.sharp_rgba, alpha_threshold=alpha_threshold) + evidence_score = adjacency_strength - reference.dispersion * 0.8 return { - "cell_dispersion": dispersion, + "cell_dispersion": reference.dispersion, "adjacency_strength": adjacency_strength, "score": evidence_score, } @@ -443,60 +444,12 @@ def lattice_source_rgba( phase_y: float, alpha_threshold: float = 0.05, ) -> tuple[np.ndarray, float]: - indices = _lattice_indices( - height=source_rgba.shape[0], - width=source_rgba.shape[1], + reference = build_source_lattice_reference( + source_rgba, target_width=target_width, target_height=target_height, phase_x=phase_x, phase_y=phase_y, + alpha_threshold=alpha_threshold, ) - cell_count = max(1, target_width * target_height) - premul = premultiply(source_rgba) - flat_idx = indices.reshape(-1) - flat_premul = premul.reshape(-1, premul.shape[-1]) - counts = np.bincount(flat_idx, minlength=cell_count).astype(np.float32) - safe_counts = np.maximum(counts, 1.0) - channel_sums = [ - np.bincount(flat_idx, weights=flat_premul[:, channel], minlength=cell_count).astype(np.float32) - for channel in range(flat_premul.shape[-1]) - ] - mean_premul = np.stack(channel_sums, axis=-1) / safe_counts[:, None] - mean_premul = mean_premul.reshape(target_height, target_width, premul.shape[-1]) - mean_rgba = _unpremultiply_array(mean_premul) - - per_pixel_mean = mean_premul.reshape(-1, premul.shape[-1])[flat_idx] - pixel_diff = np.mean(np.abs(flat_premul - per_pixel_mean), axis=-1) - source_alpha = source_rgba.reshape(-1, source_rgba.shape[-1])[:, 3] - lattice_alpha = mean_rgba.reshape(-1, mean_rgba.shape[-1])[flat_idx, 3] - support_mask = np.maximum(source_alpha, lattice_alpha) >= alpha_threshold - if np.any(support_mask): - dispersion = float(np.mean(pixel_diff[support_mask])) - else: - dispersion = float(np.mean(pixel_diff)) if pixel_diff.size else 0.0 - return mean_rgba.astype(np.float32), dispersion - - -def _lattice_indices( - *, - height: int, - width: int, - target_width: int, - target_height: int, - phase_x: float, - phase_y: float, -) -> np.ndarray: - cell_x = width / max(1, target_width) - cell_y = height / max(1, target_height) - xs = (np.arange(width, dtype=np.float32) + 0.5) / cell_x - phase_x - ys = (np.arange(height, dtype=np.float32) + 0.5) / cell_y - phase_y - x_idx = np.clip(np.floor(xs).astype(np.int32), 0, max(0, target_width - 1)) - y_idx = np.clip(np.floor(ys).astype(np.int32), 0, max(0, target_height - 1)) - return y_idx[:, None] * target_width + x_idx[None, :] - - -def _unpremultiply_array(premul: np.ndarray, eps: float = 1e-6) -> np.ndarray: - out = premul.copy() - alpha = np.maximum(out[..., 3:4], eps) - out[..., :3] = np.where(out[..., 3:4] > eps, out[..., :3] / alpha, 0.0) - return np.clip(out, 0.0, 1.0) + return reference.mean_rgba, reference.dispersion diff --git a/src/repixelizer/params.py b/src/repixelizer/params.py index f1474e8..6e499b3 100644 --- a/src/repixelizer/params.py +++ b/src/repixelizer/params.py @@ -11,10 +11,16 @@ class SolverHyperParams: boundary_direction_weight: float = 0.30 boundary_magnitude_weight: float = 0.15 snap_base_match_weight: float = 0.65 + snap_representative_match_weight: float = 0.15 + snap_source_match_weight: float = 0.85 snap_neighbor_weight: float = 0.35 snap_diagonal_weight: float = 0.14 + snap_representative_delta_weight: float = 0.20 + snap_source_delta_weight: float = 0.80 refine_anchor_weight: float = 0.60 refine_representative_weight: float = 0.20 + refine_representative_match_weight: float = 0.15 + refine_source_match_weight: float = 0.85 refine_alpha_weight: float = 0.10 refine_distance_weight: float = 0.10 refine_source_delta_weight: float = 0.25 @@ -22,6 +28,7 @@ class SolverHyperParams: refine_diagonal_weight: float = 0.08 refine_motif_weight: float = 0.12 refine_line_weight: float = 0.10 + refine_relaxed_mode_weight: float = 0.14 refine_candidate_extent: float = 0.70 refine_candidate_levels: int = 7 relax_iterations: int = 24 @@ -44,7 +51,7 @@ class SolverHyperParams: structure_source_adjacency_weight: float = 0.35 structure_source_motif_weight: float = 0.40 structure_source_line_weight: float = 0.20 - structure_representative_weight: float = 0.05 + structure_representative_weight: float = 0.02 alpha_foreground_threshold: float = 0.50 alpha_representative_foreground_threshold: float = 0.60 alpha_opaque_threshold: float = 0.95 @@ -53,9 +60,10 @@ class SolverHyperParams: phase_rerank_edge_position_weight: float = 0.20 phase_rerank_wobble_weight: float = 0.20 phase_rerank_edge_concentration_weight: float = 0.10 + phase_rerank_size_penalty_weight: float = 0.18 phase_rerank_inference_penalty_weight: float = 0.05 phase_rerank_confidence_threshold: float = 0.12 - phase_rerank_max_size_delta_ratio: float = 0.15 + phase_rerank_max_size_delta_ratio: float = 0.40 phase_rerank_margin: float = 0.004 def to_dict(self) -> dict[str, float]: diff --git a/src/repixelizer/pipeline.py b/src/repixelizer/pipeline.py index 773b83d..a2d8140 100644 --- a/src/repixelizer/pipeline.py +++ b/src/repixelizer/pipeline.py @@ -181,32 +181,75 @@ def _select_phase_candidate( (record["edge_concentration"] for record in candidate_records), higher_is_better=True, ) + size_penalty = [ + min(1.0, float(record["size_delta_ratio"]) / max(solver_params.phase_rerank_max_size_delta_ratio, 1e-6)) + for record in candidate_records + ] inference_penalty = _normalize_penalty(record["inference_penalty"] for record in candidate_records) baseline_rank: float | None = None best_rank = float("inf") best_candidate = inference + annotated_candidates = [] for index, record in enumerate(candidate_records): candidate_inference = record["inference"] - size_delta_ratio = float(record["size_delta_ratio"]) - if ( - candidate_inference.target_width != inference.target_width - or candidate_inference.target_height != inference.target_height - ) and size_delta_ratio > solver_params.phase_rerank_max_size_delta_ratio: + if float(record["size_delta_ratio"]) > solver_params.phase_rerank_max_size_delta_ratio: continue rank = ( solver_params.phase_rerank_support_weight * support_penalty[index] + solver_params.phase_rerank_edge_position_weight * edge_position_penalty[index] + solver_params.phase_rerank_wobble_weight * wobble_penalty[index] + solver_params.phase_rerank_edge_concentration_weight * edge_concentration_penalty[index] + + solver_params.phase_rerank_size_penalty_weight * size_penalty[index] + solver_params.phase_rerank_inference_penalty_weight * inference_penalty[index] ) + source_candidate = inference.top_candidates[index] + breakdown = dict(source_candidate.breakdown) + breakdown["phase_rerank_support_score"] = float(record["support_score"]) + breakdown["phase_rerank_edge_position_error"] = float(record["edge_position_error"]) + breakdown["phase_rerank_stroke_wobble_error"] = float(record["stroke_wobble_error"]) + breakdown["phase_rerank_edge_concentration"] = float(record["edge_concentration"]) + breakdown["phase_rerank_size_delta_ratio"] = float(record["size_delta_ratio"]) + breakdown["phase_rerank_size_penalty"] = float(size_penalty[index]) + breakdown["phase_rerank_inference_penalty"] = float(record["inference_penalty"]) + breakdown["phase_rerank_score"] = float(rank) + annotated_candidates.append( + source_candidate.__class__( + target_width=source_candidate.target_width, + target_height=source_candidate.target_height, + phase_x=source_candidate.phase_x, + phase_y=source_candidate.phase_y, + score=source_candidate.score, + breakdown=breakdown, + ) + ) if baseline_rank is None: baseline_rank = rank if rank < best_rank: best_rank = rank best_candidate = record["inference"] + annotated_candidates.sort(key=lambda candidate: float(candidate.breakdown.get("phase_rerank_score", float("inf")))) + for rank, candidate in enumerate(annotated_candidates, start=1): + candidate.breakdown["phase_rerank_rank"] = float(rank) + if annotated_candidates: + best_candidate = InferenceResult( + target_width=best_candidate.target_width, + target_height=best_candidate.target_height, + phase_x=best_candidate.phase_x, + phase_y=best_candidate.phase_y, + confidence=inference.confidence, + top_candidates=annotated_candidates, + ) if baseline_rank is None or best_rank > baseline_rank - solver_params.phase_rerank_margin: + if annotated_candidates: + return InferenceResult( + target_width=inference.target_width, + target_height=inference.target_height, + phase_x=inference.phase_x, + phase_y=inference.phase_y, + confidence=inference.confidence, + top_candidates=annotated_candidates, + ) return inference return best_candidate diff --git a/src/repixelizer/source_reference.py b/src/repixelizer/source_reference.py new file mode 100644 index 0000000..adb0902 --- /dev/null +++ b/src/repixelizer/source_reference.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import numpy as np + +from .io import premultiply, unpremultiply +from .types import SourceLatticeReference + + +def lattice_indices( + *, + height: int, + width: int, + target_width: int, + target_height: int, + phase_x: float, + phase_y: float, +) -> np.ndarray: + cell_x = width / max(1, target_width) + cell_y = height / max(1, target_height) + xs = (np.arange(width, dtype=np.float32) + 0.5) / cell_x - phase_x + ys = (np.arange(height, dtype=np.float32) + 0.5) / cell_y - phase_y + x_idx = np.clip(np.floor(xs).astype(np.int32), 0, max(0, target_width - 1)) + y_idx = np.clip(np.floor(ys).astype(np.int32), 0, max(0, target_height - 1)) + return y_idx[:, None] * target_width + x_idx[None, :] + + +def _reference_deltas(reference: np.ndarray) -> tuple[np.ndarray | None, np.ndarray | None, np.ndarray | None, np.ndarray | None]: + height = reference.shape[0] + width = reference.shape[1] + delta_x = reference[:, 1:, :] - reference[:, :-1, :] if width > 1 else None + delta_y = reference[1:, :, :] - reference[:-1, :, :] if height > 1 else None + delta_diag = reference[1:, 1:, :] - reference[:-1, :-1, :] if height > 1 and width > 1 else None + delta_anti = reference[1:, :-1, :] - reference[:-1, 1:, :] if height > 1 and width > 1 else None + return delta_x, delta_y, delta_diag, delta_anti + + +def build_source_lattice_reference( + source_rgba: np.ndarray, + *, + target_width: int, + target_height: int, + phase_x: float, + phase_y: float, + alpha_threshold: float = 0.05, +) -> SourceLatticeReference: + indices = lattice_indices( + height=source_rgba.shape[0], + width=source_rgba.shape[1], + target_width=target_width, + target_height=target_height, + phase_x=phase_x, + phase_y=phase_y, + ) + cell_count = max(1, target_width * target_height) + premul = premultiply(source_rgba) + flat_idx = indices.reshape(-1) + flat_premul = premul.reshape(-1, premul.shape[-1]) + flat_alpha = source_rgba.reshape(-1, source_rgba.shape[-1])[:, 3] + + counts = np.bincount(flat_idx, minlength=cell_count).astype(np.float32) + safe_counts = np.maximum(counts, 1.0) + channel_sums = [ + np.bincount(flat_idx, weights=flat_premul[:, channel], minlength=cell_count).astype(np.float32) + for channel in range(flat_premul.shape[-1]) + ] + mean_premul_flat = np.stack(channel_sums, axis=-1) / safe_counts[:, None] + mean_premul = mean_premul_flat.reshape(target_height, target_width, premul.shape[-1]) + mean_rgba = unpremultiply(mean_premul) + + per_pixel_mean = mean_premul_flat[flat_idx] + pixel_diff = np.mean(np.abs(flat_premul - per_pixel_mean), axis=-1) + cell_diff_sums = np.bincount(flat_idx, weights=pixel_diff, minlength=cell_count).astype(np.float32) + cell_dispersion = (cell_diff_sums / safe_counts).reshape(target_height, target_width) + + lattice_alpha = mean_rgba.reshape(-1, mean_rgba.shape[-1])[flat_idx, 3] + support_mask = np.maximum(flat_alpha, lattice_alpha) >= alpha_threshold + if np.any(support_mask): + dispersion = float(np.mean(pixel_diff[support_mask])) + else: + dispersion = float(np.mean(pixel_diff)) if pixel_diff.size else 0.0 + + exemplar_cost = pixel_diff + np.maximum(0.0, per_pixel_mean[:, 3] - flat_premul[:, 3]) * 1e-3 + order = np.lexsort((exemplar_cost.astype(np.float32), flat_idx.astype(np.int64))) + sharp_premul_flat = mean_premul_flat.copy() + if order.size > 0: + sorted_cells = flat_idx[order] + first_positions = np.concatenate(([0], np.flatnonzero(np.diff(sorted_cells)) + 1)) + best_order = order[first_positions] + best_cells = flat_idx[best_order] + sharp_premul_flat[best_cells] = flat_premul[best_order] + sharp_premul = sharp_premul_flat.reshape(target_height, target_width, premul.shape[-1]) + sharp_rgba = unpremultiply(sharp_premul) + + cell_alpha_max = np.zeros(cell_count, dtype=np.float32) + np.maximum.at(cell_alpha_max, flat_idx, flat_alpha) + expected_pixels_per_cell = float(source_rgba.shape[0] * source_rgba.shape[1]) / float(cell_count) + cell_support = (counts / max(expected_pixels_per_cell, 1.0)).reshape(target_height, target_width) + + delta_x, delta_y, delta_diag, delta_anti = _reference_deltas(sharp_premul) + return SourceLatticeReference( + mean_rgba=mean_rgba.astype(np.float32), + sharp_rgba=sharp_rgba.astype(np.float32), + dispersion=dispersion, + cell_dispersion=cell_dispersion.astype(np.float32), + cell_counts=counts.reshape(target_height, target_width).astype(np.float32), + cell_support=cell_support.astype(np.float32), + cell_alpha_max=cell_alpha_max.reshape(target_height, target_width).astype(np.float32), + delta_x=delta_x.astype(np.float32) if delta_x is not None else None, + delta_y=delta_y.astype(np.float32) if delta_y is not None else None, + delta_diag=delta_diag.astype(np.float32) if delta_diag is not None else None, + delta_anti=delta_anti.astype(np.float32) if delta_anti is not None else None, + ) diff --git a/src/repixelizer/types.py b/src/repixelizer/types.py index 72dfbdd..65d90f4 100644 --- a/src/repixelizer/types.py +++ b/src/repixelizer/types.py @@ -36,6 +36,21 @@ class SourceAnalysis: cluster_preview: np.ndarray +@dataclass(slots=True) +class SourceLatticeReference: + mean_rgba: np.ndarray + sharp_rgba: np.ndarray + dispersion: float + cell_dispersion: np.ndarray + cell_counts: np.ndarray + cell_support: np.ndarray + cell_alpha_max: np.ndarray + delta_x: np.ndarray | None + delta_y: np.ndarray | None + delta_diag: np.ndarray | None + delta_anti: np.ndarray | None + + @dataclass(slots=True) class SolverArtifacts: target_rgba: np.ndarray diff --git a/tests/test_continuous.py b/tests/test_continuous.py index e01cec4..d848f23 100644 --- a/tests/test_continuous.py +++ b/tests/test_continuous.py @@ -1,8 +1,13 @@ from __future__ import annotations +import numpy as np import torch -from repixelizer.continuous import _exemplar_colors, _line_pattern_loss +from repixelizer.analysis import analyze_source +from repixelizer.continuous import _exemplar_colors, _line_pattern_loss, optimize_uv_field +from repixelizer.metrics import source_lattice_consistency_breakdown +from repixelizer.synthetic import fake_pixelize +from repixelizer.types import InferenceResult def test_line_pattern_loss_penalizes_wobbling_line() -> None: @@ -41,3 +46,58 @@ def test_exemplar_colors_selects_an_actual_patch_sample() -> None: options = patches[0, 0, 0] assert any(torch.allclose(exemplar, option) for option in options) + + +def test_optimize_uv_field_does_not_regress_from_snap_on_thin_feature_case() -> None: + source = np.zeros((16, 16, 4), dtype=np.float32) + source[4:12, 8] = np.asarray([0.95, 0.95, 0.95, 1.0], dtype=np.float32) + source[4, 7:10] = np.asarray([0.95, 0.95, 0.95, 1.0], dtype=np.float32) + source[11, 7] = np.asarray([0.95, 0.95, 0.95, 1.0], dtype=np.float32) + source[10, 6] = np.asarray([0.95, 0.95, 0.95, 1.0], dtype=np.float32) + source[9, 5] = np.asarray([0.95, 0.95, 0.95, 1.0], dtype=np.float32) + + fake = fake_pixelize( + source, + upscale=9, + phase_x=0.18, + phase_y=-0.12, + blur_radius=0.45, + warp_strength=0.2, + warp_detail=5, + seed=11, + ) + inference = InferenceResult( + target_width=16, + target_height=16, + phase_x=0.18, + phase_y=-0.12, + confidence=1.0, + top_candidates=[], + ) + artifacts = optimize_uv_field( + fake, + inference=inference, + analysis=analyze_source(fake, seed=7), + steps=4, + seed=7, + device="cpu", + ) + + snap_score = source_lattice_consistency_breakdown( + fake, + artifacts.initial_rgba, + target_width=16, + target_height=16, + phase_x=0.18, + phase_y=-0.12, + )["score"] + final_score = source_lattice_consistency_breakdown( + fake, + artifacts.target_rgba, + target_width=16, + target_height=16, + phase_x=0.18, + phase_y=-0.12, + )["score"] + + assert final_score <= snap_score + 1e-6 diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index ed7e91b..2284d25 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -4,10 +4,12 @@ from pathlib import Path import numpy as np +from repixelizer.analysis import analyze_source from repixelizer.baselines import naive_resize_baseline -from repixelizer.metrics import coherence_breakdown, foreground_reconstruction_error +from repixelizer.continuous import optimize_uv_field +from repixelizer.metrics import coherence_breakdown, foreground_reconstruction_error, source_lattice_consistency_breakdown from repixelizer.pipeline import _select_phase_candidate, run_pipeline -from repixelizer.io import nearest_resize +from repixelizer.io import load_rgba, nearest_resize from repixelizer.synthetic import fake_pixelize, make_emblem, make_sprite from repixelizer.types import InferenceCandidate, InferenceResult @@ -26,6 +28,11 @@ def test_pipeline_writes_output_and_diagnostics(tmp_path: Path) -> None: assert output_path.exists() assert (diagnostics_dir / "run.json").exists() assert (diagnostics_dir / "output-preview.png").exists() + import json + + run_json = json.loads((diagnostics_dir / "run.json").read_text(encoding="utf-8")) + assert set(run_json["source_fidelity"].keys()) == {"snap_initial", "solver_target", "final_output"} + assert "phase_rerank_candidates" in run_json assert result.output_rgba.shape[0] == result.inference.target_height assert result.output_rgba.shape[1] == result.inference.target_width @@ -214,6 +221,76 @@ def test_phase_rerank_rejects_large_size_jump(monkeypatch) -> None: assert selected.target_height == candidate_a.target_height +def test_phase_rerank_can_accept_low_confidence_size_jump_with_strong_support(monkeypatch) -> None: + source = np.zeros((16, 16, 4), dtype=np.float32) + candidate_a = InferenceCandidate(target_width=16, target_height=16, phase_x=0.0, phase_y=0.0, score=0.91, breakdown={}) + candidate_b = InferenceCandidate(target_width=21, target_height=21, phase_x=0.2, phase_y=-0.2, score=0.89, breakdown={}) + inference = InferenceResult( + target_width=16, + target_height=16, + phase_x=0.0, + phase_y=0.0, + confidence=0.0, + top_candidates=[candidate_a, candidate_b], + ) + + class DummyArtifacts: + def __init__(self, rgba: np.ndarray) -> None: + self.target_rgba = rgba + + outputs = { + candidate_a.target_width: np.zeros((candidate_a.target_height, candidate_a.target_width, 4), dtype=np.float32), + candidate_b.target_width: np.ones((candidate_b.target_height, candidate_b.target_width, 4), dtype=np.float32), + } + + def fake_optimize_uv_field(source_rgba, inference, analysis, steps, seed, device, solver_params=None): + return DummyArtifacts(outputs[inference.target_width]) + + def fake_support(source_rgba, output_rgba, *, target_width, target_height, phase_x, phase_y): + return {"score": 0.22 if target_width == 16 else 0.03} + + def fake_edge_position(preview, source_rgba): + return 0.07 if np.mean(preview) < 0.1 else 0.02 + + def fake_wobble(preview, source_rgba): + return 1.25 if np.mean(preview) < 0.1 else 0.55 + + def fake_concentration(rgba): + return 0.18 if np.mean(rgba) < 0.1 else 0.31 + + monkeypatch.setattr("repixelizer.pipeline.optimize_uv_field", fake_optimize_uv_field) + monkeypatch.setattr("repixelizer.pipeline.source_lattice_consistency_breakdown", fake_support) + monkeypatch.setattr("repixelizer.pipeline.foreground_edge_position_error", fake_edge_position) + monkeypatch.setattr("repixelizer.pipeline.foreground_stroke_wobble_error", fake_wobble) + monkeypatch.setattr("repixelizer.pipeline.foreground_edge_concentration", fake_concentration) + + selected = _select_phase_candidate(source, inference, analysis=object(), seed=7, device="cpu") + assert selected.target_width == candidate_b.target_width + assert selected.target_height == candidate_b.target_height + + +def test_phase_rerank_keeps_high_confidence_candidate_without_probe(monkeypatch) -> None: + source = np.zeros((8, 8, 4), dtype=np.float32) + candidate_a = InferenceCandidate(target_width=8, target_height=8, phase_x=0.0, phase_y=0.0, score=0.9, breakdown={}) + candidate_b = InferenceCandidate(target_width=10, target_height=10, phase_x=0.2, phase_y=0.2, score=0.89, breakdown={}) + inference = InferenceResult( + target_width=8, + target_height=8, + phase_x=0.0, + phase_y=0.0, + confidence=0.5, + top_candidates=[candidate_a, candidate_b], + ) + + def fail(*args, **kwargs): + raise AssertionError("rerank probe should not run for high-confidence inference") + + monkeypatch.setattr("repixelizer.pipeline.optimize_uv_field", fail) + selected = _select_phase_candidate(source, inference, analysis=object(), seed=7, device="cpu") + assert selected.target_width == candidate_a.target_width + assert selected.target_height == candidate_a.target_height + + def test_phase_rerank_can_prefer_better_line_metrics(monkeypatch) -> None: source = np.zeros((4, 4, 4), dtype=np.float32) candidate_a = InferenceCandidate(target_width=2, target_height=2, phase_x=-0.2, phase_y=0.0, score=0.91, breakdown={}) @@ -260,3 +337,44 @@ def test_phase_rerank_can_prefer_better_line_metrics(monkeypatch) -> None: selected = _select_phase_candidate(source, inference, analysis=object(), seed=7, device="cpu") assert selected.phase_x == candidate_b.phase_x assert selected.phase_y == candidate_b.phase_y + + +def test_badge_fixture_candidate_beats_previous_source_fidelity_baselines() -> None: + source = load_rgba("tests/fixtures/real/ai-badge-cleaned.png") + inference = InferenceResult( + target_width=170, + target_height=170, + phase_x=-0.2, + phase_y=0.2, + confidence=0.0, + top_candidates=[], + ) + artifacts = optimize_uv_field( + source, + inference=inference, + analysis=analyze_source(source, seed=7), + steps=2, + seed=7, + device="cpu", + ) + + snap_consistency = source_lattice_consistency_breakdown( + source, + artifacts.initial_rgba, + target_width=170, + target_height=170, + phase_x=-0.2, + phase_y=0.2, + )["score"] + final_consistency = source_lattice_consistency_breakdown( + source, + artifacts.target_rgba, + target_width=170, + target_height=170, + phase_x=-0.2, + phase_y=0.2, + )["score"] + + assert final_consistency <= 0.1494 + assert final_consistency <= 0.1369 + assert final_consistency <= snap_consistency + 1e-6 diff --git a/tests/test_source_reference.py b/tests/test_source_reference.py new file mode 100644 index 0000000..b5d3345 --- /dev/null +++ b/tests/test_source_reference.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import numpy as np + +from repixelizer.io import premultiply +from repixelizer.source_reference import build_source_lattice_reference + + +def test_source_lattice_reference_recovers_stable_cell_colors_and_deltas() -> None: + lowres = np.asarray( + [ + [[1.0, 0.0, 0.0, 1.0], [0.0, 0.0, 1.0, 1.0]], + [[0.0, 1.0, 0.0, 1.0], [1.0, 1.0, 0.0, 1.0]], + ], + dtype=np.float32, + ) + source = np.repeat(np.repeat(lowres, 4, axis=0), 4, axis=1) + source[1, 1, :3] = np.asarray([0.6, 0.1, 0.1], dtype=np.float32) + + reference = build_source_lattice_reference( + source, + target_width=2, + target_height=2, + phase_x=0.0, + phase_y=0.0, + ) + + expected_premul = premultiply(lowres) + assert np.allclose(reference.sharp_rgba, lowres, atol=1e-5) + assert np.allclose(reference.mean_rgba[0, 1], lowres[0, 1], atol=1e-5) + assert reference.delta_x is not None + assert reference.delta_y is not None + assert reference.delta_diag is not None + assert reference.delta_anti is not None + assert np.allclose(reference.delta_x, expected_premul[:, 1:, :] - expected_premul[:, :-1, :], atol=1e-5) + assert np.allclose(reference.delta_y, expected_premul[1:, :, :] - expected_premul[:-1, :, :], atol=1e-5) + assert reference.dispersion >= 0.0