diff --git a/README.md b/README.md index de3eb36..6dbefb5 100644 --- a/README.md +++ b/README.md @@ -6,15 +6,15 @@ It is aimed at the annoying middle ground: sprites, emblems, and logos that look Instead of pretending this is a resize problem, Repixelizer treats it as lattice inference plus local structure preservation: infer the implied grid, choose real output cells, and snap them back onto a coherent pixel lattice while preserving the source's local adjacency patterns. -## Real Example +## Examples -This input was not chosen because it was easy. It has glossy shading, chunky highlights, and enough large-scale structure to confuse size inference in ways that are frankly a little rude. +Repixelizer was built to rescue fake pixel art, but it can also be used to generate pixel art directly from non-pixel source art when the shapes are clean and the local structure is doing something useful. -The cleanest result here came from forcing `--target-size 128`: +Top row: glossy non-pixel badge art repixelized at `128x128`. Bottom row: an AI fake-pixel interpretation of the same badge cleaned up with automatic lattice inference, which landed on `122x122` for this image. -![Real repixelize example](docs/readme-assets/badge-example-sheet.png) +![Repixelizer example comparison](docs/readme-assets/badge-example-sheet.png) -Automatic lattice inference on this badge is much less wrong than it used to be, but this is still a good example of the kind of image where a manual target size can help. +That is the two-headed pitch in one image: force non-pixel art onto a coherent grid, or take AI pixel art that only respects the grid locally and make it commit. ## Current Status diff --git a/docs/readme-assets/badge-example-sheet.png b/docs/readme-assets/badge-example-sheet.png index 32cfe30..1ecaacd 100644 Binary files a/docs/readme-assets/badge-example-sheet.png and b/docs/readme-assets/badge-example-sheet.png differ diff --git a/docs/readme-assets/badge-repixelized.png b/docs/readme-assets/badge-repixelized.png deleted file mode 100644 index 559eb3e..0000000 Binary files a/docs/readme-assets/badge-repixelized.png and /dev/null differ diff --git a/docs/readme-assets/badge-source.png b/docs/readme-assets/badge-source.png deleted file mode 100644 index f6982f7..0000000 Binary files a/docs/readme-assets/badge-source.png and /dev/null differ diff --git a/src/repixelizer/inference.py b/src/repixelizer/inference.py index 48d7d19..04b5dff 100644 --- a/src/repixelizer/inference.py +++ b/src/repixelizer/inference.py @@ -3,6 +3,7 @@ from __future__ import annotations import numpy as np from .io import premultiply +from .metrics import source_lattice_evidence_breakdown from .types import InferenceCandidate, InferenceResult @@ -28,7 +29,13 @@ def _resolve_device(torch, requested: str) -> str: return requested -def _candidate_dims(width: int, height: int, target_size: int | None) -> list[tuple[int, int]]: +def _candidate_dims( + width: int, + height: int, + target_size: int | None, + *, + hinted_sizes: list[int] | None = None, +) -> list[tuple[int, int]]: if target_size is not None: if width >= height: return [(target_size, max(1, round(height * target_size / width)))] @@ -37,8 +44,15 @@ def _candidate_dims(width: int, height: int, target_size: int | None) -> list[tu min_size = max(12, int(round(max_dim / 48))) max_size = min(256, max(24, int(round(max_dim / 2.5)))) step = 2 if max_size <= 96 else 4 + size_values = set(range(min_size, max_size + 1, step)) + if hinted_sizes: + for hinted_size in hinted_sizes: + for delta in range(-2, 3): + candidate = int(hinted_size) + delta + if min_size <= candidate <= max_size: + size_values.add(candidate) dims: list[tuple[int, int]] = [] - for size in range(min_size, max_size + 1, step): + for size in sorted(size_values): if width >= height: dims.append((size, max(1, round(height * size / width)))) else: @@ -195,6 +209,99 @@ def _estimate_lattice_prior_details(rgba: np.ndarray) -> tuple[float, float, flo return shared_prior, shared_prior, reliability +def _hint_target_sizes_from_spacing( + width: int, + height: int, + spacing_x: tuple[float | None, float], + spacing_y: tuple[float | None, float], +) -> list[int]: + hinted_sizes: set[int] = set() + if width >= height: + if spacing_x[0] is not None and spacing_x[1] >= 0.15: + hinted_sizes.add(max(1, int(round(width / max(1e-6, spacing_x[0]))))) + if spacing_y[0] is not None and spacing_y[1] >= 0.15: + hinted_sizes.add(max(1, int(round(height / max(1e-6, spacing_y[0]))))) + else: + if spacing_y[0] is not None and spacing_y[1] >= 0.15: + hinted_sizes.add(max(1, int(round(height / max(1e-6, spacing_y[0]))))) + if spacing_x[0] is not None and spacing_x[1] >= 0.15: + hinted_sizes.add(max(1, int(round(width / max(1e-6, spacing_x[0]))))) + return sorted(hinted_sizes) + + +def _top_candidates_by_size(candidates: list[InferenceCandidate], limit: int = 8) -> list[InferenceCandidate]: + selected: list[InferenceCandidate] = [] + seen_sizes: set[tuple[int, int]] = set() + for candidate in candidates: + size_key = (candidate.target_width, candidate.target_height) + if size_key in seen_sizes: + continue + selected.append(candidate) + seen_sizes.add(size_key) + if len(selected) >= limit: + break + return selected + + +def _normalize_candidate_scores(values: np.ndarray) -> np.ndarray: + if values.size == 0: + return values + if values.size == 1: + return np.ones_like(values, dtype=np.float32) + minimum = float(np.min(values)) + maximum = float(np.max(values)) + span = maximum - minimum + if span <= 1e-6: + return np.full_like(values, 0.5, dtype=np.float32) + return ((values - minimum) / span).astype(np.float32) + + +def _rerank_size_candidates_with_source_evidence( + rgba: np.ndarray, + candidates: list[InferenceCandidate], +) -> list[InferenceCandidate]: + if len(candidates) <= 1: + return candidates + + base_scores = np.asarray([candidate.score for candidate in candidates], dtype=np.float32) + evidence_breakdowns = [ + source_lattice_evidence_breakdown( + rgba, + target_width=candidate.target_width, + target_height=candidate.target_height, + phase_x=candidate.phase_x, + phase_y=candidate.phase_y, + ) + for candidate in candidates + ] + evidence_scores = np.asarray([breakdown["score"] for breakdown in evidence_breakdowns], dtype=np.float32) + base_norm = _normalize_candidate_scores(base_scores) + evidence_norm = _normalize_candidate_scores(evidence_scores) + + reranked: list[InferenceCandidate] = [] + for candidate, evidence, base_component, evidence_component in zip(candidates, evidence_breakdowns, base_norm, evidence_norm): + breakdown = dict(candidate.breakdown) + breakdown["base_score"] = float(candidate.score) + breakdown["source_cell_dispersion"] = float(evidence["cell_dispersion"]) + breakdown["source_adjacency_strength"] = float(evidence["adjacency_strength"]) + breakdown["source_lattice_evidence"] = float(evidence["score"]) + breakdown["base_score_norm"] = float(base_component) + breakdown["source_lattice_evidence_norm"] = float(evidence_component) + combined_score = float(base_component * 0.35 + evidence_component * 0.65) + reranked.append( + InferenceCandidate( + target_width=candidate.target_width, + target_height=candidate.target_height, + phase_x=candidate.phase_x, + phase_y=candidate.phase_y, + score=combined_score, + breakdown=breakdown, + ) + ) + reranked.sort(key=lambda item: item.score, reverse=True) + return reranked + + def _estimate_lattice_prior(rgba: np.ndarray) -> tuple[float, float]: prior_x, prior_y, _ = _estimate_lattice_prior_details(rgba) return prior_x, prior_y @@ -367,11 +474,13 @@ def infer_lattice(rgba: np.ndarray, target_size: int | None = None, device: str torch, _ = _require_torch() resolved_device = _resolve_device(torch, device) height, width = rgba.shape[:2] + spacing_x, spacing_y = _estimate_lattice_spacing(rgba) + hinted_sizes = _hint_target_sizes_from_spacing(width, height, spacing_x, spacing_y) prior_cell_x, prior_cell_y, prior_reliability = _estimate_lattice_prior_details(rgba) phase_values = np.linspace(-0.4, 0.4, num=5, dtype=np.float32) candidates: list[InferenceCandidate] = [] - for target_width, target_height in _candidate_dims(width, height, target_size): + for target_width, target_height in _candidate_dims(width, height, target_size, hinted_sizes=hinted_sizes): candidates.extend( _score_phase_group( rgba, @@ -386,16 +495,19 @@ def infer_lattice(rgba: np.ndarray, target_size: int | None = None, device: str ) candidates.sort(key=lambda item: item.score, reverse=True) - best = candidates[0] - second = candidates[1] if len(candidates) > 1 else best + size_candidates = _top_candidates_by_size(candidates, limit=len(candidates)) + reranked_candidates = _rerank_size_candidates_with_source_evidence(rgba, size_candidates) + best = reranked_candidates[0] + second = reranked_candidates[1] if len(reranked_candidates) > 1 else best confidence = max(0.0, best.score - second.score) + top_candidates = reranked_candidates[:8] return InferenceResult( target_width=best.target_width, target_height=best.target_height, phase_x=best.phase_x, phase_y=best.phase_y, confidence=confidence, - top_candidates=candidates[:8], + top_candidates=top_candidates, ) diff --git a/src/repixelizer/metrics.py b/src/repixelizer/metrics.py index a212a79..3d82c85 100644 --- a/src/repixelizer/metrics.py +++ b/src/repixelizer/metrics.py @@ -123,6 +123,31 @@ def foreground_adjacency_error( return float(np.mean(values[weights])) +def foreground_adjacency_strength( + rgba: np.ndarray, + alpha_threshold: float = 0.05, +) -> float: + premul = premultiply(rgba) + mask = rgba[..., 3] >= alpha_threshold + strengths: list[np.ndarray] = [] + weights: list[np.ndarray] = [] + + for axis in (0, 1): + delta = np.mean(np.abs(np.diff(premul, axis=axis)), axis=-1) + if axis == 0: + edge_mask = mask[1:, :] | mask[:-1, :] + else: + edge_mask = mask[:, 1:] | mask[:, :-1] + strengths.append(delta.reshape(-1)) + weights.append(edge_mask.reshape(-1)) + + values = np.concatenate(strengths, axis=0) + support = np.concatenate(weights, axis=0) + if not np.any(support): + return 0.0 + return float(np.mean(values[support])) + + def foreground_motif_error( a: np.ndarray, b: np.ndarray, @@ -180,6 +205,32 @@ def source_lattice_consistency_breakdown( } +def source_lattice_evidence_breakdown( + source_rgba: np.ndarray, + *, + target_width: int, + target_height: int, + phase_x: float, + phase_y: float, + alpha_threshold: float = 0.05, +) -> dict[str, float]: + lattice_source, dispersion = lattice_source_rgba( + source_rgba, + target_width=target_width, + target_height=target_height, + phase_x=phase_x, + phase_y=phase_y, + alpha_threshold=alpha_threshold, + ) + adjacency_strength = foreground_adjacency_strength(lattice_source, alpha_threshold=alpha_threshold) + evidence_score = adjacency_strength - dispersion * 0.8 + return { + "cell_dispersion": dispersion, + "adjacency_strength": adjacency_strength, + "score": evidence_score, + } + + def exact_match_ratio(a: np.ndarray, b: np.ndarray) -> float: a8 = np.clip(np.rint(premultiply(a) * 255.0), 0, 255).astype(np.uint8) b8 = np.clip(np.rint(premultiply(b) * 255.0), 0, 255).astype(np.uint8) diff --git a/tests/test_inference.py b/tests/test_inference.py index f27bd1b..ab2bf55 100644 --- a/tests/test_inference.py +++ b/tests/test_inference.py @@ -2,9 +2,13 @@ from __future__ import annotations from repixelizer.inference import ( _axis_prior_from_estimates, + _candidate_dims, _combine_axis_priors, + _hint_target_sizes_from_spacing, + _top_candidates_by_size, infer_lattice, ) +from repixelizer.types import InferenceCandidate from repixelizer.synthetic import fake_pixelize, make_emblem @@ -20,6 +24,33 @@ def test_combined_axis_prior_prefers_consistent_shared_cell_size() -> None: assert 0.2 < reliability < 0.6 +def test_spacing_hints_can_add_dense_candidates_around_true_size() -> None: + hinted_sizes = _hint_target_sizes_from_spacing( + 1024, + 1024, + (8.05, 0.74), + (4.0, 0.08), + ) + dims = _candidate_dims(1024, 1024, None, hinted_sizes=hinted_sizes) + sizes = {width for width, _ in dims} + assert 128 in sizes + + +def test_top_candidates_are_diversified_by_size() -> None: + candidates = [ + InferenceCandidate(target_width=113, target_height=113, phase_x=0.0, phase_y=0.0, score=0.90, breakdown={}), + InferenceCandidate(target_width=113, target_height=113, phase_x=0.2, phase_y=0.0, score=0.89, breakdown={}), + InferenceCandidate(target_width=117, target_height=117, phase_x=0.0, phase_y=0.0, score=0.88, breakdown={}), + InferenceCandidate(target_width=128, target_height=128, phase_x=0.0, phase_y=0.0, score=0.87, breakdown={}), + ] + selected = _top_candidates_by_size(candidates, limit=8) + assert [(candidate.target_width, candidate.phase_x) for candidate in selected] == [ + (113, 0.0), + (117, 0.0), + (128, 0.0), + ] + + def test_infer_lattice_recovers_emblem_scale() -> None: source = make_emblem(32, 32) fake = fake_pixelize( diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 9309a6b..582efe6 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -32,7 +32,16 @@ def test_pipeline_writes_output_and_diagnostics(tmp_path: Path) -> None: def test_pipeline_beats_naive_on_roundtrip_fidelity_for_synthetic_emblem(tmp_path: Path) -> None: source = make_emblem(32, 32) - fake = fake_pixelize(source, upscale=12, phase_x=0.2, phase_y=0.35, blur_radius=0.65) + fake = fake_pixelize( + source, + upscale=12, + phase_x=0.2, + phase_y=0.35, + blur_radius=0.65, + warp_strength=0.28, + warp_detail=6, + seed=5, + ) input_path = tmp_path / "input.png" output_path = tmp_path / "output.png" @@ -111,3 +120,37 @@ def test_phase_rerank_can_override_low_confidence_inference_pick(monkeypatch) -> selected = _select_phase_candidate(source, inference, analysis=object(), seed=7, device="cpu") assert selected.phase_x == candidate_b.phase_x assert selected.phase_y == candidate_b.phase_y + + +def test_phase_rerank_can_override_to_better_size_candidate(monkeypatch) -> None: + source = np.zeros((4, 4, 4), dtype=np.float32) + candidate_a = InferenceCandidate(target_width=2, target_height=2, phase_x=0.0, phase_y=0.0, score=0.91, breakdown={}) + candidate_b = InferenceCandidate(target_width=3, target_height=3, phase_x=0.0, phase_y=0.0, score=0.88, breakdown={}) + inference = InferenceResult( + target_width=2, + target_height=2, + phase_x=0.0, + phase_y=0.0, + confidence=0.0, + top_candidates=[candidate_a, candidate_b], + ) + + class DummyArtifacts: + def __init__(self, rgba: np.ndarray) -> None: + self.target_rgba = rgba + + def fake_optimize_uv_field(source_rgba, inference, analysis, steps, seed, device, solver_params=None): + rgba = np.zeros((inference.target_height, inference.target_width, 4), dtype=np.float32) + return DummyArtifacts(rgba) + + def fake_support(source_rgba, output_rgba, *, target_width, target_height, phase_x, phase_y): + if target_width == 2: + return {"score": 0.10} + return {"score": 0.01} + + monkeypatch.setattr("repixelizer.pipeline.optimize_uv_field", fake_optimize_uv_field) + monkeypatch.setattr("repixelizer.pipeline.source_lattice_consistency_breakdown", fake_support) + + selected = _select_phase_candidate(source, inference, analysis=object(), seed=7, device="cpu") + assert selected.target_width == candidate_b.target_width + assert selected.target_height == candidate_b.target_height