diff --git a/README.md b/README.md index 8ac4852..8cb52c0 100644 --- a/README.md +++ b/README.md @@ -38,10 +38,11 @@ Recent adjacency-focused status: Current tile-graph status: -- it builds source-driven cell candidates from connected source clusters plus lattice-aligned fallback coverage +- it now builds literal source-pixel candidates from connected source clusters plus lattice-aligned fallback coverage - it learns directional adjacency preferences between candidates and places them with a soft discrete propagation loop +- it no longer invents averaged patch colors during candidate extraction; the current regression suite now checks that tile-graph candidates stay on real source colors - it already beats naive resize on the repo's synthetic thin-feature regression -- it is not yet good enough to replace the default continuous path on the cleaned AI badge stress case +- it still is not good enough to replace the default continuous path on the cleaned AI badge stress case, though the raw-pixel rewrite materially improved that badge probe and stopped the mode from jumping to an oversized lattice candidate ## Quickstart diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 0084c4b..9294f43 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -68,8 +68,9 @@ Current implementation note: - enriched edge candidates are currently gated to high-dispersion edge cells, and `snap` stays on the conservative local grid while `refine` gets the richer candidate pool Experimental tile-graph note: -- there is now a separate `tile-graph` reconstruction mode that extracts cell candidates from connected source clusters, fills gaps with lattice-aligned fallback candidates, learns candidate adjacency, and places tiles with a soft propagation loop -- that path is already promising on synthetic thin-feature cases, but it currently overcommits to larger lattice candidates and underperforms the default continuous path on the cleaned badge fixture +- there is now a separate `tile-graph` reconstruction mode that extracts literal source-pixel candidates from connected source clusters, fills gaps with lattice-aligned fallback pixels, learns candidate adjacency, and places tiles with a soft propagation loop +- the raw-pixel rewrite fixed an implementation mismatch where candidates had drifted into cell-averaged patch colors instead of actual source pixels +- that path is still promising on synthetic thin-feature cases and now stays on the conservative `126x126` badge lattice instead of jumping to `154x154`, but it still underperforms the default continuous path on the cleaned badge fixture (`0.1676` source-fidelity score on the latest probe versus `0.0832` for the continuous path) Next after that: - rerank low-confidence top lattice candidates with short real solver probes instead of relying only on the cheapest preview @@ -113,6 +114,7 @@ What future work should improve here: - a retuning sweep after the edge-aware/source-guided changes land - some form of coordinated local relaxation so nearby cells can move together toward a globally better contour - optional background suppression or de-weighting when a baked checkerboard is clearly not semantic content +- for the experimental tile-graph path specifically, better local candidate coverage and stronger adjacency evidence are still needed even after switching candidates back to literal source pixels Current optimizer diagnosis: - the shared source-lattice reference, source-first snap/refine scoring, and relaxed-mode handoff are now in place, so the biggest remaining losses are no longer caused by the old representative-lattice collapse diff --git a/src/repixelizer/params.py b/src/repixelizer/params.py index 8c593ee..89bb31c 100644 --- a/src/repixelizer/params.py +++ b/src/repixelizer/params.py @@ -73,7 +73,8 @@ class SolverHyperParams: phase_rerank_confidence_threshold: float = 0.12 phase_rerank_max_size_delta_ratio: float = 0.40 phase_rerank_margin: float = 0.004 - tile_graph_max_candidates: int = 160 + tile_graph_max_candidates: int = 256 + tile_graph_max_candidates_per_coord: int = 2 tile_graph_component_min_area_ratio: float = 0.10 tile_graph_large_component_ratio: float = 1.35 tile_graph_window_coverage_threshold: float = 0.16 diff --git a/src/repixelizer/tile_graph.py b/src/repixelizer/tile_graph.py index 045155e..36f762e 100644 --- a/src/repixelizer/tile_graph.py +++ b/src/repixelizer/tile_graph.py @@ -6,7 +6,6 @@ from typing import Any import numpy as np -from .io import premultiply, unpremultiply from .params import SolverHyperParams from .source_reference import build_source_lattice_reference from .types import InferenceResult, SolverArtifacts, SourceAnalysis @@ -108,12 +107,10 @@ def _source_to_cell_coord( return y_idx, x_idx -def _window_mean_premul(source_premul: np.ndarray, center_y: float, center_x: float, cell_h: float, cell_w: float) -> np.ndarray: - y0, x0, y1, x1 = _window_bounds(center_y, center_x, cell_h, cell_w, source_premul.shape[0], source_premul.shape[1]) - window = source_premul[y0:y1, x0:x1] - if window.size == 0: - return np.zeros(4, dtype=np.float32) - return window.reshape(-1, 4).mean(axis=0).astype(np.float32) +def _sample_source_pixel(source_rgba: np.ndarray, center_y: float, center_x: float) -> np.ndarray: + y = int(np.clip(np.rint(center_y), 0, max(0, source_rgba.shape[0] - 1))) + x = int(np.clip(np.rint(center_x), 0, max(0, source_rgba.shape[1] - 1))) + return source_rgba[y, x].astype(np.float32) def _window_component_coverage(component: _Component, center_y: float, center_x: float, cell_h: float, cell_w: float) -> float: @@ -158,6 +155,36 @@ def _best_stepped_center( return best +def _directional_pixel( + source_rgba: np.ndarray, + component: _Component, + center_y: float, + center_x: float, + *, + dy: int, + dx: int, + step_y: float, + step_x: float, + cell_h: float, + cell_w: float, + coverage_threshold: float, +) -> np.ndarray: + best = _best_stepped_center( + component, + center_y + dy * step_y, + center_x + dx * step_x, + step_y=step_y, + step_x=step_x, + cell_h=cell_h, + cell_w=cell_w, + ) + if best is not None: + probe_y, probe_x, coverage = best + if coverage >= coverage_threshold: + return _sample_source_pixel(source_rgba, probe_y, probe_x) + return _sample_source_pixel(source_rgba, center_y + dy * step_y, center_x + dx * step_x) + + def _extract_components(cluster_map: np.ndarray, alpha_map: np.ndarray, alpha_threshold: float) -> list[_Component]: valid = alpha_map >= alpha_threshold visited = np.zeros(valid.shape, dtype=bool) @@ -229,8 +256,6 @@ def build_tile_graph_model( alpha_threshold=solver_params.alpha_transparent_threshold, edge_hint=analysis.edge_map, ) - reference_premul = premultiply(source_reference.sharp_rgba) - source_premul = premultiply(source_rgba) height, width = source_rgba.shape[:2] cell_h = height / max(1, inference.target_height) cell_w = width / max(1, inference.target_width) @@ -244,7 +269,7 @@ def build_tile_graph_model( candidate_coverage: list[float] = [1.0] candidate_deltas: list[np.ndarray] = [np.zeros((4, 4), dtype=np.float32)] - used_coords: set[tuple[int, int]] = set() + coord_counts: dict[tuple[int, int], int] = {} step_y = max(1.0, cell_h) step_x = max(1.0, cell_w) for component in components: @@ -268,29 +293,37 @@ def build_tile_graph_model( phase_y=inference.phase_y, phase_x=inference.phase_x, ) - if coord in local_seen or coord in used_coords: + if coord in local_seen: + continue + if coord_counts.get(coord, 0) >= solver_params.tile_graph_max_candidates_per_coord: continue local_seen.add(coord) coverage = _window_component_coverage(component, center_y, center_x, cell_h, cell_w) if coverage < solver_params.tile_graph_window_coverage_threshold and added > 0: continue - center_premul = _window_mean_premul(source_premul, center_y, center_x, cell_h, cell_w) + center_rgba = _sample_source_pixel(source_rgba, center_y, center_x) delta_features = np.zeros((4, 4), dtype=np.float32) for direction, dy, dx in _DIRS: - neighbor = _window_mean_premul( - source_premul, - center_y + dy * step_y, - center_x + dx * step_x, - cell_h, - cell_w, + neighbor_rgba = _directional_pixel( + source_rgba, + component, + center_y, + center_x, + dy=dy, + dx=dx, + step_y=step_y, + step_x=step_x, + cell_h=cell_h, + cell_w=cell_w, + coverage_threshold=solver_params.tile_graph_window_coverage_threshold, ) - delta_features[direction] = neighbor - center_premul - candidate_rgba.append(unpremultiply(center_premul[None, None, :])[0, 0].astype(np.float32)) + delta_features[direction] = neighbor_rgba - center_rgba + candidate_rgba.append(center_rgba) candidate_coords.append(coord) candidate_area_ratio.append(area_ratio) candidate_coverage.append(coverage) candidate_deltas.append(delta_features) - used_coords.add(coord) + coord_counts[coord] = coord_counts.get(coord, 0) + 1 added += 1 if area_ratio < solver_params.tile_graph_large_component_ratio: continue @@ -319,26 +352,30 @@ def build_tile_graph_model( phase_y=inference.phase_y, phase_x=inference.phase_x, ) - if probe_coord not in local_seen and probe_coord not in used_coords: + if ( + probe_coord not in local_seen + and coord_counts.get(probe_coord, 0) < solver_params.tile_graph_max_candidates_per_coord + ): queue.append((probe_y, probe_x)) def add_reference_candidate(coord_y: int, coord_x: int) -> None: if len(candidate_rgba) >= solver_params.tile_graph_max_candidates: return - if (coord_y, coord_x) in used_coords: + coord = (coord_y, coord_x) + if coord_counts.get(coord, 0) >= max(1, solver_params.tile_graph_max_candidates_per_coord): return delta_features = np.zeros((4, 4), dtype=np.float32) - center_premul = reference_premul[coord_y, coord_x] + center_rgba = source_reference.sharp_rgba[coord_y, coord_x].astype(np.float32) for direction, dy, dx in _DIRS: neighbor_y = min(max(coord_y + dy, 0), inference.target_height - 1) neighbor_x = min(max(coord_x + dx, 0), inference.target_width - 1) - delta_features[direction] = reference_premul[neighbor_y, neighbor_x] - center_premul - candidate_rgba.append(source_reference.sharp_rgba[coord_y, coord_x].astype(np.float32)) - candidate_coords.append((coord_y, coord_x)) + delta_features[direction] = source_reference.sharp_rgba[neighbor_y, neighbor_x] - center_rgba + candidate_rgba.append(center_rgba) + candidate_coords.append(coord) candidate_area_ratio.append(1.0) candidate_coverage.append(1.0) candidate_deltas.append(delta_features) - used_coords.add((coord_y, coord_x)) + coord_counts[coord] = coord_counts.get(coord, 0) + 1 if len(candidate_rgba) < solver_params.tile_graph_max_candidates: remaining = solver_params.tile_graph_max_candidates - len(candidate_rgba) @@ -413,8 +450,8 @@ def build_tile_graph_model( def _tile_graph_unary_cost(model: TileGraphModel, solver_params: SolverHyperParams) -> np.ndarray: height, width = model.reference_sharp_rgba.shape[:2] color_error = ( - np.mean(np.abs(model.reference_sharp_rgba[..., None, :] - model.candidate_rgba[None, None, :, :]), axis=-1) * 0.6 - + np.mean(np.abs(model.reference_mean_rgba[..., None, :] - model.candidate_rgba[None, None, :, :]), axis=-1) * 0.4 + np.mean(np.abs(model.reference_sharp_rgba[..., None, :] - model.candidate_rgba[None, None, :, :]), axis=-1) * 0.85 + + np.mean(np.abs(model.reference_mean_rgba[..., None, :] - model.candidate_rgba[None, None, :, :]), axis=-1) * 0.15 ) grid_y, grid_x = np.indices((height, width), dtype=np.float32) coord_error = ( diff --git a/tests/test_tile_graph.py b/tests/test_tile_graph.py index 3800868..3158515 100644 --- a/tests/test_tile_graph.py +++ b/tests/test_tile_graph.py @@ -8,6 +8,7 @@ from repixelizer.analysis import analyze_source from repixelizer.baselines import naive_resize_baseline from repixelizer.io import save_rgba from repixelizer.metrics import source_lattice_consistency_breakdown +from repixelizer.params import SolverHyperParams from repixelizer.pipeline import run_pipeline from repixelizer.synthetic import fake_pixelize, make_emblem from repixelizer.tile_graph import build_tile_graph_model, optimize_tile_graph @@ -39,6 +40,40 @@ def test_tile_graph_model_extracts_candidates_and_adjacency_from_component_walk( assert model.edge_density > 0.0 +def test_tile_graph_candidates_use_literal_source_pixel_colors() -> None: + source = np.zeros((8, 8, 4), dtype=np.float32) + source[..., 3] = 1.0 + palette = ( + np.asarray([1.0, 0.1, 0.1, 1.0], dtype=np.float32), + np.asarray([0.1, 1.0, 0.1, 1.0], dtype=np.float32), + np.asarray([0.1, 0.1, 1.0, 1.0], dtype=np.float32), + np.asarray([1.0, 1.0, 0.1, 1.0], dtype=np.float32), + ) + for y in range(source.shape[0]): + for x in range(source.shape[1]): + source[y, x] = palette[(2 * (y % 2)) + (x % 2)] + + inference = InferenceResult( + target_width=4, + target_height=4, + phase_x=0.0, + phase_y=0.0, + confidence=1.0, + top_candidates=[], + ) + model = build_tile_graph_model( + source, + inference=inference, + analysis=analyze_source(source, seed=13), + solver_params=SolverHyperParams(tile_graph_max_candidates=32, tile_graph_max_candidates_per_coord=2), + ) + + source_colors = {tuple(np.round(color, 4)) for color in source.reshape(-1, 4)} + candidate_colors = {tuple(np.round(color, 4)) for color in model.candidate_rgba} + + assert candidate_colors.issubset(source_colors | {tuple(np.zeros(4, dtype=np.float32))}) + + def test_tile_graph_reconstructs_synthetic_thin_feature_better_than_naive() -> None: source = np.zeros((16, 16, 4), dtype=np.float32) source[4:12, 8] = np.asarray([0.95, 0.95, 0.95, 1.0], dtype=np.float32)