diff --git a/README.md b/README.md index e6ade72..229f209 100644 --- a/README.md +++ b/README.md @@ -43,10 +43,12 @@ Current tile-graph status: - the full tile-graph path now honors `--device`: source analysis, source-lattice reference construction, proposal scoring, and the local solver all run through Torch - it no longer invents averaged patch colors during candidate extraction; the current regression suite now checks that tile-graph candidates stay on real source colors - each output cell now scores a small raw-pixel proposal pool built from its inferred lattice assignment, sharp exemplar, and edge peak instead of borrowing candidates from distant regions +- edge-heavy cells now widen their local candidate set with same-cell edge neighbors and strongest same-cell edge pixels, so the mode can at least consider harder contour colors instead of only a single medoid-like pick - it already beats naive resize on the repo's synthetic thin-feature regression - this fixes the core design mismatch that had allowed repeated distant labels to create big same-color patches and opaque black background blocks - on the current `24x24` emblem smoke case, an end-to-end `tile-graph` run dropped from about `2.57s` on CPU to `0.61s` on CUDA on this machine - the current badge CUDA probe under `artifacts/tile-graph-cuda-pass/badge-cuda/` now completes end-to-end on the selected `126x126` lattice and lands at `0.0283` final source-fidelity +- the latest hard-edge candidate pass under `artifacts/full-emblem-tile-graph-hard-edge-v2-cuda/` improves local candidate diversity and slightly sharpens some internal contour cells, but it still regresses full-emblem source-fidelity (`0.0377` vs `0.0283` for the previous tile-graph pass), so it should be treated as an experiment rather than a settled default ## Quickstart diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index df61124..cca177d 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -77,10 +77,13 @@ Experimental tile-graph note: - the later per-cell-local rewrite fixed the repeated-label and opaque-black-background failures by making candidates output-coord-scoped - the newest GPU-friendly rewrite keeps that local-candidate behavior but replaces the old component walk with per-cell top-k proposal scoring, which is fast enough to run end-to-end on CUDA - the current badge CUDA probe under `artifacts/tile-graph-cuda-pass/badge-cuda/` completes end-to-end at the selected `126x126` lattice with `24142` candidates and `0.0283` final source-fidelity +- the latest hard-edge candidate pass widens edge-cell proposal sets with same-cell edge neighbors and strongest same-cell edge pixels, and it does reduce some medoid-like contour picks locally +- that hard-edge pass is not a clean win yet: on the full badge fixture it increases candidate count to about `30k` and average choices to about `1.92`, but regresses full-image source-fidelity to about `0.0377`, so it remains exploratory Next after that: - rerank low-confidence top lattice candidates with short real solver probes instead of relying only on the cheapest preview - improve per-cell tile-graph proposal diversity so thin contours get more than “sharp vs edge-peak” in hard cells +- separate “edge-side selection” from “more edge candidates” so we can test contour-targeted scoring without paying the current whole-image fidelity penalty - evaluate coordinated local moves when single-cell refinement still breaks thin contours - rerun tuning now that the tile-graph proposal and device path have changed materially - decide whether the tile-graph path should stay a diagnostics-only experiment, become a refine-stage candidate generator, or mature into a full alternate solver @@ -122,6 +125,7 @@ What future work should improve here: - some form of coordinated local relaxation so nearby cells can move together toward a globally better contour - optional background suppression or de-weighting when a baked checkerboard is clearly not semantic content - for the experimental tile-graph path specifically, better local proposal diversity and stronger adjacency evidence are still needed even after switching candidates back to literal source pixels +- for the experimental tile-graph path specifically, edge-cell diversity alone is not enough; the current hard-edge candidate pass shows that simply widening edge-cell choices can sharpen some contour pixels while still making the global arrangement worse - for the experimental tile-graph path specifically, the next useful step is pruning and richer proposal families rather than more component-walk optimization, because the old CPU walk is now gone Current optimizer diagnosis: diff --git a/src/repixelizer/params.py b/src/repixelizer/params.py index c9014d1..b5f791a 100644 --- a/src/repixelizer/params.py +++ b/src/repixelizer/params.py @@ -75,6 +75,7 @@ class SolverHyperParams: phase_rerank_margin: float = 0.004 tile_graph_max_candidates: int = 768 tile_graph_max_candidates_per_coord: int = 2 + tile_graph_edge_candidates_per_coord: int = 6 tile_graph_component_min_area_ratio: float = 0.10 tile_graph_large_component_ratio: float = 1.35 tile_graph_window_coverage_threshold: float = 0.16 @@ -82,6 +83,10 @@ class SolverHyperParams: tile_graph_proposal_edge_weight: float = 0.40 tile_graph_proposal_distance_weight: float = 0.22 tile_graph_proposal_cluster_weight: float = 0.12 + tile_graph_proposal_color_weight: float = 0.10 + tile_graph_nonedge_sharp_weight: float = 0.85 + tile_graph_nonedge_mean_weight: float = 0.15 + tile_graph_edge_mean_weight: float = 0.03 tile_graph_area_weight: float = 0.03 tile_graph_alpha_weight: float = 0.25 tile_graph_coverage_weight: float = 0.05 diff --git a/src/repixelizer/tile_graph.py b/src/repixelizer/tile_graph.py index 3534f6b..bd40c25 100644 --- a/src/repixelizer/tile_graph.py +++ b/src/repixelizer/tile_graph.py @@ -59,6 +59,8 @@ class TileGraphModel: cell_candidate_indices: np.ndarray reference_mean_rgba: np.ndarray reference_sharp_rgba: np.ndarray + reference_edge_rgba: np.ndarray + edge_strength: np.ndarray component_count: int edge_density: float average_choices: float @@ -157,6 +159,7 @@ def build_tile_graph_model( flat_y_t = pixel_y_t.reshape(-1) flat_x_t = pixel_x_t.reshape(-1) sharp_rgba_flat_t = flat_rgba_t[sharp_linear_t] + edge_rgba_flat_t = flat_rgba_t[edge_linear_t] sharp_cluster_flat_t = flat_cluster_t[sharp_linear_t] sharp_dist_t = ( (flat_y_t.to(dtype=torch.float32) - sharp_y_flat_t[lattice_indices_t].to(dtype=torch.float32)).abs() / max(cell_h, 1.0) @@ -169,10 +172,17 @@ def build_tile_graph_model( same_cluster_t = ( (flat_cluster_t >= 0) & (flat_cluster_t == sharp_cluster_flat_t[lattice_indices_t]) ).to(dtype=torch.float32) - color_error_t = (flat_rgba_t - sharp_rgba_flat_t[lattice_indices_t]).abs().mean(dim=-1) + sharp_color_error_t = (flat_rgba_t - sharp_rgba_flat_t[lattice_indices_t]).abs().mean(dim=-1) + edge_color_error_t = (flat_rgba_t - edge_rgba_flat_t[lattice_indices_t]).abs().mean(dim=-1) + edge_cell_mask_t = edge_strength_flat_t[lattice_indices_t] >= solver_params.source_edge_detail_threshold + proposal_color_error_t = torch.where( + edge_cell_mask_t, + torch.minimum(sharp_color_error_t, edge_color_error_t), + sharp_color_error_t, + ) edge_cell_boost_t = ( (1.0 - edge_focus_dist_t.clamp(0.0, 1.0)) - * (edge_strength_flat_t[lattice_indices_t] >= solver_params.source_edge_detail_threshold).to(dtype=torch.float32) + * edge_cell_mask_t.to(dtype=torch.float32) ) proposal_score_t = ( flat_alpha_t @@ -181,7 +191,7 @@ def build_tile_graph_model( + edge_cell_boost_t * getattr(solver_params, "tile_graph_proposal_edge_weight", 0.40) * 0.5 + cell_support_flat_t[lattice_indices_t] * 0.08 - sharp_dist_t * getattr(solver_params, "tile_graph_proposal_distance_weight", 0.22) - - color_error_t * 0.10 + - proposal_color_error_t * getattr(solver_params, "tile_graph_proposal_color_weight", 0.10) ) ranked_pool_t = _build_ranked_proposal_pool_torch( torch, @@ -190,6 +200,13 @@ def build_tile_graph_model( cell_count=output_area, pool_size=proposal_pool_size, ) + edge_ranked_pool_t = _build_ranked_proposal_pool_torch( + torch, + lattice_indices_t, + flat_edge_t + flat_alpha_t * 0.05, + cell_count=output_area, + pool_size=proposal_pool_size, + ) cluster_count = max(1, int(max(analysis.cluster_centers.shape[0], 1))) pixel_coverage_t = torch.maximum(flat_alpha_t.clamp(0.0, 1.0), flat_alpha_t.new_zeros(flat_alpha_t.shape)) @@ -205,7 +222,21 @@ def build_tile_graph_model( sharp_linear_np = sharp_linear_t.detach().cpu().numpy().astype(np.int64) edge_linear_np = edge_linear_t.detach().cpu().numpy().astype(np.int64) + edge_neighbor_linear_np: list[np.ndarray] = [] + cell_ids_t = torch.arange(output_area, device=resolved_device, dtype=torch.long) + for _, dy, dx in _DIRS: + neighbor_y_t = (edge_peak_y_flat_t + dy).clamp(0, max(0, height - 1)) + neighbor_x_t = (edge_peak_x_flat_t + dx).clamp(0, max(0, width - 1)) + neighbor_linear_t = neighbor_y_t * width + neighbor_x_t + same_cell_neighbor_t = lattice_indices_t[neighbor_linear_t] == cell_ids_t + masked_neighbor_linear_t = torch.where( + same_cell_neighbor_t, + neighbor_linear_t, + torch.full_like(neighbor_linear_t, -1), + ) + edge_neighbor_linear_np.append(masked_neighbor_linear_t.detach().cpu().numpy().astype(np.int64)) ranked_pool_np = ranked_pool_t.detach().cpu().numpy().astype(np.int64) + edge_ranked_pool_np = edge_ranked_pool_t.detach().cpu().numpy().astype(np.int64) pixel_coverage_np = pixel_coverage_t.detach().cpu().numpy().astype(np.float32) edge_strength_np = source_reference.edge_strength.reshape(-1).astype(np.float32) flat_alpha_np = source_rgba.reshape(-1, source_rgba.shape[-1])[:, 3].astype(np.float32) @@ -215,14 +246,20 @@ def build_tile_graph_model( candidate_area_ratio: list[float] = [] candidate_coverage: list[float] = [] choice_counts_list: list[int] = [] + edge_candidate_cap = max(max_candidates_per_coord, int(getattr(solver_params, "tile_graph_edge_candidates_per_coord", max_candidates_per_coord))) for flat_index in range(output_area): coord_y = flat_index // inference.target_width coord_x = flat_index % inference.target_width seen_pixels: set[int] = set() cell_start = len(candidate_linear) + edge_cell = edge_strength_np[flat_index] >= solver_params.source_edge_detail_threshold + allowed_candidates = edge_candidate_cap if edge_cell else max_candidates_per_coord proposal_pixels = [int(sharp_linear_np[flat_index])] - if edge_strength_np[flat_index] >= solver_params.source_edge_detail_threshold: + if edge_cell: proposal_pixels.append(int(edge_linear_np[flat_index])) + for neighbor_linear in edge_neighbor_linear_np: + proposal_pixels.append(int(neighbor_linear[flat_index])) + proposal_pixels.extend(int(pixel_idx) for pixel_idx in edge_ranked_pool_np[flat_index].tolist() if int(pixel_idx) >= 0) proposal_pixels.extend(int(pixel_idx) for pixel_idx in ranked_pool_np[flat_index].tolist() if int(pixel_idx) >= 0) for pixel_index in proposal_pixels: if pixel_index < 0 or pixel_index in seen_pixels: @@ -232,7 +269,7 @@ def build_tile_graph_model( if ( not is_primary and coverage < solver_params.tile_graph_window_coverage_threshold - and edge_strength_np[flat_index] < solver_params.source_edge_detail_threshold + and not edge_cell ): continue seen_pixels.add(pixel_index) @@ -240,7 +277,7 @@ def build_tile_graph_model( candidate_coords.append((coord_y, coord_x)) candidate_area_ratio.append(1.0) candidate_coverage.append(max(coverage, float(flat_alpha_np[pixel_index]))) - if len(seen_pixels) >= max_candidates_per_coord: + if len(seen_pixels) >= allowed_candidates: break if len(candidate_linear) == cell_start: pixel_index = int(sharp_linear_np[flat_index]) @@ -303,6 +340,8 @@ def build_tile_graph_model( cell_candidate_indices=cell_candidate_indices_np, reference_mean_rgba=source_reference.mean_rgba.astype(np.float32), reference_sharp_rgba=source_reference.sharp_rgba.astype(np.float32), + reference_edge_rgba=edge_rgba_flat_t.reshape(inference.target_height, inference.target_width, -1).detach().cpu().numpy().astype(np.float32), + edge_strength=source_reference.edge_strength.astype(np.float32), component_count=int(np.unique(analysis.cluster_map[analysis.alpha_map >= solver_params.alpha_transparent_threshold]).size), edge_density=edge_density, average_choices=average_choices, @@ -345,11 +384,18 @@ def _tile_graph_unary_cost(model: TileGraphModel, solver_params: SolverHyperPara indices = model.cell_candidate_indices[start:end] reference_sharp = model.reference_sharp_rgba[y, x] reference_mean = model.reference_mean_rgba[y, x] + reference_edge = model.reference_edge_rgba[y, x] candidates = model.candidate_rgba[indices] - color_error = ( - np.mean(np.abs(reference_sharp[None, :] - candidates), axis=-1) * 0.85 - + np.mean(np.abs(reference_mean[None, :] - candidates), axis=-1) * 0.15 - ) + sharp_error = np.mean(np.abs(reference_sharp[None, :] - candidates), axis=-1) + mean_error = np.mean(np.abs(reference_mean[None, :] - candidates), axis=-1) + edge_error = np.mean(np.abs(reference_edge[None, :] - candidates), axis=-1) + if float(model.edge_strength[y, x]) >= solver_params.source_edge_detail_threshold: + color_error = np.minimum(sharp_error, edge_error) + mean_error * solver_params.tile_graph_edge_mean_weight + else: + color_error = ( + sharp_error * solver_params.tile_graph_nonedge_sharp_weight + + mean_error * solver_params.tile_graph_nonedge_mean_weight + ) area_error = np.abs(np.log(np.clip(model.candidate_area_ratio[indices], 1e-4, None) + 1e-4)) alpha_error = np.abs(candidates[:, 3] - reference_mean[3]) coverage_error = 1.0 - np.clip(model.candidate_coverage[indices], 0.0, 1.0) @@ -377,12 +423,20 @@ def _tile_graph_unary_cost_torch( candidate_coverage_t = torch.from_numpy(model.candidate_coverage).to(device=device, dtype=torch.float32) reference_sharp_t = torch.from_numpy(model.reference_sharp_rgba).to(device=device, dtype=torch.float32) reference_mean_t = torch.from_numpy(model.reference_mean_rgba).to(device=device, dtype=torch.float32) + reference_edge_t = torch.from_numpy(model.reference_edge_rgba).to(device=device, dtype=torch.float32) + edge_strength_t = torch.from_numpy(model.edge_strength).to(device=device, dtype=torch.float32) choice_rgba_t = candidate_rgba_t[choice_indices_t] choice_deltas_t = candidate_deltas_t[choice_indices_t] - color_error = ( - (reference_sharp_t[..., None, :] - choice_rgba_t).abs().mean(dim=-1) * 0.85 - + (reference_mean_t[..., None, :] - choice_rgba_t).abs().mean(dim=-1) * 0.15 + sharp_error_t = (reference_sharp_t[..., None, :] - choice_rgba_t).abs().mean(dim=-1) + mean_error_t = (reference_mean_t[..., None, :] - choice_rgba_t).abs().mean(dim=-1) + edge_error_t = (reference_edge_t[..., None, :] - choice_rgba_t).abs().mean(dim=-1) + edge_cell_mask_t = edge_strength_t[..., None] >= solver_params.source_edge_detail_threshold + color_error = torch.where( + edge_cell_mask_t, + torch.minimum(sharp_error_t, edge_error_t) + mean_error_t * solver_params.tile_graph_edge_mean_weight, + sharp_error_t * solver_params.tile_graph_nonedge_sharp_weight + + mean_error_t * solver_params.tile_graph_nonedge_mean_weight, ) area_error = torch.log(candidate_area_t[choice_indices_t].clamp_min(1e-4) + 1e-4).abs() alpha_error = (choice_rgba_t[..., 3] - reference_mean_t[..., None, 3]).abs() diff --git a/tests/test_tile_graph.py b/tests/test_tile_graph.py index 72ac334..5a22459 100644 --- a/tests/test_tile_graph.py +++ b/tests/test_tile_graph.py @@ -105,6 +105,44 @@ def test_tile_graph_candidates_are_scoped_to_their_output_coord() -> None: assert np.all(coords[:, 1] == x) +def test_tile_graph_edge_cells_include_multiple_same_cell_contour_colors() -> None: + source = np.zeros((8, 8, 4), dtype=np.float32) + source[..., 3] = 1.0 + source[:4, 0] = np.asarray([1.0, 0.9, 0.2, 1.0], dtype=np.float32) + source[:4, 1] = np.asarray([0.85, 0.6, 0.1, 1.0], dtype=np.float32) + source[:4, 2] = np.asarray([0.0, 0.0, 0.0, 1.0], dtype=np.float32) + source[:4, 3] = np.asarray([0.9, 0.2, 0.1, 1.0], dtype=np.float32) + source[:4, 4:] = np.asarray([0.2, 0.8, 0.2, 1.0], dtype=np.float32) + source[4:, :] = np.asarray([0.2, 0.2, 0.8, 1.0], dtype=np.float32) + + inference = InferenceResult( + target_width=2, + target_height=2, + phase_x=0.0, + phase_y=0.0, + confidence=1.0, + top_candidates=[], + ) + model = build_tile_graph_model( + source, + inference=inference, + analysis=analyze_source(source, seed=17), + device="cpu", + ) + + start = int(model.cell_candidate_offsets[0]) + end = int(model.cell_candidate_offsets[1]) + candidates = { + tuple(np.round(color, 4)) + for color in model.candidate_rgba[model.cell_candidate_indices[start:end]] + } + non_black_candidates = {color for color in candidates if color[:3] != (np.float32(0.0), np.float32(0.0), np.float32(0.0))} + + assert len(candidates) >= 3 + assert tuple(np.round(np.asarray([0.0, 0.0, 0.0, 1.0], dtype=np.float32), 4)) in candidates + assert len(non_black_candidates) >= 2 + + def test_tile_graph_reconstructs_synthetic_thin_feature_better_than_naive() -> None: source = np.zeros((16, 16, 4), dtype=np.float32) source[4:12, 8] = np.asarray([0.95, 0.95, 0.95, 1.0], dtype=np.float32)