Probe hard-edge tile graph candidates
This commit is contained in:
parent
124d31fa8a
commit
4c2f435f2f
5 changed files with 116 additions and 13 deletions
|
|
@ -43,10 +43,12 @@ Current tile-graph status:
|
|||
- the full tile-graph path now honors `--device`: source analysis, source-lattice reference construction, proposal scoring, and the local solver all run through Torch
|
||||
- it no longer invents averaged patch colors during candidate extraction; the current regression suite now checks that tile-graph candidates stay on real source colors
|
||||
- each output cell now scores a small raw-pixel proposal pool built from its inferred lattice assignment, sharp exemplar, and edge peak instead of borrowing candidates from distant regions
|
||||
- edge-heavy cells now widen their local candidate set with same-cell edge neighbors and strongest same-cell edge pixels, so the mode can at least consider harder contour colors instead of only a single medoid-like pick
|
||||
- it already beats naive resize on the repo's synthetic thin-feature regression
|
||||
- this fixes the core design mismatch that had allowed repeated distant labels to create big same-color patches and opaque black background blocks
|
||||
- on the current `24x24` emblem smoke case, an end-to-end `tile-graph` run dropped from about `2.57s` on CPU to `0.61s` on CUDA on this machine
|
||||
- the current badge CUDA probe under `artifacts/tile-graph-cuda-pass/badge-cuda/` now completes end-to-end on the selected `126x126` lattice and lands at `0.0283` final source-fidelity
|
||||
- the latest hard-edge candidate pass under `artifacts/full-emblem-tile-graph-hard-edge-v2-cuda/` improves local candidate diversity and slightly sharpens some internal contour cells, but it still regresses full-emblem source-fidelity (`0.0377` vs `0.0283` for the previous tile-graph pass), so it should be treated as an experiment rather than a settled default
|
||||
|
||||
## Quickstart
|
||||
|
||||
|
|
|
|||
|
|
@ -77,10 +77,13 @@ Experimental tile-graph note:
|
|||
- the later per-cell-local rewrite fixed the repeated-label and opaque-black-background failures by making candidates output-coord-scoped
|
||||
- the newest GPU-friendly rewrite keeps that local-candidate behavior but replaces the old component walk with per-cell top-k proposal scoring, which is fast enough to run end-to-end on CUDA
|
||||
- the current badge CUDA probe under `artifacts/tile-graph-cuda-pass/badge-cuda/` completes end-to-end at the selected `126x126` lattice with `24142` candidates and `0.0283` final source-fidelity
|
||||
- the latest hard-edge candidate pass widens edge-cell proposal sets with same-cell edge neighbors and strongest same-cell edge pixels, and it does reduce some medoid-like contour picks locally
|
||||
- that hard-edge pass is not a clean win yet: on the full badge fixture it increases candidate count to about `30k` and average choices to about `1.92`, but regresses full-image source-fidelity to about `0.0377`, so it remains exploratory
|
||||
|
||||
Next after that:
|
||||
- rerank low-confidence top lattice candidates with short real solver probes instead of relying only on the cheapest preview
|
||||
- improve per-cell tile-graph proposal diversity so thin contours get more than “sharp vs edge-peak” in hard cells
|
||||
- separate “edge-side selection” from “more edge candidates” so we can test contour-targeted scoring without paying the current whole-image fidelity penalty
|
||||
- evaluate coordinated local moves when single-cell refinement still breaks thin contours
|
||||
- rerun tuning now that the tile-graph proposal and device path have changed materially
|
||||
- decide whether the tile-graph path should stay a diagnostics-only experiment, become a refine-stage candidate generator, or mature into a full alternate solver
|
||||
|
|
@ -122,6 +125,7 @@ What future work should improve here:
|
|||
- some form of coordinated local relaxation so nearby cells can move together toward a globally better contour
|
||||
- optional background suppression or de-weighting when a baked checkerboard is clearly not semantic content
|
||||
- for the experimental tile-graph path specifically, better local proposal diversity and stronger adjacency evidence are still needed even after switching candidates back to literal source pixels
|
||||
- for the experimental tile-graph path specifically, edge-cell diversity alone is not enough; the current hard-edge candidate pass shows that simply widening edge-cell choices can sharpen some contour pixels while still making the global arrangement worse
|
||||
- for the experimental tile-graph path specifically, the next useful step is pruning and richer proposal families rather than more component-walk optimization, because the old CPU walk is now gone
|
||||
|
||||
Current optimizer diagnosis:
|
||||
|
|
|
|||
|
|
@ -75,6 +75,7 @@ class SolverHyperParams:
|
|||
phase_rerank_margin: float = 0.004
|
||||
tile_graph_max_candidates: int = 768
|
||||
tile_graph_max_candidates_per_coord: int = 2
|
||||
tile_graph_edge_candidates_per_coord: int = 6
|
||||
tile_graph_component_min_area_ratio: float = 0.10
|
||||
tile_graph_large_component_ratio: float = 1.35
|
||||
tile_graph_window_coverage_threshold: float = 0.16
|
||||
|
|
@ -82,6 +83,10 @@ class SolverHyperParams:
|
|||
tile_graph_proposal_edge_weight: float = 0.40
|
||||
tile_graph_proposal_distance_weight: float = 0.22
|
||||
tile_graph_proposal_cluster_weight: float = 0.12
|
||||
tile_graph_proposal_color_weight: float = 0.10
|
||||
tile_graph_nonedge_sharp_weight: float = 0.85
|
||||
tile_graph_nonedge_mean_weight: float = 0.15
|
||||
tile_graph_edge_mean_weight: float = 0.03
|
||||
tile_graph_area_weight: float = 0.03
|
||||
tile_graph_alpha_weight: float = 0.25
|
||||
tile_graph_coverage_weight: float = 0.05
|
||||
|
|
|
|||
|
|
@ -59,6 +59,8 @@ class TileGraphModel:
|
|||
cell_candidate_indices: np.ndarray
|
||||
reference_mean_rgba: np.ndarray
|
||||
reference_sharp_rgba: np.ndarray
|
||||
reference_edge_rgba: np.ndarray
|
||||
edge_strength: np.ndarray
|
||||
component_count: int
|
||||
edge_density: float
|
||||
average_choices: float
|
||||
|
|
@ -157,6 +159,7 @@ def build_tile_graph_model(
|
|||
flat_y_t = pixel_y_t.reshape(-1)
|
||||
flat_x_t = pixel_x_t.reshape(-1)
|
||||
sharp_rgba_flat_t = flat_rgba_t[sharp_linear_t]
|
||||
edge_rgba_flat_t = flat_rgba_t[edge_linear_t]
|
||||
sharp_cluster_flat_t = flat_cluster_t[sharp_linear_t]
|
||||
sharp_dist_t = (
|
||||
(flat_y_t.to(dtype=torch.float32) - sharp_y_flat_t[lattice_indices_t].to(dtype=torch.float32)).abs() / max(cell_h, 1.0)
|
||||
|
|
@ -169,10 +172,17 @@ def build_tile_graph_model(
|
|||
same_cluster_t = (
|
||||
(flat_cluster_t >= 0) & (flat_cluster_t == sharp_cluster_flat_t[lattice_indices_t])
|
||||
).to(dtype=torch.float32)
|
||||
color_error_t = (flat_rgba_t - sharp_rgba_flat_t[lattice_indices_t]).abs().mean(dim=-1)
|
||||
sharp_color_error_t = (flat_rgba_t - sharp_rgba_flat_t[lattice_indices_t]).abs().mean(dim=-1)
|
||||
edge_color_error_t = (flat_rgba_t - edge_rgba_flat_t[lattice_indices_t]).abs().mean(dim=-1)
|
||||
edge_cell_mask_t = edge_strength_flat_t[lattice_indices_t] >= solver_params.source_edge_detail_threshold
|
||||
proposal_color_error_t = torch.where(
|
||||
edge_cell_mask_t,
|
||||
torch.minimum(sharp_color_error_t, edge_color_error_t),
|
||||
sharp_color_error_t,
|
||||
)
|
||||
edge_cell_boost_t = (
|
||||
(1.0 - edge_focus_dist_t.clamp(0.0, 1.0))
|
||||
* (edge_strength_flat_t[lattice_indices_t] >= solver_params.source_edge_detail_threshold).to(dtype=torch.float32)
|
||||
* edge_cell_mask_t.to(dtype=torch.float32)
|
||||
)
|
||||
proposal_score_t = (
|
||||
flat_alpha_t
|
||||
|
|
@ -181,7 +191,7 @@ def build_tile_graph_model(
|
|||
+ edge_cell_boost_t * getattr(solver_params, "tile_graph_proposal_edge_weight", 0.40) * 0.5
|
||||
+ cell_support_flat_t[lattice_indices_t] * 0.08
|
||||
- sharp_dist_t * getattr(solver_params, "tile_graph_proposal_distance_weight", 0.22)
|
||||
- color_error_t * 0.10
|
||||
- proposal_color_error_t * getattr(solver_params, "tile_graph_proposal_color_weight", 0.10)
|
||||
)
|
||||
ranked_pool_t = _build_ranked_proposal_pool_torch(
|
||||
torch,
|
||||
|
|
@ -190,6 +200,13 @@ def build_tile_graph_model(
|
|||
cell_count=output_area,
|
||||
pool_size=proposal_pool_size,
|
||||
)
|
||||
edge_ranked_pool_t = _build_ranked_proposal_pool_torch(
|
||||
torch,
|
||||
lattice_indices_t,
|
||||
flat_edge_t + flat_alpha_t * 0.05,
|
||||
cell_count=output_area,
|
||||
pool_size=proposal_pool_size,
|
||||
)
|
||||
|
||||
cluster_count = max(1, int(max(analysis.cluster_centers.shape[0], 1)))
|
||||
pixel_coverage_t = torch.maximum(flat_alpha_t.clamp(0.0, 1.0), flat_alpha_t.new_zeros(flat_alpha_t.shape))
|
||||
|
|
@ -205,7 +222,21 @@ def build_tile_graph_model(
|
|||
|
||||
sharp_linear_np = sharp_linear_t.detach().cpu().numpy().astype(np.int64)
|
||||
edge_linear_np = edge_linear_t.detach().cpu().numpy().astype(np.int64)
|
||||
edge_neighbor_linear_np: list[np.ndarray] = []
|
||||
cell_ids_t = torch.arange(output_area, device=resolved_device, dtype=torch.long)
|
||||
for _, dy, dx in _DIRS:
|
||||
neighbor_y_t = (edge_peak_y_flat_t + dy).clamp(0, max(0, height - 1))
|
||||
neighbor_x_t = (edge_peak_x_flat_t + dx).clamp(0, max(0, width - 1))
|
||||
neighbor_linear_t = neighbor_y_t * width + neighbor_x_t
|
||||
same_cell_neighbor_t = lattice_indices_t[neighbor_linear_t] == cell_ids_t
|
||||
masked_neighbor_linear_t = torch.where(
|
||||
same_cell_neighbor_t,
|
||||
neighbor_linear_t,
|
||||
torch.full_like(neighbor_linear_t, -1),
|
||||
)
|
||||
edge_neighbor_linear_np.append(masked_neighbor_linear_t.detach().cpu().numpy().astype(np.int64))
|
||||
ranked_pool_np = ranked_pool_t.detach().cpu().numpy().astype(np.int64)
|
||||
edge_ranked_pool_np = edge_ranked_pool_t.detach().cpu().numpy().astype(np.int64)
|
||||
pixel_coverage_np = pixel_coverage_t.detach().cpu().numpy().astype(np.float32)
|
||||
edge_strength_np = source_reference.edge_strength.reshape(-1).astype(np.float32)
|
||||
flat_alpha_np = source_rgba.reshape(-1, source_rgba.shape[-1])[:, 3].astype(np.float32)
|
||||
|
|
@ -215,14 +246,20 @@ def build_tile_graph_model(
|
|||
candidate_area_ratio: list[float] = []
|
||||
candidate_coverage: list[float] = []
|
||||
choice_counts_list: list[int] = []
|
||||
edge_candidate_cap = max(max_candidates_per_coord, int(getattr(solver_params, "tile_graph_edge_candidates_per_coord", max_candidates_per_coord)))
|
||||
for flat_index in range(output_area):
|
||||
coord_y = flat_index // inference.target_width
|
||||
coord_x = flat_index % inference.target_width
|
||||
seen_pixels: set[int] = set()
|
||||
cell_start = len(candidate_linear)
|
||||
edge_cell = edge_strength_np[flat_index] >= solver_params.source_edge_detail_threshold
|
||||
allowed_candidates = edge_candidate_cap if edge_cell else max_candidates_per_coord
|
||||
proposal_pixels = [int(sharp_linear_np[flat_index])]
|
||||
if edge_strength_np[flat_index] >= solver_params.source_edge_detail_threshold:
|
||||
if edge_cell:
|
||||
proposal_pixels.append(int(edge_linear_np[flat_index]))
|
||||
for neighbor_linear in edge_neighbor_linear_np:
|
||||
proposal_pixels.append(int(neighbor_linear[flat_index]))
|
||||
proposal_pixels.extend(int(pixel_idx) for pixel_idx in edge_ranked_pool_np[flat_index].tolist() if int(pixel_idx) >= 0)
|
||||
proposal_pixels.extend(int(pixel_idx) for pixel_idx in ranked_pool_np[flat_index].tolist() if int(pixel_idx) >= 0)
|
||||
for pixel_index in proposal_pixels:
|
||||
if pixel_index < 0 or pixel_index in seen_pixels:
|
||||
|
|
@ -232,7 +269,7 @@ def build_tile_graph_model(
|
|||
if (
|
||||
not is_primary
|
||||
and coverage < solver_params.tile_graph_window_coverage_threshold
|
||||
and edge_strength_np[flat_index] < solver_params.source_edge_detail_threshold
|
||||
and not edge_cell
|
||||
):
|
||||
continue
|
||||
seen_pixels.add(pixel_index)
|
||||
|
|
@ -240,7 +277,7 @@ def build_tile_graph_model(
|
|||
candidate_coords.append((coord_y, coord_x))
|
||||
candidate_area_ratio.append(1.0)
|
||||
candidate_coverage.append(max(coverage, float(flat_alpha_np[pixel_index])))
|
||||
if len(seen_pixels) >= max_candidates_per_coord:
|
||||
if len(seen_pixels) >= allowed_candidates:
|
||||
break
|
||||
if len(candidate_linear) == cell_start:
|
||||
pixel_index = int(sharp_linear_np[flat_index])
|
||||
|
|
@ -303,6 +340,8 @@ def build_tile_graph_model(
|
|||
cell_candidate_indices=cell_candidate_indices_np,
|
||||
reference_mean_rgba=source_reference.mean_rgba.astype(np.float32),
|
||||
reference_sharp_rgba=source_reference.sharp_rgba.astype(np.float32),
|
||||
reference_edge_rgba=edge_rgba_flat_t.reshape(inference.target_height, inference.target_width, -1).detach().cpu().numpy().astype(np.float32),
|
||||
edge_strength=source_reference.edge_strength.astype(np.float32),
|
||||
component_count=int(np.unique(analysis.cluster_map[analysis.alpha_map >= solver_params.alpha_transparent_threshold]).size),
|
||||
edge_density=edge_density,
|
||||
average_choices=average_choices,
|
||||
|
|
@ -345,11 +384,18 @@ def _tile_graph_unary_cost(model: TileGraphModel, solver_params: SolverHyperPara
|
|||
indices = model.cell_candidate_indices[start:end]
|
||||
reference_sharp = model.reference_sharp_rgba[y, x]
|
||||
reference_mean = model.reference_mean_rgba[y, x]
|
||||
reference_edge = model.reference_edge_rgba[y, x]
|
||||
candidates = model.candidate_rgba[indices]
|
||||
color_error = (
|
||||
np.mean(np.abs(reference_sharp[None, :] - candidates), axis=-1) * 0.85
|
||||
+ np.mean(np.abs(reference_mean[None, :] - candidates), axis=-1) * 0.15
|
||||
)
|
||||
sharp_error = np.mean(np.abs(reference_sharp[None, :] - candidates), axis=-1)
|
||||
mean_error = np.mean(np.abs(reference_mean[None, :] - candidates), axis=-1)
|
||||
edge_error = np.mean(np.abs(reference_edge[None, :] - candidates), axis=-1)
|
||||
if float(model.edge_strength[y, x]) >= solver_params.source_edge_detail_threshold:
|
||||
color_error = np.minimum(sharp_error, edge_error) + mean_error * solver_params.tile_graph_edge_mean_weight
|
||||
else:
|
||||
color_error = (
|
||||
sharp_error * solver_params.tile_graph_nonedge_sharp_weight
|
||||
+ mean_error * solver_params.tile_graph_nonedge_mean_weight
|
||||
)
|
||||
area_error = np.abs(np.log(np.clip(model.candidate_area_ratio[indices], 1e-4, None) + 1e-4))
|
||||
alpha_error = np.abs(candidates[:, 3] - reference_mean[3])
|
||||
coverage_error = 1.0 - np.clip(model.candidate_coverage[indices], 0.0, 1.0)
|
||||
|
|
@ -377,12 +423,20 @@ def _tile_graph_unary_cost_torch(
|
|||
candidate_coverage_t = torch.from_numpy(model.candidate_coverage).to(device=device, dtype=torch.float32)
|
||||
reference_sharp_t = torch.from_numpy(model.reference_sharp_rgba).to(device=device, dtype=torch.float32)
|
||||
reference_mean_t = torch.from_numpy(model.reference_mean_rgba).to(device=device, dtype=torch.float32)
|
||||
reference_edge_t = torch.from_numpy(model.reference_edge_rgba).to(device=device, dtype=torch.float32)
|
||||
edge_strength_t = torch.from_numpy(model.edge_strength).to(device=device, dtype=torch.float32)
|
||||
|
||||
choice_rgba_t = candidate_rgba_t[choice_indices_t]
|
||||
choice_deltas_t = candidate_deltas_t[choice_indices_t]
|
||||
color_error = (
|
||||
(reference_sharp_t[..., None, :] - choice_rgba_t).abs().mean(dim=-1) * 0.85
|
||||
+ (reference_mean_t[..., None, :] - choice_rgba_t).abs().mean(dim=-1) * 0.15
|
||||
sharp_error_t = (reference_sharp_t[..., None, :] - choice_rgba_t).abs().mean(dim=-1)
|
||||
mean_error_t = (reference_mean_t[..., None, :] - choice_rgba_t).abs().mean(dim=-1)
|
||||
edge_error_t = (reference_edge_t[..., None, :] - choice_rgba_t).abs().mean(dim=-1)
|
||||
edge_cell_mask_t = edge_strength_t[..., None] >= solver_params.source_edge_detail_threshold
|
||||
color_error = torch.where(
|
||||
edge_cell_mask_t,
|
||||
torch.minimum(sharp_error_t, edge_error_t) + mean_error_t * solver_params.tile_graph_edge_mean_weight,
|
||||
sharp_error_t * solver_params.tile_graph_nonedge_sharp_weight
|
||||
+ mean_error_t * solver_params.tile_graph_nonedge_mean_weight,
|
||||
)
|
||||
area_error = torch.log(candidate_area_t[choice_indices_t].clamp_min(1e-4) + 1e-4).abs()
|
||||
alpha_error = (choice_rgba_t[..., 3] - reference_mean_t[..., None, 3]).abs()
|
||||
|
|
|
|||
|
|
@ -105,6 +105,44 @@ def test_tile_graph_candidates_are_scoped_to_their_output_coord() -> None:
|
|||
assert np.all(coords[:, 1] == x)
|
||||
|
||||
|
||||
def test_tile_graph_edge_cells_include_multiple_same_cell_contour_colors() -> None:
|
||||
source = np.zeros((8, 8, 4), dtype=np.float32)
|
||||
source[..., 3] = 1.0
|
||||
source[:4, 0] = np.asarray([1.0, 0.9, 0.2, 1.0], dtype=np.float32)
|
||||
source[:4, 1] = np.asarray([0.85, 0.6, 0.1, 1.0], dtype=np.float32)
|
||||
source[:4, 2] = np.asarray([0.0, 0.0, 0.0, 1.0], dtype=np.float32)
|
||||
source[:4, 3] = np.asarray([0.9, 0.2, 0.1, 1.0], dtype=np.float32)
|
||||
source[:4, 4:] = np.asarray([0.2, 0.8, 0.2, 1.0], dtype=np.float32)
|
||||
source[4:, :] = np.asarray([0.2, 0.2, 0.8, 1.0], dtype=np.float32)
|
||||
|
||||
inference = InferenceResult(
|
||||
target_width=2,
|
||||
target_height=2,
|
||||
phase_x=0.0,
|
||||
phase_y=0.0,
|
||||
confidence=1.0,
|
||||
top_candidates=[],
|
||||
)
|
||||
model = build_tile_graph_model(
|
||||
source,
|
||||
inference=inference,
|
||||
analysis=analyze_source(source, seed=17),
|
||||
device="cpu",
|
||||
)
|
||||
|
||||
start = int(model.cell_candidate_offsets[0])
|
||||
end = int(model.cell_candidate_offsets[1])
|
||||
candidates = {
|
||||
tuple(np.round(color, 4))
|
||||
for color in model.candidate_rgba[model.cell_candidate_indices[start:end]]
|
||||
}
|
||||
non_black_candidates = {color for color in candidates if color[:3] != (np.float32(0.0), np.float32(0.0), np.float32(0.0))}
|
||||
|
||||
assert len(candidates) >= 3
|
||||
assert tuple(np.round(np.asarray([0.0, 0.0, 0.0, 1.0], dtype=np.float32), 4)) in candidates
|
||||
assert len(non_black_candidates) >= 2
|
||||
|
||||
|
||||
def test_tile_graph_reconstructs_synthetic_thin_feature_better_than_naive() -> None:
|
||||
source = np.zeros((16, 16, 4), dtype=np.float32)
|
||||
source[4:12, 8] = np.asarray([0.95, 0.95, 0.95, 1.0], dtype=np.float32)
|
||||
|
|
|
|||
Loading…
Reference in a new issue