reikhelm/reikhelm-web/fpv.js
Parley Hatch 018909b2cd fix(render): full-range per-frame depth normalization for FPV ControlNet
Diagnosis: every baked FPV depth map used only the upper-mid grays
(typ. 160–214, 0/68 frames reaching true black, only 35/68 reaching
near-white). A fixed 0..maxView=12 scale meant the nearest floor (~1
cell) never hit white and a wall 3 cells ahead never went past mid-gray,
so a small room collapsed into a low-contrast band. A depth ControlNet
trained on full-range MiDaS/depth-anything maps gets almost no structural
signal from that — the root cause of "controlnet barely followed / hit
and miss" renders.

Fix: two-pass raycast. Pass 1 records each column's wall distance and the
frame's true near/far. Pass 2 maps depth through a per-frame window so
near surfaces read white and the deepest visible recess reads black every
frame. The far plane tracks the deepest real surface but is clamped to a
minSpan floor so a shallow dead-end nook is NOT stretched into a fake
tunnel — its faced wall stays an honest mid-gray slab (no black recess
for the model to paint a phantom door into). gamma default 2.0→1.5 since
normalization now owns the range. nearPlane/minSpan exposed as opts.

Verified offline (headless bake, seed 7, 68 frames): median dynamic range
193→255; frames reaching true-white near surfaces 35/68→68/68; worst-frame
range 54→106. Visual before/after confirms open frames now frame a black
passage and closed frames a clean mid-gray wall.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-15 20:24:29 -06:00

241 lines
12 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// fpv.js — first-person (Eye-of-the-Beholder / Daggerfall style) depth exporter.
//
// A renderer concern, like depth.js: it reads the same envelope and produces an
// 8-bit depth map for the AI-render pipeline — but viewed from *inside* the
// dungeon rather than top-down. A grid dungeon is a 2D map extruded: walls are
// full-height blocks, floor and ceiling are flat planes, the eye sits at mid
// height, and you face one of four cardinal directions. So we don't need any 3D
// data from the core — a classic Wolfenstein-style raycaster turns the grid into
// a perspective depth map directly.
//
// Per screen column we cast one ray, march the grid (DDA) to the first blocking
// cell, and fill the column: a wall slice at that distance, floor ramping toward
// the camera below it, ceiling above. Distance → gray with NEAR=white/FAR=black
// (the same convention comfy.py expects). Doorways are non-blocking, so they
// read as dark recesses leading deeper — which is exactly what we want.
import { getStage, distanceToWall } from './render.js';
import { depthToCanvas } from './depth.js';
const WALL = 0, PILLAR = 3;
// Cardinal facings as (dx, dy) on the grid (y grows downward / "south").
export const DIRS = { N: [0, -1], E: [1, 0], S: [0, 1], W: [-1, 0] };
const clamp = (v, lo, hi) => v < lo ? lo : v > hi ? hi : v;
const blocks = (t) => t === WALL || t === PILLAR; // line-of-sight blockers
// Centroid of a cell list as [cx, cy].
export function centroidOf(cells) {
let cx = 0, cy = 0;
for (const [x, y] of cells) { cx += x; cy += y; }
return [cx / cells.length, cy / cells.length];
}
// The *most open* cell (max distance-to-wall) among `cells`, so a camera stands
// in open space rather than against a wall (a room centroid can land near a wall
// when the room is non-convex). Ties break toward the cell nearest the centroid,
// keeping it visually centred. `dist` is a precomputed distanceToWall field.
export function mostOpenCell(dist, w, cells) {
const [cx, cy] = centroidOf(cells);
let pick = cells[0], pmax = -1, pcen = Infinity;
for (const [x, y] of cells) {
const d = dist[y * w + x];
const cen = (x - cx) ** 2 + (y - cy) ** 2;
if (d > pmax || (d === pmax && cen < pcen)) { pmax = d; pcen = cen; pick = [x, y]; }
}
return pick;
}
// Vantage cell for one room region (its most-open cell).
export function roomVantage(env, region, opts = {}) {
const { tiles } = getStage(env, opts.stage);
return mostOpenCell(distanceToWall(tiles, env.width, env.height), env.width, region.cells);
}
// Per-(room, facing) camera cell, chosen so the faced geometry reads
// UNAMBIGUOUSLY in the depth map. The room-wide most-open vantage stands far
// from everything: the faced wall lands near max-view (≈ black), and the
// diffusion model decorates that ambiguity with hallucinated doors — while a
// real opening renders off-center because the camera isn't aligned with the
// gap. Instead:
// open ahead → stand in a cell whose straight-ahead ray ESCAPES the room
// through the gap, a comfortable 27 cells back: the passage
// you'd walk through is centered and clearly deep.
// wall ahead → stand ~3 cells from the wall: it renders mid-bright and
// solid, leaving the model nothing to invent.
export function directedVantage(env, region, dirName, opts = {}) {
const { tiles } = getStage(env, opts.stage);
const w = env.width, h = env.height;
const dist = distanceToWall(tiles, w, h);
const [dx, dy] = DIRS[dirName];
const cellSet = new Set(region.cells.map(([x, y]) => x + ',' + y));
const [cx, cy] = centroidOf(region.cells);
let best = null, bestScore = Infinity, anyEscape = false;
const cand = [];
for (const [x, y] of region.cells) {
let px = x, py = y, run = 0, escaped = false;
for (let s = 0; s < 24; s++) {
px += dx; py += dy;
if (px < 0 || py < 0 || px >= w || py >= h) break;
if (blocks(tiles[py][px])) break;
run++;
if (!cellSet.has(px + ',' + py)) escaped = true; // ray left the room through a gap
}
if (escaped) anyEscape = true;
cand.push({ x, y, run, escaped });
}
for (const c of cand) {
if (anyEscape && !c.escaped) continue; // when a gap is sightable, only gap-aligned cells qualify
const clearance = dist[c.y * w + c.x];
// Sweet bands: gaps read best 27 cells out; a blank wall at ~2.
const band = anyEscape
? (c.run < 2 ? (2 - c.run) * 6 : c.run > 7 ? (c.run - 7) * 2 : 0)
: Math.abs(c.run - 2) * 3; // closed: stand close — a bright dominant slab leaves no room for phantom doors
const hug = clearance <= 1 ? 5 : 0; // don't press against a side wall
// Open: center on the gap. Closed: deliberately stand OFF the room's
// perpendicular axis — a symmetric stage begs the model to paint a
// centered focal door; an off-axis dead-end corner doesn't.
const perp = dirName === 'N' || dirName === 'S' ? c.x - cx : c.y - cy;
const cen = ((c.x - cx) ** 2 + (c.y - cy) ** 2) * 0.05;
const offAxis = anyEscape ? 0 : (Math.abs(perp) < 1.5 ? 4 : 0);
const score = band + hug + (anyEscape ? cen : offAxis + cen * 0.4);
if (score < bestScore) { bestScore = score; best = [c.x, c.y]; }
}
return best ?? mostOpenCell(dist, w, region.cells);
}
// Default demo vantage: the most-open cell of the largest room (or, failing that,
// the most open floor cell anywhere).
export function vantage(env, opts = {}) {
const { tiles, regions } = getStage(env, opts.stage);
const w = env.width, h = env.height;
const dist = distanceToWall(tiles, w, h);
let best = null, bestArea = 0;
for (const r of regions) {
if (r.kind !== 'Room' || !r.cells || r.cells.length === 0) continue;
if (r.cells.length > bestArea) { bestArea = r.cells.length; best = r; }
}
if (best) return mostOpenCell(dist, w, best.cells);
const all = [];
for (let y = 0; y < h; y++) for (let x = 0; x < w; x++) if (!blocks(tiles[y][x])) all.push([x, y]);
return all.length ? mostOpenCell(dist, w, all) : [w >> 1, h >> 1];
}
// Build a first-person depth field. Returns { width, height, gray, at, dir }.
// opts: { stage?, at?[x,y], dir?'N'|'E'|'S'|'W', width?, height?, fov?deg,
// maxView?, gamma?, nearPlane?, minSpan? }
export function computeFPVDepth(env, opts = {}) {
const { tiles } = getStage(env, opts.stage);
const w = env.width, h = env.height;
const W = opts.width ?? 1280, H = opts.height ?? 720; // 16:9 — integer-scales to 1080p/1440p
const fov = (opts.fov ?? 80) * Math.PI / 180; // wider horizontal FOV to fill the widescreen frame
const planeLen = Math.tan(fov / 2);
const maxView = opts.maxView ?? 12; // how far the eye sees (cells) before it's pure dark
const gamma = opts.gamma ?? 1.5; // >1 deepens the falloff to black; normalization owns the range now
const at = opts.at ?? vantage(env, opts);
const dirName = opts.dir ?? 'N';
const [dirX, dirY] = DIRS[dirName];
const posX = at[0] + 0.5, posY = at[1] + 0.5;
// Camera plane ⟂ to the view direction (rotate dir 90°), scaled to the FOV.
const planeX = -dirY * planeLen, planeY = dirX * planeLen;
const gray = new Uint8ClampedArray(W * H);
const half = H / 2;
const centerX = W >> 1;
let ahead = maxView; // straight-ahead wall distance (cells) — big = an opening leads onward
// Pass 1 — cast every column, recording the wall-hit distance (or maxView when
// the ray escapes off-map / through a gap). We need the frame's true near/far
// span BEFORE mapping to gray. A fixed 0..maxView scale wastes most of the
// tonal range on small rooms: the nearest floor sits ~1 cell off (never quite
// white) and a wall 3 cells ahead lands at mid-gray (never black), so a whole
// depth map collapses into the 160214 band — almost no contrast for the depth
// ControlNet to honor. Normalizing each frame to its own near/far instead
// makes near surfaces white and the deepest recess black every time.
const colDist = new Float32Array(W);
let nearRaw = 1.0; // the floor at the screen's bottom edge always sits ~1 cell away
let farRaw = 0;
for (let x = 0; x < W; x++) {
const camX = 2 * x / W - 1; // 1 … 1 across the screen
const rayX = dirX + planeX * camX;
const rayY = dirY + planeY * camX;
// DDA grid march.
let mapX = posX | 0, mapY = posY | 0;
const deltaX = Math.abs(1 / rayX), deltaY = Math.abs(1 / rayY); // Infinity when axis-aligned — fine
let stepX, stepY, sideX, sideY;
if (rayX < 0) { stepX = -1; sideX = (posX - mapX) * deltaX; } else { stepX = 1; sideX = (mapX + 1 - posX) * deltaX; }
if (rayY < 0) { stepY = -1; sideY = (posY - mapY) * deltaY; } else { stepY = 1; sideY = (mapY + 1 - posY) * deltaY; }
let hit = false, side = 0;
const maxSteps = maxView * 2 + 4;
for (let s = 0; s < maxSteps; s++) {
if (sideX < sideY) { sideX += deltaX; mapX += stepX; side = 0; }
else { sideY += deltaY; mapY += stepY; side = 1; }
if (mapX < 0 || mapY < 0 || mapX >= w || mapY >= h) break; // off-map → open/far
if (blocks(tiles[mapY][mapX])) { hit = true; break; }
}
const perp = side === 0 ? sideX - deltaX : sideY - deltaY;
const dist = hit ? clamp(perp, 0.02, maxView) : maxView;
colDist[x] = dist;
if (dist < nearRaw) nearRaw = dist;
if (dist > farRaw) farRaw = dist;
if (x === centerX) ahead = dist;
}
// Per-frame normalization window. The far plane tracks the deepest surface
// actually visible (so a real passage reads black), but is clamped to a floor
// (minSpan) so a shallow dead-end nook isn't stretched into a dramatic fake
// tunnel — its faced wall stays an honest mid-gray slab, not a black recess
// the model would paint a phantom door into. `gamma` then shapes the falloff
// within that window.
const nearPlane = opts.nearPlane ?? nearRaw;
const minSpan = opts.minSpan ?? 5;
const farPlane = clamp(farRaw, nearPlane + minSpan, maxView);
const span = Math.max(farPlane - nearPlane, 1e-3);
// Pass 2 — paint each column: a wall slice at its distance, floor/ceiling
// ramping toward the camera, every depth mapped through the per-frame window.
for (let x = 0; x < W; x++) {
const dist = colDist[x];
// Project the 1-cell-tall wall: lineH = H/dist (fills the screen at dist 1).
const lineH = H / dist;
const ds = clamp(Math.floor(half - lineH / 2), 0, H);
const de = clamp(Math.floor(half + lineH / 2), 0, H);
for (let y = 0; y < H; y++) {
let dpix;
if (y >= ds && y < de) {
dpix = dist; // wall slice
} else {
// Floor (below) / ceiling (above). (0.5·H)/p is continuous with the
// wall distance exactly at the slice edge, so the column has no seam.
const p = y < half ? (half - y) : (y - half + 1);
dpix = Math.min((0.5 * H) / p, maxView);
}
const t = clamp((dpix - nearPlane) / span, 0, 1);
gray[y * W + x] = Math.round(255 * Math.pow(1 - t, gamma));
}
}
return { width: W, height: H, gray, at, dir: dirName, ahead };
}
// --- export glue (mirrors depth.js) --------------------------------------
export function exportFPVDataURL(env, opts = {}) {
const field = computeFPVDepth(env, opts);
return { url: depthToCanvas(field).toDataURL('image/png'), width: field.width, height: field.height, at: field.at, dir: field.dir, ahead: field.ahead };
}
export async function exportFPVToServer(env, name, opts = {}) {
const field = computeFPVDepth(env, opts);
const blob = await new Promise((res) => depthToCanvas(field).toBlob(res, 'image/png'));
const resp = await fetch('/save/' + encodeURIComponent(name), { method: 'POST', body: blob });
if (!resp.ok) throw new Error('save failed: ' + resp.status);
const info = await resp.json();
return { ...info, width: field.width, height: field.height, at: field.at, dir: field.dir, ahead: field.ahead };
}