From 53cd1023e02ecb011bc2c7134617bb4edb64c633 Mon Sep 17 00:00:00 2001 From: Parley Hatch Date: Mon, 15 Jun 2026 20:25:45 -0600 Subject: [PATCH] feat(pipeline): strength-sweep harness + reliability findings doc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sweep_strength.py renders one open + one closed FPV frame across a ControlNet strength matrix at fixed seed/prompt and montages them beside the depth map — the GPU experiment to find the right strength now that rev-3 depth maps carry a full-range signal (018909b). Findings/plan in .dev/2026-06-15-diffusion-pipeline-reliability.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...26-06-15-diffusion-pipeline-reliability.md | 64 ++++++++++ tools/comfy-spike/sweep_strength.py | 117 ++++++++++++++++++ 2 files changed, 181 insertions(+) create mode 100644 .dev/2026-06-15-diffusion-pipeline-reliability.md create mode 100644 tools/comfy-spike/sweep_strength.py diff --git a/.dev/2026-06-15-diffusion-pipeline-reliability.md b/.dev/2026-06-15-diffusion-pipeline-reliability.md new file mode 100644 index 0000000..9398929 --- /dev/null +++ b/.dev/2026-06-15-diffusion-pipeline-reliability.md @@ -0,0 +1,64 @@ +# Diffusion pipeline reliability — making the depth ControlNet actually followed + +**Started:** 2026-06-15 · ongoing (`/loop`) +**Problem (user):** "controlnet maps generated are barely followed; images of the +dungeons are very hit and miss." + +## Root cause found (iteration 1) + +The FPV depth maps fed to the Z-Image Fun depth ControlNet used only a sliver of +the tonal range. Histogram over all 68 baked seed-7 frames: + +- median dynamic range **193 / 255**, worst frame only **54** +- **0/68** frames reached true black (`<32`); the deepest recesses topped out ~mid-gray +- only **35/68** reached near-white at the near surfaces + +Cause in `reikhelm-web/fpv.js`: a fixed `t = dpix / maxView` with `maxView = 12`. +The nearest floor sits ~1 cell away (never hit white) and a wall 3 cells ahead +landed at mid-gray (never black), so small rooms collapsed into the 160–214 band. +Depth ControlNets are trained on full-range MiDaS / depth-anything maps, so a +washed-out map gives almost no structural signal → the model freelances. + +## Fix shipped — commit `018909b` + +Two-pass raycast in `computeFPVDepth`: +- Pass 1 records each column's wall distance + the frame's true near/far. +- Pass 2 maps depth through a **per-frame window**: near → white, deepest visible + recess → black, every frame. +- Far plane tracks the deepest *real* surface but is clamped to `minSpan` (5 cells) + so a shallow dead-end nook is NOT stretched into a fake tunnel — its faced wall + stays an honest mid-gray slab (no black recess → no phantom-door bait). +- `gamma` 2.0 → 1.5 (normalization now owns the range). `nearPlane`/`minSpan` are opts. + +**Verified offline** (headless `bakeAtlasDepths(7)`, 68 frames): +median range 193 → **255**; near-white frames 35/68 → **68/68**; worst range 54 → 106. +Visual before/after (`/tmp/ba.png` during the session): open frames now frame a +black passage; closed frames a clean mid-gray wall. + +## Next — needs the render workstation (192.168.1.26:8188, was OFFLINE this iteration) + +The depth signal is now strong, so the old strength is almost certainly wrong. +With a weak signal you crank strength to force adherence; a strong signal wants +*less*. `bake_atlas.py` defaults `--strength 0.85`. + +1. Re-probe the workstation: `python3 tools/comfy-spike/comfy.py probe http://192.168.1.26:8188` +2. **Strength sweep** (tool already written, validated): + ``` + python3 tools/comfy-spike/sweep_strength.py http://192.168.1.26:8188 \ + --seed-dir tools/out/atlas/7 --open r8_N --closed r0_N \ + --strengths 0.55,0.7,0.85,1.0 + ``` + → `tools/out/compare/sweep_.png`. Read adherence: does the rendered passage + land on the black recess? does the closed wall stay a wall (no phantom door)? + Pick the lowest strength that still honors geometry, update `bake_atlas.py` default. +3. Then re-bake a full atlas at the chosen strength and spot-check a montage vs the + old `acc8137`/`a12bcbb` bakes. + +## Open levers if still hit-and-miss after strength tuning +- `nearPlane`/`minSpan`/`gamma` are now tunable per-export — A/B if near surfaces + read too blown-out (the AFTER maps are ~47% near-white; may want a slightly + deeper nearPlane to retain near-field texture). +- Subtle masonry-block relief on closed-wall slices so flat walls render as detailed + stone (offline-buildable, but verify on GPU it doesn't read as structure). +- Strength `start_percent`/`end_percent` scheduling (currently 0→1 full) — easing + control off in late steps frees texture while locking composition. diff --git a/tools/comfy-spike/sweep_strength.py b/tools/comfy-spike/sweep_strength.py new file mode 100644 index 0000000..80674d1 --- /dev/null +++ b/tools/comfy-spike/sweep_strength.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Sweep ControlNet strength for the Z-Image FPV depth pipeline and montage it. + +With the rev-3 depth maps (full per-frame normalization, commit 018909b) the +control signal is now high-contrast, so the right strength almost certainly +shifted. This renders ONE open frame + ONE closed frame across a strength matrix +at a fixed seed/prompt/sampler, then stacks them next to their depth map so you +can read adherence (does the passage land where the black recess is? does a +closed wall stay a wall?) at a glance. + + python3 tools/comfy-spike/sweep_strength.py http://192.168.1.26:8188 \ + --seed-dir tools/out/atlas/7 --open r8_N --closed r0_N \ + --strengths 0.55,0.7,0.85,1.0 + +Output: tools/out/compare/sweep_.png (rows = frames, cols = depth + each strength) +Re-run after re-baking depth maps to compare pipeline revisions head to head. +""" +import argparse +import glob +import os +import subprocess +import sys + +REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +COMFY = os.path.join(REPO, "tools", "comfy-spike", "comfy.py") +VENV_PY = os.path.join(REPO, "tools", "pixelart", ".venv", "bin", "python") + +# Mirror bake_atlas.py's proven prompt skeleton so the sweep reflects production. +STYLE = ("dark fantasy, ancient torchlit stone, warm orange torchlight, deep black shadows, " + "wet stone, volumetric haze, architectural photography of an empty ruin, " + "empty, deserted, no people, uninhabited, atmospheric, highly detailed, ") +OPEN_BODY = "grand pillared hall, rows of carved stone columns, a dark arched passage leads onward into shadow ahead" +WALL_BODY = ("dungeon gatehouse of carved stone, the way ahead ends at a blank unbroken wall of stone blocks, " + "sealed solid masonry with no door and no opening, a dead end") +NEGATIVE = ("person, people, human, figure, character, hands, fingers, hand, arm, arms, " + "holding weapon, sword, feet, legs, body, silhouette, UI, HUD, text, watermark, modern") + + +def render(url, depth, prompt, strength, seed, prefix, wait): + ok = subprocess.run(["python3", COMFY, "zrender", url, "--depth", depth, + "--width", "1280", "--height", "720", "--strength", str(strength), + "--seed", str(seed), "--prompt", prompt, "--negative", NEGATIVE, + "--prefix", prefix, "--wait", str(wait)], + capture_output=True, text=True) + if ok.returncode != 0: + sys.stderr.write(ok.stdout + ok.stderr + "\n") + return None + hits = glob.glob(os.path.join(os.path.dirname(depth), f"out_{prefix}_*.png")) + return hits[0] if hits else None + + +def montage(rows, strengths, out): + from PIL import Image, ImageDraw + cw, ch, pad = 360, 203, 6 + ncol = 1 + len(strengths) + sheet = Image.new("RGB", (cw * ncol + pad * (ncol + 1), (ch + 20) * len(rows) + pad), (16, 16, 16)) + dr = ImageDraw.Draw(sheet) + y = pad + for label, depth, imgs in rows: + cells = [("depth", depth)] + [(f"str {s}", imgs.get(s)) for s in strengths] + x = pad + for cap, path in cells: + if path and os.path.exists(path): + sheet.paste(Image.open(path).convert("RGB").resize((cw, ch)), (x, y + 18)) + dr.text((x, y + 4), f"{label} · {cap}", fill=(0, 230, 120)) + x += cw + pad + y += ch + 20 + sheet.save(out) + return out + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("url") + ap.add_argument("--seed-dir", default=os.path.join(REPO, "tools", "out", "atlas", "7")) + ap.add_argument("--open", default="r8_N", help="frame id for the open-passage test") + ap.add_argument("--closed", default="r0_N", help="frame id for the closed-wall test") + ap.add_argument("--strengths", default="0.55,0.7,0.85,1.0") + ap.add_argument("--seed", type=int, default=7, help="fixed diffusion seed (style lock)") + ap.add_argument("--wait", type=int, default=300) + a = ap.parse_args() + + work = os.path.join(a.seed_dir, "_work") + strengths = [float(s) for s in a.strengths.split(",") if s.strip()] + run = os.getpid() + cases = [(a.open, OPEN_BODY), (a.closed, WALL_BODY)] + + rows = [] + for frame, body in cases: + depth = os.path.join(work, f"{frame}_depth.png") + if not os.path.exists(depth): + sys.exit(f"!! missing depth map {depth} — bake the atlas first") + prompt = STYLE + body + imgs = {} + for s in strengths: + prefix = f"sweep_{frame}_{int(s*100)}_{run}" + print(f"[{frame} @ strength {s}] rendering ...", flush=True) + out = render(a.url, depth, prompt, s, a.seed, prefix, a.wait) + if out: + imgs[s] = out + rows.append((frame, depth, imgs)) + + out_dir = os.path.join(REPO, "tools", "out", "compare") + os.makedirs(out_dir, exist_ok=True) + out = montage(rows, strengths, os.path.join(out_dir, f"sweep_{run}.png")) + # tidy the per-strength intermediates; the montage is the artifact + for _, _, imgs in rows: + for p in imgs.values(): + try: + os.remove(p) + except OSError: + pass + print(f"\nmontage → {os.path.relpath(out, REPO)}") + + +if __name__ == "__main__": + main()