From 86024464c209db9db00908a35b3790fbb0facd28 Mon Sep 17 00:00:00 2001 From: Hiren Kangad Date: Tue, 10 Mar 2026 10:53:51 +0530 Subject: [PATCH] feat(refboard): viewport-based texture loading with hysteresis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Images no longer load textures eagerly on creation. Instead, the culling ticker checks viewport proximity every 200ms: - Load margin: 1x screen size beyond viewport (preload before visible) - Unload margin: 2x screen size (hysteresis prevents thrash during pan) ImageSprite changes: - Constructor no longer calls loadTexture() — deferred to culling system - New unloadTexture() restores placeholder and frees GPU memory - Exposed texture getter for clipboard resolution detection - loaded field now public for culling system to check state This should allow hundreds of images without GPU OOM, since only nearby textures consume GPU memory at any time. --- frontend/src/canvas/PixiCanvas.tsx | 42 ++++++++++++++++++---- frontend/src/canvas/sprites/ImageSprite.ts | 35 +++++++++++++++--- 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/frontend/src/canvas/PixiCanvas.tsx b/frontend/src/canvas/PixiCanvas.tsx index 5b4818b..4056ef4 100644 --- a/frontend/src/canvas/PixiCanvas.tsx +++ b/frontend/src/canvas/PixiCanvas.tsx @@ -18,6 +18,7 @@ import { Application } from 'pixi.js'; import { Viewport } from 'pixi-viewport'; import { SceneManager, getItemWorldBounds } from './SceneManager'; import { TextureManager } from './TextureManager'; +import { ImageSprite } from './sprites/ImageSprite'; import { SpringManager } from './spring'; import { convertFabricToV2 } from './scene-format'; import type { SceneData } from './scene-format'; @@ -165,6 +166,9 @@ const PixiCanvas = forwardRef( }); // -- Visibility culling ticker (runs every 200ms, not every frame) -- + // Images: load textures when near viewport, unload when far away. + // Uses preload margin (1 screen) and larger unload margin (2 screens) + // to avoid thrashing during panning. let lastCullCheck = 0; app.ticker.add((ticker) => { @@ -173,16 +177,42 @@ const PixiCanvas = forwardRef( lastCullCheck = 0; const bounds = viewport.getVisibleBounds(); - const margin = 200; + // Preload margin: 1x screen size beyond viewport edges + const loadMargin = Math.max(bounds.width, bounds.height); + // Unload margin: 2x screen size — hysteresis prevents thrash + const unloadMargin = loadMargin * 2; for (const item of scene.getAllItems()) { + const { x: ix, y: iy, w: iw, h: ih } = getItemWorldBounds(item); + + if (item.type === 'image' && item.displayObject instanceof ImageSprite) { + const nearViewport = + ix + iw > bounds.x - loadMargin && + ix < bounds.x + bounds.width + loadMargin && + iy + ih > bounds.y - loadMargin && + iy < bounds.y + bounds.height + loadMargin; + + if (nearViewport && !item.displayObject.loaded) { + item.displayObject.loadTexture(); + } else if (!nearViewport) { + // Only unload if well outside viewport (hysteresis) + const farFromViewport = + ix + iw < bounds.x - unloadMargin || + ix > bounds.x + bounds.width + unloadMargin || + iy + ih < bounds.y - unloadMargin || + iy > bounds.y + bounds.height + unloadMargin; + if (farFromViewport && item.displayObject.loaded) { + item.displayObject.unloadTexture(); + } + } + } + if (item.type === 'video' && 'onVisibilityChange' in item.displayObject) { - const { x: ix, y: iy, w: iw, h: ih } = getItemWorldBounds(item); const inView = - ix + iw > bounds.x - margin && - ix < bounds.x + bounds.width + margin && - iy + ih > bounds.y - margin && - iy < bounds.y + bounds.height + margin; + ix + iw > bounds.x - loadMargin && + ix < bounds.x + bounds.width + loadMargin && + iy + ih > bounds.y - loadMargin && + iy < bounds.y + bounds.height + loadMargin; (item.displayObject as any).onVisibilityChange(inView); } } diff --git a/frontend/src/canvas/sprites/ImageSprite.ts b/frontend/src/canvas/sprites/ImageSprite.ts index 9146677..28d885d 100644 --- a/frontend/src/canvas/sprites/ImageSprite.ts +++ b/frontend/src/canvas/sprites/ImageSprite.ts @@ -4,6 +4,9 @@ import { TextureManager } from "../TextureManager"; /** * A Container holding a shadow graphic + sprite with lazy texture loading. * Uses a lightweight Graphics shadow instead of DropShadowFilter (GPU-heavy). + * + * Textures are loaded/unloaded by the viewport culling system (PixiCanvas). + * The constructor does NOT start loading — call loadTexture() when near viewport. */ // Shadow defaults (resting state) const SHADOW_REST = { offsetX: 3, offsetY: 3, alpha: 0.2 }; @@ -14,7 +17,7 @@ export class ImageSprite extends Container { readonly assetKey: string; private textures: TextureManager; - private loaded = false; + loaded = false; private loading = false; private placeholder: Graphics | null = null; private _sprite: Sprite; @@ -46,14 +49,13 @@ export class ImageSprite extends Container { this._sprite.height = h; this.addChild(this._sprite); - // Create placeholder: dark rect shown until first texture loads + // Placeholder: dark rect shown until texture is loaded by culling system const placeholder = new Graphics(); placeholder.rect(0, 0, w, h).fill(0x2a2a2a); this.placeholder = placeholder; this.addChild(placeholder); - // Immediately start loading the full texture - this.loadTexture(); + // NOTE: texture loading is deferred — PixiCanvas culling ticker calls loadTexture() } private _drawShadow(cfg: { offsetX: number; offsetY: number; alpha: number }): void { @@ -72,13 +74,19 @@ export class ImageSprite extends Container { this._drawShadow(SHADOW_REST); } - /** Load the full-res texture. GPU handles scaling natively. */ + /** The underlying sprite's texture (used by clipboard for resolution detection). */ + get texture(): Texture { + return this._sprite.texture; + } + + /** Load the full-res texture. Called by viewport culling when near viewport. */ async loadTexture(): Promise { if (this.loaded || this.loading) return; this.loading = true; try { const tex = await this.textures.load(this.assetKey); + if (this.destroyed) return; // component may have been removed while loading this._sprite.texture = tex; this._sprite.width = this._naturalWidth; this._sprite.height = this._naturalHeight; @@ -99,4 +107,21 @@ export class ImageSprite extends Container { this.loading = false; } } + + /** Unload texture to free GPU memory. Called by viewport culling when far from viewport. */ + unloadTexture(): void { + if (!this.loaded) return; + + this.textures.unload(this.assetKey); + this._sprite.texture = Texture.EMPTY; + this.loaded = false; + + // Restore placeholder + if (!this.placeholder) { + const placeholder = new Graphics(); + placeholder.rect(0, 0, this._naturalWidth, this._naturalHeight).fill(0x2a2a2a); + this.placeholder = placeholder; + this.addChild(placeholder); + } + } }