feat: media processing pipeline, spatial indexing, canvas-based video rendering
- Background media worker: polls media_jobs table, runs ffprobe+ffmpeg with concurrency limit, generates video posters, emits socket events - Non-blocking video upload: stores file + enqueues job, returns immediately - Poster hydration on board load: GET /boards/:id injects poster/dimensions from DB into canvas_state video objects - SpatialGrid: fixed-cell (512px) spatial hash for O(nearby) culling instead of O(all) item scanning, eliminates setTimeout violations - Canvas-based video rendering: draws video frames to offscreen canvas then uploads to GPU, completely eliminates GL_INVALID_OPERATION errors from PixiJS VideoSource auto-update mechanism - Server poster upgrade path: culling ticker and applyProcessedMedia() both upgrade client-captured posters to server posters when available - Pause restores server poster (paused video behaves like an image) - Selection drag-end persistence: onObjectDragEnd broadcasts + saves + undo - Live media:job:update socket handler patches scene data + VideoSprite dimensions without broadcast fanout
This commit is contained in:
Hiren Kangad
@@ -175,68 +175,77 @@ const PixiCanvas = forwardRef<PixiCanvasHandle, PixiCanvasProps>(
|
||||
const MAX_POSTER_VIDEOS = 4; // max poster textures in GPU (tier 2)
|
||||
|
||||
let lastCullCheck = 0;
|
||||
// Track loaded resources so we can unload without scanning all items
|
||||
const loadedImages = new Set<string>(); // IDs of images with loaded textures
|
||||
const loadedVideos = new Set<string>(); // IDs of videos with init/poster/playing state
|
||||
app.ticker.add((ticker) => {
|
||||
lastCullCheck += ticker.deltaMS;
|
||||
if (lastCullCheck < 200) return;
|
||||
if (lastCullCheck < 250) return;
|
||||
lastCullCheck = 0;
|
||||
|
||||
const bounds = viewport.getVisibleBounds();
|
||||
const loadMargin = Math.max(bounds.width, bounds.height);
|
||||
const loadMargin = Math.max(bounds.width, bounds.height) * 0.5;
|
||||
const vcx = bounds.x + bounds.width / 2;
|
||||
const vcy = bounds.y + bounds.height / 2;
|
||||
|
||||
const imageItems: { item: SceneItem; sprite: ImageSprite; dist: number; near: boolean }[] = [];
|
||||
const videoItems: { item: SceneItem; sprite: VideoSprite; dist: number; near: boolean }[] = [];
|
||||
|
||||
for (const item of scene.getAllItems()) {
|
||||
const { x: ix, y: iy, w: iw, h: ih } = getItemWorldBounds(item);
|
||||
const cx = ix + iw / 2;
|
||||
const cy = iy + ih / 2;
|
||||
// Spatial query: only check items within the extended viewport region
|
||||
const qx = bounds.x - loadMargin;
|
||||
const qy = bounds.y - loadMargin;
|
||||
const qw = bounds.width + loadMargin * 2;
|
||||
const qh = bounds.height + loadMargin * 2;
|
||||
const nearbyItems = scene.queryRegion(qx, qy, qw, qh);
|
||||
|
||||
for (const item of nearbyItems) {
|
||||
const cx = item.data.x + (item.data.w * Math.abs(item.data.sx)) / 2;
|
||||
const cy = item.data.y + (item.data.h * Math.abs(item.data.sy)) / 2;
|
||||
const dist = (cx - vcx) ** 2 + (cy - vcy) ** 2;
|
||||
const near =
|
||||
ix + iw > bounds.x - loadMargin &&
|
||||
ix < bounds.x + bounds.width + loadMargin &&
|
||||
iy + ih > bounds.y - loadMargin &&
|
||||
iy < bounds.y + bounds.height + loadMargin;
|
||||
|
||||
if (item.type === 'image' && item.displayObject instanceof ImageSprite) {
|
||||
imageItems.push({ item, sprite: item.displayObject, dist, near });
|
||||
imageItems.push({ item, sprite: item.displayObject, dist, near: true });
|
||||
} else if (item.type === 'video' && item.displayObject instanceof VideoSprite) {
|
||||
videoItems.push({ item, sprite: item.displayObject, dist, near });
|
||||
videoItems.push({ item, sprite: item.displayObject, dist, near: true });
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Image budget management ----
|
||||
imageItems.sort((a, b) => a.dist - b.dist);
|
||||
const nearImages = imageItems.filter(i => i.near);
|
||||
const shouldLoadImage = new Set(
|
||||
nearImages.slice(0, MAX_LOADED_TEXTURES).map(i => i.item.id)
|
||||
imageItems.slice(0, MAX_LOADED_TEXTURES).map(i => i.item.id)
|
||||
);
|
||||
|
||||
let imgLoadsThisTick = 0;
|
||||
for (const { item, sprite } of imageItems) {
|
||||
if (shouldLoadImage.has(item.id) && !sprite.loaded && imgLoadsThisTick < 5) {
|
||||
if (shouldLoadImage.has(item.id) && !sprite.loaded && imgLoadsThisTick < 3) {
|
||||
sprite.loadTexture();
|
||||
loadedImages.add(item.id);
|
||||
imgLoadsThisTick++;
|
||||
}
|
||||
}
|
||||
for (const { item, sprite, near } of imageItems) {
|
||||
if (!sprite.loaded) continue;
|
||||
if (!near || !shouldLoadImage.has(item.id)) {
|
||||
|
||||
// Unload images that left the viewport (check tracked set, not all items)
|
||||
for (const id of loadedImages) {
|
||||
if (shouldLoadImage.has(id)) continue;
|
||||
const item = scene.items.get(id);
|
||||
if (!item) { loadedImages.delete(id); continue; }
|
||||
const sprite = item.displayObject;
|
||||
if (sprite instanceof ImageSprite && sprite.loaded) {
|
||||
sprite.unloadTexture();
|
||||
}
|
||||
loadedImages.delete(id);
|
||||
}
|
||||
|
||||
// ---- Video budget management ----
|
||||
videoItems.sort((a, b) => a.dist - b.dist);
|
||||
|
||||
// Determine budget sets (sorted by distance, nearest first)
|
||||
const nearVideos = videoItems.filter(i => i.near);
|
||||
const shouldInit = new Set(
|
||||
nearVideos.slice(0, MAX_INITIALIZED_VIDEOS).map(i => i.item.id)
|
||||
videoItems.slice(0, MAX_INITIALIZED_VIDEOS).map(i => i.item.id)
|
||||
);
|
||||
const shouldPoster = new Set(
|
||||
nearVideos.slice(0, MAX_POSTER_VIDEOS).map(i => i.item.id)
|
||||
videoItems.slice(0, MAX_POSTER_VIDEOS).map(i => i.item.id)
|
||||
);
|
||||
|
||||
// Initialize nearest videos within budget
|
||||
@@ -244,40 +253,38 @@ const PixiCanvas = forwardRef<PixiCanvasHandle, PixiCanvasProps>(
|
||||
for (const { item, sprite } of videoItems) {
|
||||
if (shouldInit.has(item.id) && !sprite.isInitialized && vidInitsThisTick < 2) {
|
||||
sprite.initVideo();
|
||||
loadedVideos.add(item.id);
|
||||
vidInitsThisTick++;
|
||||
}
|
||||
}
|
||||
|
||||
// Load posters for nearest subset within budget
|
||||
for (const { item, sprite } of videoItems) {
|
||||
if (!shouldPoster.has(item.id) || sprite.hasPoster) continue;
|
||||
if (sprite.hasServerPoster) {
|
||||
// Server poster: load as image texture (no <video> needed)
|
||||
sprite.loadServerPoster();
|
||||
} else if (sprite.isInitialized) {
|
||||
// Fallback: client-capture from video element
|
||||
sprite.capturePoster();
|
||||
// Load posters within budget
|
||||
// Prefer server poster over client capture; upgrade if server poster becomes available
|
||||
if (shouldPoster.has(item.id)) {
|
||||
if (!sprite.hasPoster) {
|
||||
if (sprite.hasServerPoster) {
|
||||
sprite.loadServerPoster();
|
||||
} else if (sprite.isInitialized) {
|
||||
sprite.capturePoster();
|
||||
}
|
||||
} else if (sprite.hasServerPoster && !sprite.isServerPosterLoaded) {
|
||||
// Client poster exists but server poster now available — upgrade
|
||||
sprite.loadServerPoster();
|
||||
}
|
||||
loadedVideos.add(item.id);
|
||||
}
|
||||
}
|
||||
|
||||
// Tear down videos outside budgets (farthest first = reverse iteration)
|
||||
for (let i = videoItems.length - 1; i >= 0; i--) {
|
||||
const { item, sprite, near } = videoItems[i];
|
||||
// Tear down videos that left the viewport (check tracked set)
|
||||
for (const id of loadedVideos) {
|
||||
if (shouldInit.has(id) || shouldPoster.has(id)) continue;
|
||||
const item = scene.items.get(id);
|
||||
if (!item) { loadedVideos.delete(id); continue; }
|
||||
const sprite = item.displayObject;
|
||||
if (!(sprite instanceof VideoSprite)) { loadedVideos.delete(id); continue; }
|
||||
|
||||
// Pause offscreen playing videos
|
||||
if (!near && sprite.isPlaying) {
|
||||
sprite.onVisibilityChange(false);
|
||||
}
|
||||
|
||||
// Destroy poster if outside poster budget
|
||||
if (!shouldPoster.has(item.id) && sprite.hasPoster && !sprite.isPlaying) {
|
||||
sprite.destroyPoster();
|
||||
}
|
||||
|
||||
// Tear down video element if outside init budget
|
||||
if (!shouldInit.has(item.id) && sprite.isInitialized && !sprite.isPlaying) {
|
||||
sprite.teardownVideo();
|
||||
}
|
||||
if (sprite.isPlaying) sprite.onVisibilityChange(false);
|
||||
if (sprite.hasPoster && !sprite.isPlaying) sprite.destroyPoster();
|
||||
if (sprite.isInitialized && !sprite.isPlaying) sprite.teardownVideo();
|
||||
loadedVideos.delete(id);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ import { DrawingSprite } from './sprites/DrawingSprite';
|
||||
import { FrameSprite } from './sprites/FrameSprite';
|
||||
import { SpringManager, Spring, PRESETS } from './spring';
|
||||
import { reparentGroupChildren } from './grouping';
|
||||
import { SpatialGrid } from './SpatialGrid';
|
||||
import type {
|
||||
SceneData,
|
||||
AnySceneObject,
|
||||
@@ -140,6 +141,7 @@ export class SceneManager {
|
||||
readonly viewport: Viewport;
|
||||
readonly textures: TextureManager;
|
||||
readonly springs: SpringManager;
|
||||
readonly spatialGrid: SpatialGrid<SceneItem> = new SpatialGrid<SceneItem>(512);
|
||||
|
||||
private _onChange: (() => void) | null = null;
|
||||
private _onItemDimensionsChanged: ((itemId: string) => void) | null = null;
|
||||
@@ -151,6 +153,17 @@ export class SceneManager {
|
||||
this.springs = springs;
|
||||
}
|
||||
|
||||
/** Update an item's spatial index entry from its current data. */
|
||||
updateSpatialEntry(item: SceneItem): void {
|
||||
const { x, y, w, h } = getItemWorldBounds(item);
|
||||
this.spatialGrid.upsert(item.id, item, x, y, w, h);
|
||||
}
|
||||
|
||||
/** Query items overlapping the given world rectangle. */
|
||||
queryRegion(rx: number, ry: number, rw: number, rh: number): SceneItem[] {
|
||||
return this.spatialGrid.query(rx, ry, rw, rh).map(e => e.data);
|
||||
}
|
||||
|
||||
// -- onChange callback ----------------------------------------------------
|
||||
|
||||
set onChange(fn: (() => void) | null) {
|
||||
@@ -220,6 +233,7 @@ export class SceneManager {
|
||||
}
|
||||
|
||||
item.displayObject.destroy({ children: true });
|
||||
this.spatialGrid.remove(id);
|
||||
this.items.delete(id);
|
||||
}
|
||||
|
||||
@@ -332,6 +346,7 @@ export class SceneManager {
|
||||
};
|
||||
|
||||
this.items.set(data.id, item);
|
||||
this.updateSpatialEntry(item);
|
||||
|
||||
// Wire video dimension auto-correction to the STORED item.data (not the original param)
|
||||
if (data.type === 'video' && displayObject instanceof VideoSprite) {
|
||||
@@ -429,6 +444,7 @@ export class SceneManager {
|
||||
// Update stored data
|
||||
item.data = { ...data };
|
||||
item.type = data.type;
|
||||
this.updateSpatialEntry(item);
|
||||
}
|
||||
|
||||
// -- Z-Ordering ----------------------------------------------------------
|
||||
@@ -505,12 +521,14 @@ export class SceneManager {
|
||||
if (item.data.type === 'group') {
|
||||
const groupData = item.data as import('./scene-format').GroupObject;
|
||||
for (const childId of groupData.children) {
|
||||
this.spatialGrid.remove(childId);
|
||||
this.items.delete(childId);
|
||||
}
|
||||
}
|
||||
|
||||
if (!animate) {
|
||||
item.displayObject.destroy({ children: true });
|
||||
this.spatialGrid.remove(id);
|
||||
this.items.delete(id);
|
||||
this._onChange?.();
|
||||
return;
|
||||
@@ -519,6 +537,7 @@ export class SceneManager {
|
||||
const obj = item.displayObject;
|
||||
|
||||
// Remove from map immediately to prevent double-remove
|
||||
this.spatialGrid.remove(id);
|
||||
this.items.delete(id);
|
||||
|
||||
// Scale toward center on removal
|
||||
@@ -710,6 +729,7 @@ export class SceneManager {
|
||||
data: groupData,
|
||||
};
|
||||
this.items.set(groupData.id, groupItem);
|
||||
this.updateSpatialEntry(groupItem);
|
||||
|
||||
// Animate each child ~25px toward center, then reparent into container
|
||||
const CONVERGE_PX = 25;
|
||||
@@ -842,6 +862,7 @@ export class SceneManager {
|
||||
if (!container.destroyed) {
|
||||
container.destroy();
|
||||
}
|
||||
this.spatialGrid.remove(groupId);
|
||||
this.items.delete(groupId);
|
||||
|
||||
this._applyZOrder();
|
||||
|
||||
@@ -41,6 +41,7 @@ export class SelectionManager {
|
||||
private _onSelectionChange: ((ids: string[]) => void) | null = null;
|
||||
private _onItemTransform: ((item: SceneItem) => void) | null = null;
|
||||
private _onItemsTransform: ((items: SceneItem[]) => void) | null = null;
|
||||
private _onObjectDragEnd: ((ids: string[]) => void) | null = null;
|
||||
|
||||
// Pointer state
|
||||
private _pointerDown = false;
|
||||
@@ -122,6 +123,11 @@ export class SelectionManager {
|
||||
this._onItemsTransform = fn;
|
||||
}
|
||||
|
||||
/** Called when object drag ends (for persist/save/spatial refresh). */
|
||||
set onObjectDragEnd(fn: (ids: string[]) => void) {
|
||||
this._onObjectDragEnd = fn;
|
||||
}
|
||||
|
||||
/** Called on double-click of a text item (for inline editing). */
|
||||
set onDoubleClickText(fn: (item: SceneItem) => void) {
|
||||
this._onDoubleClickText = fn;
|
||||
@@ -313,6 +319,7 @@ export class SelectionManager {
|
||||
|
||||
// Notify change (positions updated)
|
||||
this._emitChange();
|
||||
this._onObjectDragEnd?.(Array.from(this.selectedIds));
|
||||
} else if (this._rubberBanding) {
|
||||
// Finish rubber band selection
|
||||
const currentWorld = this._viewport.toWorld(e.global.x, e.global.y);
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
/**
|
||||
* SpatialGrid — fixed-cell spatial index for fast region queries.
|
||||
*
|
||||
* Each item occupies one or more grid cells based on its axis-aligned bounds.
|
||||
* Query returns all items whose cells overlap the query rectangle.
|
||||
*
|
||||
* Cell size should be ~1-2x the median item size for best performance.
|
||||
* Too small = many cells per item. Too large = too many items per cell.
|
||||
*/
|
||||
|
||||
export interface SpatialEntry<T> {
|
||||
id: string;
|
||||
data: T;
|
||||
x: number;
|
||||
y: number;
|
||||
w: number;
|
||||
h: number;
|
||||
}
|
||||
|
||||
export class SpatialGrid<T> {
|
||||
private cellSize: number;
|
||||
private cells: Map<string, Set<string>> = new Map();
|
||||
private entries: Map<string, SpatialEntry<T>> = new Map();
|
||||
|
||||
constructor(cellSize = 512) {
|
||||
this.cellSize = cellSize;
|
||||
}
|
||||
|
||||
/** Insert or update an entry. */
|
||||
upsert(id: string, data: T, x: number, y: number, w: number, h: number): void {
|
||||
// Remove old cells if updating
|
||||
if (this.entries.has(id)) {
|
||||
this._removeCells(id);
|
||||
}
|
||||
|
||||
const entry: SpatialEntry<T> = { id, data, x, y, w, h };
|
||||
this.entries.set(id, entry);
|
||||
this._insertCells(id, x, y, w, h);
|
||||
}
|
||||
|
||||
/** Remove an entry. */
|
||||
remove(id: string): void {
|
||||
if (!this.entries.has(id)) return;
|
||||
this._removeCells(id);
|
||||
this.entries.delete(id);
|
||||
}
|
||||
|
||||
/** Query all entries that overlap the given rectangle. */
|
||||
query(rx: number, ry: number, rw: number, rh: number): SpatialEntry<T>[] {
|
||||
const seen = new Set<string>();
|
||||
const results: SpatialEntry<T>[] = [];
|
||||
|
||||
const c0 = Math.floor(rx / this.cellSize);
|
||||
const r0 = Math.floor(ry / this.cellSize);
|
||||
const c1 = Math.floor((rx + rw) / this.cellSize);
|
||||
const r1 = Math.floor((ry + rh) / this.cellSize);
|
||||
|
||||
for (let r = r0; r <= r1; r++) {
|
||||
for (let c = c0; c <= c1; c++) {
|
||||
const key = `${c},${r}`;
|
||||
const cell = this.cells.get(key);
|
||||
if (!cell) continue;
|
||||
|
||||
for (const id of cell) {
|
||||
if (seen.has(id)) continue;
|
||||
seen.add(id);
|
||||
|
||||
const entry = this.entries.get(id)!;
|
||||
// AABB overlap check
|
||||
if (
|
||||
entry.x < rx + rw &&
|
||||
entry.x + entry.w > rx &&
|
||||
entry.y < ry + rh &&
|
||||
entry.y + entry.h > ry
|
||||
) {
|
||||
results.push(entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/** Get a specific entry by ID. */
|
||||
get(id: string): SpatialEntry<T> | undefined {
|
||||
return this.entries.get(id);
|
||||
}
|
||||
|
||||
/** Number of entries in the index. */
|
||||
get size(): number {
|
||||
return this.entries.size;
|
||||
}
|
||||
|
||||
/** Clear the entire index. */
|
||||
clear(): void {
|
||||
this.cells.clear();
|
||||
this.entries.clear();
|
||||
}
|
||||
|
||||
// ---- Internal ----
|
||||
|
||||
private _cellKeys(x: number, y: number, w: number, h: number): string[] {
|
||||
const c0 = Math.floor(x / this.cellSize);
|
||||
const r0 = Math.floor(y / this.cellSize);
|
||||
const c1 = Math.floor((x + w) / this.cellSize);
|
||||
const r1 = Math.floor((y + h) / this.cellSize);
|
||||
|
||||
const keys: string[] = [];
|
||||
for (let r = r0; r <= r1; r++) {
|
||||
for (let c = c0; c <= c1; c++) {
|
||||
keys.push(`${c},${r}`);
|
||||
}
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
private _insertCells(id: string, x: number, y: number, w: number, h: number): void {
|
||||
for (const key of this._cellKeys(x, y, w, h)) {
|
||||
let cell = this.cells.get(key);
|
||||
if (!cell) {
|
||||
cell = new Set();
|
||||
this.cells.set(key, cell);
|
||||
}
|
||||
cell.add(id);
|
||||
}
|
||||
}
|
||||
|
||||
private _removeCells(id: string): void {
|
||||
const entry = this.entries.get(id);
|
||||
if (!entry) return;
|
||||
for (const key of this._cellKeys(entry.x, entry.y, entry.w, entry.h)) {
|
||||
const cell = this.cells.get(key);
|
||||
if (cell) {
|
||||
cell.delete(id);
|
||||
if (cell.size === 0) this.cells.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,8 +11,10 @@ import { TextureManager } from '../TextureManager';
|
||||
* Tier 2 — client-captured poster from video frame (fallback if no server poster)
|
||||
* Tier 3 — actively playing with live video texture (user-initiated)
|
||||
*
|
||||
* If the server provides a poster asset key at upload time, the video renders
|
||||
* as a plain image until explicitly played. No <video> element needed for thumbnails.
|
||||
* Playback uses a canvas intermediary: video frames are drawn to an offscreen
|
||||
* canvas, then uploaded to GPU via a standard image texture. This avoids
|
||||
* GL_INVALID_OPERATION errors from PixiJS's VideoSource auto-update mechanism
|
||||
* which tries to copy video frames before the GPU texture is allocated.
|
||||
*/
|
||||
|
||||
const MAX_PLAYING_VIDEOS = 1;
|
||||
@@ -24,7 +26,7 @@ const SHADOW_LIFT = { offsetX: 6, offsetY: 8, alpha: 0.3 };
|
||||
export class VideoSprite extends Container {
|
||||
readonly assetKey: string;
|
||||
readonly videoUrl: string;
|
||||
readonly posterAssetKey: string | null;
|
||||
posterAssetKey: string | null;
|
||||
|
||||
private textures: TextureManager | null;
|
||||
private videoEl: HTMLVideoElement | null = null;
|
||||
@@ -41,6 +43,11 @@ export class VideoSprite extends Container {
|
||||
private _isPlaying = false;
|
||||
private _videoInitialized = false;
|
||||
private _hasPoster = false;
|
||||
// Canvas-based video rendering (avoids VideoSource GL errors)
|
||||
private _frameCanvas: HTMLCanvasElement | null = null;
|
||||
private _frameCtx: CanvasRenderingContext2D | null = null;
|
||||
private _rafId: number | null = null;
|
||||
private _useRVFC = false; // true if requestVideoFrameCallback is available
|
||||
muted = true;
|
||||
loop = true;
|
||||
|
||||
@@ -66,6 +73,7 @@ export class VideoSprite extends Container {
|
||||
this.textures = textures ?? null;
|
||||
this._naturalWidth = w;
|
||||
this._naturalHeight = h;
|
||||
this._useRVFC = 'requestVideoFrameCallback' in HTMLVideoElement.prototype;
|
||||
|
||||
// Shadow
|
||||
this._shadow = new Graphics();
|
||||
@@ -96,6 +104,50 @@ export class VideoSprite extends Container {
|
||||
/** Whether this video has a server-generated poster available (not yet loaded). */
|
||||
get hasServerPoster(): boolean { return !!this.posterAssetKey && !!this.textures; }
|
||||
|
||||
/** Whether the server-generated poster has been loaded (vs client-captured). */
|
||||
get isServerPosterLoaded(): boolean { return this._serverPosterLoaded; }
|
||||
|
||||
// ---- Tier 0: Apply processed media from background worker ---------------
|
||||
|
||||
/** Update dimensions + poster key from the media worker result.
|
||||
* Resizes all internal visuals and loads poster immediately if not playing. */
|
||||
applyProcessedMedia(opts: {
|
||||
posterAssetKey?: string | null;
|
||||
nativeWidth?: number | null;
|
||||
nativeHeight?: number | null;
|
||||
}): void {
|
||||
if (this.destroyed) return;
|
||||
|
||||
if (opts.posterAssetKey) {
|
||||
this.posterAssetKey = opts.posterAssetKey;
|
||||
}
|
||||
|
||||
const nw = opts.nativeWidth;
|
||||
const nh = opts.nativeHeight;
|
||||
if (nw && nh && nw > 0 && nh > 0 && (nw !== this._naturalWidth || nh !== this._naturalHeight)) {
|
||||
this._naturalWidth = nw;
|
||||
this._naturalHeight = nh;
|
||||
|
||||
if (this._sprite) {
|
||||
this._sprite.width = nw;
|
||||
this._sprite.height = nh;
|
||||
}
|
||||
if (this._placeholder) {
|
||||
this._placeholder.clear();
|
||||
this._placeholder.rect(0, 0, nw, nh).fill(0x2a2a2a);
|
||||
}
|
||||
this._drawShadow(SHADOW_REST);
|
||||
this._drawPlayIcon();
|
||||
this.onDimensionsKnown?.(nw, nh);
|
||||
}
|
||||
|
||||
// Load server poster if not playing and server poster not yet loaded.
|
||||
// This upgrades a client-captured poster (possibly black) to the real one.
|
||||
if (!this._isPlaying && !this._serverPosterLoaded && this.hasServerPoster) {
|
||||
this.loadServerPoster();
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Tier 0.5: Server-generated poster (loaded like an image) -----------
|
||||
|
||||
/** Load server poster. Called by culling system when within poster budget. */
|
||||
@@ -109,6 +161,11 @@ export class VideoSprite extends Container {
|
||||
return;
|
||||
}
|
||||
|
||||
// Clean up any existing client-captured poster before replacing
|
||||
if (this._hasPoster && this.posterTexture && !this._serverPosterLoaded) {
|
||||
this.posterTexture.destroy(true);
|
||||
}
|
||||
|
||||
this.posterTexture = tex;
|
||||
this._hasPoster = true;
|
||||
this._serverPosterLoaded = true;
|
||||
@@ -116,7 +173,6 @@ export class VideoSprite extends Container {
|
||||
this._ensureSprite(tex);
|
||||
this._removePlaceholder();
|
||||
} catch {
|
||||
// Server poster failed — will fall back to client capture if needed
|
||||
} finally {
|
||||
this._serverPosterLoading = false;
|
||||
}
|
||||
@@ -214,31 +270,67 @@ export class VideoSprite extends Container {
|
||||
this.videoEl.muted = this.muted;
|
||||
this.videoEl.loop = this.loop;
|
||||
|
||||
if (!this.videoTexture) {
|
||||
this.videoTexture = Texture.from(this.videoEl);
|
||||
}
|
||||
|
||||
// Drop poster while playing — don't keep both resident
|
||||
if (this._hasPoster && this.posterTexture) {
|
||||
if (this._serverPosterLoaded && this.posterAssetKey && this.textures) {
|
||||
this.textures.release(this.posterAssetKey);
|
||||
} else {
|
||||
this.posterTexture.destroy(true);
|
||||
}
|
||||
this.posterTexture = null;
|
||||
this._hasPoster = false;
|
||||
this._serverPosterLoaded = false;
|
||||
}
|
||||
|
||||
this._ensureSprite(this.videoTexture);
|
||||
this._removePlaceholder();
|
||||
|
||||
this._isPlaying = true;
|
||||
this._overlay.visible = false;
|
||||
activeVideos.add(this);
|
||||
this.onStateChange?.();
|
||||
|
||||
this.videoEl.play().catch(() => {
|
||||
const videoEl = this.videoEl!;
|
||||
|
||||
// Once a decoded frame is available, create a canvas-backed texture.
|
||||
// Drawing video → canvas → GPU avoids all VideoSource GL errors.
|
||||
const swapToVideoTexture = () => {
|
||||
if (this.destroyed || !this._isPlaying || !this.videoEl) return;
|
||||
|
||||
const vw = this.videoEl.videoWidth || this._naturalWidth;
|
||||
const vh = this.videoEl.videoHeight || this._naturalHeight;
|
||||
|
||||
if (!this.videoTexture) {
|
||||
// Create offscreen canvas and draw the first frame
|
||||
this._frameCanvas = document.createElement('canvas');
|
||||
this._frameCanvas.width = vw;
|
||||
this._frameCanvas.height = vh;
|
||||
this._frameCtx = this._frameCanvas.getContext('2d');
|
||||
if (this._frameCtx) {
|
||||
this._frameCtx.drawImage(this.videoEl, 0, 0, vw, vh);
|
||||
}
|
||||
// Create texture from canvas — always has valid pixel data
|
||||
this.videoTexture = Texture.from(this._frameCanvas);
|
||||
}
|
||||
|
||||
// Drop poster while playing — don't keep both resident
|
||||
if (this._hasPoster && this.posterTexture) {
|
||||
if (this._serverPosterLoaded && this.posterAssetKey && this.textures) {
|
||||
this.textures.release(this.posterAssetKey);
|
||||
} else {
|
||||
this.posterTexture.destroy(true);
|
||||
}
|
||||
this.posterTexture = null;
|
||||
this._hasPoster = false;
|
||||
this._serverPosterLoaded = false;
|
||||
}
|
||||
|
||||
this._ensureSprite(this.videoTexture);
|
||||
this._removePlaceholder();
|
||||
this._overlay.visible = false;
|
||||
|
||||
// Start manual frame update loop
|
||||
this._startFrameLoop();
|
||||
};
|
||||
|
||||
// Wait for a decoded frame before creating the GPU texture.
|
||||
let swapped = false;
|
||||
const safeSwap = () => { if (!swapped) { swapped = true; swapToVideoTexture(); } };
|
||||
|
||||
if (videoEl.readyState >= 4) {
|
||||
safeSwap();
|
||||
} else if (this._useRVFC) {
|
||||
(videoEl as any).requestVideoFrameCallback(safeSwap);
|
||||
} else {
|
||||
videoEl.addEventListener('playing', safeSwap, { once: true });
|
||||
}
|
||||
|
||||
videoEl.play().catch(() => {
|
||||
videoEl.removeEventListener('playing', safeSwap);
|
||||
this._isPlaying = false;
|
||||
this._overlay.visible = true;
|
||||
activeVideos.delete(this);
|
||||
@@ -252,6 +344,15 @@ export class VideoSprite extends Container {
|
||||
this._isPlaying = false;
|
||||
this._overlay.visible = true;
|
||||
activeVideos.delete(this);
|
||||
this._stopFrameLoop();
|
||||
|
||||
// Restore server poster — paused video behaves like an image.
|
||||
// Drop the live video texture to free GPU memory.
|
||||
if (this.hasServerPoster && !this._hasPoster) {
|
||||
this._destroyVideoTexture();
|
||||
this.loadServerPoster();
|
||||
}
|
||||
|
||||
this.onStateChange?.();
|
||||
}
|
||||
|
||||
@@ -440,7 +541,49 @@ export class VideoSprite extends Container {
|
||||
this.videoEl = null;
|
||||
}
|
||||
|
||||
/** Draw current video frame to offscreen canvas, then tell PixiJS to re-upload. */
|
||||
private _drawFrame(): void {
|
||||
if (!this._frameCtx || !this._frameCanvas || !this.videoEl || !this.videoTexture) return;
|
||||
this._frameCtx.drawImage(this.videoEl, 0, 0, this._frameCanvas.width, this._frameCanvas.height);
|
||||
this.videoTexture.source.update();
|
||||
}
|
||||
|
||||
private _startFrameLoop(): void {
|
||||
if (this._rafId !== null) return;
|
||||
const videoEl = this.videoEl;
|
||||
if (!videoEl) return;
|
||||
|
||||
if (this._useRVFC) {
|
||||
// requestVideoFrameCallback — fires only when a new decoded frame is ready
|
||||
const onFrame = () => {
|
||||
if (!this._isPlaying || this.destroyed) { this._rafId = null; return; }
|
||||
this._drawFrame();
|
||||
this._rafId = (videoEl as any).requestVideoFrameCallback(onFrame);
|
||||
};
|
||||
this._rafId = (videoEl as any).requestVideoFrameCallback(onFrame);
|
||||
} else {
|
||||
// Fallback: requestAnimationFrame
|
||||
const onFrame = () => {
|
||||
if (!this._isPlaying || this.destroyed) { this._rafId = null; return; }
|
||||
this._drawFrame();
|
||||
this._rafId = requestAnimationFrame(onFrame);
|
||||
};
|
||||
this._rafId = requestAnimationFrame(onFrame);
|
||||
}
|
||||
}
|
||||
|
||||
private _stopFrameLoop(): void {
|
||||
if (this._rafId === null) return;
|
||||
if (this._useRVFC && this.videoEl) {
|
||||
(this.videoEl as any).cancelVideoFrameCallback(this._rafId);
|
||||
} else {
|
||||
cancelAnimationFrame(this._rafId);
|
||||
}
|
||||
this._rafId = null;
|
||||
}
|
||||
|
||||
private _destroyVideoTexture(): void {
|
||||
this._stopFrameLoop();
|
||||
if (this.videoTexture) {
|
||||
if (this._sprite && this._sprite.texture === this.videoTexture) {
|
||||
this._removeSpriteFromTree();
|
||||
@@ -449,12 +592,17 @@ export class VideoSprite extends Container {
|
||||
this.videoTexture.destroy(true);
|
||||
this.videoTexture = null;
|
||||
}
|
||||
this._frameCanvas = null;
|
||||
this._frameCtx = null;
|
||||
}
|
||||
|
||||
destroy(options?: Parameters<Container['destroy']>[0]): void {
|
||||
this._stopFrameLoop();
|
||||
this.pause();
|
||||
this._destroyVideoEl();
|
||||
if (this.videoTexture) { this.videoTexture.destroy(true); this.videoTexture = null; }
|
||||
this._frameCanvas = null;
|
||||
this._frameCtx = null;
|
||||
if (this._hasPoster && this.posterTexture) {
|
||||
if (this._serverPosterLoaded && this.posterAssetKey && this.textures) {
|
||||
this.textures.release(this.posterAssetKey);
|
||||
|
||||
@@ -7,6 +7,7 @@ import { setupDragDrop, setupPaste } from '../canvas/image-drop';
|
||||
import { UndoManager } from '../canvas/history';
|
||||
import { InboxZone } from '../canvas/InboxZone';
|
||||
import { LaserPointer } from '../canvas/LaserPointer';
|
||||
import { VideoSprite } from '../canvas/sprites/VideoSprite';
|
||||
// PresenceOverlay removed — remote selection highlighting was too heavy for minimal benefit
|
||||
import { connectSocket, disconnectSocket } from '../socket';
|
||||
|
||||
@@ -137,8 +138,11 @@ export function useCanvasSetup(deps: CanvasSetupDeps) {
|
||||
selection.transformBox.onItemTransform = (item) => {
|
||||
syncRef.current?.broadcastTransform(item);
|
||||
};
|
||||
selection.onObjectDragEnd = (itemIds) => {
|
||||
onCanvasChange(itemIds); // broadcasts elements + saves + undo + spatial refresh
|
||||
};
|
||||
selection.transformBox.onDragEnd = (itemIds) => {
|
||||
onCanvasChange(itemIds); // broadcasts elements + saves + undo
|
||||
onCanvasChange(itemIds); // broadcasts elements + saves + undo + spatial refresh
|
||||
};
|
||||
|
||||
socket.on('user:joined', (data: any) => {
|
||||
@@ -176,6 +180,39 @@ export function useCanvasSetup(deps: CanvasSetupDeps) {
|
||||
}
|
||||
});
|
||||
|
||||
// Media processing pipeline: upgrade videos when poster/metadata arrives
|
||||
socket.on('media:job:update', (data: any) => {
|
||||
if (!data || data.status !== 'done') return;
|
||||
const { imageId, posterAssetKey, nativeWidth, nativeHeight, duration } = data;
|
||||
if (!imageId) return;
|
||||
|
||||
// Find the video item by matching its DB image ID stored in scene data
|
||||
for (const item of scene.items.values()) {
|
||||
if (item.data.type !== 'video') continue;
|
||||
// The asset key contains the imageId (e.g. boards/{boardId}/{imageId}.mp4)
|
||||
if (!item.data.asset?.includes(imageId)) continue;
|
||||
|
||||
// Update scene data model
|
||||
const vidData = item.data as any;
|
||||
if (posterAssetKey) vidData.poster = posterAssetKey;
|
||||
if (nativeWidth) { vidData.nativeW = nativeWidth; vidData.w = nativeWidth; }
|
||||
if (nativeHeight) { vidData.nativeH = nativeHeight; vidData.h = nativeHeight; }
|
||||
if (duration) vidData.duration = duration;
|
||||
|
||||
// Update rendered VideoSprite (dimensions, shadow, overlay, poster key)
|
||||
if (item.displayObject instanceof VideoSprite) {
|
||||
item.displayObject.applyProcessedMedia({ posterAssetKey, nativeWidth, nativeHeight });
|
||||
}
|
||||
|
||||
// Update spatial index for changed dimensions
|
||||
scene.updateSpatialEntry(item);
|
||||
|
||||
// No broadcastElements here — all clients receive the same
|
||||
// media:job:update from the server. Echoing would cause fanout churn.
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
// Cursor tracking — throttled to ~30fps to avoid flooding the socket
|
||||
let cursorTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
const onPointerMove = (e: any) => {
|
||||
|
||||
@@ -111,6 +111,14 @@ export default function Editor({ isPublicView }: EditorProps) {
|
||||
if (changedIds && changedIds.length > 0) {
|
||||
// Incremental: broadcast only changed elements
|
||||
syncRef.current?.broadcastElements(changedIds);
|
||||
// Keep spatial index in sync for all local mutations (align/flip/arrange/etc.)
|
||||
const scene = canvasRef.current?.getScene();
|
||||
if (scene) {
|
||||
for (const id of changedIds) {
|
||||
const item = scene.items.get(id);
|
||||
if (item) scene.updateSpatialEntry(item);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Full scene sync happens via debounced SceneManager.onChange chain in sync.ts
|
||||
if (undoRef.current && !undoRef.current.isLocked()) {
|
||||
|
||||
Reference in New Issue
Block a user