feat: media processing pipeline, spatial indexing, canvas-based video rendering

- Background media worker: polls media_jobs table, runs ffprobe+ffmpeg
  with concurrency limit, generates video posters, emits socket events
- Non-blocking video upload: stores file + enqueues job, returns immediately
- Poster hydration on board load: GET /boards/:id injects poster/dimensions
  from DB into canvas_state video objects
- SpatialGrid: fixed-cell (512px) spatial hash for O(nearby) culling instead
  of O(all) item scanning, eliminates setTimeout violations
- Canvas-based video rendering: draws video frames to offscreen canvas then
  uploads to GPU, completely eliminates GL_INVALID_OPERATION errors from
  PixiJS VideoSource auto-update mechanism
- Server poster upgrade path: culling ticker and applyProcessedMedia() both
  upgrade client-captured posters to server posters when available
- Pause restores server poster (paused video behaves like an image)
- Selection drag-end persistence: onObjectDragEnd broadcasts + saves + undo
- Live media:job:update socket handler patches scene data + VideoSprite
  dimensions without broadcast fanout
This commit is contained in:
Hiren Kangad
2026-03-10 14:30:01 +05:30
parent fdcee3536f
commit de418b2ba5
12 changed files with 761 additions and 110 deletions
+38 -1
View File
@@ -7,6 +7,7 @@ import { setupDragDrop, setupPaste } from '../canvas/image-drop';
import { UndoManager } from '../canvas/history';
import { InboxZone } from '../canvas/InboxZone';
import { LaserPointer } from '../canvas/LaserPointer';
import { VideoSprite } from '../canvas/sprites/VideoSprite';
// PresenceOverlay removed — remote selection highlighting was too heavy for minimal benefit
import { connectSocket, disconnectSocket } from '../socket';
@@ -137,8 +138,11 @@ export function useCanvasSetup(deps: CanvasSetupDeps) {
selection.transformBox.onItemTransform = (item) => {
syncRef.current?.broadcastTransform(item);
};
selection.onObjectDragEnd = (itemIds) => {
onCanvasChange(itemIds); // broadcasts elements + saves + undo + spatial refresh
};
selection.transformBox.onDragEnd = (itemIds) => {
onCanvasChange(itemIds); // broadcasts elements + saves + undo
onCanvasChange(itemIds); // broadcasts elements + saves + undo + spatial refresh
};
socket.on('user:joined', (data: any) => {
@@ -176,6 +180,39 @@ export function useCanvasSetup(deps: CanvasSetupDeps) {
}
});
// Media processing pipeline: upgrade videos when poster/metadata arrives
socket.on('media:job:update', (data: any) => {
if (!data || data.status !== 'done') return;
const { imageId, posterAssetKey, nativeWidth, nativeHeight, duration } = data;
if (!imageId) return;
// Find the video item by matching its DB image ID stored in scene data
for (const item of scene.items.values()) {
if (item.data.type !== 'video') continue;
// The asset key contains the imageId (e.g. boards/{boardId}/{imageId}.mp4)
if (!item.data.asset?.includes(imageId)) continue;
// Update scene data model
const vidData = item.data as any;
if (posterAssetKey) vidData.poster = posterAssetKey;
if (nativeWidth) { vidData.nativeW = nativeWidth; vidData.w = nativeWidth; }
if (nativeHeight) { vidData.nativeH = nativeHeight; vidData.h = nativeHeight; }
if (duration) vidData.duration = duration;
// Update rendered VideoSprite (dimensions, shadow, overlay, poster key)
if (item.displayObject instanceof VideoSprite) {
item.displayObject.applyProcessedMedia({ posterAssetKey, nativeWidth, nativeHeight });
}
// Update spatial index for changed dimensions
scene.updateSpatialEntry(item);
// No broadcastElements here — all clients receive the same
// media:job:update from the server. Echoing would cause fanout churn.
break;
}
});
// Cursor tracking — throttled to ~30fps to avoid flooding the socket
let cursorTimer: ReturnType<typeof setTimeout> | null = null;
const onPointerMove = (e: any) => {