feat: media processing pipeline, spatial indexing, canvas-based video rendering

- Background media worker: polls media_jobs table, runs ffprobe+ffmpeg
  with concurrency limit, generates video posters, emits socket events
- Non-blocking video upload: stores file + enqueues job, returns immediately
- Poster hydration on board load: GET /boards/:id injects poster/dimensions
  from DB into canvas_state video objects
- SpatialGrid: fixed-cell (512px) spatial hash for O(nearby) culling instead
  of O(all) item scanning, eliminates setTimeout violations
- Canvas-based video rendering: draws video frames to offscreen canvas then
  uploads to GPU, completely eliminates GL_INVALID_OPERATION errors from
  PixiJS VideoSource auto-update mechanism
- Server poster upgrade path: culling ticker and applyProcessedMedia() both
  upgrade client-captured posters to server posters when available
- Pause restores server poster (paused video behaves like an image)
- Selection drag-end persistence: onObjectDragEnd broadcasts + saves + undo
- Live media:job:update socket handler patches scene data + VideoSprite
  dimensions without broadcast fanout
This commit is contained in:
Hiren Kangad
2026-03-10 14:30:01 +05:30
parent fdcee3536f
commit de418b2ba5
12 changed files with 761 additions and 110 deletions
+24 -1
View File
@@ -120,10 +120,33 @@ router.get('/:boardId', (req, res) => {
const images = getBoardImages(board.id);
// Hydrate video objects with poster/dimensions from DB
// (worker may have finished after canvas was last saved)
let canvasState = board.canvas_state ? JSON.parse(board.canvas_state) : {};
if (canvasState.objects && Array.isArray(canvasState.objects)) {
const videoImages = new Map();
for (const img of images) {
if (img.media_type === 'video' && img.poster_asset_key) {
videoImages.set(img.asset_key, img);
}
}
if (videoImages.size > 0) {
for (const obj of canvasState.objects) {
if (obj.type !== 'video') continue;
const dbImg = videoImages.get(obj.asset);
if (!dbImg) continue;
if (!obj.poster && dbImg.poster_asset_key) obj.poster = dbImg.poster_asset_key;
if (!obj.nativeW && dbImg.native_width) { obj.nativeW = dbImg.native_width; obj.w = dbImg.native_width; }
if (!obj.nativeH && dbImg.native_height) { obj.nativeH = dbImg.native_height; obj.h = dbImg.native_height; }
if (!obj.duration && dbImg.duration) obj.duration = dbImg.duration;
}
}
}
return res.json({
board: {
...board,
canvas_state: board.canvas_state ? JSON.parse(board.canvas_state) : {},
canvas_state: canvasState,
},
collection: {
id: collection.id,