feat: media processing pipeline, spatial indexing, canvas-based video rendering

- Background media worker: polls media_jobs table, runs ffprobe+ffmpeg
  with concurrency limit, generates video posters, emits socket events
- Non-blocking video upload: stores file + enqueues job, returns immediately
- Poster hydration on board load: GET /boards/:id injects poster/dimensions
  from DB into canvas_state video objects
- SpatialGrid: fixed-cell (512px) spatial hash for O(nearby) culling instead
  of O(all) item scanning, eliminates setTimeout violations
- Canvas-based video rendering: draws video frames to offscreen canvas then
  uploads to GPU, completely eliminates GL_INVALID_OPERATION errors from
  PixiJS VideoSource auto-update mechanism
- Server poster upgrade path: culling ticker and applyProcessedMedia() both
  upgrade client-captured posters to server posters when available
- Pause restores server poster (paused video behaves like an image)
- Selection drag-end persistence: onObjectDragEnd broadcasts + saves + undo
- Live media:job:update socket handler patches scene data + VideoSprite
  dimensions without broadcast fanout

backend/db.js

@@ -95,6 +95,24 @@ db.exec(`
   );
 
   CREATE INDEX IF NOT EXISTS idx_board_channel_links_board ON board_channel_links(board_id);
+
+  CREATE TABLE IF NOT EXISTS media_jobs (
+    id TEXT PRIMARY KEY,
+    image_id TEXT NOT NULL REFERENCES images(id) ON DELETE CASCADE,
+    board_id TEXT NOT NULL,
+    type TEXT NOT NULL DEFAULT 'poster',
+    status TEXT NOT NULL DEFAULT 'queued',
+    progress REAL DEFAULT 0,
+    error TEXT,
+    attempts INTEGER NOT NULL DEFAULT 0,
+    result_json TEXT,
+    created_at TEXT NOT NULL DEFAULT (datetime('now')),
+    started_at TEXT,
+    finished_at TEXT
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_media_jobs_status ON media_jobs(status);
+  CREATE INDEX IF NOT EXISTS idx_media_jobs_image ON media_jobs(image_id);
 `);
 
 // Migrations — add columns to existing tables
@@ -123,6 +141,22 @@ try {
 } catch {
   db.exec("ALTER TABLE images ADD COLUMN mm_file_id TEXT");
 }
+try {
+  db.prepare("SELECT poster_asset_key FROM images LIMIT 0").get();
+} catch {
+  db.exec("ALTER TABLE images ADD COLUMN poster_asset_key TEXT");
+}
+try {
+  db.prepare("SELECT duration FROM images LIMIT 0").get();
+} catch {
+  db.exec("ALTER TABLE images ADD COLUMN duration REAL");
+}
+try {
+  db.prepare("SELECT native_width FROM images LIMIT 0").get();
+} catch {
+  db.exec("ALTER TABLE images ADD COLUMN native_width INTEGER");
+  db.exec("ALTER TABLE images ADD COLUMN native_height INTEGER");
+}
 
 // ---------------------
 // User helpers
@@ -406,6 +440,46 @@ function getImageByMmFileId(boardId, mmFileId) {
   return db.prepare('SELECT * FROM images WHERE board_id = ? AND mm_file_id = ?').get(boardId, mmFileId);
 }
 
+// ---------------------
+// Media Jobs
+// ---------------------
+function createMediaJob({ id, imageId, boardId, type }) {
+  db.prepare(`
+    INSERT INTO media_jobs (id, image_id, board_id, type, status)
+    VALUES (?, ?, ?, ?, 'queued')
+  `).run(id, imageId, boardId, type || 'poster');
+  return db.prepare('SELECT * FROM media_jobs WHERE id = ?').get(id);
+}
+
+function updateMediaJob(id, updates) {
+  const sets = [];
+  const values = [];
+  for (const [key, val] of Object.entries(updates)) {
+    const col = key.replace(/([A-Z])/g, '_$1').toLowerCase(); // camelCase → snake_case
+    sets.push(`${col} = ?`);
+    values.push(val);
+  }
+  if (sets.length === 0) return;
+  values.push(id);
+  db.prepare(`UPDATE media_jobs SET ${sets.join(', ')} WHERE id = ?`).run(...values);
+}
+
+function getMediaJob(id) {
+  return db.prepare('SELECT * FROM media_jobs WHERE id = ?').get(id);
+}
+
+function getPendingMediaJobs(limit = 10) {
+  return db.prepare('SELECT * FROM media_jobs WHERE status IN (?, ?) ORDER BY created_at ASC LIMIT ?')
+    .all('queued', 'retry', limit);
+}
+
+function updateImageMedia(imageId, { posterAssetKey, duration, nativeWidth, nativeHeight }) {
+  db.prepare(`
+    UPDATE images SET poster_asset_key = ?, duration = ?, native_width = ?, native_height = ?
+    WHERE id = ?
+  `).run(posterAssetKey || null, duration || null, nativeWidth || null, nativeHeight || null, imageId);
+}
+
 module.exports = {
   db,
   // Users
@@ -423,4 +497,6 @@ module.exports = {
   // Board–Channel Links
   createBoardChannelLink, getBoardChannelLinks, getBoardChannelLink, deleteBoardChannelLink,
   getAllBoardChannelLinks, getImageByMmFileId,
+  // Media Jobs
+  createMediaJob, updateMediaJob, getMediaJob, getPendingMediaJobs, updateImageMedia,
 };

backend/routes/boards.js

@@ -120,10 +120,33 @@ router.get('/:boardId', (req, res) => {
 
     const images = getBoardImages(board.id);
 
+    // Hydrate video objects with poster/dimensions from DB
+    // (worker may have finished after canvas was last saved)
+    let canvasState = board.canvas_state ? JSON.parse(board.canvas_state) : {};
+    if (canvasState.objects && Array.isArray(canvasState.objects)) {
+      const videoImages = new Map();
+      for (const img of images) {
+        if (img.media_type === 'video' && img.poster_asset_key) {
+          videoImages.set(img.asset_key, img);
+        }
+      }
+      if (videoImages.size > 0) {
+        for (const obj of canvasState.objects) {
+          if (obj.type !== 'video') continue;
+          const dbImg = videoImages.get(obj.asset);
+          if (!dbImg) continue;
+          if (!obj.poster && dbImg.poster_asset_key) obj.poster = dbImg.poster_asset_key;
+          if (!obj.nativeW && dbImg.native_width) { obj.nativeW = dbImg.native_width; obj.w = dbImg.native_width; }
+          if (!obj.nativeH && dbImg.native_height) { obj.nativeH = dbImg.native_height; obj.h = dbImg.native_height; }
+          if (!obj.duration && dbImg.duration) obj.duration = dbImg.duration;
+        }
+      }
+    }
+
     return res.json({
       board: {
         ...board,
-        canvas_state: board.canvas_state ? JSON.parse(board.canvas_state) : {},
+        canvas_state: canvasState,
       },
       collection: {
         id: collection.id,

backend/routes/upload.js

@@ -6,9 +6,8 @@ const https = require('https');
 const http = require('http');
 const { URL } = require('url');
 const { authMiddleware } = require('../auth');
-const { getBoard, getCollectionMember, createImage } = require('../db');
+const { getBoard, getCollectionMember, createImage, createMediaJob } = require('../db');
 const { putBuffer, getImageUrl, MIME_TO_EXT, MAX_FILE_SIZE } = require('../minio');
-const { probeVideo, extractPoster } = require('../video-utils');
 
 const router = Router();
 
@@ -91,8 +90,8 @@ function classifyMedia(mimeType) {
 
 /**
  * Upload a media file (image or video) to MinIO.
- * For videos: extracts poster frame + metadata via ffmpeg at upload time
- * so the board never needs a <video> element for thumbnails.
+ * Videos: stored immediately, poster/metadata extracted async by media-worker.
+ * Images: dimensions extracted inline (fast, no ffmpeg).
  */
 async function uploadMedia(boardId, imageId, buffer, mimetype) {
   const ext = MIME_TO_EXT[mimetype] || '.bin';
@@ -100,33 +99,15 @@ async function uploadMedia(boardId, imageId, buffer, mimetype) {
   await putBuffer(minioPath, buffer, mimetype);
 
   const isVideo = VIDEO_MIME_TYPES.includes(mimetype);
-  let width = null, height = null, duration = null, posterAssetKey = null;
+  let width = null, height = null;
 
-  if (isVideo) {
-    // Extract metadata and poster frame server-side
-    const [meta, posterBuf] = await Promise.all([
-      probeVideo(buffer),
-      extractPoster(buffer),
-    ]);
-
-    if (meta) {
-      width = meta.width;
-      height = meta.height;
-      duration = meta.duration;
-    }
-
-    if (posterBuf) {
-      const posterPath = `boards/${boardId}/${imageId}_poster.jpg`;
-      await putBuffer(posterPath, posterBuf, 'image/jpeg');
-      posterAssetKey = posterPath;
-    }
-  } else {
+  if (!isVideo) {
     const dims = await getImageDimensions(buffer, mimetype);
     width = dims.width;
     height = dims.height;
   }
 
-  return { assetKey: minioPath, minioPath, width, height, duration, posterAssetKey };
+  return { assetKey: minioPath, minioPath, width, height, isVideo };
 }
 
 /**
@@ -146,10 +127,10 @@ router.post('/boards/:boardId/images', upload.single('image'), async (req, res)
     const { buffer, originalname, mimetype, size } = req.file;
     const mediaType = classifyMedia(mimetype);
 
-    const { assetKey, minioPath, width, height, duration, posterAssetKey } = await uploadMedia(board.id, imageId, buffer, mimetype);
+    const { assetKey, minioPath, width, height, isVideo } = await uploadMedia(board.id, imageId, buffer, mimetype);
     const publicUrl = getImageUrl(minioPath);
 
-    // Save record
+    // Save image record first (media_jobs FK references images)
     const image = createImage({
       id: imageId,
       boardId: board.id,
@@ -165,6 +146,13 @@ router.post('/boards/:boardId/images', upload.single('image'), async (req, res)
       mediaType,
     });
 
+    // Enqueue background processing after image record exists
+    let jobId = null;
+    if (isVideo) {
+      jobId = uuidv4();
+      createMediaJob({ id: jobId, imageId, boardId: board.id, type: 'poster' });
+    }
+
     return res.status(201).json({
       id: image.id,
       url: publicUrl,
@@ -175,8 +163,8 @@ router.post('/boards/:boardId/images', upload.single('image'), async (req, res)
       mime_type: image.mime_type,
       asset_key: image.asset_key,
       media_type: image.media_type,
-      duration: duration || undefined,
-      poster_asset_key: posterAssetKey || undefined,
+      processing: jobId ? 'queued' : undefined,
+      job_id: jobId || undefined,
     });
   } catch (err) {
     if (err.code === 'LIMIT_FILE_SIZE') {
@@ -253,10 +241,10 @@ router.post('/boards/:boardId/images/from-url', async (req, res) => {
     const imageId = uuidv4();
     const mediaType = classifyMedia(mimeType);
 
-    const { assetKey, minioPath, width, height, duration, posterAssetKey } = await uploadMedia(board.id, imageId, buffer, mimeType);
+    const { assetKey, minioPath, width, height, isVideo } = await uploadMedia(board.id, imageId, buffer, mimeType);
     const publicUrl = getImageUrl(minioPath);
 
-    // Save record
+    // Save image record first (media_jobs FK references images)
     const image = createImage({
       id: imageId,
       boardId: board.id,
@@ -272,6 +260,13 @@ router.post('/boards/:boardId/images/from-url', async (req, res) => {
       mediaType,
     });
 
+    // Enqueue background processing after image record exists
+    let jobId = null;
+    if (isVideo) {
+      jobId = uuidv4();
+      createMediaJob({ id: jobId, imageId, boardId: board.id, type: 'poster' });
+    }
+
     return res.status(201).json({
       id: image.id,
       url: publicUrl,
@@ -282,8 +277,8 @@ router.post('/boards/:boardId/images/from-url', async (req, res) => {
       mime_type: image.mime_type,
       asset_key: image.asset_key,
       media_type: image.media_type,
-      duration: duration || undefined,
-      poster_asset_key: posterAssetKey || undefined,
+      processing: jobId ? 'queued' : undefined,
+      job_id: jobId || undefined,
     });
   } catch (err) {
     console.error('[upload] from-url error:', err);

backend/server.js

@@ -174,6 +174,14 @@ async function start() {
     console.error('[server] MM watcher failed to start:', err.message);
   }
 
+  // Start media processing worker
+  try {
+    const { startMediaWorker } = require('./services/media-worker');
+    startMediaWorker(io);
+  } catch (err) {
+    console.error('[server] Media worker failed to start:', err.message);
+  }
+
   server.listen(PORT, '0.0.0.0', () => {
     console.log(`[server] RefBoard backend listening on port ${PORT}`);
   });
@@ -188,6 +196,11 @@ function shutdown(signal) {
     stopWatcher();
   } catch {}
 
+  try {
+    const { stopMediaWorker } = require('./services/media-worker');
+    stopMediaWorker();
+  } catch {}
+
   io.close(() => {
     console.log('[server] Socket.IO closed');
   });

backend/services/media-worker.js

@@ -0,0 +1,176 @@
+/**
+ * media-worker.js — Background queue worker for video processing.
+ *
+ * Polls the media_jobs table for pending jobs and runs ffprobe + ffmpeg
+ * with a concurrency limit so uploads return instantly.
+ *
+ * Emits socket events so the frontend can upgrade placeholders in real-time.
+ */
+
+const {
+  getPendingMediaJobs,
+  updateMediaJob,
+  updateImageMedia,
+  getImage,
+} = require('../db');
+const { probeVideo, extractPoster } = require('../video-utils');
+const { putBuffer, minioClient, MINIO_BUCKET } = require('../minio');
+
+const POLL_INTERVAL_MS = 3000;
+const MAX_CONCURRENCY = 2;
+
+let io = null;
+let pollTimer = null;
+let activeCount = 0;
+let stopping = false;
+
+/**
+ * Start the media worker. Pass the Socket.IO server instance for notifications.
+ */
+function startMediaWorker(ioInstance) {
+  io = ioInstance;
+  stopping = false;
+  console.log('[media-worker] Started (poll=%dms, concurrency=%d)', POLL_INTERVAL_MS, MAX_CONCURRENCY);
+  poll();
+}
+
+function stopMediaWorker() {
+  stopping = true;
+  if (pollTimer) {
+    clearTimeout(pollTimer);
+    pollTimer = null;
+  }
+  console.log('[media-worker] Stopped');
+}
+
+function schedulePoll() {
+  if (stopping) return;
+  pollTimer = setTimeout(poll, POLL_INTERVAL_MS);
+}
+
+async function poll() {
+  if (stopping) return;
+
+  const slotsAvailable = MAX_CONCURRENCY - activeCount;
+  if (slotsAvailable <= 0) {
+    schedulePoll();
+    return;
+  }
+
+  try {
+    const jobs = getPendingMediaJobs(slotsAvailable);
+    for (const job of jobs) {
+      activeCount++;
+      processJob(job).finally(() => {
+        activeCount--;
+      });
+    }
+  } catch (err) {
+    console.error('[media-worker] Poll error:', err.message);
+  }
+
+  schedulePoll();
+}
+
+async function processJob(job) {
+  const jobId = job.id;
+  const imageId = job.image_id;
+  const boardId = job.board_id;
+
+  try {
+    updateMediaJob(jobId, { status: 'processing', startedAt: new Date().toISOString() });
+    emitJobUpdate(boardId, jobId, imageId, 'processing');
+
+    // Fetch the raw video from MinIO
+    const image = getImage(imageId);
+    if (!image) {
+      updateMediaJob(jobId, { status: 'failed', error: 'Image record not found' });
+      emitJobUpdate(boardId, jobId, imageId, 'failed');
+      return;
+    }
+
+    const videoBuffer = await fetchFromMinio(image.minio_path);
+    if (!videoBuffer) {
+      updateMediaJob(jobId, { status: 'failed', error: 'Failed to fetch video from storage' });
+      emitJobUpdate(boardId, jobId, imageId, 'failed');
+      return;
+    }
+
+    // Run ffprobe + ffmpeg in parallel
+    const [meta, posterBuf] = await Promise.all([
+      probeVideo(videoBuffer),
+      extractPoster(videoBuffer),
+    ]);
+
+    let posterAssetKey = null;
+    if (posterBuf) {
+      posterAssetKey = `boards/${boardId}/${imageId}_poster.jpg`;
+      await putBuffer(posterAssetKey, posterBuf, 'image/jpeg');
+    }
+
+    const nativeWidth = meta?.width || null;
+    const nativeHeight = meta?.height || null;
+    const duration = meta?.duration || null;
+
+    // Update the image record with processed media info
+    updateImageMedia(imageId, { posterAssetKey, duration, nativeWidth, nativeHeight });
+
+    updateMediaJob(jobId, {
+      status: 'done',
+      finishedAt: new Date().toISOString(),
+    });
+
+    emitJobUpdate(boardId, jobId, imageId, 'done', {
+      posterAssetKey,
+      nativeWidth,
+      nativeHeight,
+      duration,
+    });
+
+    console.log('[media-worker] Job %s done (image=%s, poster=%s)', jobId, imageId, !!posterAssetKey);
+  } catch (err) {
+    console.error('[media-worker] Job %s failed:', jobId, err.message);
+
+    const attempts = (job.attempts || 0) + 1;
+    if (attempts < 3) {
+      updateMediaJob(jobId, { status: 'retry', attempts, error: err.message });
+      emitJobUpdate(boardId, jobId, imageId, 'retry');
+    } else {
+      updateMediaJob(jobId, { status: 'failed', attempts, error: err.message });
+      emitJobUpdate(boardId, jobId, imageId, 'failed');
+    }
+  }
+}
+
+/**
+ * Fetch object from MinIO as a Buffer.
+ */
+async function fetchFromMinio(objectPath) {
+  try {
+    const stream = await minioClient.getObject(MINIO_BUCKET, objectPath);
+    const chunks = [];
+    for await (const chunk of stream) {
+      chunks.push(chunk);
+    }
+    return Buffer.concat(chunks);
+  } catch (err) {
+    console.warn('[media-worker] MinIO fetch failed:', err.message);
+    return null;
+  }
+}
+
+/**
+ * Emit a media job update to all sockets in the board room.
+ */
+function emitJobUpdate(boardId, jobId, imageId, status, result) {
+  if (!io) return;
+  const room = `board:${boardId}`;
+  io.to(room).emit('media:job:update', {
+    jobId,
+    imageId,
+    status,
+    ...(result || {}),
+  });
+}
+
+module.exports = { startMediaWorker, stopMediaWorker };