feat: media processing pipeline, spatial indexing, canvas-based video rendering

- Background media worker: polls media_jobs table, runs ffprobe+ffmpeg
  with concurrency limit, generates video posters, emits socket events
- Non-blocking video upload: stores file + enqueues job, returns immediately
- Poster hydration on board load: GET /boards/:id injects poster/dimensions
  from DB into canvas_state video objects
- SpatialGrid: fixed-cell (512px) spatial hash for O(nearby) culling instead
  of O(all) item scanning, eliminates setTimeout violations
- Canvas-based video rendering: draws video frames to offscreen canvas then
  uploads to GPU, completely eliminates GL_INVALID_OPERATION errors from
  PixiJS VideoSource auto-update mechanism
- Server poster upgrade path: culling ticker and applyProcessedMedia() both
  upgrade client-captured posters to server posters when available
- Pause restores server poster (paused video behaves like an image)
- Selection drag-end persistence: onObjectDragEnd broadcasts + saves + undo
- Live media:job:update socket handler patches scene data + VideoSprite
  dimensions without broadcast fanout
This commit is contained in:
Hiren Kangad
2026-03-10 14:30:01 +05:30
parent fdcee3536f
commit de418b2ba5
12 changed files with 761 additions and 110 deletions
+76
View File
@@ -95,6 +95,24 @@ db.exec(`
);
CREATE INDEX IF NOT EXISTS idx_board_channel_links_board ON board_channel_links(board_id);
CREATE TABLE IF NOT EXISTS media_jobs (
id TEXT PRIMARY KEY,
image_id TEXT NOT NULL REFERENCES images(id) ON DELETE CASCADE,
board_id TEXT NOT NULL,
type TEXT NOT NULL DEFAULT 'poster',
status TEXT NOT NULL DEFAULT 'queued',
progress REAL DEFAULT 0,
error TEXT,
attempts INTEGER NOT NULL DEFAULT 0,
result_json TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
started_at TEXT,
finished_at TEXT
);
CREATE INDEX IF NOT EXISTS idx_media_jobs_status ON media_jobs(status);
CREATE INDEX IF NOT EXISTS idx_media_jobs_image ON media_jobs(image_id);
`);
// Migrations — add columns to existing tables
@@ -123,6 +141,22 @@ try {
} catch {
db.exec("ALTER TABLE images ADD COLUMN mm_file_id TEXT");
}
try {
db.prepare("SELECT poster_asset_key FROM images LIMIT 0").get();
} catch {
db.exec("ALTER TABLE images ADD COLUMN poster_asset_key TEXT");
}
try {
db.prepare("SELECT duration FROM images LIMIT 0").get();
} catch {
db.exec("ALTER TABLE images ADD COLUMN duration REAL");
}
try {
db.prepare("SELECT native_width FROM images LIMIT 0").get();
} catch {
db.exec("ALTER TABLE images ADD COLUMN native_width INTEGER");
db.exec("ALTER TABLE images ADD COLUMN native_height INTEGER");
}
// ---------------------
// User helpers
@@ -406,6 +440,46 @@ function getImageByMmFileId(boardId, mmFileId) {
return db.prepare('SELECT * FROM images WHERE board_id = ? AND mm_file_id = ?').get(boardId, mmFileId);
}
// ---------------------
// Media Jobs
// ---------------------
function createMediaJob({ id, imageId, boardId, type }) {
db.prepare(`
INSERT INTO media_jobs (id, image_id, board_id, type, status)
VALUES (?, ?, ?, ?, 'queued')
`).run(id, imageId, boardId, type || 'poster');
return db.prepare('SELECT * FROM media_jobs WHERE id = ?').get(id);
}
function updateMediaJob(id, updates) {
const sets = [];
const values = [];
for (const [key, val] of Object.entries(updates)) {
const col = key.replace(/([A-Z])/g, '_$1').toLowerCase(); // camelCase → snake_case
sets.push(`${col} = ?`);
values.push(val);
}
if (sets.length === 0) return;
values.push(id);
db.prepare(`UPDATE media_jobs SET ${sets.join(', ')} WHERE id = ?`).run(...values);
}
function getMediaJob(id) {
return db.prepare('SELECT * FROM media_jobs WHERE id = ?').get(id);
}
function getPendingMediaJobs(limit = 10) {
return db.prepare('SELECT * FROM media_jobs WHERE status IN (?, ?) ORDER BY created_at ASC LIMIT ?')
.all('queued', 'retry', limit);
}
function updateImageMedia(imageId, { posterAssetKey, duration, nativeWidth, nativeHeight }) {
db.prepare(`
UPDATE images SET poster_asset_key = ?, duration = ?, native_width = ?, native_height = ?
WHERE id = ?
`).run(posterAssetKey || null, duration || null, nativeWidth || null, nativeHeight || null, imageId);
}
module.exports = {
db,
// Users
@@ -423,4 +497,6 @@ module.exports = {
// BoardChannel Links
createBoardChannelLink, getBoardChannelLinks, getBoardChannelLink, deleteBoardChannelLink,
getAllBoardChannelLinks, getImageByMmFileId,
// Media Jobs
createMediaJob, updateMediaJob, getMediaJob, getPendingMediaJobs, updateImageMedia,
};