feat: media processing pipeline, spatial indexing, canvas-based video rendering
- Background media worker: polls media_jobs table, runs ffprobe+ffmpeg with concurrency limit, generates video posters, emits socket events - Non-blocking video upload: stores file + enqueues job, returns immediately - Poster hydration on board load: GET /boards/:id injects poster/dimensions from DB into canvas_state video objects - SpatialGrid: fixed-cell (512px) spatial hash for O(nearby) culling instead of O(all) item scanning, eliminates setTimeout violations - Canvas-based video rendering: draws video frames to offscreen canvas then uploads to GPU, completely eliminates GL_INVALID_OPERATION errors from PixiJS VideoSource auto-update mechanism - Server poster upgrade path: culling ticker and applyProcessedMedia() both upgrade client-captured posters to server posters when available - Pause restores server poster (paused video behaves like an image) - Selection drag-end persistence: onObjectDragEnd broadcasts + saves + undo - Live media:job:update socket handler patches scene data + VideoSprite dimensions without broadcast fanout
This commit is contained in:
@@ -7,6 +7,7 @@ import { setupDragDrop, setupPaste } from '../canvas/image-drop';
|
||||
import { UndoManager } from '../canvas/history';
|
||||
import { InboxZone } from '../canvas/InboxZone';
|
||||
import { LaserPointer } from '../canvas/LaserPointer';
|
||||
import { VideoSprite } from '../canvas/sprites/VideoSprite';
|
||||
// PresenceOverlay removed — remote selection highlighting was too heavy for minimal benefit
|
||||
import { connectSocket, disconnectSocket } from '../socket';
|
||||
|
||||
@@ -137,8 +138,11 @@ export function useCanvasSetup(deps: CanvasSetupDeps) {
|
||||
selection.transformBox.onItemTransform = (item) => {
|
||||
syncRef.current?.broadcastTransform(item);
|
||||
};
|
||||
selection.onObjectDragEnd = (itemIds) => {
|
||||
onCanvasChange(itemIds); // broadcasts elements + saves + undo + spatial refresh
|
||||
};
|
||||
selection.transformBox.onDragEnd = (itemIds) => {
|
||||
onCanvasChange(itemIds); // broadcasts elements + saves + undo
|
||||
onCanvasChange(itemIds); // broadcasts elements + saves + undo + spatial refresh
|
||||
};
|
||||
|
||||
socket.on('user:joined', (data: any) => {
|
||||
@@ -176,6 +180,39 @@ export function useCanvasSetup(deps: CanvasSetupDeps) {
|
||||
}
|
||||
});
|
||||
|
||||
// Media processing pipeline: upgrade videos when poster/metadata arrives
|
||||
socket.on('media:job:update', (data: any) => {
|
||||
if (!data || data.status !== 'done') return;
|
||||
const { imageId, posterAssetKey, nativeWidth, nativeHeight, duration } = data;
|
||||
if (!imageId) return;
|
||||
|
||||
// Find the video item by matching its DB image ID stored in scene data
|
||||
for (const item of scene.items.values()) {
|
||||
if (item.data.type !== 'video') continue;
|
||||
// The asset key contains the imageId (e.g. boards/{boardId}/{imageId}.mp4)
|
||||
if (!item.data.asset?.includes(imageId)) continue;
|
||||
|
||||
// Update scene data model
|
||||
const vidData = item.data as any;
|
||||
if (posterAssetKey) vidData.poster = posterAssetKey;
|
||||
if (nativeWidth) { vidData.nativeW = nativeWidth; vidData.w = nativeWidth; }
|
||||
if (nativeHeight) { vidData.nativeH = nativeHeight; vidData.h = nativeHeight; }
|
||||
if (duration) vidData.duration = duration;
|
||||
|
||||
// Update rendered VideoSprite (dimensions, shadow, overlay, poster key)
|
||||
if (item.displayObject instanceof VideoSprite) {
|
||||
item.displayObject.applyProcessedMedia({ posterAssetKey, nativeWidth, nativeHeight });
|
||||
}
|
||||
|
||||
// Update spatial index for changed dimensions
|
||||
scene.updateSpatialEntry(item);
|
||||
|
||||
// No broadcastElements here — all clients receive the same
|
||||
// media:job:update from the server. Echoing would cause fanout churn.
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
// Cursor tracking — throttled to ~30fps to avoid flooding the socket
|
||||
let cursorTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
const onPointerMove = (e: any) => {
|
||||
|
||||
Reference in New Issue
Block a user