feat: media processing pipeline, spatial indexing, canvas-based video rendering
- Background media worker: polls media_jobs table, runs ffprobe+ffmpeg with concurrency limit, generates video posters, emits socket events - Non-blocking video upload: stores file + enqueues job, returns immediately - Poster hydration on board load: GET /boards/:id injects poster/dimensions from DB into canvas_state video objects - SpatialGrid: fixed-cell (512px) spatial hash for O(nearby) culling instead of O(all) item scanning, eliminates setTimeout violations - Canvas-based video rendering: draws video frames to offscreen canvas then uploads to GPU, completely eliminates GL_INVALID_OPERATION errors from PixiJS VideoSource auto-update mechanism - Server poster upgrade path: culling ticker and applyProcessedMedia() both upgrade client-captured posters to server posters when available - Pause restores server poster (paused video behaves like an image) - Selection drag-end persistence: onObjectDragEnd broadcasts + saves + undo - Live media:job:update socket handler patches scene data + VideoSprite dimensions without broadcast fanout
This commit is contained in:
Hiren Kangad
@@ -95,6 +95,24 @@ db.exec(`
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_board_channel_links_board ON board_channel_links(board_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS media_jobs (
|
||||
id TEXT PRIMARY KEY,
|
||||
image_id TEXT NOT NULL REFERENCES images(id) ON DELETE CASCADE,
|
||||
board_id TEXT NOT NULL,
|
||||
type TEXT NOT NULL DEFAULT 'poster',
|
||||
status TEXT NOT NULL DEFAULT 'queued',
|
||||
progress REAL DEFAULT 0,
|
||||
error TEXT,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
result_json TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
started_at TEXT,
|
||||
finished_at TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_media_jobs_status ON media_jobs(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_media_jobs_image ON media_jobs(image_id);
|
||||
`);
|
||||
|
||||
// Migrations — add columns to existing tables
|
||||
@@ -123,6 +141,22 @@ try {
|
||||
} catch {
|
||||
db.exec("ALTER TABLE images ADD COLUMN mm_file_id TEXT");
|
||||
}
|
||||
try {
|
||||
db.prepare("SELECT poster_asset_key FROM images LIMIT 0").get();
|
||||
} catch {
|
||||
db.exec("ALTER TABLE images ADD COLUMN poster_asset_key TEXT");
|
||||
}
|
||||
try {
|
||||
db.prepare("SELECT duration FROM images LIMIT 0").get();
|
||||
} catch {
|
||||
db.exec("ALTER TABLE images ADD COLUMN duration REAL");
|
||||
}
|
||||
try {
|
||||
db.prepare("SELECT native_width FROM images LIMIT 0").get();
|
||||
} catch {
|
||||
db.exec("ALTER TABLE images ADD COLUMN native_width INTEGER");
|
||||
db.exec("ALTER TABLE images ADD COLUMN native_height INTEGER");
|
||||
}
|
||||
|
||||
// ---------------------
|
||||
// User helpers
|
||||
@@ -406,6 +440,46 @@ function getImageByMmFileId(boardId, mmFileId) {
|
||||
return db.prepare('SELECT * FROM images WHERE board_id = ? AND mm_file_id = ?').get(boardId, mmFileId);
|
||||
}
|
||||
|
||||
// ---------------------
|
||||
// Media Jobs
|
||||
// ---------------------
|
||||
function createMediaJob({ id, imageId, boardId, type }) {
|
||||
db.prepare(`
|
||||
INSERT INTO media_jobs (id, image_id, board_id, type, status)
|
||||
VALUES (?, ?, ?, ?, 'queued')
|
||||
`).run(id, imageId, boardId, type || 'poster');
|
||||
return db.prepare('SELECT * FROM media_jobs WHERE id = ?').get(id);
|
||||
}
|
||||
|
||||
function updateMediaJob(id, updates) {
|
||||
const sets = [];
|
||||
const values = [];
|
||||
for (const [key, val] of Object.entries(updates)) {
|
||||
const col = key.replace(/([A-Z])/g, '_$1').toLowerCase(); // camelCase → snake_case
|
||||
sets.push(`${col} = ?`);
|
||||
values.push(val);
|
||||
}
|
||||
if (sets.length === 0) return;
|
||||
values.push(id);
|
||||
db.prepare(`UPDATE media_jobs SET ${sets.join(', ')} WHERE id = ?`).run(...values);
|
||||
}
|
||||
|
||||
function getMediaJob(id) {
|
||||
return db.prepare('SELECT * FROM media_jobs WHERE id = ?').get(id);
|
||||
}
|
||||
|
||||
function getPendingMediaJobs(limit = 10) {
|
||||
return db.prepare('SELECT * FROM media_jobs WHERE status IN (?, ?) ORDER BY created_at ASC LIMIT ?')
|
||||
.all('queued', 'retry', limit);
|
||||
}
|
||||
|
||||
function updateImageMedia(imageId, { posterAssetKey, duration, nativeWidth, nativeHeight }) {
|
||||
db.prepare(`
|
||||
UPDATE images SET poster_asset_key = ?, duration = ?, native_width = ?, native_height = ?
|
||||
WHERE id = ?
|
||||
`).run(posterAssetKey || null, duration || null, nativeWidth || null, nativeHeight || null, imageId);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
db,
|
||||
// Users
|
||||
@@ -423,4 +497,6 @@ module.exports = {
|
||||
// Board–Channel Links
|
||||
createBoardChannelLink, getBoardChannelLinks, getBoardChannelLink, deleteBoardChannelLink,
|
||||
getAllBoardChannelLinks, getImageByMmFileId,
|
||||
// Media Jobs
|
||||
createMediaJob, updateMediaJob, getMediaJob, getPendingMediaJobs, updateImageMedia,
|
||||
};
|
||||
|
||||
@@ -120,10 +120,33 @@ router.get('/:boardId', (req, res) => {
|
||||
|
||||
const images = getBoardImages(board.id);
|
||||
|
||||
// Hydrate video objects with poster/dimensions from DB
|
||||
// (worker may have finished after canvas was last saved)
|
||||
let canvasState = board.canvas_state ? JSON.parse(board.canvas_state) : {};
|
||||
if (canvasState.objects && Array.isArray(canvasState.objects)) {
|
||||
const videoImages = new Map();
|
||||
for (const img of images) {
|
||||
if (img.media_type === 'video' && img.poster_asset_key) {
|
||||
videoImages.set(img.asset_key, img);
|
||||
}
|
||||
}
|
||||
if (videoImages.size > 0) {
|
||||
for (const obj of canvasState.objects) {
|
||||
if (obj.type !== 'video') continue;
|
||||
const dbImg = videoImages.get(obj.asset);
|
||||
if (!dbImg) continue;
|
||||
if (!obj.poster && dbImg.poster_asset_key) obj.poster = dbImg.poster_asset_key;
|
||||
if (!obj.nativeW && dbImg.native_width) { obj.nativeW = dbImg.native_width; obj.w = dbImg.native_width; }
|
||||
if (!obj.nativeH && dbImg.native_height) { obj.nativeH = dbImg.native_height; obj.h = dbImg.native_height; }
|
||||
if (!obj.duration && dbImg.duration) obj.duration = dbImg.duration;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return res.json({
|
||||
board: {
|
||||
...board,
|
||||
canvas_state: board.canvas_state ? JSON.parse(board.canvas_state) : {},
|
||||
canvas_state: canvasState,
|
||||
},
|
||||
collection: {
|
||||
id: collection.id,
|
||||
|
||||
+28
-33
@@ -6,9 +6,8 @@ const https = require('https');
|
||||
const http = require('http');
|
||||
const { URL } = require('url');
|
||||
const { authMiddleware } = require('../auth');
|
||||
const { getBoard, getCollectionMember, createImage } = require('../db');
|
||||
const { getBoard, getCollectionMember, createImage, createMediaJob } = require('../db');
|
||||
const { putBuffer, getImageUrl, MIME_TO_EXT, MAX_FILE_SIZE } = require('../minio');
|
||||
const { probeVideo, extractPoster } = require('../video-utils');
|
||||
|
||||
const router = Router();
|
||||
|
||||
@@ -91,8 +90,8 @@ function classifyMedia(mimeType) {
|
||||
|
||||
/**
|
||||
* Upload a media file (image or video) to MinIO.
|
||||
* For videos: extracts poster frame + metadata via ffmpeg at upload time
|
||||
* so the board never needs a <video> element for thumbnails.
|
||||
* Videos: stored immediately, poster/metadata extracted async by media-worker.
|
||||
* Images: dimensions extracted inline (fast, no ffmpeg).
|
||||
*/
|
||||
async function uploadMedia(boardId, imageId, buffer, mimetype) {
|
||||
const ext = MIME_TO_EXT[mimetype] || '.bin';
|
||||
@@ -100,33 +99,15 @@ async function uploadMedia(boardId, imageId, buffer, mimetype) {
|
||||
await putBuffer(minioPath, buffer, mimetype);
|
||||
|
||||
const isVideo = VIDEO_MIME_TYPES.includes(mimetype);
|
||||
let width = null, height = null, duration = null, posterAssetKey = null;
|
||||
let width = null, height = null;
|
||||
|
||||
if (isVideo) {
|
||||
// Extract metadata and poster frame server-side
|
||||
const [meta, posterBuf] = await Promise.all([
|
||||
probeVideo(buffer),
|
||||
extractPoster(buffer),
|
||||
]);
|
||||
|
||||
if (meta) {
|
||||
width = meta.width;
|
||||
height = meta.height;
|
||||
duration = meta.duration;
|
||||
}
|
||||
|
||||
if (posterBuf) {
|
||||
const posterPath = `boards/${boardId}/${imageId}_poster.jpg`;
|
||||
await putBuffer(posterPath, posterBuf, 'image/jpeg');
|
||||
posterAssetKey = posterPath;
|
||||
}
|
||||
} else {
|
||||
if (!isVideo) {
|
||||
const dims = await getImageDimensions(buffer, mimetype);
|
||||
width = dims.width;
|
||||
height = dims.height;
|
||||
}
|
||||
|
||||
return { assetKey: minioPath, minioPath, width, height, duration, posterAssetKey };
|
||||
return { assetKey: minioPath, minioPath, width, height, isVideo };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -146,10 +127,10 @@ router.post('/boards/:boardId/images', upload.single('image'), async (req, res)
|
||||
const { buffer, originalname, mimetype, size } = req.file;
|
||||
const mediaType = classifyMedia(mimetype);
|
||||
|
||||
const { assetKey, minioPath, width, height, duration, posterAssetKey } = await uploadMedia(board.id, imageId, buffer, mimetype);
|
||||
const { assetKey, minioPath, width, height, isVideo } = await uploadMedia(board.id, imageId, buffer, mimetype);
|
||||
const publicUrl = getImageUrl(minioPath);
|
||||
|
||||
// Save record
|
||||
// Save image record first (media_jobs FK references images)
|
||||
const image = createImage({
|
||||
id: imageId,
|
||||
boardId: board.id,
|
||||
@@ -165,6 +146,13 @@ router.post('/boards/:boardId/images', upload.single('image'), async (req, res)
|
||||
mediaType,
|
||||
});
|
||||
|
||||
// Enqueue background processing after image record exists
|
||||
let jobId = null;
|
||||
if (isVideo) {
|
||||
jobId = uuidv4();
|
||||
createMediaJob({ id: jobId, imageId, boardId: board.id, type: 'poster' });
|
||||
}
|
||||
|
||||
return res.status(201).json({
|
||||
id: image.id,
|
||||
url: publicUrl,
|
||||
@@ -175,8 +163,8 @@ router.post('/boards/:boardId/images', upload.single('image'), async (req, res)
|
||||
mime_type: image.mime_type,
|
||||
asset_key: image.asset_key,
|
||||
media_type: image.media_type,
|
||||
duration: duration || undefined,
|
||||
poster_asset_key: posterAssetKey || undefined,
|
||||
processing: jobId ? 'queued' : undefined,
|
||||
job_id: jobId || undefined,
|
||||
});
|
||||
} catch (err) {
|
||||
if (err.code === 'LIMIT_FILE_SIZE') {
|
||||
@@ -253,10 +241,10 @@ router.post('/boards/:boardId/images/from-url', async (req, res) => {
|
||||
const imageId = uuidv4();
|
||||
const mediaType = classifyMedia(mimeType);
|
||||
|
||||
const { assetKey, minioPath, width, height, duration, posterAssetKey } = await uploadMedia(board.id, imageId, buffer, mimeType);
|
||||
const { assetKey, minioPath, width, height, isVideo } = await uploadMedia(board.id, imageId, buffer, mimeType);
|
||||
const publicUrl = getImageUrl(minioPath);
|
||||
|
||||
// Save record
|
||||
// Save image record first (media_jobs FK references images)
|
||||
const image = createImage({
|
||||
id: imageId,
|
||||
boardId: board.id,
|
||||
@@ -272,6 +260,13 @@ router.post('/boards/:boardId/images/from-url', async (req, res) => {
|
||||
mediaType,
|
||||
});
|
||||
|
||||
// Enqueue background processing after image record exists
|
||||
let jobId = null;
|
||||
if (isVideo) {
|
||||
jobId = uuidv4();
|
||||
createMediaJob({ id: jobId, imageId, boardId: board.id, type: 'poster' });
|
||||
}
|
||||
|
||||
return res.status(201).json({
|
||||
id: image.id,
|
||||
url: publicUrl,
|
||||
@@ -282,8 +277,8 @@ router.post('/boards/:boardId/images/from-url', async (req, res) => {
|
||||
mime_type: image.mime_type,
|
||||
asset_key: image.asset_key,
|
||||
media_type: image.media_type,
|
||||
duration: duration || undefined,
|
||||
poster_asset_key: posterAssetKey || undefined,
|
||||
processing: jobId ? 'queued' : undefined,
|
||||
job_id: jobId || undefined,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('[upload] from-url error:', err);
|
||||
|
||||
@@ -174,6 +174,14 @@ async function start() {
|
||||
console.error('[server] MM watcher failed to start:', err.message);
|
||||
}
|
||||
|
||||
// Start media processing worker
|
||||
try {
|
||||
const { startMediaWorker } = require('./services/media-worker');
|
||||
startMediaWorker(io);
|
||||
} catch (err) {
|
||||
console.error('[server] Media worker failed to start:', err.message);
|
||||
}
|
||||
|
||||
server.listen(PORT, '0.0.0.0', () => {
|
||||
console.log(`[server] RefBoard backend listening on port ${PORT}`);
|
||||
});
|
||||
@@ -188,6 +196,11 @@ function shutdown(signal) {
|
||||
stopWatcher();
|
||||
} catch {}
|
||||
|
||||
try {
|
||||
const { stopMediaWorker } = require('./services/media-worker');
|
||||
stopMediaWorker();
|
||||
} catch {}
|
||||
|
||||
io.close(() => {
|
||||
console.log('[server] Socket.IO closed');
|
||||
});
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
/**
|
||||
* media-worker.js — Background queue worker for video processing.
|
||||
*
|
||||
* Polls the media_jobs table for pending jobs and runs ffprobe + ffmpeg
|
||||
* with a concurrency limit so uploads return instantly.
|
||||
*
|
||||
* Emits socket events so the frontend can upgrade placeholders in real-time.
|
||||
*/
|
||||
|
||||
const {
|
||||
getPendingMediaJobs,
|
||||
updateMediaJob,
|
||||
updateImageMedia,
|
||||
getImage,
|
||||
} = require('../db');
|
||||
const { probeVideo, extractPoster } = require('../video-utils');
|
||||
const { putBuffer, minioClient, MINIO_BUCKET } = require('../minio');
|
||||
|
||||
const POLL_INTERVAL_MS = 3000;
|
||||
const MAX_CONCURRENCY = 2;
|
||||
|
||||
let io = null;
|
||||
let pollTimer = null;
|
||||
let activeCount = 0;
|
||||
let stopping = false;
|
||||
|
||||
/**
|
||||
* Start the media worker. Pass the Socket.IO server instance for notifications.
|
||||
*/
|
||||
function startMediaWorker(ioInstance) {
|
||||
io = ioInstance;
|
||||
stopping = false;
|
||||
console.log('[media-worker] Started (poll=%dms, concurrency=%d)', POLL_INTERVAL_MS, MAX_CONCURRENCY);
|
||||
poll();
|
||||
}
|
||||
|
||||
function stopMediaWorker() {
|
||||
stopping = true;
|
||||
if (pollTimer) {
|
||||
clearTimeout(pollTimer);
|
||||
pollTimer = null;
|
||||
}
|
||||
console.log('[media-worker] Stopped');
|
||||
}
|
||||
|
||||
function schedulePoll() {
|
||||
if (stopping) return;
|
||||
pollTimer = setTimeout(poll, POLL_INTERVAL_MS);
|
||||
}
|
||||
|
||||
async function poll() {
|
||||
if (stopping) return;
|
||||
|
||||
const slotsAvailable = MAX_CONCURRENCY - activeCount;
|
||||
if (slotsAvailable <= 0) {
|
||||
schedulePoll();
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const jobs = getPendingMediaJobs(slotsAvailable);
|
||||
for (const job of jobs) {
|
||||
activeCount++;
|
||||
processJob(job).finally(() => {
|
||||
activeCount--;
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[media-worker] Poll error:', err.message);
|
||||
}
|
||||
|
||||
schedulePoll();
|
||||
}
|
||||
|
||||
async function processJob(job) {
|
||||
const jobId = job.id;
|
||||
const imageId = job.image_id;
|
||||
const boardId = job.board_id;
|
||||
|
||||
try {
|
||||
updateMediaJob(jobId, { status: 'processing', startedAt: new Date().toISOString() });
|
||||
emitJobUpdate(boardId, jobId, imageId, 'processing');
|
||||
|
||||
// Fetch the raw video from MinIO
|
||||
const image = getImage(imageId);
|
||||
if (!image) {
|
||||
updateMediaJob(jobId, { status: 'failed', error: 'Image record not found' });
|
||||
emitJobUpdate(boardId, jobId, imageId, 'failed');
|
||||
return;
|
||||
}
|
||||
|
||||
const videoBuffer = await fetchFromMinio(image.minio_path);
|
||||
if (!videoBuffer) {
|
||||
updateMediaJob(jobId, { status: 'failed', error: 'Failed to fetch video from storage' });
|
||||
emitJobUpdate(boardId, jobId, imageId, 'failed');
|
||||
return;
|
||||
}
|
||||
|
||||
// Run ffprobe + ffmpeg in parallel
|
||||
const [meta, posterBuf] = await Promise.all([
|
||||
probeVideo(videoBuffer),
|
||||
extractPoster(videoBuffer),
|
||||
]);
|
||||
|
||||
let posterAssetKey = null;
|
||||
if (posterBuf) {
|
||||
posterAssetKey = `boards/${boardId}/${imageId}_poster.jpg`;
|
||||
await putBuffer(posterAssetKey, posterBuf, 'image/jpeg');
|
||||
}
|
||||
|
||||
const nativeWidth = meta?.width || null;
|
||||
const nativeHeight = meta?.height || null;
|
||||
const duration = meta?.duration || null;
|
||||
|
||||
// Update the image record with processed media info
|
||||
updateImageMedia(imageId, { posterAssetKey, duration, nativeWidth, nativeHeight });
|
||||
|
||||
updateMediaJob(jobId, {
|
||||
status: 'done',
|
||||
finishedAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
emitJobUpdate(boardId, jobId, imageId, 'done', {
|
||||
posterAssetKey,
|
||||
nativeWidth,
|
||||
nativeHeight,
|
||||
duration,
|
||||
});
|
||||
|
||||
console.log('[media-worker] Job %s done (image=%s, poster=%s)', jobId, imageId, !!posterAssetKey);
|
||||
} catch (err) {
|
||||
console.error('[media-worker] Job %s failed:', jobId, err.message);
|
||||
|
||||
const attempts = (job.attempts || 0) + 1;
|
||||
if (attempts < 3) {
|
||||
updateMediaJob(jobId, { status: 'retry', attempts, error: err.message });
|
||||
emitJobUpdate(boardId, jobId, imageId, 'retry');
|
||||
} else {
|
||||
updateMediaJob(jobId, { status: 'failed', attempts, error: err.message });
|
||||
emitJobUpdate(boardId, jobId, imageId, 'failed');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch object from MinIO as a Buffer.
|
||||
*/
|
||||
async function fetchFromMinio(objectPath) {
|
||||
try {
|
||||
const stream = await minioClient.getObject(MINIO_BUCKET, objectPath);
|
||||
const chunks = [];
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
return Buffer.concat(chunks);
|
||||
} catch (err) {
|
||||
console.warn('[media-worker] MinIO fetch failed:', err.message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit a media job update to all sockets in the board room.
|
||||
*/
|
||||
function emitJobUpdate(boardId, jobId, imageId, status, result) {
|
||||
if (!io) return;
|
||||
const room = `board:${boardId}`;
|
||||
io.to(room).emit('media:job:update', {
|
||||
jobId,
|
||||
imageId,
|
||||
status,
|
||||
...(result || {}),
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { startMediaWorker, stopMediaWorker };
|
||||
@@ -175,68 +175,77 @@ const PixiCanvas = forwardRef<PixiCanvasHandle, PixiCanvasProps>(
|
||||
const MAX_POSTER_VIDEOS = 4; // max poster textures in GPU (tier 2)
|
||||
|
||||
let lastCullCheck = 0;
|
||||
// Track loaded resources so we can unload without scanning all items
|
||||
const loadedImages = new Set<string>(); // IDs of images with loaded textures
|
||||
const loadedVideos = new Set<string>(); // IDs of videos with init/poster/playing state
|
||||
app.ticker.add((ticker) => {
|
||||
lastCullCheck += ticker.deltaMS;
|
||||
if (lastCullCheck < 200) return;
|
||||
if (lastCullCheck < 250) return;
|
||||
lastCullCheck = 0;
|
||||
|
||||
const bounds = viewport.getVisibleBounds();
|
||||
const loadMargin = Math.max(bounds.width, bounds.height);
|
||||
const loadMargin = Math.max(bounds.width, bounds.height) * 0.5;
|
||||
const vcx = bounds.x + bounds.width / 2;
|
||||
const vcy = bounds.y + bounds.height / 2;
|
||||
|
||||
const imageItems: { item: SceneItem; sprite: ImageSprite; dist: number; near: boolean }[] = [];
|
||||
const videoItems: { item: SceneItem; sprite: VideoSprite; dist: number; near: boolean }[] = [];
|
||||
|
||||
for (const item of scene.getAllItems()) {
|
||||
const { x: ix, y: iy, w: iw, h: ih } = getItemWorldBounds(item);
|
||||
const cx = ix + iw / 2;
|
||||
const cy = iy + ih / 2;
|
||||
// Spatial query: only check items within the extended viewport region
|
||||
const qx = bounds.x - loadMargin;
|
||||
const qy = bounds.y - loadMargin;
|
||||
const qw = bounds.width + loadMargin * 2;
|
||||
const qh = bounds.height + loadMargin * 2;
|
||||
const nearbyItems = scene.queryRegion(qx, qy, qw, qh);
|
||||
|
||||
for (const item of nearbyItems) {
|
||||
const cx = item.data.x + (item.data.w * Math.abs(item.data.sx)) / 2;
|
||||
const cy = item.data.y + (item.data.h * Math.abs(item.data.sy)) / 2;
|
||||
const dist = (cx - vcx) ** 2 + (cy - vcy) ** 2;
|
||||
const near =
|
||||
ix + iw > bounds.x - loadMargin &&
|
||||
ix < bounds.x + bounds.width + loadMargin &&
|
||||
iy + ih > bounds.y - loadMargin &&
|
||||
iy < bounds.y + bounds.height + loadMargin;
|
||||
|
||||
if (item.type === 'image' && item.displayObject instanceof ImageSprite) {
|
||||
imageItems.push({ item, sprite: item.displayObject, dist, near });
|
||||
imageItems.push({ item, sprite: item.displayObject, dist, near: true });
|
||||
} else if (item.type === 'video' && item.displayObject instanceof VideoSprite) {
|
||||
videoItems.push({ item, sprite: item.displayObject, dist, near });
|
||||
videoItems.push({ item, sprite: item.displayObject, dist, near: true });
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Image budget management ----
|
||||
imageItems.sort((a, b) => a.dist - b.dist);
|
||||
const nearImages = imageItems.filter(i => i.near);
|
||||
const shouldLoadImage = new Set(
|
||||
nearImages.slice(0, MAX_LOADED_TEXTURES).map(i => i.item.id)
|
||||
imageItems.slice(0, MAX_LOADED_TEXTURES).map(i => i.item.id)
|
||||
);
|
||||
|
||||
let imgLoadsThisTick = 0;
|
||||
for (const { item, sprite } of imageItems) {
|
||||
if (shouldLoadImage.has(item.id) && !sprite.loaded && imgLoadsThisTick < 5) {
|
||||
if (shouldLoadImage.has(item.id) && !sprite.loaded && imgLoadsThisTick < 3) {
|
||||
sprite.loadTexture();
|
||||
loadedImages.add(item.id);
|
||||
imgLoadsThisTick++;
|
||||
}
|
||||
}
|
||||
for (const { item, sprite, near } of imageItems) {
|
||||
if (!sprite.loaded) continue;
|
||||
if (!near || !shouldLoadImage.has(item.id)) {
|
||||
|
||||
// Unload images that left the viewport (check tracked set, not all items)
|
||||
for (const id of loadedImages) {
|
||||
if (shouldLoadImage.has(id)) continue;
|
||||
const item = scene.items.get(id);
|
||||
if (!item) { loadedImages.delete(id); continue; }
|
||||
const sprite = item.displayObject;
|
||||
if (sprite instanceof ImageSprite && sprite.loaded) {
|
||||
sprite.unloadTexture();
|
||||
}
|
||||
loadedImages.delete(id);
|
||||
}
|
||||
|
||||
// ---- Video budget management ----
|
||||
videoItems.sort((a, b) => a.dist - b.dist);
|
||||
|
||||
// Determine budget sets (sorted by distance, nearest first)
|
||||
const nearVideos = videoItems.filter(i => i.near);
|
||||
const shouldInit = new Set(
|
||||
nearVideos.slice(0, MAX_INITIALIZED_VIDEOS).map(i => i.item.id)
|
||||
videoItems.slice(0, MAX_INITIALIZED_VIDEOS).map(i => i.item.id)
|
||||
);
|
||||
const shouldPoster = new Set(
|
||||
nearVideos.slice(0, MAX_POSTER_VIDEOS).map(i => i.item.id)
|
||||
videoItems.slice(0, MAX_POSTER_VIDEOS).map(i => i.item.id)
|
||||
);
|
||||
|
||||
// Initialize nearest videos within budget
|
||||
@@ -244,40 +253,38 @@ const PixiCanvas = forwardRef<PixiCanvasHandle, PixiCanvasProps>(
|
||||
for (const { item, sprite } of videoItems) {
|
||||
if (shouldInit.has(item.id) && !sprite.isInitialized && vidInitsThisTick < 2) {
|
||||
sprite.initVideo();
|
||||
loadedVideos.add(item.id);
|
||||
vidInitsThisTick++;
|
||||
}
|
||||
}
|
||||
|
||||
// Load posters for nearest subset within budget
|
||||
for (const { item, sprite } of videoItems) {
|
||||
if (!shouldPoster.has(item.id) || sprite.hasPoster) continue;
|
||||
if (sprite.hasServerPoster) {
|
||||
// Server poster: load as image texture (no <video> needed)
|
||||
sprite.loadServerPoster();
|
||||
} else if (sprite.isInitialized) {
|
||||
// Fallback: client-capture from video element
|
||||
sprite.capturePoster();
|
||||
// Load posters within budget
|
||||
// Prefer server poster over client capture; upgrade if server poster becomes available
|
||||
if (shouldPoster.has(item.id)) {
|
||||
if (!sprite.hasPoster) {
|
||||
if (sprite.hasServerPoster) {
|
||||
sprite.loadServerPoster();
|
||||
} else if (sprite.isInitialized) {
|
||||
sprite.capturePoster();
|
||||
}
|
||||
} else if (sprite.hasServerPoster && !sprite.isServerPosterLoaded) {
|
||||
// Client poster exists but server poster now available — upgrade
|
||||
sprite.loadServerPoster();
|
||||
}
|
||||
loadedVideos.add(item.id);
|
||||
}
|
||||
}
|
||||
|
||||
// Tear down videos outside budgets (farthest first = reverse iteration)
|
||||
for (let i = videoItems.length - 1; i >= 0; i--) {
|
||||
const { item, sprite, near } = videoItems[i];
|
||||
// Tear down videos that left the viewport (check tracked set)
|
||||
for (const id of loadedVideos) {
|
||||
if (shouldInit.has(id) || shouldPoster.has(id)) continue;
|
||||
const item = scene.items.get(id);
|
||||
if (!item) { loadedVideos.delete(id); continue; }
|
||||
const sprite = item.displayObject;
|
||||
if (!(sprite instanceof VideoSprite)) { loadedVideos.delete(id); continue; }
|
||||
|
||||
// Pause offscreen playing videos
|
||||
if (!near && sprite.isPlaying) {
|
||||
sprite.onVisibilityChange(false);
|
||||
}
|
||||
|
||||
// Destroy poster if outside poster budget
|
||||
if (!shouldPoster.has(item.id) && sprite.hasPoster && !sprite.isPlaying) {
|
||||
sprite.destroyPoster();
|
||||
}
|
||||
|
||||
// Tear down video element if outside init budget
|
||||
if (!shouldInit.has(item.id) && sprite.isInitialized && !sprite.isPlaying) {
|
||||
sprite.teardownVideo();
|
||||
}
|
||||
if (sprite.isPlaying) sprite.onVisibilityChange(false);
|
||||
if (sprite.hasPoster && !sprite.isPlaying) sprite.destroyPoster();
|
||||
if (sprite.isInitialized && !sprite.isPlaying) sprite.teardownVideo();
|
||||
loadedVideos.delete(id);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ import { DrawingSprite } from './sprites/DrawingSprite';
|
||||
import { FrameSprite } from './sprites/FrameSprite';
|
||||
import { SpringManager, Spring, PRESETS } from './spring';
|
||||
import { reparentGroupChildren } from './grouping';
|
||||
import { SpatialGrid } from './SpatialGrid';
|
||||
import type {
|
||||
SceneData,
|
||||
AnySceneObject,
|
||||
@@ -140,6 +141,7 @@ export class SceneManager {
|
||||
readonly viewport: Viewport;
|
||||
readonly textures: TextureManager;
|
||||
readonly springs: SpringManager;
|
||||
readonly spatialGrid: SpatialGrid<SceneItem> = new SpatialGrid<SceneItem>(512);
|
||||
|
||||
private _onChange: (() => void) | null = null;
|
||||
private _onItemDimensionsChanged: ((itemId: string) => void) | null = null;
|
||||
@@ -151,6 +153,17 @@ export class SceneManager {
|
||||
this.springs = springs;
|
||||
}
|
||||
|
||||
/** Update an item's spatial index entry from its current data. */
|
||||
updateSpatialEntry(item: SceneItem): void {
|
||||
const { x, y, w, h } = getItemWorldBounds(item);
|
||||
this.spatialGrid.upsert(item.id, item, x, y, w, h);
|
||||
}
|
||||
|
||||
/** Query items overlapping the given world rectangle. */
|
||||
queryRegion(rx: number, ry: number, rw: number, rh: number): SceneItem[] {
|
||||
return this.spatialGrid.query(rx, ry, rw, rh).map(e => e.data);
|
||||
}
|
||||
|
||||
// -- onChange callback ----------------------------------------------------
|
||||
|
||||
set onChange(fn: (() => void) | null) {
|
||||
@@ -220,6 +233,7 @@ export class SceneManager {
|
||||
}
|
||||
|
||||
item.displayObject.destroy({ children: true });
|
||||
this.spatialGrid.remove(id);
|
||||
this.items.delete(id);
|
||||
}
|
||||
|
||||
@@ -332,6 +346,7 @@ export class SceneManager {
|
||||
};
|
||||
|
||||
this.items.set(data.id, item);
|
||||
this.updateSpatialEntry(item);
|
||||
|
||||
// Wire video dimension auto-correction to the STORED item.data (not the original param)
|
||||
if (data.type === 'video' && displayObject instanceof VideoSprite) {
|
||||
@@ -429,6 +444,7 @@ export class SceneManager {
|
||||
// Update stored data
|
||||
item.data = { ...data };
|
||||
item.type = data.type;
|
||||
this.updateSpatialEntry(item);
|
||||
}
|
||||
|
||||
// -- Z-Ordering ----------------------------------------------------------
|
||||
@@ -505,12 +521,14 @@ export class SceneManager {
|
||||
if (item.data.type === 'group') {
|
||||
const groupData = item.data as import('./scene-format').GroupObject;
|
||||
for (const childId of groupData.children) {
|
||||
this.spatialGrid.remove(childId);
|
||||
this.items.delete(childId);
|
||||
}
|
||||
}
|
||||
|
||||
if (!animate) {
|
||||
item.displayObject.destroy({ children: true });
|
||||
this.spatialGrid.remove(id);
|
||||
this.items.delete(id);
|
||||
this._onChange?.();
|
||||
return;
|
||||
@@ -519,6 +537,7 @@ export class SceneManager {
|
||||
const obj = item.displayObject;
|
||||
|
||||
// Remove from map immediately to prevent double-remove
|
||||
this.spatialGrid.remove(id);
|
||||
this.items.delete(id);
|
||||
|
||||
// Scale toward center on removal
|
||||
@@ -710,6 +729,7 @@ export class SceneManager {
|
||||
data: groupData,
|
||||
};
|
||||
this.items.set(groupData.id, groupItem);
|
||||
this.updateSpatialEntry(groupItem);
|
||||
|
||||
// Animate each child ~25px toward center, then reparent into container
|
||||
const CONVERGE_PX = 25;
|
||||
@@ -842,6 +862,7 @@ export class SceneManager {
|
||||
if (!container.destroyed) {
|
||||
container.destroy();
|
||||
}
|
||||
this.spatialGrid.remove(groupId);
|
||||
this.items.delete(groupId);
|
||||
|
||||
this._applyZOrder();
|
||||
|
||||
@@ -41,6 +41,7 @@ export class SelectionManager {
|
||||
private _onSelectionChange: ((ids: string[]) => void) | null = null;
|
||||
private _onItemTransform: ((item: SceneItem) => void) | null = null;
|
||||
private _onItemsTransform: ((items: SceneItem[]) => void) | null = null;
|
||||
private _onObjectDragEnd: ((ids: string[]) => void) | null = null;
|
||||
|
||||
// Pointer state
|
||||
private _pointerDown = false;
|
||||
@@ -122,6 +123,11 @@ export class SelectionManager {
|
||||
this._onItemsTransform = fn;
|
||||
}
|
||||
|
||||
/** Called when object drag ends (for persist/save/spatial refresh). */
|
||||
set onObjectDragEnd(fn: (ids: string[]) => void) {
|
||||
this._onObjectDragEnd = fn;
|
||||
}
|
||||
|
||||
/** Called on double-click of a text item (for inline editing). */
|
||||
set onDoubleClickText(fn: (item: SceneItem) => void) {
|
||||
this._onDoubleClickText = fn;
|
||||
@@ -313,6 +319,7 @@ export class SelectionManager {
|
||||
|
||||
// Notify change (positions updated)
|
||||
this._emitChange();
|
||||
this._onObjectDragEnd?.(Array.from(this.selectedIds));
|
||||
} else if (this._rubberBanding) {
|
||||
// Finish rubber band selection
|
||||
const currentWorld = this._viewport.toWorld(e.global.x, e.global.y);
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
/**
|
||||
* SpatialGrid — fixed-cell spatial index for fast region queries.
|
||||
*
|
||||
* Each item occupies one or more grid cells based on its axis-aligned bounds.
|
||||
* Query returns all items whose cells overlap the query rectangle.
|
||||
*
|
||||
* Cell size should be ~1-2x the median item size for best performance.
|
||||
* Too small = many cells per item. Too large = too many items per cell.
|
||||
*/
|
||||
|
||||
export interface SpatialEntry<T> {
|
||||
id: string;
|
||||
data: T;
|
||||
x: number;
|
||||
y: number;
|
||||
w: number;
|
||||
h: number;
|
||||
}
|
||||
|
||||
export class SpatialGrid<T> {
|
||||
private cellSize: number;
|
||||
private cells: Map<string, Set<string>> = new Map();
|
||||
private entries: Map<string, SpatialEntry<T>> = new Map();
|
||||
|
||||
constructor(cellSize = 512) {
|
||||
this.cellSize = cellSize;
|
||||
}
|
||||
|
||||
/** Insert or update an entry. */
|
||||
upsert(id: string, data: T, x: number, y: number, w: number, h: number): void {
|
||||
// Remove old cells if updating
|
||||
if (this.entries.has(id)) {
|
||||
this._removeCells(id);
|
||||
}
|
||||
|
||||
const entry: SpatialEntry<T> = { id, data, x, y, w, h };
|
||||
this.entries.set(id, entry);
|
||||
this._insertCells(id, x, y, w, h);
|
||||
}
|
||||
|
||||
/** Remove an entry. */
|
||||
remove(id: string): void {
|
||||
if (!this.entries.has(id)) return;
|
||||
this._removeCells(id);
|
||||
this.entries.delete(id);
|
||||
}
|
||||
|
||||
/** Query all entries that overlap the given rectangle. */
|
||||
query(rx: number, ry: number, rw: number, rh: number): SpatialEntry<T>[] {
|
||||
const seen = new Set<string>();
|
||||
const results: SpatialEntry<T>[] = [];
|
||||
|
||||
const c0 = Math.floor(rx / this.cellSize);
|
||||
const r0 = Math.floor(ry / this.cellSize);
|
||||
const c1 = Math.floor((rx + rw) / this.cellSize);
|
||||
const r1 = Math.floor((ry + rh) / this.cellSize);
|
||||
|
||||
for (let r = r0; r <= r1; r++) {
|
||||
for (let c = c0; c <= c1; c++) {
|
||||
const key = `${c},${r}`;
|
||||
const cell = this.cells.get(key);
|
||||
if (!cell) continue;
|
||||
|
||||
for (const id of cell) {
|
||||
if (seen.has(id)) continue;
|
||||
seen.add(id);
|
||||
|
||||
const entry = this.entries.get(id)!;
|
||||
// AABB overlap check
|
||||
if (
|
||||
entry.x < rx + rw &&
|
||||
entry.x + entry.w > rx &&
|
||||
entry.y < ry + rh &&
|
||||
entry.y + entry.h > ry
|
||||
) {
|
||||
results.push(entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/** Get a specific entry by ID. */
|
||||
get(id: string): SpatialEntry<T> | undefined {
|
||||
return this.entries.get(id);
|
||||
}
|
||||
|
||||
/** Number of entries in the index. */
|
||||
get size(): number {
|
||||
return this.entries.size;
|
||||
}
|
||||
|
||||
/** Clear the entire index. */
|
||||
clear(): void {
|
||||
this.cells.clear();
|
||||
this.entries.clear();
|
||||
}
|
||||
|
||||
// ---- Internal ----
|
||||
|
||||
private _cellKeys(x: number, y: number, w: number, h: number): string[] {
|
||||
const c0 = Math.floor(x / this.cellSize);
|
||||
const r0 = Math.floor(y / this.cellSize);
|
||||
const c1 = Math.floor((x + w) / this.cellSize);
|
||||
const r1 = Math.floor((y + h) / this.cellSize);
|
||||
|
||||
const keys: string[] = [];
|
||||
for (let r = r0; r <= r1; r++) {
|
||||
for (let c = c0; c <= c1; c++) {
|
||||
keys.push(`${c},${r}`);
|
||||
}
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
private _insertCells(id: string, x: number, y: number, w: number, h: number): void {
|
||||
for (const key of this._cellKeys(x, y, w, h)) {
|
||||
let cell = this.cells.get(key);
|
||||
if (!cell) {
|
||||
cell = new Set();
|
||||
this.cells.set(key, cell);
|
||||
}
|
||||
cell.add(id);
|
||||
}
|
||||
}
|
||||
|
||||
private _removeCells(id: string): void {
|
||||
const entry = this.entries.get(id);
|
||||
if (!entry) return;
|
||||
for (const key of this._cellKeys(entry.x, entry.y, entry.w, entry.h)) {
|
||||
const cell = this.cells.get(key);
|
||||
if (cell) {
|
||||
cell.delete(id);
|
||||
if (cell.size === 0) this.cells.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,8 +11,10 @@ import { TextureManager } from '../TextureManager';
|
||||
* Tier 2 — client-captured poster from video frame (fallback if no server poster)
|
||||
* Tier 3 — actively playing with live video texture (user-initiated)
|
||||
*
|
||||
* If the server provides a poster asset key at upload time, the video renders
|
||||
* as a plain image until explicitly played. No <video> element needed for thumbnails.
|
||||
* Playback uses a canvas intermediary: video frames are drawn to an offscreen
|
||||
* canvas, then uploaded to GPU via a standard image texture. This avoids
|
||||
* GL_INVALID_OPERATION errors from PixiJS's VideoSource auto-update mechanism
|
||||
* which tries to copy video frames before the GPU texture is allocated.
|
||||
*/
|
||||
|
||||
const MAX_PLAYING_VIDEOS = 1;
|
||||
@@ -24,7 +26,7 @@ const SHADOW_LIFT = { offsetX: 6, offsetY: 8, alpha: 0.3 };
|
||||
export class VideoSprite extends Container {
|
||||
readonly assetKey: string;
|
||||
readonly videoUrl: string;
|
||||
readonly posterAssetKey: string | null;
|
||||
posterAssetKey: string | null;
|
||||
|
||||
private textures: TextureManager | null;
|
||||
private videoEl: HTMLVideoElement | null = null;
|
||||
@@ -41,6 +43,11 @@ export class VideoSprite extends Container {
|
||||
private _isPlaying = false;
|
||||
private _videoInitialized = false;
|
||||
private _hasPoster = false;
|
||||
// Canvas-based video rendering (avoids VideoSource GL errors)
|
||||
private _frameCanvas: HTMLCanvasElement | null = null;
|
||||
private _frameCtx: CanvasRenderingContext2D | null = null;
|
||||
private _rafId: number | null = null;
|
||||
private _useRVFC = false; // true if requestVideoFrameCallback is available
|
||||
muted = true;
|
||||
loop = true;
|
||||
|
||||
@@ -66,6 +73,7 @@ export class VideoSprite extends Container {
|
||||
this.textures = textures ?? null;
|
||||
this._naturalWidth = w;
|
||||
this._naturalHeight = h;
|
||||
this._useRVFC = 'requestVideoFrameCallback' in HTMLVideoElement.prototype;
|
||||
|
||||
// Shadow
|
||||
this._shadow = new Graphics();
|
||||
@@ -96,6 +104,50 @@ export class VideoSprite extends Container {
|
||||
/** Whether this video has a server-generated poster available (not yet loaded). */
|
||||
get hasServerPoster(): boolean { return !!this.posterAssetKey && !!this.textures; }
|
||||
|
||||
/** Whether the server-generated poster has been loaded (vs client-captured). */
|
||||
get isServerPosterLoaded(): boolean { return this._serverPosterLoaded; }
|
||||
|
||||
// ---- Tier 0: Apply processed media from background worker ---------------
|
||||
|
||||
/** Update dimensions + poster key from the media worker result.
|
||||
* Resizes all internal visuals and loads poster immediately if not playing. */
|
||||
applyProcessedMedia(opts: {
|
||||
posterAssetKey?: string | null;
|
||||
nativeWidth?: number | null;
|
||||
nativeHeight?: number | null;
|
||||
}): void {
|
||||
if (this.destroyed) return;
|
||||
|
||||
if (opts.posterAssetKey) {
|
||||
this.posterAssetKey = opts.posterAssetKey;
|
||||
}
|
||||
|
||||
const nw = opts.nativeWidth;
|
||||
const nh = opts.nativeHeight;
|
||||
if (nw && nh && nw > 0 && nh > 0 && (nw !== this._naturalWidth || nh !== this._naturalHeight)) {
|
||||
this._naturalWidth = nw;
|
||||
this._naturalHeight = nh;
|
||||
|
||||
if (this._sprite) {
|
||||
this._sprite.width = nw;
|
||||
this._sprite.height = nh;
|
||||
}
|
||||
if (this._placeholder) {
|
||||
this._placeholder.clear();
|
||||
this._placeholder.rect(0, 0, nw, nh).fill(0x2a2a2a);
|
||||
}
|
||||
this._drawShadow(SHADOW_REST);
|
||||
this._drawPlayIcon();
|
||||
this.onDimensionsKnown?.(nw, nh);
|
||||
}
|
||||
|
||||
// Load server poster if not playing and server poster not yet loaded.
|
||||
// This upgrades a client-captured poster (possibly black) to the real one.
|
||||
if (!this._isPlaying && !this._serverPosterLoaded && this.hasServerPoster) {
|
||||
this.loadServerPoster();
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Tier 0.5: Server-generated poster (loaded like an image) -----------
|
||||
|
||||
/** Load server poster. Called by culling system when within poster budget. */
|
||||
@@ -109,6 +161,11 @@ export class VideoSprite extends Container {
|
||||
return;
|
||||
}
|
||||
|
||||
// Clean up any existing client-captured poster before replacing
|
||||
if (this._hasPoster && this.posterTexture && !this._serverPosterLoaded) {
|
||||
this.posterTexture.destroy(true);
|
||||
}
|
||||
|
||||
this.posterTexture = tex;
|
||||
this._hasPoster = true;
|
||||
this._serverPosterLoaded = true;
|
||||
@@ -116,7 +173,6 @@ export class VideoSprite extends Container {
|
||||
this._ensureSprite(tex);
|
||||
this._removePlaceholder();
|
||||
} catch {
|
||||
// Server poster failed — will fall back to client capture if needed
|
||||
} finally {
|
||||
this._serverPosterLoading = false;
|
||||
}
|
||||
@@ -214,31 +270,67 @@ export class VideoSprite extends Container {
|
||||
this.videoEl.muted = this.muted;
|
||||
this.videoEl.loop = this.loop;
|
||||
|
||||
if (!this.videoTexture) {
|
||||
this.videoTexture = Texture.from(this.videoEl);
|
||||
}
|
||||
|
||||
// Drop poster while playing — don't keep both resident
|
||||
if (this._hasPoster && this.posterTexture) {
|
||||
if (this._serverPosterLoaded && this.posterAssetKey && this.textures) {
|
||||
this.textures.release(this.posterAssetKey);
|
||||
} else {
|
||||
this.posterTexture.destroy(true);
|
||||
}
|
||||
this.posterTexture = null;
|
||||
this._hasPoster = false;
|
||||
this._serverPosterLoaded = false;
|
||||
}
|
||||
|
||||
this._ensureSprite(this.videoTexture);
|
||||
this._removePlaceholder();
|
||||
|
||||
this._isPlaying = true;
|
||||
this._overlay.visible = false;
|
||||
activeVideos.add(this);
|
||||
this.onStateChange?.();
|
||||
|
||||
this.videoEl.play().catch(() => {
|
||||
const videoEl = this.videoEl!;
|
||||
|
||||
// Once a decoded frame is available, create a canvas-backed texture.
|
||||
// Drawing video → canvas → GPU avoids all VideoSource GL errors.
|
||||
const swapToVideoTexture = () => {
|
||||
if (this.destroyed || !this._isPlaying || !this.videoEl) return;
|
||||
|
||||
const vw = this.videoEl.videoWidth || this._naturalWidth;
|
||||
const vh = this.videoEl.videoHeight || this._naturalHeight;
|
||||
|
||||
if (!this.videoTexture) {
|
||||
// Create offscreen canvas and draw the first frame
|
||||
this._frameCanvas = document.createElement('canvas');
|
||||
this._frameCanvas.width = vw;
|
||||
this._frameCanvas.height = vh;
|
||||
this._frameCtx = this._frameCanvas.getContext('2d');
|
||||
if (this._frameCtx) {
|
||||
this._frameCtx.drawImage(this.videoEl, 0, 0, vw, vh);
|
||||
}
|
||||
// Create texture from canvas — always has valid pixel data
|
||||
this.videoTexture = Texture.from(this._frameCanvas);
|
||||
}
|
||||
|
||||
// Drop poster while playing — don't keep both resident
|
||||
if (this._hasPoster && this.posterTexture) {
|
||||
if (this._serverPosterLoaded && this.posterAssetKey && this.textures) {
|
||||
this.textures.release(this.posterAssetKey);
|
||||
} else {
|
||||
this.posterTexture.destroy(true);
|
||||
}
|
||||
this.posterTexture = null;
|
||||
this._hasPoster = false;
|
||||
this._serverPosterLoaded = false;
|
||||
}
|
||||
|
||||
this._ensureSprite(this.videoTexture);
|
||||
this._removePlaceholder();
|
||||
this._overlay.visible = false;
|
||||
|
||||
// Start manual frame update loop
|
||||
this._startFrameLoop();
|
||||
};
|
||||
|
||||
// Wait for a decoded frame before creating the GPU texture.
|
||||
let swapped = false;
|
||||
const safeSwap = () => { if (!swapped) { swapped = true; swapToVideoTexture(); } };
|
||||
|
||||
if (videoEl.readyState >= 4) {
|
||||
safeSwap();
|
||||
} else if (this._useRVFC) {
|
||||
(videoEl as any).requestVideoFrameCallback(safeSwap);
|
||||
} else {
|
||||
videoEl.addEventListener('playing', safeSwap, { once: true });
|
||||
}
|
||||
|
||||
videoEl.play().catch(() => {
|
||||
videoEl.removeEventListener('playing', safeSwap);
|
||||
this._isPlaying = false;
|
||||
this._overlay.visible = true;
|
||||
activeVideos.delete(this);
|
||||
@@ -252,6 +344,15 @@ export class VideoSprite extends Container {
|
||||
this._isPlaying = false;
|
||||
this._overlay.visible = true;
|
||||
activeVideos.delete(this);
|
||||
this._stopFrameLoop();
|
||||
|
||||
// Restore server poster — paused video behaves like an image.
|
||||
// Drop the live video texture to free GPU memory.
|
||||
if (this.hasServerPoster && !this._hasPoster) {
|
||||
this._destroyVideoTexture();
|
||||
this.loadServerPoster();
|
||||
}
|
||||
|
||||
this.onStateChange?.();
|
||||
}
|
||||
|
||||
@@ -440,7 +541,49 @@ export class VideoSprite extends Container {
|
||||
this.videoEl = null;
|
||||
}
|
||||
|
||||
/** Draw current video frame to offscreen canvas, then tell PixiJS to re-upload. */
|
||||
private _drawFrame(): void {
|
||||
if (!this._frameCtx || !this._frameCanvas || !this.videoEl || !this.videoTexture) return;
|
||||
this._frameCtx.drawImage(this.videoEl, 0, 0, this._frameCanvas.width, this._frameCanvas.height);
|
||||
this.videoTexture.source.update();
|
||||
}
|
||||
|
||||
private _startFrameLoop(): void {
|
||||
if (this._rafId !== null) return;
|
||||
const videoEl = this.videoEl;
|
||||
if (!videoEl) return;
|
||||
|
||||
if (this._useRVFC) {
|
||||
// requestVideoFrameCallback — fires only when a new decoded frame is ready
|
||||
const onFrame = () => {
|
||||
if (!this._isPlaying || this.destroyed) { this._rafId = null; return; }
|
||||
this._drawFrame();
|
||||
this._rafId = (videoEl as any).requestVideoFrameCallback(onFrame);
|
||||
};
|
||||
this._rafId = (videoEl as any).requestVideoFrameCallback(onFrame);
|
||||
} else {
|
||||
// Fallback: requestAnimationFrame
|
||||
const onFrame = () => {
|
||||
if (!this._isPlaying || this.destroyed) { this._rafId = null; return; }
|
||||
this._drawFrame();
|
||||
this._rafId = requestAnimationFrame(onFrame);
|
||||
};
|
||||
this._rafId = requestAnimationFrame(onFrame);
|
||||
}
|
||||
}
|
||||
|
||||
private _stopFrameLoop(): void {
|
||||
if (this._rafId === null) return;
|
||||
if (this._useRVFC && this.videoEl) {
|
||||
(this.videoEl as any).cancelVideoFrameCallback(this._rafId);
|
||||
} else {
|
||||
cancelAnimationFrame(this._rafId);
|
||||
}
|
||||
this._rafId = null;
|
||||
}
|
||||
|
||||
private _destroyVideoTexture(): void {
|
||||
this._stopFrameLoop();
|
||||
if (this.videoTexture) {
|
||||
if (this._sprite && this._sprite.texture === this.videoTexture) {
|
||||
this._removeSpriteFromTree();
|
||||
@@ -449,12 +592,17 @@ export class VideoSprite extends Container {
|
||||
this.videoTexture.destroy(true);
|
||||
this.videoTexture = null;
|
||||
}
|
||||
this._frameCanvas = null;
|
||||
this._frameCtx = null;
|
||||
}
|
||||
|
||||
destroy(options?: Parameters<Container['destroy']>[0]): void {
|
||||
this._stopFrameLoop();
|
||||
this.pause();
|
||||
this._destroyVideoEl();
|
||||
if (this.videoTexture) { this.videoTexture.destroy(true); this.videoTexture = null; }
|
||||
this._frameCanvas = null;
|
||||
this._frameCtx = null;
|
||||
if (this._hasPoster && this.posterTexture) {
|
||||
if (this._serverPosterLoaded && this.posterAssetKey && this.textures) {
|
||||
this.textures.release(this.posterAssetKey);
|
||||
|
||||
@@ -7,6 +7,7 @@ import { setupDragDrop, setupPaste } from '../canvas/image-drop';
|
||||
import { UndoManager } from '../canvas/history';
|
||||
import { InboxZone } from '../canvas/InboxZone';
|
||||
import { LaserPointer } from '../canvas/LaserPointer';
|
||||
import { VideoSprite } from '../canvas/sprites/VideoSprite';
|
||||
// PresenceOverlay removed — remote selection highlighting was too heavy for minimal benefit
|
||||
import { connectSocket, disconnectSocket } from '../socket';
|
||||
|
||||
@@ -137,8 +138,11 @@ export function useCanvasSetup(deps: CanvasSetupDeps) {
|
||||
selection.transformBox.onItemTransform = (item) => {
|
||||
syncRef.current?.broadcastTransform(item);
|
||||
};
|
||||
selection.onObjectDragEnd = (itemIds) => {
|
||||
onCanvasChange(itemIds); // broadcasts elements + saves + undo + spatial refresh
|
||||
};
|
||||
selection.transformBox.onDragEnd = (itemIds) => {
|
||||
onCanvasChange(itemIds); // broadcasts elements + saves + undo
|
||||
onCanvasChange(itemIds); // broadcasts elements + saves + undo + spatial refresh
|
||||
};
|
||||
|
||||
socket.on('user:joined', (data: any) => {
|
||||
@@ -176,6 +180,39 @@ export function useCanvasSetup(deps: CanvasSetupDeps) {
|
||||
}
|
||||
});
|
||||
|
||||
// Media processing pipeline: upgrade videos when poster/metadata arrives
|
||||
socket.on('media:job:update', (data: any) => {
|
||||
if (!data || data.status !== 'done') return;
|
||||
const { imageId, posterAssetKey, nativeWidth, nativeHeight, duration } = data;
|
||||
if (!imageId) return;
|
||||
|
||||
// Find the video item by matching its DB image ID stored in scene data
|
||||
for (const item of scene.items.values()) {
|
||||
if (item.data.type !== 'video') continue;
|
||||
// The asset key contains the imageId (e.g. boards/{boardId}/{imageId}.mp4)
|
||||
if (!item.data.asset?.includes(imageId)) continue;
|
||||
|
||||
// Update scene data model
|
||||
const vidData = item.data as any;
|
||||
if (posterAssetKey) vidData.poster = posterAssetKey;
|
||||
if (nativeWidth) { vidData.nativeW = nativeWidth; vidData.w = nativeWidth; }
|
||||
if (nativeHeight) { vidData.nativeH = nativeHeight; vidData.h = nativeHeight; }
|
||||
if (duration) vidData.duration = duration;
|
||||
|
||||
// Update rendered VideoSprite (dimensions, shadow, overlay, poster key)
|
||||
if (item.displayObject instanceof VideoSprite) {
|
||||
item.displayObject.applyProcessedMedia({ posterAssetKey, nativeWidth, nativeHeight });
|
||||
}
|
||||
|
||||
// Update spatial index for changed dimensions
|
||||
scene.updateSpatialEntry(item);
|
||||
|
||||
// No broadcastElements here — all clients receive the same
|
||||
// media:job:update from the server. Echoing would cause fanout churn.
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
// Cursor tracking — throttled to ~30fps to avoid flooding the socket
|
||||
let cursorTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
const onPointerMove = (e: any) => {
|
||||
|
||||
@@ -111,6 +111,14 @@ export default function Editor({ isPublicView }: EditorProps) {
|
||||
if (changedIds && changedIds.length > 0) {
|
||||
// Incremental: broadcast only changed elements
|
||||
syncRef.current?.broadcastElements(changedIds);
|
||||
// Keep spatial index in sync for all local mutations (align/flip/arrange/etc.)
|
||||
const scene = canvasRef.current?.getScene();
|
||||
if (scene) {
|
||||
for (const id of changedIds) {
|
||||
const item = scene.items.get(id);
|
||||
if (item) scene.updateSpatialEntry(item);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Full scene sync happens via debounced SceneManager.onChange chain in sync.ts
|
||||
if (undoRef.current && !undoRef.current.isLocked()) {
|
||||
|
||||
Reference in New Issue
Block a user