feat(refboard): server-generated video posters via ffmpeg
Upload pipeline now extracts poster frame (JPEG) and metadata (width, height, duration) from videos at upload time using ffmpeg. Server: - Add ffmpeg to Docker image (Alpine) - video-utils.js: probeVideo() and extractPoster() using ffprobe/ffmpeg - Upload response includes poster_asset_key and duration Frontend: - VideoObject gains poster and duration fields in scene format - VideoSprite accepts posterAssetKey + TextureManager, loads poster as a regular image texture on construction (no <video> needed) - Server poster loaded/released via TextureManager ref counting - addVideoFromUpload passes poster and duration through to scene data - image-drop.ts forwards poster_asset_key from upload response Result: videos with server posters render as images by default. Zero <video> elements needed for thumbnails. Only explicit play creates a media element.
This commit is contained in:
Hiren Kangad
@@ -8,6 +8,7 @@ const { URL } = require('url');
|
||||
const { authMiddleware } = require('../auth');
|
||||
const { getBoard, getCollectionMember, createImage } = require('../db');
|
||||
const { putBuffer, getImageUrl, MIME_TO_EXT, MAX_FILE_SIZE } = require('../minio');
|
||||
const { probeVideo, extractPoster } = require('../video-utils');
|
||||
|
||||
const router = Router();
|
||||
|
||||
@@ -89,17 +90,43 @@ function classifyMedia(mimeType) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a media file (image or video) to MinIO as a single file.
|
||||
* GPU handles all image scaling natively — no LOD tiers needed.
|
||||
* Upload a media file (image or video) to MinIO.
|
||||
* For videos: extracts poster frame + metadata via ffmpeg at upload time
|
||||
* so the board never needs a <video> element for thumbnails.
|
||||
*/
|
||||
async function uploadMedia(boardId, imageId, buffer, mimetype) {
|
||||
const ext = MIME_TO_EXT[mimetype] || '.bin';
|
||||
const minioPath = `boards/${boardId}/${imageId}${ext}`;
|
||||
await putBuffer(minioPath, buffer, mimetype);
|
||||
|
||||
const dims = await getImageDimensions(buffer, mimetype);
|
||||
// assetKey = minioPath so frontend can build direct URLs
|
||||
return { assetKey: minioPath, minioPath, width: dims.width, height: dims.height };
|
||||
const isVideo = VIDEO_MIME_TYPES.includes(mimetype);
|
||||
let width = null, height = null, duration = null, posterAssetKey = null;
|
||||
|
||||
if (isVideo) {
|
||||
// Extract metadata and poster frame server-side
|
||||
const [meta, posterBuf] = await Promise.all([
|
||||
probeVideo(buffer),
|
||||
extractPoster(buffer),
|
||||
]);
|
||||
|
||||
if (meta) {
|
||||
width = meta.width;
|
||||
height = meta.height;
|
||||
duration = meta.duration;
|
||||
}
|
||||
|
||||
if (posterBuf) {
|
||||
const posterPath = `boards/${boardId}/${imageId}_poster.jpg`;
|
||||
await putBuffer(posterPath, posterBuf, 'image/jpeg');
|
||||
posterAssetKey = posterPath;
|
||||
}
|
||||
} else {
|
||||
const dims = await getImageDimensions(buffer, mimetype);
|
||||
width = dims.width;
|
||||
height = dims.height;
|
||||
}
|
||||
|
||||
return { assetKey: minioPath, minioPath, width, height, duration, posterAssetKey };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -119,7 +146,7 @@ router.post('/boards/:boardId/images', upload.single('image'), async (req, res)
|
||||
const { buffer, originalname, mimetype, size } = req.file;
|
||||
const mediaType = classifyMedia(mimetype);
|
||||
|
||||
const { assetKey, minioPath, width, height } = await uploadMedia(board.id, imageId, buffer, mimetype);
|
||||
const { assetKey, minioPath, width, height, duration, posterAssetKey } = await uploadMedia(board.id, imageId, buffer, mimetype);
|
||||
const publicUrl = getImageUrl(minioPath);
|
||||
|
||||
// Save record
|
||||
@@ -148,6 +175,8 @@ router.post('/boards/:boardId/images', upload.single('image'), async (req, res)
|
||||
mime_type: image.mime_type,
|
||||
asset_key: image.asset_key,
|
||||
media_type: image.media_type,
|
||||
duration: duration || undefined,
|
||||
poster_asset_key: posterAssetKey || undefined,
|
||||
});
|
||||
} catch (err) {
|
||||
if (err.code === 'LIMIT_FILE_SIZE') {
|
||||
@@ -224,7 +253,7 @@ router.post('/boards/:boardId/images/from-url', async (req, res) => {
|
||||
const imageId = uuidv4();
|
||||
const mediaType = classifyMedia(mimeType);
|
||||
|
||||
const { assetKey, minioPath, width, height } = await uploadMedia(board.id, imageId, buffer, mimeType);
|
||||
const { assetKey, minioPath, width, height, duration, posterAssetKey } = await uploadMedia(board.id, imageId, buffer, mimeType);
|
||||
const publicUrl = getImageUrl(minioPath);
|
||||
|
||||
// Save record
|
||||
@@ -253,6 +282,8 @@ router.post('/boards/:boardId/images/from-url', async (req, res) => {
|
||||
mime_type: image.mime_type,
|
||||
asset_key: image.asset_key,
|
||||
media_type: image.media_type,
|
||||
duration: duration || undefined,
|
||||
poster_asset_key: posterAssetKey || undefined,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('[upload] from-url error:', err);
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
/**
|
||||
* video-utils.js — Extract poster frame and metadata from video buffers using ffmpeg.
|
||||
*
|
||||
* Called at upload time so every video gets a poster image and cached metadata.
|
||||
* The board never needs to create a <video> element just to discover dimensions
|
||||
* or capture a first frame.
|
||||
*/
|
||||
|
||||
const { execFile } = require('child_process');
|
||||
const { writeFileSync, readFileSync, unlinkSync, mkdtempSync } = require('fs');
|
||||
const path = require('path');
|
||||
const os = require('os');
|
||||
|
||||
/**
|
||||
* Extract video metadata (dimensions, duration, hasAudio) using ffprobe.
|
||||
* Returns { width, height, duration, hasAudio } or null on failure.
|
||||
*/
|
||||
function probeVideo(buffer) {
|
||||
return new Promise((resolve) => {
|
||||
let tmpDir, tmpFile;
|
||||
try {
|
||||
tmpDir = mkdtempSync(path.join(os.tmpdir(), 'refboard-vid-'));
|
||||
tmpFile = path.join(tmpDir, 'input.vid');
|
||||
writeFileSync(tmpFile, buffer);
|
||||
} catch (err) {
|
||||
console.warn('[video-utils] Failed to write temp file:', err.message);
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
|
||||
execFile('ffprobe', [
|
||||
'-v', 'quiet',
|
||||
'-print_format', 'json',
|
||||
'-show_format',
|
||||
'-show_streams',
|
||||
tmpFile,
|
||||
], { timeout: 15000 }, (err, stdout) => {
|
||||
cleanup(tmpFile, tmpDir);
|
||||
if (err) {
|
||||
console.warn('[video-utils] ffprobe failed:', err.message);
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const info = JSON.parse(stdout);
|
||||
const videoStream = (info.streams || []).find(s => s.codec_type === 'video');
|
||||
const audioStream = (info.streams || []).find(s => s.codec_type === 'audio');
|
||||
|
||||
if (!videoStream) {
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
|
||||
resolve({
|
||||
width: videoStream.width || null,
|
||||
height: videoStream.height || null,
|
||||
duration: info.format?.duration ? parseFloat(info.format.duration) : null,
|
||||
hasAudio: !!audioStream,
|
||||
});
|
||||
} catch (parseErr) {
|
||||
console.warn('[video-utils] ffprobe parse failed:', parseErr.message);
|
||||
resolve(null);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a poster frame (JPEG) from a video buffer.
|
||||
* Returns a Buffer containing JPEG data, or null on failure.
|
||||
*/
|
||||
function extractPoster(buffer) {
|
||||
return new Promise((resolve) => {
|
||||
let tmpDir, tmpInput, tmpOutput;
|
||||
try {
|
||||
tmpDir = mkdtempSync(path.join(os.tmpdir(), 'refboard-poster-'));
|
||||
tmpInput = path.join(tmpDir, 'input.vid');
|
||||
tmpOutput = path.join(tmpDir, 'poster.jpg');
|
||||
writeFileSync(tmpInput, buffer);
|
||||
} catch (err) {
|
||||
console.warn('[video-utils] Failed to write temp file:', err.message);
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
|
||||
execFile('ffmpeg', [
|
||||
'-i', tmpInput,
|
||||
'-vframes', '1', // single frame
|
||||
'-ss', '0.1', // skip 0.1s (avoid black leader)
|
||||
'-q:v', '3', // JPEG quality (2=best, 31=worst)
|
||||
'-y', // overwrite
|
||||
tmpOutput,
|
||||
], { timeout: 15000 }, (err) => {
|
||||
if (err) {
|
||||
console.warn('[video-utils] ffmpeg poster extraction failed:', err.message);
|
||||
cleanup(tmpInput, tmpDir);
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const posterBuffer = readFileSync(tmpOutput);
|
||||
cleanup(tmpInput, tmpDir, tmpOutput);
|
||||
resolve(posterBuffer);
|
||||
} catch (readErr) {
|
||||
console.warn('[video-utils] Failed to read poster:', readErr.message);
|
||||
cleanup(tmpInput, tmpDir, tmpOutput);
|
||||
resolve(null);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function cleanup(...files) {
|
||||
for (const f of files) {
|
||||
try { unlinkSync(f); } catch { /* ignore */ }
|
||||
}
|
||||
// Try removing parent dirs (they're temp dirs we created)
|
||||
for (const f of files) {
|
||||
try {
|
||||
const dir = path.dirname(f);
|
||||
if (dir.includes('refboard-')) {
|
||||
require('fs').rmdirSync(dir);
|
||||
}
|
||||
} catch { /* ignore — dir may not be empty or already removed */ }
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { probeVideo, extractPoster };
|
||||
Reference in New Issue
Block a user