From 198497381f68967c1b0ab060e3b2bedf01655726 Mon Sep 17 00:00:00 2001
From: Hiren Kangad <hir.012612@gmail.com>
Date: Tue, 10 Mar 2026 12:10:18 +0530
Subject: [PATCH] feat(refboard): server-generated video posters via ffmpeg

Upload pipeline now extracts poster frame (JPEG) and metadata
(width, height, duration) from videos at upload time using ffmpeg.

Server:
- Add ffmpeg to Docker image (Alpine)
- video-utils.js: probeVideo() and extractPoster() using ffprobe/ffmpeg
- Upload response includes poster_asset_key and duration

Frontend:
- VideoObject gains poster and duration fields in scene format
- VideoSprite accepts posterAssetKey + TextureManager, loads poster
  as a regular image texture on construction (no <video> needed)
- Server poster loaded/released via TextureManager ref counting
- addVideoFromUpload passes poster and duration through to scene data
- image-drop.ts forwards poster_asset_key from upload response

Result: videos with server posters render as images by default.
Zero <video> elements needed for thumbnails. Only explicit play
creates a media element.
---
 Dockerfile                                 |   3 +-
 backend/routes/upload.js                   |  45 +++++--
 backend/video-utils.js                     | 130 +++++++++++++++++++++
 frontend/src/canvas/SceneManager.ts        |   8 +-
 frontend/src/canvas/image-drop.ts          |   4 +-
 frontend/src/canvas/scene-format.ts        |   4 +
 frontend/src/canvas/sprites/VideoSprite.ts | 107 +++++++++++------
 7 files changed, 256 insertions(+), 45 deletions(-)
 create mode 100644 backend/video-utils.js

diff --git a/Dockerfile b/Dockerfile
index e947adf..7c6c366 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -27,8 +27,9 @@ COPY --from=frontend-build /build/frontend/dist ./frontend/dist
 # Data directory for SQLite
 RUN mkdir -p /app/data
 
+# ffmpeg for video poster/metadata extraction at upload time
 # Health check utility
-RUN apk add --no-cache wget
+RUN apk add --no-cache wget ffmpeg
 
 EXPOSE 8000
 
diff --git a/backend/routes/upload.js b/backend/routes/upload.js
index 76ccfdd..3c9ed41 100644
--- a/backend/routes/upload.js
+++ b/backend/routes/upload.js
@@ -8,6 +8,7 @@ const { URL } = require('url');
 const { authMiddleware } = require('../auth');
 const { getBoard, getCollectionMember, createImage } = require('../db');
 const { putBuffer, getImageUrl, MIME_TO_EXT, MAX_FILE_SIZE } = require('../minio');
+const { probeVideo, extractPoster } = require('../video-utils');
 
 const router = Router();
 
@@ -89,17 +90,43 @@ function classifyMedia(mimeType) {
 }
 
 /**
- * Upload a media file (image or video) to MinIO as a single file.
- * GPU handles all image scaling natively — no LOD tiers needed.
+ * Upload a media file (image or video) to MinIO.
+ * For videos: extracts poster frame + metadata via ffmpeg at upload time
+ * so the board never needs a <video> element for thumbnails.
  */
 async function uploadMedia(boardId, imageId, buffer, mimetype) {
   const ext = MIME_TO_EXT[mimetype] || '.bin';
   const minioPath = `boards/${boardId}/${imageId}${ext}`;
   await putBuffer(minioPath, buffer, mimetype);
 
-  const dims = await getImageDimensions(buffer, mimetype);
-  // assetKey = minioPath so frontend can build direct URLs
-  return { assetKey: minioPath, minioPath, width: dims.width, height: dims.height };
+  const isVideo = VIDEO_MIME_TYPES.includes(mimetype);
+  let width = null, height = null, duration = null, posterAssetKey = null;
+
+  if (isVideo) {
+    // Extract metadata and poster frame server-side
+    const [meta, posterBuf] = await Promise.all([
+      probeVideo(buffer),
+      extractPoster(buffer),
+    ]);
+
+    if (meta) {
+      width = meta.width;
+      height = meta.height;
+      duration = meta.duration;
+    }
+
+    if (posterBuf) {
+      const posterPath = `boards/${boardId}/${imageId}_poster.jpg`;
+      await putBuffer(posterPath, posterBuf, 'image/jpeg');
+      posterAssetKey = posterPath;
+    }
+  } else {
+    const dims = await getImageDimensions(buffer, mimetype);
+    width = dims.width;
+    height = dims.height;
+  }
+
+  return { assetKey: minioPath, minioPath, width, height, duration, posterAssetKey };
 }
 
 /**
@@ -119,7 +146,7 @@ router.post('/boards/:boardId/images', upload.single('image'), async (req, res)
     const { buffer, originalname, mimetype, size } = req.file;
     const mediaType = classifyMedia(mimetype);
 
-    const { assetKey, minioPath, width, height } = await uploadMedia(board.id, imageId, buffer, mimetype);
+    const { assetKey, minioPath, width, height, duration, posterAssetKey } = await uploadMedia(board.id, imageId, buffer, mimetype);
     const publicUrl = getImageUrl(minioPath);
 
     // Save record
@@ -148,6 +175,8 @@ router.post('/boards/:boardId/images', upload.single('image'), async (req, res)
       mime_type: image.mime_type,
       asset_key: image.asset_key,
       media_type: image.media_type,
+      duration: duration || undefined,
+      poster_asset_key: posterAssetKey || undefined,
     });
   } catch (err) {
     if (err.code === 'LIMIT_FILE_SIZE') {
@@ -224,7 +253,7 @@ router.post('/boards/:boardId/images/from-url', async (req, res) => {
     const imageId = uuidv4();
     const mediaType = classifyMedia(mimeType);
 
-    const { assetKey, minioPath, width, height } = await uploadMedia(board.id, imageId, buffer, mimeType);
+    const { assetKey, minioPath, width, height, duration, posterAssetKey } = await uploadMedia(board.id, imageId, buffer, mimeType);
     const publicUrl = getImageUrl(minioPath);
 
     // Save record
@@ -253,6 +282,8 @@ router.post('/boards/:boardId/images/from-url', async (req, res) => {
       mime_type: image.mime_type,
       asset_key: image.asset_key,
       media_type: image.media_type,
+      duration: duration || undefined,
+      poster_asset_key: posterAssetKey || undefined,
     });
   } catch (err) {
     console.error('[upload] from-url error:', err);
diff --git a/backend/video-utils.js b/backend/video-utils.js
new file mode 100644
index 0000000..91c93f9
--- /dev/null
+++ b/backend/video-utils.js
@@ -0,0 +1,130 @@
+/**
+ * video-utils.js — Extract poster frame and metadata from video buffers using ffmpeg.
+ *
+ * Called at upload time so every video gets a poster image and cached metadata.
+ * The board never needs to create a <video> element just to discover dimensions
+ * or capture a first frame.
+ */
+
+const { execFile } = require('child_process');
+const { writeFileSync, readFileSync, unlinkSync, mkdtempSync } = require('fs');
+const path = require('path');
+const os = require('os');
+
+/**
+ * Extract video metadata (dimensions, duration, hasAudio) using ffprobe.
+ * Returns { width, height, duration, hasAudio } or null on failure.
+ */
+function probeVideo(buffer) {
+  return new Promise((resolve) => {
+    let tmpDir, tmpFile;
+    try {
+      tmpDir = mkdtempSync(path.join(os.tmpdir(), 'refboard-vid-'));
+      tmpFile = path.join(tmpDir, 'input.vid');
+      writeFileSync(tmpFile, buffer);
+    } catch (err) {
+      console.warn('[video-utils] Failed to write temp file:', err.message);
+      resolve(null);
+      return;
+    }
+
+    execFile('ffprobe', [
+      '-v', 'quiet',
+      '-print_format', 'json',
+      '-show_format',
+      '-show_streams',
+      tmpFile,
+    ], { timeout: 15000 }, (err, stdout) => {
+      cleanup(tmpFile, tmpDir);
+      if (err) {
+        console.warn('[video-utils] ffprobe failed:', err.message);
+        resolve(null);
+        return;
+      }
+
+      try {
+        const info = JSON.parse(stdout);
+        const videoStream = (info.streams || []).find(s => s.codec_type === 'video');
+        const audioStream = (info.streams || []).find(s => s.codec_type === 'audio');
+
+        if (!videoStream) {
+          resolve(null);
+          return;
+        }
+
+        resolve({
+          width: videoStream.width || null,
+          height: videoStream.height || null,
+          duration: info.format?.duration ? parseFloat(info.format.duration) : null,
+          hasAudio: !!audioStream,
+        });
+      } catch (parseErr) {
+        console.warn('[video-utils] ffprobe parse failed:', parseErr.message);
+        resolve(null);
+      }
+    });
+  });
+}
+
+/**
+ * Extract a poster frame (JPEG) from a video buffer.
+ * Returns a Buffer containing JPEG data, or null on failure.
+ */
+function extractPoster(buffer) {
+  return new Promise((resolve) => {
+    let tmpDir, tmpInput, tmpOutput;
+    try {
+      tmpDir = mkdtempSync(path.join(os.tmpdir(), 'refboard-poster-'));
+      tmpInput = path.join(tmpDir, 'input.vid');
+      tmpOutput = path.join(tmpDir, 'poster.jpg');
+      writeFileSync(tmpInput, buffer);
+    } catch (err) {
+      console.warn('[video-utils] Failed to write temp file:', err.message);
+      resolve(null);
+      return;
+    }
+
+    execFile('ffmpeg', [
+      '-i', tmpInput,
+      '-vframes', '1',       // single frame
+      '-ss', '0.1',          // skip 0.1s (avoid black leader)
+      '-q:v', '3',           // JPEG quality (2=best, 31=worst)
+      '-y',                  // overwrite
+      tmpOutput,
+    ], { timeout: 15000 }, (err) => {
+      if (err) {
+        console.warn('[video-utils] ffmpeg poster extraction failed:', err.message);
+        cleanup(tmpInput, tmpDir);
+        resolve(null);
+        return;
+      }
+
+      try {
+        const posterBuffer = readFileSync(tmpOutput);
+        cleanup(tmpInput, tmpDir, tmpOutput);
+        resolve(posterBuffer);
+      } catch (readErr) {
+        console.warn('[video-utils] Failed to read poster:', readErr.message);
+        cleanup(tmpInput, tmpDir, tmpOutput);
+        resolve(null);
+      }
+    });
+  });
+}
+
+function cleanup(...files) {
+  for (const f of files) {
+    try { unlinkSync(f); } catch { /* ignore */ }
+  }
+  // Try removing parent dirs (they're temp dirs we created)
+  for (const f of files) {
+    try {
+      const dir = path.dirname(f);
+      if (dir.includes('refboard-')) {
+        require('fs').rmdirSync(dir);
+      }
+    } catch { /* ignore — dir may not be empty or already removed */ }
+  }
+}
+
+module.exports = { probeVideo, extractPoster };
diff --git a/frontend/src/canvas/SceneManager.ts b/frontend/src/canvas/SceneManager.ts
index ab9953e..caff069 100644
--- a/frontend/src/canvas/SceneManager.ts
+++ b/frontend/src/canvas/SceneManager.ts
@@ -271,7 +271,7 @@ export class SceneManager {
         // Use cached native dimensions if available (avoids re-init just to discover size)
         const vw = vidData.nativeW || vidData.w;
         const vh = vidData.nativeH || vidData.h;
-        const videoSprite = new VideoSprite(vidData.asset, vw, vh, videoUrl);
+        const videoSprite = new VideoSprite(vidData.asset, vw, vh, videoUrl, vidData.poster, this.textures);
         // Apply cached native dims to scene data so display matches
         if (vidData.nativeW && vidData.nativeH) {
           data.w = vw;
@@ -607,6 +607,8 @@ export class SceneManager {
     h: number,
     x: number,
     y: number,
+    posterAssetKey?: string,
+    duration?: number,
   ): SceneItem {
     const data: VideoObject = {
       id: crypto.randomUUID(),
@@ -626,6 +628,10 @@ export class SceneManager {
       asset: assetKey,
       muted: true,
       loop: true,
+      nativeW: w,
+      nativeH: h,
+      poster: posterAssetKey,
+      duration,
     };
 
     this._createItem(data, true);
diff --git a/frontend/src/canvas/image-drop.ts b/frontend/src/canvas/image-drop.ts
index 3062cfa..9fc0e72 100644
--- a/frontend/src/canvas/image-drop.ts
+++ b/frontend/src/canvas/image-drop.ts
@@ -55,7 +55,9 @@ function handleUploadResult(
   }
 
   if (mediaType === 'video' && assetKey) {
-    sceneManager.addVideoFromUpload(assetKey, finalW, finalH, x, y);
+    const posterKey: string | undefined = imgData.poster_asset_key;
+    const duration: number | undefined = imgData.duration;
+    sceneManager.addVideoFromUpload(assetKey, finalW, finalH, x, y, posterKey, duration);
   } else if (assetKey) {
     sceneManager.addImageFromUpload(assetKey, finalW, finalH, x, y);
   } else {
diff --git a/frontend/src/canvas/scene-format.ts b/frontend/src/canvas/scene-format.ts
index 39183f4..2eac7bf 100644
--- a/frontend/src/canvas/scene-format.ts
+++ b/frontend/src/canvas/scene-format.ts
@@ -35,6 +35,10 @@ export interface VideoObject extends SceneObject {
   /** Native video dimensions, cached after first metadata load. */
   nativeW?: number;
   nativeH?: number;
+  /** Server-generated poster asset key (JPEG). */
+  poster?: string;
+  /** Video duration in seconds. */
+  duration?: number;
 }
 
 export interface TextObject extends SceneObject {
diff --git a/frontend/src/canvas/sprites/VideoSprite.ts b/frontend/src/canvas/sprites/VideoSprite.ts
index ac420be..c08f82c 100644
--- a/frontend/src/canvas/sprites/VideoSprite.ts
+++ b/frontend/src/canvas/sprites/VideoSprite.ts
@@ -1,20 +1,18 @@
 import { Container, Sprite, Texture, Graphics } from 'pixi.js';
+import { TextureManager } from '../TextureManager';
 
 /**
  * VideoSprite — Container with budget-aware lazy video lifecycle.
  *
  * Memory tiers (managed by external culling system):
  *   Tier 0 — placeholder only (offscreen, default state)
+ *   Tier 0.5 — server poster loaded as image texture (no <video> needed!)
  *   Tier 1 — initialized <video> with preload='metadata' (near viewport)
- *   Tier 2 — poster texture captured (nearest subset / selected)
+ *   Tier 2 — client-captured poster from video frame (fallback if no server poster)
  *   Tier 3 — actively playing with live video texture (user-initiated)
  *
- * The culling system in PixiCanvas manages budgets:
- *   - MAX_INITIALIZED_VIDEOS (tier 1+)
- *   - MAX_POSTER_VIDEOS (tier 2+)
- *   - MAX_PLAYING_VIDEOS (tier 3)
- *
- * Constructor creates only shadow + placeholder + overlay. No <video> element.
+ * If the server provides a poster asset key at upload time, the video renders
+ * as a plain image until explicitly played. No <video> element needed for thumbnails.
  */
 
 const MAX_PLAYING_VIDEOS = 1;
@@ -26,10 +24,13 @@ const SHADOW_LIFT = { offsetX: 6, offsetY: 8, alpha: 0.3 };
 export class VideoSprite extends Container {
   readonly assetKey: string;
   readonly videoUrl: string;
+  readonly posterAssetKey: string | null;
 
+  private textures: TextureManager | null;
   private videoEl: HTMLVideoElement | null = null;
   private videoTexture: Texture | null = null;
   private posterTexture: Texture | null = null;
+  private _serverPosterLoaded = false;
   private _sprite: Sprite | null = null;
   private _shadow: Graphics;
   private _overlay: Graphics;
@@ -48,11 +49,20 @@ export class VideoSprite extends Container {
   /** Called when playback state changes (for external controls). */
   onStateChange: (() => void) | null = null;
 
-  constructor(assetKey: string, w: number, h: number, videoUrl: string) {
+  constructor(
+    assetKey: string,
+    w: number,
+    h: number,
+    videoUrl: string,
+    posterAssetKey?: string | null,
+    textures?: TextureManager | null,
+  ) {
     super();
 
     this.assetKey = assetKey;
     this.videoUrl = videoUrl;
+    this.posterAssetKey = posterAssetKey ?? null;
+    this.textures = textures ?? null;
     this._naturalWidth = w;
     this._naturalHeight = h;
 
@@ -70,6 +80,12 @@ export class VideoSprite extends Container {
     this._overlay = new Graphics();
     this._drawPlayIcon();
     this.addChild(this._overlay);
+
+    // If server poster available, load it immediately as an image texture
+    // This is the key optimization: video thumbnail = image, no <video> needed
+    if (this.posterAssetKey && this.textures) {
+      this._loadServerPoster();
+    }
   }
 
   get isPlaying(): boolean { return this._isPlaying; }
@@ -79,6 +95,25 @@ export class VideoSprite extends Container {
   /** Expose the underlying HTMLVideoElement for external controls. Null if not initialized. */
   get videoElement(): HTMLVideoElement | null { return this.videoEl; }
 
+  // ---- Tier 0.5: Server-generated poster (loaded like an image) -----------
+
+  private async _loadServerPoster(): Promise<void> {
+    if (!this.posterAssetKey || !this.textures || this.destroyed) return;
+    try {
+      const tex = await this.textures.load(this.posterAssetKey);
+      if (this.destroyed || this._isPlaying) return;
+
+      this.posterTexture = tex;
+      this._hasPoster = true;
+      this._serverPosterLoaded = true;
+
+      this._ensureSprite(tex);
+      this._removePlaceholder();
+    } catch {
+      // Server poster failed — will fall back to client capture if needed
+    }
+  }
+
   // ---- Tier 1: Initialize <video> element ---------------------------------
 
   /** Create <video> with preload='metadata'. Called by culling when within init budget. */
@@ -97,35 +132,32 @@ export class VideoSprite extends Container {
 
     this.videoEl = video;
     video.src = this.videoUrl;
-    this.onStateChange?.(); // notify controls of new element
+    this.onStateChange?.();
   }
 
   /** Tear down the <video> element. Keeps poster if present. */
   teardownVideo(): void {
     if (!this._videoInitialized) return;
-    if (this._isPlaying) return; // don't tear down while user is watching
+    if (this._isPlaying) return;
 
     this._destroyVideoEl();
     this._destroyVideoTexture();
     this._videoInitialized = false;
-    this.onStateChange?.(); // notify controls element is gone
+    this.onStateChange?.();
   }
 
-  // ---- Tier 2: Poster texture ---------------------------------------------
+  // ---- Tier 2: Client-captured poster (fallback) --------------------------
 
-  /** Capture a poster frame. Requires initialized video. */
+  /** Capture a poster frame from video. Only needed if no server poster. */
   capturePoster(): void {
     if (this._hasPoster || !this.videoEl || this.destroyed) return;
 
     if (this.videoEl.readyState >= 2) {
-      // Frame data already available — capture directly
       this._captureFirstFrame();
     } else if (this.videoEl.readyState >= 1) {
-      // Metadata loaded but no frame decoded (preload='metadata').
-      // Force a seek to trigger frame decode, capture on 'seeked'.
+      // preload='metadata' may not decode frames. Force a seek.
       this._trySeekCapture();
     } else {
-      // Not even metadata yet — wait for loadeddata
       this.videoEl.addEventListener('loadeddata', this._onLoadedData, { once: true });
     }
   }
@@ -134,14 +166,19 @@ export class VideoSprite extends Container {
   destroyPoster(): void {
     if (!this._hasPoster) return;
     if (this.posterTexture) {
-      // If sprite is showing poster, switch back to nothing
       if (this._sprite && this._sprite.texture === this.posterTexture) {
         this._removeSpriteFromTree();
       }
-      this.posterTexture.destroy(true);
+      // Release via TextureManager if server poster, otherwise destroy directly
+      if (this._serverPosterLoaded && this.posterAssetKey && this.textures) {
+        this.textures.release(this.posterAssetKey);
+      } else {
+        this.posterTexture.destroy(true);
+      }
       this.posterTexture = null;
     }
     this._hasPoster = false;
+    this._serverPosterLoaded = false;
     this._restorePlaceholder();
   }
 
@@ -157,31 +194,32 @@ export class VideoSprite extends Container {
   play(): void {
     if (this._isPlaying) return;
 
-    // Ensure video element exists
     if (!this._videoInitialized) this.initVideo();
     if (!this.videoEl) return;
 
-    // Evict oldest if at capacity
     if (activeVideos.size >= MAX_PLAYING_VIDEOS) {
       const oldest = activeVideos.values().next().value as VideoSprite;
       oldest.pause();
     }
 
-    // Promote to full load for playback
     this.videoEl.preload = 'auto';
     this.videoEl.muted = this.muted;
     this.videoEl.loop = this.loop;
 
-    // Create live video texture
     if (!this.videoTexture) {
       this.videoTexture = Texture.from(this.videoEl);
     }
 
     // Drop poster while playing — don't keep both resident
-    if (this.posterTexture && this._hasPoster) {
-      this.posterTexture.destroy(true);
+    if (this._hasPoster && this.posterTexture) {
+      if (this._serverPosterLoaded && this.posterAssetKey && this.textures) {
+        this.textures.release(this.posterAssetKey);
+      } else {
+        this.posterTexture.destroy(true);
+      }
       this.posterTexture = null;
       this._hasPoster = false;
+      this._serverPosterLoaded = false;
     }
 
     this._ensureSprite(this.videoTexture);
@@ -193,7 +231,6 @@ export class VideoSprite extends Container {
     this.onStateChange?.();
 
     this.videoEl.play().catch(() => {
-      // Autoplay blocked — show placeholder
       this._isPlaying = false;
       this._overlay.visible = true;
       activeVideos.delete(this);
@@ -208,7 +245,6 @@ export class VideoSprite extends Container {
     this._overlay.visible = true;
     activeVideos.delete(this);
     this.onStateChange?.();
-    // Keep video texture showing last frame
   }
 
   togglePlayPause(): void {
@@ -228,13 +264,10 @@ export class VideoSprite extends Container {
 
   // ---- Visibility (called by culling system) ------------------------------
 
-  /** Simple visibility callback — culling system manages budgets externally. */
   onVisibilityChange(visible: boolean): void {
     if (!visible && this._isPlaying) {
       this.pause();
     }
-    // NOTE: init/teardown is now managed by the culling budget system,
-    // not by individual visibility changes.
   }
 
   // ---- Shadow / overlay ---------------------------------------------------
@@ -267,7 +300,7 @@ export class VideoSprite extends Container {
     this._overlay.fill({ color: 0xffffff, alpha: 0.9 });
   }
 
-  // ---- Media event handlers (arrow fns for stable `this`) -----------------
+  // ---- Media event handlers -----------------------------------------------
 
   private _onLoadedMetadata = (): void => {
     if (!this.videoEl || this.destroyed) return;
@@ -357,7 +390,6 @@ export class VideoSprite extends Container {
       this._sprite = new Sprite(texture);
       this._sprite.width = this._naturalWidth;
       this._sprite.height = this._naturalHeight;
-      // Insert after shadow (index 0), before placeholder/overlay
       this.addChildAt(this._sprite, 1);
     } else {
       this._sprite.texture = texture;
@@ -387,7 +419,6 @@ export class VideoSprite extends Container {
     const p = new Graphics();
     p.rect(0, 0, this._naturalWidth, this._naturalHeight).fill(0x2a2a2a);
     this._placeholder = p;
-    // Insert after shadow
     this.addChildAt(p, 1);
   }
 
@@ -403,7 +434,6 @@ export class VideoSprite extends Container {
 
   private _destroyVideoTexture(): void {
     if (this.videoTexture) {
-      // If sprite is showing live texture, remove it
       if (this._sprite && this._sprite.texture === this.videoTexture) {
         this._removeSpriteFromTree();
         this._restorePlaceholder();
@@ -417,7 +447,14 @@ export class VideoSprite extends Container {
     this.pause();
     this._destroyVideoEl();
     if (this.videoTexture) { this.videoTexture.destroy(true); this.videoTexture = null; }
-    if (this.posterTexture) { this.posterTexture.destroy(true); this.posterTexture = null; }
+    if (this._hasPoster && this.posterTexture) {
+      if (this._serverPosterLoaded && this.posterAssetKey && this.textures) {
+        this.textures.release(this.posterAssetKey);
+      } else {
+        this.posterTexture.destroy(true);
+      }
+      this.posterTexture = null;
+    }
     super.destroy(options);
   }
 }