Hermes/refboard-ayon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(refboard): video budget system, lazy lifecycle, incremental sync fixes

Browse Source 
Video memory:
- Lazy <video> creation: constructor makes placeholder only, initVideo()
  creates element with preload='metadata' when near viewport
- Tiered budgets: 1 playing / 6 initialized / 4 poster textures max
- Aggressive teardown: offscreen videos lose <video> element, poster,
  and all textures — zero memory for offscreen videos
- Poster textures dropped while playing (don't keep both resident)
- Dimension caching in scene data (nativeW/nativeH) avoids re-init

Server:
- HTTP Range support (206 Partial Content) for video seeking
- Proper end clamping and invalid range rejection (416)

Sync:
- Remove redundant broadcastSceneDebounced() from broadcastTransform()
- Remove duplicate broadcastElements() from drag-end handler

VideoControls:
- Event-driven (timeupdate/play/pause/seeked) instead of rAF polling
- Tracks actual HTMLVideoElement reference, rebinds on init/teardown
- Seek uses ref-based commit on pointerUp, not conditional onChange
This commit is contained in:
Hiren Kangad
2026-03-10 11:56:33 +05:30
parent ae050d2119
commit 89010335be
8 changed files with 523 additions and 311 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/canvas/PixiCanvas.tsx
+76 -38
View File
@@ -19,6 +19,7 @@ import { Viewport } from 'pixi-viewport';
import { SceneManager, getItemWorldBounds, type SceneItem } from './SceneManager';
import { TextureManager } from './TextureManager';
import { ImageSprite } from './sprites/ImageSprite';
import { VideoSprite } from './sprites/VideoSprite';
import { SpringManager } from './spring';
import { convertFabricToV2 } from './scene-format';
import type { SceneData } from './scene-format';
@@ -166,11 +167,13 @@ const PixiCanvas = forwardRef<PixiCanvasHandle, PixiCanvasProps>(
});
// -- Visibility culling ticker (runs every 200ms, not every frame) --
// Images: load textures when near viewport, unload when far away.
// Uses preload margin and hysteresis. Caps concurrent loaded textures
// to prevent GPU OOM when zoomed out (all items visible at once).
// Manages GPU/memory budgets for both images and videos.
// Distance-based priority: nearest to viewport center get resources first.
const MAX_LOADED_TEXTURES = 60; // max image textures in GPU
const MAX_INITIALIZED_VIDEOS = 6; // max <video> elements (tier 1+)
const MAX_POSTER_VIDEOS = 4; // max poster textures in GPU (tier 2)
const MAX_LOADED_TEXTURES = 60; // GPU budget — only this many images loaded at once
let lastCullCheck = 0;
app.ticker.add((ticker) => {
lastCullCheck += ticker.deltaMS;
@@ -179,61 +182,96 @@ const PixiCanvas = forwardRef<PixiCanvasHandle, PixiCanvasProps>(
const bounds = viewport.getVisibleBounds();
const loadMargin = Math.max(bounds.width, bounds.height);
const unloadMargin = loadMargin * 2;
const vcx = bounds.x + bounds.width / 2;
const vcy = bounds.y + bounds.height / 2;
// Collect image items with distance from viewport center
const imageItems: { item: SceneItem; sprite: ImageSprite; dist: number; near: boolean }[] = [];
const videoItems: { item: SceneItem; sprite: VideoSprite; dist: number; near: boolean }[] = [];
for (const item of scene.getAllItems()) {
const { x: ix, y: iy, w: iw, h: ih } = getItemWorldBounds(item);
const cx = ix + iw / 2;
const cy = iy + ih / 2;
const dist = (cx - vcx) ** 2 + (cy - vcy) ** 2;
const near =
ix + iw > bounds.x - loadMargin &&
ix < bounds.x + bounds.width + loadMargin &&
iy + ih > bounds.y - loadMargin &&
iy < bounds.y + bounds.height + loadMargin;
if (item.type === 'image' && item.displayObject instanceof ImageSprite) {
const near =
ix + iw > bounds.x - loadMargin &&
ix < bounds.x + bounds.width + loadMargin &&
iy + ih > bounds.y - loadMargin &&
iy < bounds.y + bounds.height + loadMargin;
const cx = ix + iw / 2;
const cy = iy + ih / 2;
const dist = (cx - vcx) ** 2 + (cy - vcy) ** 2;
imageItems.push({ item, sprite: item.displayObject, dist, near });
}
if (item.type === 'video' && 'onVisibilityChange' in item.displayObject) {
const inView =
ix + iw > bounds.x - loadMargin &&
ix < bounds.x + bounds.width + loadMargin &&
iy + ih > bounds.y - loadMargin &&
iy < bounds.y + bounds.height + loadMargin;
(item.displayObject as any).onVisibilityChange(inView);
} else if (item.type === 'video' && item.displayObject instanceof VideoSprite) {
videoItems.push({ item, sprite: item.displayObject, dist, near });
}
}
// Sort by distance — closest to viewport center first
// ---- Image budget management ----
imageItems.sort((a, b) => a.dist - b.dist);
// Determine which should be loaded: nearest items up to budget
const nearItems = imageItems.filter(i => i.near);
const shouldBeLoaded = new Set(
nearItems.slice(0, MAX_LOADED_TEXTURES).map(i => i.item.id)
const nearImages = imageItems.filter(i => i.near);
const shouldLoadImage = new Set(
nearImages.slice(0, MAX_LOADED_TEXTURES).map(i => i.item.id)
);
// Load nearest unloaded items (cap at 5 per tick to avoid spike)
let loadsThisTick = 0;
let imgLoadsThisTick = 0;
for (const { item, sprite } of imageItems) {
if (shouldBeLoaded.has(item.id) && !sprite.loaded && loadsThisTick < 5) {
if (shouldLoadImage.has(item.id) && !sprite.loaded && imgLoadsThisTick < 5) {
sprite.loadTexture();
loadsThisTick++;
imgLoadsThisTick++;
}
}
for (const { item, sprite, near } of imageItems) {
if (!sprite.loaded) continue;
if (!near || !shouldLoadImage.has(item.id)) {
sprite.unloadTexture();
}
}
// Unload items that are either far away or over budget
for (const { item, sprite, near } of imageItems) {
if (!sprite.loaded) continue;
if (!near || !shouldBeLoaded.has(item.id)) {
sprite.unloadTexture();
// ---- Video budget management ----
videoItems.sort((a, b) => a.dist - b.dist);
// Determine budget sets (sorted by distance, nearest first)
const nearVideos = videoItems.filter(i => i.near);
const shouldInit = new Set(
nearVideos.slice(0, MAX_INITIALIZED_VIDEOS).map(i => i.item.id)
);
const shouldPoster = new Set(
nearVideos.slice(0, MAX_POSTER_VIDEOS).map(i => i.item.id)
);
// Initialize nearest videos within budget
let vidInitsThisTick = 0;
for (const { item, sprite } of videoItems) {
if (shouldInit.has(item.id) && !sprite.isInitialized && vidInitsThisTick < 2) {
sprite.initVideo();
vidInitsThisTick++;
}
}
// Capture posters for nearest subset (only if already initialized)
for (const { item, sprite } of videoItems) {
if (shouldPoster.has(item.id) && sprite.isInitialized && !sprite.hasPoster) {
sprite.capturePoster();
}
}
// Tear down videos outside budgets (farthest first = reverse iteration)
for (let i = videoItems.length - 1; i >= 0; i--) {
const { item, sprite, near } = videoItems[i];
// Pause offscreen playing videos
if (!near && sprite.isPlaying) {
sprite.onVisibilityChange(false);
}
// Destroy poster if outside poster budget
if (!shouldPoster.has(item.id) && sprite.hasPoster && !sprite.isPlaying) {
sprite.destroyPoster();
}
// Tear down video element if outside init budget
if (!shouldInit.has(item.id) && sprite.isInitialized && !sprite.isPlaying) {
sprite.teardownVideo();
}
}
});
frontend/src/canvas/SceneManager.ts
+12 -1
View File
@@ -268,7 +268,15 @@ export class SceneManager {
case 'video': {
const vidData = data as VideoObject;
const videoUrl = this.textures.urlForAsset(vidData.asset);
const videoSprite = new VideoSprite(vidData.asset, vidData.w, vidData.h, videoUrl);
// Use cached native dimensions if available (avoids re-init just to discover size)
const vw = vidData.nativeW || vidData.w;
const vh = vidData.nativeH || vidData.h;
const videoSprite = new VideoSprite(vidData.asset, vw, vh, videoUrl);
// Apply cached native dims to scene data so display matches
if (vidData.nativeW && vidData.nativeH) {
data.w = vw;
data.h = vh;
}
// Auto-correct dimensions when video metadata loads
// NOTE: callback must update item.data (the stored copy), not the original `data` param.
// We wire this after the item is created below (see post-creation video wiring).
@@ -330,6 +338,9 @@ export class SceneManager {
displayObject.onDimensionsKnown = (realW, realH) => {
item.data.w = realW;
item.data.h = realH;
// Cache native dimensions so future scene loads skip re-init for size
(item.data as VideoObject).nativeW = realW;
(item.data as VideoObject).nativeH = realH;
this._onChange?.();
this._onItemDimensionsChanged?.(item.id);
};
frontend/src/canvas/scene-format.ts
+3
View File
@@ -32,6 +32,9 @@ export interface VideoObject extends SceneObject {
asset: string;
muted: boolean;
loop: boolean;
/** Native video dimensions, cached after first metadata load. */
nativeW?: number;
nativeH?: number;
}
export interface TextObject extends SceneObject {
frontend/src/canvas/sprites/VideoSprite.ts
+319 -214
View File
@@ -1,23 +1,23 @@
import { Container, Sprite, Texture, Graphics } from 'pixi.js';
/**
* VideoSprite — Container holding shadow + video sprite + play overlay.
* VideoSprite — Container with budget-aware lazy video lifecycle.
*
* Matches ImageSprite pattern: extends Container (not Sprite) so shadow
* and overlay children don't affect bounds for transform box.
* Does NOT auto-play. User clicks to play/pause.
* Auto-corrects dimensions from video metadata if initial w/h were fallbacks.
* Memory tiers (managed by external culling system):
* Tier 0 — placeholder only (offscreen, default state)
* Tier 1 — initialized <video> with preload='metadata' (near viewport)
* Tier 2 — poster texture captured (nearest subset / selected)
* Tier 3 — actively playing with live video texture (user-initiated)
*
* Key design:
* - preload='auto' so the browser fetches enough data for the first frame
* - On 'loadeddata' we briefly play+pause to force the first frame to decode,
* then capture it as a static canvas texture (poster). This avoids the
* "black box" problem where PixiJS video textures only render while playing.
* - When the user hits play, we swap to the live video texture.
* - On pause, we keep the live texture (shows last frame).
* The culling system in PixiCanvas manages budgets:
* - MAX_INITIALIZED_VIDEOS (tier 1+)
* - MAX_POSTER_VIDEOS (tier 2+)
* - MAX_PLAYING_VIDEOS (tier 3)
*
* Constructor creates only shadow + placeholder + overlay. No <video> element.
*/
const MAX_CONCURRENT_VIDEOS = 3;
const MAX_PLAYING_VIDEOS = 1;
const activeVideos: Set<VideoSprite> = new Set();
const SHADOW_REST = { offsetX: 3, offsetY: 3, alpha: 0.2 };
@@ -25,27 +25,34 @@ const SHADOW_LIFT = { offsetX: 6, offsetY: 8, alpha: 0.3 };
export class VideoSprite extends Container {
readonly assetKey: string;
private videoEl: HTMLVideoElement;
readonly videoUrl: string;
private videoEl: HTMLVideoElement | null = null;
private videoTexture: Texture | null = null;
private posterTexture: Texture | null = null;
private _sprite: Sprite;
private _sprite: Sprite | null = null;
private _shadow: Graphics;
private _overlay: Graphics;
private _placeholder: Graphics | null = null;
private _naturalWidth: number;
private _naturalHeight: number;
private _isPlaying = false;
private _videoReady = false;
private _videoInitialized = false;
private _hasPoster = false;
muted = true;
loop = true;
/** Called when video metadata reveals the real dimensions. */
onDimensionsKnown: ((w: number, h: number) => void) | null = null;
/** Called when playback state changes (for external controls). */
onStateChange: (() => void) | null = null;
constructor(assetKey: string, w: number, h: number, videoUrl: string) {
super();
this.assetKey = assetKey;
this.videoUrl = videoUrl;
this._naturalWidth = w;
this._naturalHeight = h;
@@ -54,13 +61,7 @@ export class VideoSprite extends Container {
this._drawShadow(SHADOW_REST);
this.addChild(this._shadow);
// Main sprite (starts empty)
this._sprite = new Sprite(Texture.EMPTY);
this._sprite.width = w;
this._sprite.height = h;
this.addChild(this._sprite);
// Dark placeholder with film icon
// Dark placeholder
this._placeholder = new Graphics();
this._placeholder.rect(0, 0, w, h).fill(0x2a2a2a);
this.addChild(this._placeholder);
@@ -69,128 +70,173 @@ export class VideoSprite extends Container {
this._overlay = new Graphics();
this._drawPlayIcon();
this.addChild(this._overlay);
// Video element
this.videoEl = document.createElement('video');
this.videoEl.crossOrigin = 'anonymous';
this.videoEl.muted = true;
this.videoEl.loop = true;
this.videoEl.playsInline = true;
this.videoEl.preload = 'auto'; // load enough for first frame
// Auto-correct dimensions from video metadata
this.videoEl.addEventListener('loadedmetadata', () => {
const vw = this.videoEl.videoWidth;
const vh = this.videoEl.videoHeight;
if (vw > 0 && vh > 0) {
// Cap to 600px max dimension, preserving aspect ratio
const maxDim = 600;
let fw = vw, fh = vh;
if (vw > maxDim || vh > maxDim) {
const s = maxDim / Math.max(vw, vh);
fw = Math.round(vw * s);
fh = Math.round(vh * s);
}
if (fw !== this._naturalWidth || fh !== this._naturalHeight) {
this._naturalWidth = fw;
this._naturalHeight = fh;
this._sprite.width = fw;
this._sprite.height = fh;
// Update placeholder size too
if (this._placeholder) {
this._placeholder.clear();
this._placeholder.rect(0, 0, fw, fh).fill(0x2a2a2a);
}
this._drawShadow(SHADOW_REST);
this._drawPlayIcon();
this.onDimensionsKnown?.(fw, fh);
}
}
}, { once: true });
// When first frame data is available, capture a poster
this.videoEl.addEventListener('loadeddata', () => {
this._captureFirstFrame();
}, { once: true });
// Set src last to start loading
this.videoEl.src = videoUrl;
}
/**
* Capture the first frame as a static canvas texture.
* This ensures the video shows content even when paused/not-yet-played.
*/
private _captureFirstFrame(): void {
const vw = this.videoEl.videoWidth;
const vh = this.videoEl.videoHeight;
if (vw <= 0 || vh <= 0) return;
get isPlaying(): boolean { return this._isPlaying; }
get isInitialized(): boolean { return this._videoInitialized; }
get hasPoster(): boolean { return this._hasPoster; }
try {
const canvas = document.createElement('canvas');
canvas.width = vw;
canvas.height = vh;
const ctx = canvas.getContext('2d');
if (!ctx) return;
/** Expose the underlying HTMLVideoElement for external controls. Null if not initialized. */
get videoElement(): HTMLVideoElement | null { return this.videoEl; }
ctx.drawImage(this.videoEl, 0, 0, vw, vh);
this.posterTexture = Texture.from(canvas);
this._sprite.texture = this.posterTexture;
this._sprite.width = this._naturalWidth;
this._sprite.height = this._naturalHeight;
this._videoReady = true;
// ---- Tier 1: Initialize <video> element ---------------------------------
// Remove placeholder
if (this._placeholder) {
this.removeChild(this._placeholder);
this._placeholder.destroy();
this._placeholder = null;
}
} catch (err) {
// Cross-origin or other issue — try the seeked approach
this._trySeekCapture();
/** Create <video> with preload='metadata'. Called by culling when within init budget. */
initVideo(): void {
if (this._videoInitialized || this.destroyed) return;
this._videoInitialized = true;
const video = document.createElement('video');
video.crossOrigin = 'anonymous';
video.muted = true;
video.loop = true;
video.playsInline = true;
video.preload = 'metadata';
video.addEventListener('loadedmetadata', this._onLoadedMetadata);
this.videoEl = video;
video.src = this.videoUrl;
this.onStateChange?.(); // notify controls of new element
}
/** Tear down the <video> element. Keeps poster if present. */
teardownVideo(): void {
if (!this._videoInitialized) return;
if (this._isPlaying) return; // don't tear down while user is watching
this._destroyVideoEl();
this._destroyVideoTexture();
this._videoInitialized = false;
this.onStateChange?.(); // notify controls element is gone
}
// ---- Tier 2: Poster texture ---------------------------------------------
/** Capture a poster frame. Requires initialized video. */
capturePoster(): void {
if (this._hasPoster || !this.videoEl || this.destroyed) return;
// Need loadeddata to capture a frame — listen if not yet ready
if (this.videoEl.readyState >= 2) {
this._captureFirstFrame();
} else {
this.videoEl.addEventListener('loadeddata', this._onLoadedData, { once: true });
}
}
/**
* Fallback: seek to 0.1s and capture on 'seeked' event.
* Some browsers need a seek to decode the first frame.
*/
private _trySeekCapture(): void {
const onSeeked = () => {
this.videoEl.removeEventListener('seeked', onSeeked);
const vw = this.videoEl.videoWidth;
const vh = this.videoEl.videoHeight;
if (vw <= 0 || vh <= 0) return;
try {
const canvas = document.createElement('canvas');
canvas.width = vw;
canvas.height = vh;
const ctx = canvas.getContext('2d');
if (!ctx) return;
ctx.drawImage(this.videoEl, 0, 0, vw, vh);
this.posterTexture = Texture.from(canvas);
this._sprite.texture = this.posterTexture;
this._sprite.width = this._naturalWidth;
this._sprite.height = this._naturalHeight;
this._videoReady = true;
if (this._placeholder) {
this.removeChild(this._placeholder);
this._placeholder.destroy();
this._placeholder = null;
}
} catch {
// Give up on poster — user will see placeholder until play
/** Destroy the poster texture to free GPU memory. */
destroyPoster(): void {
if (!this._hasPoster) return;
if (this.posterTexture) {
// If sprite is showing poster, switch back to nothing
if (this._sprite && this._sprite.texture === this.posterTexture) {
this._removeSpriteFromTree();
}
};
this.videoEl.addEventListener('seeked', onSeeked);
this.videoEl.currentTime = 0.1;
this.posterTexture.destroy(true);
this.posterTexture = null;
}
this._hasPoster = false;
this._restorePlaceholder();
}
/** Full aggressive teardown: destroy video element + poster + all textures. */
teardownFull(): void {
if (this._isPlaying) this.pause();
this.destroyPoster();
this.teardownVideo();
}
// ---- Tier 3: Playback ---------------------------------------------------
play(): void {
if (this._isPlaying) return;
// Ensure video element exists
if (!this._videoInitialized) this.initVideo();
if (!this.videoEl) return;
// Evict oldest if at capacity
if (activeVideos.size >= MAX_PLAYING_VIDEOS) {
const oldest = activeVideos.values().next().value as VideoSprite;
oldest.pause();
}
// Promote to full load for playback
this.videoEl.preload = 'auto';
this.videoEl.muted = this.muted;
this.videoEl.loop = this.loop;
// Create live video texture
if (!this.videoTexture) {
this.videoTexture = Texture.from(this.videoEl);
}
// Drop poster while playing — don't keep both resident
if (this.posterTexture && this._hasPoster) {
this.posterTexture.destroy(true);
this.posterTexture = null;
this._hasPoster = false;
}
this._ensureSprite(this.videoTexture);
this._removePlaceholder();
this._isPlaying = true;
this._overlay.visible = false;
activeVideos.add(this);
this.onStateChange?.();
this.videoEl.play().catch(() => {
// Autoplay blocked — show placeholder
this._isPlaying = false;
this._overlay.visible = true;
activeVideos.delete(this);
this.onStateChange?.();
});
}
pause(): void {
if (!this._isPlaying) return;
this.videoEl?.pause();
this._isPlaying = false;
this._overlay.visible = true;
activeVideos.delete(this);
this.onStateChange?.();
// Keep video texture showing last frame
}
togglePlayPause(): void {
if (this._isPlaying) this.pause(); else this.play();
}
toggleMute(): void {
this.muted = !this.muted;
if (this.videoEl) this.videoEl.muted = this.muted;
this.onStateChange?.();
}
seek(time: number): void {
if (!this.videoEl) return;
this.videoEl.currentTime = Math.max(0, Math.min(time, this.videoEl.duration || 0));
}
// ---- Visibility (called by culling system) ------------------------------
/** Simple visibility callback — culling system manages budgets externally. */
onVisibilityChange(visible: boolean): void {
if (!visible && this._isPlaying) {
this.pause();
}
// NOTE: init/teardown is now managed by the culling budget system,
// not by individual visibility changes.
}
// ---- Shadow / overlay ---------------------------------------------------
liftShadow(): void { this._drawShadow(SHADOW_LIFT); }
dropShadow(): void { this._drawShadow(SHADOW_REST); }
private _drawShadow(cfg: { offsetX: number; offsetY: number; alpha: number }): void {
this._shadow.clear();
this._shadow.rect(cfg.offsetX, cfg.offsetY, this._naturalWidth, this._naturalHeight);
@@ -205,11 +251,9 @@ export class VideoSprite extends Container {
const cx = w / 2;
const cy = h / 2;
// Semi-transparent circle
this._overlay.circle(cx, cy, triSize * 0.8);
this._overlay.fill({ color: 0x000000, alpha: 0.5 });
// Play triangle
this._overlay.poly([
cx - triSize * 0.3, cy - triSize * 0.4,
cx + triSize * 0.4, cy,
@@ -218,96 +262,157 @@ export class VideoSprite extends Container {
this._overlay.fill({ color: 0xffffff, alpha: 0.9 });
}
liftShadow(): void {
this._drawShadow(SHADOW_LIFT);
}
// ---- Media event handlers (arrow fns for stable `this`) -----------------
dropShadow(): void {
this._drawShadow(SHADOW_REST);
}
play(): void {
if (this._isPlaying) return;
// Evict oldest if at capacity
if (activeVideos.size >= MAX_CONCURRENT_VIDEOS) {
const oldest = activeVideos.values().next().value as VideoSprite;
oldest.pause();
}
this.videoEl.muted = this.muted;
// Create live video texture for playback
if (!this.videoTexture) {
this.videoTexture = Texture.from(this.videoEl);
}
this._sprite.texture = this.videoTexture;
this._sprite.width = this._naturalWidth;
this._sprite.height = this._naturalHeight;
this.videoEl.play().catch(() => {
// Autoplay blocked — revert to poster
if (this.posterTexture) {
this._sprite.texture = this.posterTexture;
private _onLoadedMetadata = (): void => {
if (!this.videoEl || this.destroyed) return;
const vw = this.videoEl.videoWidth;
const vh = this.videoEl.videoHeight;
if (vw > 0 && vh > 0 && (vw !== this._naturalWidth || vh !== this._naturalHeight)) {
this._naturalWidth = vw;
this._naturalHeight = vh;
if (this._sprite) {
this._sprite.width = vw;
this._sprite.height = vh;
}
this._overlay.visible = true;
});
this._isPlaying = true;
this._overlay.visible = false;
activeVideos.add(this);
if (this._placeholder) {
this._placeholder.clear();
this._placeholder.rect(0, 0, vw, vh).fill(0x2a2a2a);
}
this._drawShadow(SHADOW_REST);
this._drawPlayIcon();
this.onDimensionsKnown?.(vw, vh);
}
};
private _onLoadedData = (): void => {
this._captureFirstFrame();
};
// ---- Internal helpers ---------------------------------------------------
private _captureFirstFrame(): void {
if (!this.videoEl || this.destroyed || this._hasPoster) return;
const vw = this.videoEl.videoWidth;
const vh = this.videoEl.videoHeight;
if (vw <= 0 || vh <= 0) return;
try {
const canvas = document.createElement('canvas');
canvas.width = vw;
canvas.height = vh;
const ctx = canvas.getContext('2d');
if (!ctx) return;
ctx.drawImage(this.videoEl, 0, 0, vw, vh);
this.posterTexture = Texture.from(canvas);
this._hasPoster = true;
this._ensureSprite(this.posterTexture);
this._removePlaceholder();
} catch {
this._trySeekCapture();
}
}
pause(): void {
if (!this._isPlaying) return;
private _trySeekCapture(): void {
if (!this.videoEl) return;
const onSeeked = () => {
if (!this.videoEl) return;
this.videoEl.removeEventListener('seeked', onSeeked);
if (this._hasPoster || this.destroyed) return;
const vw = this.videoEl.videoWidth;
const vh = this.videoEl.videoHeight;
if (vw <= 0 || vh <= 0) return;
try {
const canvas = document.createElement('canvas');
canvas.width = vw;
canvas.height = vh;
const ctx = canvas.getContext('2d');
if (!ctx) return;
ctx.drawImage(this.videoEl, 0, 0, vw, vh);
this.posterTexture = Texture.from(canvas);
this._hasPoster = true;
this._ensureSprite(this.posterTexture);
this._removePlaceholder();
} catch {
// Give up on poster
}
};
this.videoEl.addEventListener('seeked', onSeeked);
this.videoEl.currentTime = 0.1;
}
private _ensureSprite(texture: Texture): void {
if (!this._sprite) {
this._sprite = new Sprite(texture);
this._sprite.width = this._naturalWidth;
this._sprite.height = this._naturalHeight;
// Insert after shadow (index 0), before placeholder/overlay
this.addChildAt(this._sprite, 1);
} else {
this._sprite.texture = texture;
this._sprite.width = this._naturalWidth;
this._sprite.height = this._naturalHeight;
}
}
private _removeSpriteFromTree(): void {
if (this._sprite) {
this.removeChild(this._sprite);
this._sprite.destroy();
this._sprite = null;
}
}
private _removePlaceholder(): void {
if (this._placeholder) {
this.removeChild(this._placeholder);
this._placeholder.destroy();
this._placeholder = null;
}
}
private _restorePlaceholder(): void {
if (this._placeholder) return;
const p = new Graphics();
p.rect(0, 0, this._naturalWidth, this._naturalHeight).fill(0x2a2a2a);
this._placeholder = p;
// Insert after shadow
this.addChildAt(p, 1);
}
private _destroyVideoEl(): void {
if (!this.videoEl) return;
this.videoEl.removeEventListener('loadedmetadata', this._onLoadedMetadata);
this.videoEl.removeEventListener('loadeddata', this._onLoadedData);
this.videoEl.pause();
this._isPlaying = false;
this._overlay.visible = true;
activeVideos.delete(this);
// Keep the video texture showing (last frame) — don't revert to poster
this.videoEl.src = '';
this.videoEl.load();
this.videoEl = null;
}
togglePlayPause(): void {
if (this._isPlaying) this.pause();
else this.play();
}
toggleMute(): void {
this.muted = !this.muted;
this.videoEl.muted = this.muted;
}
get isPlaying(): boolean {
return this._isPlaying;
}
/** Expose the underlying HTMLVideoElement for external controls. */
get videoElement(): HTMLVideoElement {
return this.videoEl;
}
/** Seek to a specific time (in seconds). */
seek(time: number): void {
this.videoEl.currentTime = Math.max(0, Math.min(time, this.videoEl.duration || 0));
}
onVisibilityChange(visible: boolean): void {
if (!visible && this._isPlaying) {
this.pause();
private _destroyVideoTexture(): void {
if (this.videoTexture) {
// If sprite is showing live texture, remove it
if (this._sprite && this._sprite.texture === this.videoTexture) {
this._removeSpriteFromTree();
this._restorePlaceholder();
}
this.videoTexture.destroy(true);
this.videoTexture = null;
}
}
destroy(options?: Parameters<Container['destroy']>[0]): void {
this.pause();
this.videoEl.src = '';
this.videoEl.load();
if (this.videoTexture) {
this.videoTexture.destroy(true);
this.videoTexture = null;
}
if (this.posterTexture) {
this.posterTexture.destroy(true);
this.posterTexture = null;
}
this._destroyVideoEl();
if (this.videoTexture) { this.videoTexture.destroy(true); this.videoTexture = null; }
if (this.posterTexture) { this.posterTexture.destroy(true); this.posterTexture = null; }
super.destroy(options);
}
}
frontend/src/canvas/sync.ts
-1
View File
@@ -134,7 +134,6 @@ export function setupSync(
}));
socket.emit('object:transform', { boardId, transforms });
moveTimer = setTimeout(() => { moveTimer = null; }, MOVE_THROTTLE);
broadcastSceneDebounced();
}
// ---- RECEIVE: full scene --------------------------------------------------
frontend/src/components/VideoControls.tsx
+78 -49
View File
@@ -15,75 +15,104 @@ function formatTime(seconds: number): string {
}
export default function VideoControls({ videoSprite, screenRect }: VideoControlsProps) {
const video = videoSprite.videoElement;
const [playing, setPlaying] = useState(videoSprite.isPlaying);
const [muted, setMuted] = useState(videoSprite.muted);
const [currentTime, setCurrentTime] = useState(video.currentTime || 0);
const [duration, setDuration] = useState(video.duration || 0);
const [currentTime, setCurrentTime] = useState(0);
const [duration, setDuration] = useState(0);
const [seeking, setSeeking] = useState(false);
const pollRef = useRef<number | null>(null);
// Poll currentTime while playing
// Track the actual HTMLVideoElement reference so the effect rebinds
// when the sprite creates/destroys the underlying element.
const [videoEl, setVideoEl] = useState<HTMLVideoElement | null>(videoSprite.videoElement);
const seekValueRef = useRef(0); // stable ref for commit value during drag
// Poll the videoElement reference via onStateChange (fires on init/teardown/play/pause)
useEffect(() => {
if (pollRef.current) {
cancelAnimationFrame(pollRef.current);
pollRef.current = null;
}
if (!playing || seeking) return;
let lastUpdate = 0;
const poll = (now: number) => {
if (now - lastUpdate >= 250) {
setCurrentTime(video.currentTime || 0);
setDuration(video.duration || 0);
lastUpdate = now;
}
pollRef.current = requestAnimationFrame(poll);
const syncState = () => {
setPlaying(videoSprite.isPlaying);
setMuted(videoSprite.muted);
setVideoEl(videoSprite.videoElement);
};
pollRef.current = requestAnimationFrame(poll);
const prev = videoSprite.onStateChange;
videoSprite.onStateChange = () => {
syncState();
prev?.();
};
// Also sync on mount
syncState();
return () => { videoSprite.onStateChange = prev ?? null; };
}, [videoSprite]);
// Subscribe to media events on the actual <video> element.
// Keyed off `videoEl` so it rebinds when the element changes.
useEffect(() => {
if (!videoEl) return;
const onTimeUpdate = () => {
if (!seeking) setCurrentTime(videoEl.currentTime || 0);
};
const onDurationChange = () => setDuration(videoEl.duration || 0);
const onPlay = () => setPlaying(true);
const onPause = () => setPlaying(false);
const onEnded = () => setPlaying(false);
const onVolumeChange = () => setMuted(videoEl.muted);
const onSeeked = () => {
if (!seeking) setCurrentTime(videoEl.currentTime || 0);
};
videoEl.addEventListener('timeupdate', onTimeUpdate);
videoEl.addEventListener('durationchange', onDurationChange);
videoEl.addEventListener('loadedmetadata', onDurationChange);
videoEl.addEventListener('play', onPlay);
videoEl.addEventListener('pause', onPause);
videoEl.addEventListener('ended', onEnded);
videoEl.addEventListener('volumechange', onVolumeChange);
videoEl.addEventListener('seeked', onSeeked);
// Sync initial state from this element
setDuration(videoEl.duration || 0);
setCurrentTime(videoEl.currentTime || 0);
return () => {
if (pollRef.current) {
cancelAnimationFrame(pollRef.current);
pollRef.current = null;
}
videoEl.removeEventListener('timeupdate', onTimeUpdate);
videoEl.removeEventListener('durationchange', onDurationChange);
videoEl.removeEventListener('loadedmetadata', onDurationChange);
videoEl.removeEventListener('play', onPlay);
videoEl.removeEventListener('pause', onPause);
videoEl.removeEventListener('ended', onEnded);
videoEl.removeEventListener('volumechange', onVolumeChange);
videoEl.removeEventListener('seeked', onSeeked);
};
}, [playing, seeking, video]);
// Sync duration on metadata load
useEffect(() => {
const onMeta = () => setDuration(video.duration || 0);
video.addEventListener('loadedmetadata', onMeta);
// Also grab current values
setDuration(video.duration || 0);
setCurrentTime(video.currentTime || 0);
return () => video.removeEventListener('loadedmetadata', onMeta);
}, [video]);
}, [videoEl, seeking]);
const handlePlayPause = useCallback(() => {
videoSprite.togglePlayPause();
setPlaying(videoSprite.isPlaying);
// After play, the video element may have just been created
setVideoEl(videoSprite.videoElement);
}, [videoSprite]);
const handleMuteToggle = useCallback(() => {
videoSprite.toggleMute();
setMuted(videoSprite.muted);
}, [videoSprite]);
// Seek: onChange only updates local preview state + ref.
// Commit happens on pointerUp/touchEnd/blur using the ref value.
const handleSeekStart = useCallback(() => {
setSeeking(true);
}, []);
const handleSeekChange = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
const handleSeekPreview = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
const t = parseFloat(e.target.value);
seekValueRef.current = t;
setCurrentTime(t);
}, []);
const handleSeekEnd = useCallback((e: React.MouseEvent<HTMLInputElement> | React.TouchEvent<HTMLInputElement>) => {
const t = parseFloat((e.target as HTMLInputElement).value);
videoSprite.seek(t);
setCurrentTime(t);
const commitSeek = useCallback(() => {
videoSprite.seek(seekValueRef.current);
setCurrentTime(seekValueRef.current);
setSeeking(false);
}, [videoSprite]);
@@ -113,6 +142,7 @@ export default function VideoControls({ videoSprite, screenRect }: VideoControls
zIndex: 100,
pointerEvents: 'auto',
boxSizing: 'border-box',
opacity: videoEl ? 1 : 0.5,
}}
onPointerDown={(e) => e.stopPropagation()}
>
@@ -140,18 +170,17 @@ export default function VideoControls({ videoSprite, screenRect }: VideoControls
{playing ? <IcoPause /> : <IcoPlay />}
</button>
{/* Seekbar */}
{/* Seekbar — onChange previews, pointerUp/blur commits */}
<input
type="range"
min={0}
max={duration || 1}
step={0.1}
value={currentTime}
onMouseDown={handleSeekStart}
onTouchStart={handleSeekStart}
onChange={handleSeekChange}
onMouseUp={handleSeekEnd}
onTouchEnd={handleSeekEnd}
onPointerDown={handleSeekStart}
onChange={handleSeekPreview}
onPointerUp={commitSeek}
onBlur={commitSeek}
style={{
flex: 1,
height: '4px',
frontend/src/hooks/useCanvasSetup.ts
+1 -2
View File
@@ -138,8 +138,7 @@ export function useCanvasSetup(deps: CanvasSetupDeps) {
syncRef.current?.broadcastTransform(item);
};
selection.transformBox.onDragEnd = (itemIds) => {
syncRef.current?.broadcastElements(itemIds);
onCanvasChange(itemIds);
onCanvasChange(itemIds); // broadcasts elements + saves + undo
};
socket.on('user:joined', (data: any) => {