diff --git a/src/components/canvas/players/audio-player.ts b/src/components/canvas/players/audio-player.ts index 56ebeefa..6f3b0763 100644 --- a/src/components/canvas/players/audio-player.ts +++ b/src/components/canvas/players/audio-player.ts @@ -1,5 +1,6 @@ import { KeyframeBuilder } from "@animations/keyframe-builder"; import type { Edit } from "@core/edit-session"; +import { sec } from "@core/timing/types"; import { type Size } from "@layouts/geometry"; import { AudioLoadParser } from "@loaders/audio-load-parser"; import { type AudioAsset, type ResolvedClip, type Keyframe } from "@schemas"; @@ -11,6 +12,7 @@ import { Player, PlayerType } from "./player"; export class AudioPlayer extends Player { private audioResource: howler.Howl | null; private isPlaying: boolean; + private assetAcquisitions = new Map(); private volumeKeyframeBuilder!: KeyframeBuilder; @@ -25,24 +27,37 @@ export class AudioPlayer extends Player { } public override async load(): Promise { + const revision = this.beginMediaTimingLoad(); await super.load(); + if (!this.isMediaTimingLoadCurrent(revision)) return; const audioClipConfiguration = this.clipConfiguration.asset as AudioAsset; const identifier = audioClipConfiguration.src; if (!identifier) { + this.completeMediaTimingLoad(revision, null); // Prompt-bearing assets route to pending placeholder players — reaching here without a src is invalid data throw new Error("Audio asset has no src to load."); } const loadOptions: pixi.UnresolvedAsset = { src: identifier, parser: AudioLoadParser.Name }; + this.assetAcquisitions.set(revision, identifier); const audioResource = await this.edit.assetLoader.load(identifier, loadOptions); + if (!this.isMediaTimingLoadCurrent(revision)) { + if (audioResource) this.releaseAssetAcquisition(revision); + else this.assetAcquisitions.delete(revision); + return; + } const isValidAudioSource = audioResource instanceof howler.Howl; if (!isValidAudioSource) { + if (audioResource) this.releaseAssetAcquisition(revision); + else this.assetAcquisitions.delete(revision); + this.completeMediaTimingLoad(revision, null); throw new Error(`Invalid audio source '${audioClipConfiguration.src}'.`); } this.audioResource = audioResource; + this.completeMediaTimingLoad(revision, sec(audioResource.duration())); // Create volume keyframes after timing is resolved (not in constructor) const baseVolume = typeof audioClipConfiguration.volume === "number" ? audioClipConfiguration.volume : 1; @@ -107,6 +122,8 @@ export class AudioPlayer extends Player { } public override dispose(): void { + const { src } = this.clipConfiguration.asset as AudioAsset; + this.releaseAssetAcquisitions(src); if (this.audioResource) { this.audioResource.stop(); this.audioResource.unload(); @@ -118,6 +135,8 @@ export class AudioPlayer extends Player { /** Reload the audio asset when asset.src changes (e.g., merge field update or loadEdit) */ public override async reloadAsset(): Promise { + const revision = this.beginMediaTimingLoad(); + this.releaseAssetAcquisitions(); if (this.audioResource) { this.audioResource.stop(); this.audioResource.unload(); @@ -129,19 +148,49 @@ export class AudioPlayer extends Player { const audioAsset = this.clipConfiguration.asset as AudioAsset; const { src } = audioAsset; if (!src) { + this.completeMediaTimingLoad(revision, null); throw new Error("Audio asset has no src to load."); } const loadOptions: pixi.UnresolvedAsset = { src, parser: AudioLoadParser.Name }; + this.assetAcquisitions.set(revision, src); const audioResource = await this.edit.assetLoader.load(src, loadOptions); + if (!this.isMediaTimingLoadCurrent(revision)) { + if (audioResource) this.releaseAssetAcquisition(revision); + else this.assetAcquisitions.delete(revision); + return; + } if (!(audioResource instanceof howler.Howl)) { + if (audioResource) this.releaseAssetAcquisition(revision); + else this.assetAcquisitions.delete(revision); + this.completeMediaTimingLoad(revision, null); throw new Error(`Invalid audio source '${audioAsset.src}'.`); } this.audioResource = audioResource; + this.completeMediaTimingLoad(revision, sec(audioResource.duration())); this.audioResource.volume(this.getVolume()); } + private releaseAssetAcquisition(revision: number): void { + const identifier = this.assetAcquisitions.get(revision); + if (!identifier) return; + this.assetAcquisitions.delete(revision); + this.edit.assetLoader.release(identifier); + } + + private releaseAssetAcquisitions(alreadyReleasedIdentifier?: string): void { + let skipIdentifier = alreadyReleasedIdentifier; + for (const [revision, identifier] of this.assetAcquisitions) { + this.assetAcquisitions.delete(revision); + if (identifier === skipIdentifier) { + skipIdentifier = undefined; + } else { + this.edit.assetLoader.release(identifier); + } + } + } + public override reconfigureAfterRestore(): void { super.reconfigureAfterRestore(); diff --git a/src/components/canvas/players/image-player.ts b/src/components/canvas/players/image-player.ts index db47746d..9a9f4362 100644 --- a/src/components/canvas/players/image-player.ts +++ b/src/components/canvas/players/image-player.ts @@ -1,4 +1,6 @@ import type { Edit } from "@core/edit-session"; +import { appendCorsQuery, type GifImageSource, isGifUrl } from "@core/loaders/gif-image-source"; +import { sec } from "@core/timing/types"; import { type Size } from "@layouts/geometry"; import { type ResolvedClip, type ImageAsset } from "@schemas"; import * as pixi from "pixi.js"; @@ -7,9 +9,14 @@ import { createPlaceholderGraphic } from "./placeholder-graphic"; import { Player, PlayerType } from "./player"; export class ImagePlayer extends Player { - private texture: pixi.Texture | null; + private texture: pixi.Texture | null; private sprite: pixi.Sprite | null; private placeholder: pixi.Graphics | null; + private gifSource: GifImageSource | null = null; + private gifFrameTextures: pixi.Texture[] = []; + private ownedTextureWrappers: pixi.Texture[] = []; + private currentGifFrame = -1; + private assetAcquisitions = new Map(); constructor(edit: Edit, clipConfiguration: ResolvedClip) { super(edit, clipConfiguration, PlayerType.Image); @@ -20,11 +27,17 @@ export class ImagePlayer extends Player { } public override async load(): Promise { + const revision = this.beginMediaTimingLoad(); await super.load(); + if (!this.isMediaTimingLoadCurrent(revision)) return; + try { - await this.loadTexture(); + if (!(await this.loadTexture(revision))) return; + this.completeMediaTimingLoad(revision, this.gifSource && this.gifSource.frames.length > 1 ? sec(this.gifSource.duration / 1000) : null); this.configureKeyframes(); } catch { + if (!this.isMediaTimingLoadCurrent(revision)) return; + this.completeMediaTimingLoad(revision, null); this.createFallbackGraphic(); } } @@ -40,9 +53,12 @@ export class ImagePlayer extends Player { public override update(deltaTime: number, elapsed: number): void { super.update(deltaTime, elapsed); + this.updateGifFrame(); } public override dispose(): void { + const { src } = this.clipConfiguration.asset as ImageAsset; + this.releaseAssetAcquisitions(src); this.disposeTexture(); this.clearPlaceholder(); super.dispose(); @@ -71,19 +87,28 @@ export class ImagePlayer extends Player { return this.placeholder ? this.getDisplaySize() : { width: 0, height: 0 }; } + public override async prepareStaticRender(): Promise { + this.updateGifFrame(); + } + /** Reload the image asset when asset.src changes (e.g., merge field update) */ public override async reloadAsset(): Promise { + const revision = this.beginMediaTimingLoad(); this.disposeTexture(); this.clearPlaceholder(); + this.releaseAssetAcquisitions(); try { - await this.loadTexture(); + if (!(await this.loadTexture(revision))) return; + this.completeMediaTimingLoad(revision, this.gifSource && this.gifSource.frames.length > 1 ? sec(this.gifSource.duration / 1000) : null); } catch { + if (!this.isMediaTimingLoadCurrent(revision)) return; + this.completeMediaTimingLoad(revision, null); this.createFallbackGraphic(); } } - private async loadTexture(): Promise { + private async loadTexture(revision: number): Promise { const imageAsset = this.clipConfiguration.asset as ImageAsset; const { src } = imageAsset; if (!src) { @@ -91,15 +116,39 @@ export class ImagePlayer extends Player { throw new Error("Image asset has no src to load."); } - const corsUrl = `${src}${src.includes("?") ? "&" : "?"}x-cors=1`; - const loadOptions: pixi.UnresolvedAsset = { src: corsUrl, crossorigin: "anonymous", data: {} }; - const texture = await this.edit.assetLoader.load>(corsUrl, loadOptions); + const requestUrl = appendCorsQuery(src); + this.assetAcquisitions.set(revision, src); + if (isGifUrl(src)) { + const source = await this.edit.assetLoader.loadGif(src, requestUrl); + if (!this.isMediaTimingLoadCurrent(revision)) { + if (source) this.releaseAssetAcquisition(revision); + else this.assetAcquisitions.delete(revision); + return false; + } + if (!source) { + this.assetAcquisitions.delete(revision); + throw new Error(`Unable to decode GIF image source '${src}'.`); + } + try { + this.configureGif(source); + return true; + } catch (error) { + this.disposeTexture(); + this.releaseAssetAcquisition(revision); + throw error; + } + } + const loadOptions: pixi.UnresolvedAsset = { alias: src, src: requestUrl, crossorigin: "anonymous", data: {} }; + const texture = await this.edit.assetLoader.load>(src, loadOptions); + if (!this.isMediaTimingLoadCurrent(revision)) { + if (texture) this.releaseAssetAcquisition(revision); + else this.assetAcquisitions.delete(revision); + return false; + } if (!(texture?.source instanceof pixi.ImageSource)) { - if (texture) { - texture.destroy(true); - await this.edit.assetLoader.rejectAsset(corsUrl); - } + this.assetAcquisitions.delete(revision); + if (texture) await this.edit.assetLoader.rejectAsset(src); throw new Error(`Invalid image source '${src}'.`); } @@ -111,6 +160,32 @@ export class ImagePlayer extends Player { if (this.clipConfiguration.width && this.clipConfiguration.height) { this.applyFixedDimensions(); } + return true; + } + + private configureGif(source: GifImageSource): void { + this.clearPlaceholder(); + this.gifSource = source; + this.gifFrameTextures = source.frames.map(frame => this.createCroppedTexture(frame.texture)); + this.texture = this.gifFrameTextures[0] ?? null; + if (!this.texture) throw new Error("GIF contains no renderable frames."); + + this.sprite = new pixi.Sprite(this.texture); + this.contentContainer.addChild(this.sprite); + this.currentGifFrame = 0; + if (this.clipConfiguration.width && this.clipConfiguration.height) this.applyFixedDimensions(); + this.updateGifFrame(); + } + + private updateGifFrame(): void { + if (!this.gifSource || !this.sprite || !this.isActive()) return; + const frameIndex = this.gifSource.frameIndexAt(this.getPlaybackTime() * 1000); + if (frameIndex === this.currentGifFrame) return; + const texture = this.gifFrameTextures[frameIndex]; + if (!texture) return; + this.sprite.texture = texture; + this.texture = texture; + this.currentGifFrame = frameIndex; } private disposeTexture(): void { @@ -119,9 +194,31 @@ export class ImagePlayer extends Player { this.sprite.destroy(); this.sprite = null; } - // DON'T destroy the texture - it's managed by Assets - // The unloadClipAssets() method handles proper cleanup via Assets.unload() + for (const texture of this.ownedTextureWrappers) texture.destroy(false); + this.ownedTextureWrappers = []; this.texture = null; + this.gifFrameTextures = []; + this.gifSource = null; + this.currentGifFrame = -1; + } + + private releaseAssetAcquisition(revision: number): void { + const identifier = this.assetAcquisitions.get(revision); + if (!identifier) return; + this.assetAcquisitions.delete(revision); + this.edit.assetLoader.release(identifier); + } + + private releaseAssetAcquisitions(alreadyReleasedIdentifier?: string): void { + let skipIdentifier = alreadyReleasedIdentifier; + for (const [revision, identifier] of this.assetAcquisitions) { + this.assetAcquisitions.delete(revision); + if (identifier === skipIdentifier) { + skipIdentifier = undefined; + } else { + this.edit.assetLoader.release(identifier); + } + } } private clearPlaceholder(): void { @@ -136,7 +233,7 @@ export class ImagePlayer extends Player { return true; } - private createCroppedTexture(texture: pixi.Texture): pixi.Texture { + private createCroppedTexture(texture: pixi.Texture): pixi.Texture { const imageAsset = this.clipConfiguration.asset as ImageAsset; if (!imageAsset.crop) { @@ -157,6 +254,8 @@ export class ImagePlayer extends Player { const height = originalHeight - top - bottom; const crop = new pixi.Rectangle(x, y, width, height); - return new pixi.Texture({ source: texture.source, frame: crop }); + const croppedTexture = new pixi.Texture({ source: texture.source, frame: crop }); + this.ownedTextureWrappers.push(croppedTexture); + return croppedTexture; } } diff --git a/src/components/canvas/players/luma-player.ts b/src/components/canvas/players/luma-player.ts index 02564373..d5d03a02 100644 --- a/src/components/canvas/players/luma-player.ts +++ b/src/components/canvas/players/luma-player.ts @@ -1,4 +1,5 @@ import type { Edit } from "@core/edit-session"; +import { sec } from "@core/timing/types"; import { type Size } from "@layouts/geometry"; import { type ResolvedClip, type LumaAsset } from "@schemas"; import * as pixi from "pixi.js"; @@ -21,6 +22,7 @@ export class LumaPlayer extends Player { } public override async load(): Promise { + const revision = this.beginMediaTimingLoad(); await super.load(); const lumaAsset = this.clipConfiguration.asset as LumaAsset; @@ -31,6 +33,7 @@ export class LumaPlayer extends Player { const isValidLumaSource = texture?.source instanceof pixi.ImageSource || texture?.source instanceof pixi.VideoSource; if (!isValidLumaSource) { + this.completeMediaTimingLoad(revision, null); if (texture) { texture.destroy(true); await this.edit.assetLoader.rejectAsset(identifier); @@ -45,6 +48,7 @@ export class LumaPlayer extends Player { } this.texture = texture; + this.completeMediaTimingLoad(revision, texture.source instanceof pixi.VideoSource ? sec(texture.source.resource.duration) : null); this.sprite = new pixi.Sprite(this.texture); this.contentContainer.addChild(this.sprite); diff --git a/src/components/canvas/players/player.ts b/src/components/canvas/players/player.ts index f9032722..325f079c 100644 --- a/src/components/canvas/players/player.ts +++ b/src/components/canvas/players/player.ts @@ -79,6 +79,8 @@ export abstract class Player extends Entity { public clipConfiguration: ResolvedClip; private resolvedTiming: ResolvedTiming; + private mediaDuration: Seconds | null = null; + private mediaTimingRevision = 0; private offsetXKeyframeBuilder?: ComposedKeyframeBuilder; private offsetYKeyframeBuilder?: ComposedKeyframeBuilder; @@ -340,6 +342,7 @@ export abstract class Player extends Entity { } public override dispose(): void { + this.mediaTimingRevision += 1; this.wipeMask?.destroy(); this.wipeMask = null; this.wipeFilter?.destroy(); @@ -401,6 +404,25 @@ export abstract class Player extends Entity { return { ...this.resolvedTiming }; } + public getMediaDuration(): Seconds | null { + return this.mediaDuration; + } + + protected beginMediaTimingLoad(): number { + this.mediaTimingRevision += 1; + this.mediaDuration = null; + return this.mediaTimingRevision; + } + + protected isMediaTimingLoadCurrent(revision: number): boolean { + return revision === this.mediaTimingRevision; + } + + protected completeMediaTimingLoad(revision: number, duration: Seconds | null): void { + if (!this.isMediaTimingLoadCurrent(revision)) return; + this.mediaDuration = duration !== null && Number.isFinite(duration) && duration > 0 ? duration : null; + } + public setResolvedTiming(timing: ResolvedTiming): void { this.resolvedTiming = { ...timing }; this.clipConfiguration.start = timing.start; diff --git a/src/components/canvas/players/video-player.ts b/src/components/canvas/players/video-player.ts index 1c29b873..397c2944 100644 --- a/src/components/canvas/players/video-player.ts +++ b/src/components/canvas/players/video-player.ts @@ -1,5 +1,6 @@ import { KeyframeBuilder } from "@animations/keyframe-builder"; import type { Edit } from "@core/edit-session"; +import { sec } from "@core/timing/types"; import { type Size } from "@layouts/geometry"; import { type ResolvedClip, type VideoAsset } from "@schemas"; import * as pixi from "pixi.js"; @@ -36,11 +37,16 @@ export class VideoPlayer extends Player { } public override async load(): Promise { + const revision = this.beginMediaTimingLoad(); await super.load(); + if (!this.isMediaTimingLoadCurrent(revision)) return; try { - await this.loadVideo(); + if (!(await this.loadVideo(revision))) return; + this.completeMediaTimingLoad(revision, sec(this.texture?.source.resource.duration ?? 0)); this.configureKeyframes(); } catch (error) { + if (!this.isMediaTimingLoadCurrent(revision)) return; + this.completeMediaTimingLoad(revision, null); console.warn(`[VideoPlayer.load] FAILED clipId=${this.clipId}:`, error); this.createFallbackGraphic(); } @@ -141,6 +147,7 @@ export class VideoPlayer extends Player { /** Reload the video asset when asset.src changes (e.g., merge field update) */ public override async reloadAsset(): Promise { + const revision = this.beginMediaTimingLoad(); this.skipVideoUpdate = true; this.isPlaying = false; this.syncTimer = 0; @@ -149,12 +156,15 @@ export class VideoPlayer extends Player { try { this.disposeVideo(); this.clearPlaceholder(); - await this.loadVideo(); + if (!(await this.loadVideo(revision))) return; + this.completeMediaTimingLoad(revision, sec(this.texture?.source.resource.duration ?? 0)); } catch (error) { + if (!this.isMediaTimingLoadCurrent(revision)) return; + this.completeMediaTimingLoad(revision, null); console.warn(`[VideoPlayer.reloadAsset] FAILED clipId=${this.clipId}:`, error); this.createFallbackGraphic(); } finally { - this.skipVideoUpdate = false; + if (this.isMediaTimingLoadCurrent(revision)) this.skipVideoUpdate = false; } } @@ -165,7 +175,7 @@ export class VideoPlayer extends Player { this.volumeKeyframeBuilder = new KeyframeBuilder(videoAsset.volume ?? 1, this.getLength()); } - private async loadVideo(): Promise { + private async loadVideo(revision: number): Promise { const videoAsset = this.clipConfiguration.asset as VideoAsset; const { src } = videoAsset; if (!src) { @@ -187,17 +197,17 @@ export class VideoPlayer extends Player { if (!texture || !(texture.source instanceof pixi.VideoSource)) { throw new Error(`Invalid video source '${src}'.`); } - - this.clearPlaceholder(); + if (!this.isMediaTimingLoadCurrent(revision)) { + this.destroyVideoTexture(texture); + return false; + } // Fix alpha channel rendering for WebM VP9 videos (PixiJS 8 auto-detection is buggy) texture.source.alphaMode = "no-premultiply-alpha"; - this.texture = this.createCroppedTexture(texture); - // Ensure the video has at least one decoded frame before adding to render tree // This prevents WebGL errors when GPU tries to upload uninitialized texture data - const video = (this.texture.source as pixi.VideoSource).resource; + const video = texture.source.resource; if (video instanceof HTMLVideoElement && video.readyState < 2) { await new Promise(resolve => { const onReady = () => { @@ -208,21 +218,22 @@ export class VideoPlayer extends Player { if (video.readyState >= 2) resolve(); }); } + if (!this.isMediaTimingLoadCurrent(revision)) { + this.destroyVideoTexture(texture); + return false; + } + this.clearPlaceholder(); + this.texture = this.createCroppedTexture(texture); this.sprite = new pixi.Sprite(this.texture); this.contentContainer.addChild(this.sprite); // Set initial volume immediately so the element never sits at the browser default of 1.0 this.texture.source.resource.volume = this.getVolume(); + return true; } private disposeVideo(): void { - if (this.texture?.source?.resource) { - this.texture.source.resource.pause(); - // Release video resource - each player owns its own video element - this.texture.source.resource.src = ""; - this.texture.source.resource.load(); - } if (this.sprite) { this.contentContainer.removeChild(this.sprite); this.sprite.destroy(); @@ -230,11 +241,19 @@ export class VideoPlayer extends Player { } // Destroy the texture since we own it (created via loadVideoUnique) if (this.texture) { - this.texture.destroy(true); + this.destroyVideoTexture(this.texture); this.texture = null; } } + private destroyVideoTexture(texture: pixi.Texture): void { + const { resource } = texture.source; + resource.pause(); + resource.src = ""; + resource.load(); + texture.destroy(true); + } + private clearPlaceholder(): void { if (!this.placeholder) { return; diff --git a/src/core/edit-session.ts b/src/core/edit-session.ts index 6e7bb4e4..30e6a04f 100644 --- a/src/core/edit-session.ts +++ b/src/core/edit-session.ts @@ -31,7 +31,7 @@ import { calculateSizeFromPreset, OutputSettingsManager } from "@core/output-set import { SelectionManager } from "@core/selection-manager"; import { findEligibleSourceClips, ensureClipAlias } from "@core/shared/source-clip-finder"; import { deepMerge, nextFrame, setNestedValue } from "@core/shared/utils"; -import { calculateTimelineEnd, resolveAutoLength, resolveAutoStart } from "@core/timing/resolver"; +import { calculateTimelineEnd, resolveAutoLength } from "@core/timing/resolver"; import { type Milliseconds, type ResolutionContext, type Seconds, sec, isAliasReference } from "@core/timing/types"; import { TimingManager } from "@core/timing-manager"; import type { Size } from "@layouts/geometry"; @@ -58,7 +58,7 @@ import { CommandQueue } from "./commands/command-queue"; import { CommandNoop, type EditCommand, type CommandContext, type CommandResult } from "./commands/types"; import { EditDocument } from "./edit-document"; import { PlayerReconciler } from "./player-reconciler"; -import { resolve as resolveDocument, resolveClip as resolveClipById, type SingleClipContext } from "./resolver"; +import { resolve as resolveDocument, resolveClip as resolveClipById, type ResolveContext, type SingleClipContext } from "./resolver"; import { InvalidAssetUrlError, extractClipUrls, extractTrackUrls } from "./url-validation"; /** Internal type for clips with hydrated IDs during edit updates */ @@ -212,23 +212,24 @@ export class Edit { // 7. Create players await this.playerReconciler.reconcileInitial(resolvedEdit); - // 7.5 Establish luma→content relationships before timeline renders + // 8. Resolve media timing before luma matching uses clip overlap and duration + this.lastResolved = null; + await this.timingManager.resolveAllTiming(); + + // 9. Establish luma→content relationships before timeline renders this.normalizeLumaAttachments(); - // 8. Set up clip bindings for merge field tracking + // 10. Set up clip bindings for merge field tracking for (const [clipId, bindings] of bindingsPerClip) { if (bindings.size > 0) { this.document.setClipBindingsForClip(clipId, bindings); } } - // 9. Resolve async timing (auto-length for videos, etc.) - await this.timingManager.resolveAllTiming(); - - // 10. Update total duration + // 11. Update total duration this.updateTotalDuration(); - // 11. Load soundtrack if present + // 12. Load soundtrack if present if (parsedEdit.timeline.soundtrack) { await this.loadSoundtrack(parsedEdit.timeline.soundtrack); } @@ -496,13 +497,21 @@ export class Edit { */ public getResolvedEdit(): ResolvedEdit { if (!this.lastResolved) { - this.lastResolved = resolveDocument(this.document, { - mergeFields: this.mergeFieldService - }); + this.lastResolved = resolveDocument(this.document, this.getResolverContext()); } return this.lastResolved; } + private getResolverContext(): ResolveContext { + const mediaDurationByClipId = new Map(); + for (const [clipId, player] of this.playerByClipId) { + if (player.getTimingIntent().length === "auto") { + mediaDurationByClipId.set(clipId, player.getMediaDuration()); + } + } + return { mergeFields: this.mergeFieldService, mediaDurationByClipId }; + } + /** * Get a specific clip from the resolved edit. * @internal @@ -553,9 +562,7 @@ export class Edit { /** @internal Resolve the document to a ResolvedEdit and emit the Resolved event. */ public resolve(): ResolvedEdit { - this.lastResolved = resolveDocument(this.document, { - mergeFields: this.mergeFieldService - }); + this.lastResolved = resolveDocument(this.document, this.getResolverContext()); // Emit event for components to react this.internalEvents.emit(InternalEvent.Resolved, { edit: this.lastResolved }); @@ -598,12 +605,12 @@ export class Edit { // Get previous clip's end time (for "auto" start resolution) const previousPlayer = clipIndex > 0 ? track[clipIndex - 1] : null; const previousClipEnd = previousPlayer ? previousPlayer.getEnd() : sec(0); - // Build single-clip context const context: SingleClipContext = { mergeFields: this.mergeFieldService, previousClipEnd, - cachedTimelineEnd: this.timingManager.getTimelineEnd() + cachedTimelineEnd: this.timingManager.getTimelineEnd(), + intrinsicDuration: player.getMediaDuration() }; // Resolve just this one clip @@ -1699,8 +1706,7 @@ export class Edit { const intent = player.getTimingIntent(); // Only lookup intrinsic duration if the clip uses "auto" length if (intent.length === "auto") { - // The player's resolved length IS the intrinsic duration after async load - intrinsicDuration = player.getLength(); + intrinsicDuration = resolveAutoLength(player.clipConfiguration.asset, player.getMediaDuration()); } } @@ -1857,10 +1863,7 @@ export class Edit { private unloadClipAssets(clip: Player): void { const { asset } = clip.clipConfiguration; if (asset && "src" in asset && typeof asset.src === "string") { - const safeToUnload = this.assetLoader.decrementRef(asset.src); - if (safeToUnload && pixi.Assets.cache.has(asset.src)) { - pixi.Assets.unload(asset.src); - } + this.assetLoader.release(asset.src); } } @@ -1909,25 +1912,16 @@ export class Edit { public async resolveClipAutoLength(clip: Player): Promise { const intent = clip.getTimingIntent(); if (intent.length !== "auto") return; + await this.playerReconciler.whenPlayerSettled(clip); - // Find clip indices first (needed if start is also auto) - const indices = this.findClipIndices(clip); - - // Resolve auto start if needed, otherwise use current start - let resolvedStart = clip.getStart(); - if (intent.start === "auto" && indices) { - resolvedStart = resolveAutoStart(indices.trackIndex, indices.clipIndex, this.tracks); - } - - const newLength = await resolveAutoLength(clip.clipConfiguration.asset); - clip.setResolvedTiming({ - start: resolvedStart, - length: newLength - }); - clip.reconfigureAfterRestore(); + this.lastResolved = null; + await this.timingManager.resolveAllTiming(); + const indices = this.findClipIndices(clip); if (indices) { this.propagateTimingChanges(indices.trackIndex, indices.clipIndex); + } else { + this.updateTotalDuration(); } } diff --git a/src/core/loaders/asset-loader.ts b/src/core/loaders/asset-loader.ts index e68bc5c9..0aa0ee33 100644 --- a/src/core/loaders/asset-loader.ts +++ b/src/core/loaders/asset-loader.ts @@ -2,6 +2,8 @@ import * as pixi from "pixi.js"; import { AssetLoadTracker, type AssetLoadInfoStatus } from "../events/asset-load-tracker"; +import { GifImageSource } from "./gif-image-source"; + export class AssetLoader { private static readonly VIDEO_EXTENSIONS = [".mp4", ".m4v", ".webm", ".ogg", ".ogv"]; private static readonly VIDEO_MIME: Record = { @@ -15,6 +17,8 @@ export class AssetLoader { /** Reference counts for loaded assets - prevents premature unloading during transforms */ private refCounts = new Map(); + private assetLoads = new Map>(); + private gifSources = new Map>(); /** * Increment reference count for an asset. @@ -29,8 +33,9 @@ export class AssetLoader { * @returns true if asset can be safely unloaded (count reached zero) */ public decrementRef(src: string): boolean { - const count = this.refCounts.get(src) ?? 0; - if (count <= 1) { + const count = this.refCounts.get(src); + if (!count) return false; + if (count === 1) { this.refCounts.delete(src); return true; // Safe to unload } @@ -38,6 +43,24 @@ export class AssetLoader { return false; // Still in use } + public release(identifier: string): void { + if (!this.decrementRef(identifier)) return; + + const gifSource = this.gifSources.get(identifier); + if (gifSource) { + this.gifSources.delete(identifier); + gifSource.then(source => source.destroy()).catch(() => undefined); + } + + const unload = (): void => { + if (this.refCounts.has(identifier) || !pixi.Assets.cache.has(identifier)) return; + Promise.resolve(pixi.Assets.unload(identifier)).catch(() => undefined); + }; + const assetLoad = this.assetLoads.get(identifier); + if (assetLoad) assetLoad.then(unload, unload); + else unload(); + } + constructor() { pixi.Assets.setPreferences({ crossOrigin: "anonymous" }); } @@ -55,15 +78,17 @@ export class AssetLoader { public async load(identifier: string, loadOptions: pixi.UnresolvedAsset): Promise { this.updateAssetLoadMetadata(identifier, "pending", 0); this.incrementRef(identifier); + const loadPromise = this.shouldUseSafariVideoLoader(loadOptions).then(useSafari => + useSafari + ? this.loadVideoForSafari(identifier, loadOptions) + : pixi.Assets.load(loadOptions, progress => { + this.updateAssetLoadMetadata(identifier, "loading", progress); + }) + ); + this.assetLoads.set(identifier, loadPromise); try { - const useSafari = await this.shouldUseSafariVideoLoader(loadOptions); - - const resolvedAsset = useSafari - ? await this.loadVideoForSafari(identifier, loadOptions) - : await pixi.Assets.load(loadOptions, progress => { - this.updateAssetLoadMetadata(identifier, "loading", progress); - }); + const resolvedAsset = await loadPromise; if (resolvedAsset == null) { console.warn(`[AssetLoader.load] Empty asset returned for "${identifier}"`); @@ -79,6 +104,32 @@ export class AssetLoader { this.updateAssetLoadMetadata(identifier, "failed", 1); await this.cleanupFailedLoad(identifier); return null; + } finally { + if (this.assetLoads.get(identifier) === loadPromise) this.assetLoads.delete(identifier); + } + } + + public async loadGif(identifier: string, requestUrl: string): Promise { + this.updateAssetLoadMetadata(identifier, "pending", 0); + this.incrementRef(identifier); + + let sourcePromise = this.gifSources.get(identifier); + if (!sourcePromise) { + sourcePromise = GifImageSource.fetch(requestUrl); + this.gifSources.set(identifier, sourcePromise); + } + + try { + this.updateAssetLoadMetadata(identifier, "loading", 0.5); + const source = await sourcePromise; + this.updateAssetLoadMetadata(identifier, "success", 1); + return source; + } catch (error) { + console.warn(`[AssetLoader.loadGif] Failed to load "${identifier}":`, error); + this.updateAssetLoadMetadata(identifier, "failed", 1); + if (this.gifSources.get(identifier) === sourcePromise) this.gifSources.delete(identifier); + this.decrementRef(identifier); + return null; } } diff --git a/src/core/loaders/gif-image-source.ts b/src/core/loaders/gif-image-source.ts new file mode 100644 index 00000000..fd7cf74d --- /dev/null +++ b/src/core/loaders/gif-image-source.ts @@ -0,0 +1,233 @@ +import * as pixi from "pixi.js"; +import { GifSource as PixiGifSource } from "pixi.js/gif"; + +const MAX_COMPRESSED_BYTES = 50 * 1024 * 1024; +const MAX_DECODED_BYTES = 64 * 1024 * 1024; +const MAX_DIMENSION = 4096; +const MAX_FRAMES = 1000; +const MAX_DURATION_MS = 300_000; +const DEFAULT_FRAME_DELAY_MS = 100; +const REQUEST_TIMEOUT_MS = 15_000; + +export interface GifFrame { + readonly start: number; + readonly end: number; + readonly texture: pixi.Texture; +} + +function assertAvailable(bytes: Uint8Array, offset: number, length: number): void { + if (!Number.isSafeInteger(length) || length < 0 || offset + length > bytes.length) { + throw new Error("GIF data is truncated or malformed."); + } +} + +function inspectGif(buffer: ArrayBuffer): void { + if (buffer.byteLength === 0 || buffer.byteLength > MAX_COMPRESSED_BYTES) { + throw new Error("GIF compressed size is outside the supported range."); + } + + const bytes = new Uint8Array(buffer); + let offset = 0; + const readByte = (): number => { + assertAvailable(bytes, offset, 1); + const value = bytes[offset]; + offset += 1; + return value; + }; + const readUint16 = (): number => readByte() + readByte() * 256; + const skip = (length: number): void => { + assertAvailable(bytes, offset, length); + offset += length; + }; + const skipSubBlocks = (): void => { + for (;;) { + const length = readByte(); + if (length === 0) return; + skip(length); + } + }; + const colorTableSize = (packed: number): number => 3 * 2 ** ((packed % 8) + 1); + + assertAvailable(bytes, 0, 6); + const signature = String.fromCharCode(...bytes.subarray(0, 6)); + if (signature !== "GIF87a" && signature !== "GIF89a") throw new Error("Invalid GIF signature."); + offset = 6; + + const width = readUint16(); + const height = readUint16(); + const packed = readByte(); + skip(2); + if (width === 0 || height === 0 || width > MAX_DIMENSION || height > MAX_DIMENSION) { + throw new Error("GIF dimensions are outside the supported range."); + } + if (packed >= 0x80) skip(colorTableSize(packed)); + + const fullFrameBytes = width * height * 4; + let frameCount = 0; + let patchBytes = 0; + let durationMs = 0; + let nextDelayMs: number | null = null; + let foundTrailer = false; + + while (offset < bytes.length) { + const introducer = readByte(); + if (introducer === 0x3b) { + foundTrailer = true; + break; + } + + if (introducer === 0x21) { + const label = readByte(); + if (label === 0xf9) { + if (readByte() !== 4) throw new Error("Malformed GIF control extension."); + readByte(); + nextDelayMs = (readUint16() || 10) * 10; + readByte(); + if (readByte() !== 0) throw new Error("Malformed GIF control extension."); + } else { + skipSubBlocks(); + if (label === 0x01) nextDelayMs = null; + } + } else if (introducer === 0x2c) { + const left = readUint16(); + const top = readUint16(); + const frameWidth = readUint16(); + const frameHeight = readUint16(); + if (frameWidth === 0 || frameHeight === 0 || left + frameWidth > width || top + frameHeight > height) { + throw new Error("GIF frame exceeds its logical screen."); + } + + const framePacked = readByte(); + if (framePacked >= 0x80) skip(colorTableSize(framePacked)); + readByte(); + skipSubBlocks(); + + frameCount += 1; + patchBytes += frameWidth * frameHeight * 4; + durationMs += nextDelayMs ?? DEFAULT_FRAME_DELAY_MS; + nextDelayMs = null; + const peakDecodedBytes = fullFrameBytes * (frameCount + 2) + patchBytes; + if (frameCount > MAX_FRAMES || !Number.isSafeInteger(peakDecodedBytes) || peakDecodedBytes > MAX_DECODED_BYTES) { + throw new Error("GIF decoded size is outside the supported range."); + } + if (durationMs > MAX_DURATION_MS) throw new Error("GIF duration is outside the supported range."); + } else { + throw new Error("Unsupported GIF block."); + } + } + + if (!foundTrailer || frameCount === 0) throw new Error("GIF has no complete image frames."); +} + +async function readBoundedResponse(response: Response): Promise { + const contentLength = Number(response.headers.get("content-length")); + if (Number.isFinite(contentLength) && contentLength > MAX_COMPRESSED_BYTES) { + await response.body?.cancel().catch(() => undefined); + throw new Error("GIF compressed size is outside the supported range."); + } + if (!response.body) { + const buffer = await response.arrayBuffer(); + if (buffer.byteLength > MAX_COMPRESSED_BYTES) throw new Error("GIF compressed size is outside the supported range."); + return buffer; + } + + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + if (value) { + size += value.byteLength; + if (size > MAX_COMPRESSED_BYTES) throw new Error("GIF compressed size is outside the supported range."); + chunks.push(value); + } + } + } catch (error) { + await reader.cancel(error).catch(() => undefined); + throw error; + } finally { + reader.releaseLock(); + } + + const bytes = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return bytes.buffer; +} + +export function isGifUrl(src: string): boolean { + if (/^data:image\/gif(?:;|,)/i.test(src)) return true; + try { + return new URL(src, typeof window === "undefined" ? "http://localhost" : window.location.origin).pathname.toLowerCase().endsWith(".gif"); + } catch { + return false; + } +} + +export function appendCorsQuery(src: string): string { + if (/^(?:data|blob):/i.test(src)) return src; + return `${src}${src.includes("?") ? "&" : "?"}x-cors=1`; +} + +export class GifImageSource { + public readonly width: number; + public readonly height: number; + public readonly duration: number; + public readonly frames: GifFrame[]; + + private constructor(source: PixiGifSource) { + this.width = source.width; + this.height = source.height; + this.duration = source.duration; + this.frames = source.frames.map(frame => { + const { resource } = frame.texture.source; + if (!resource) throw new Error("GIF decoder returned an empty frame."); + return { + start: frame.start, + end: frame.end, + texture: new pixi.Texture({ source: new pixi.CanvasSource({ resource }) }) + }; + }); + } + + public static async fetch(url: string): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS); + try { + const response = await fetch(url, { signal: controller.signal }); + if (!response.ok) throw new Error(`GIF request failed with HTTP ${response.status}.`); + return GifImageSource.from(await readBoundedResponse(response)); + } finally { + clearTimeout(timeout); + } + } + + public static from(buffer: ArrayBuffer): GifImageSource { + inspectGif(buffer); + const source = PixiGifSource.from(buffer, { fps: 10 }); + try { + if (!Number.isFinite(source.duration) || source.duration <= 0 || source.duration > MAX_DURATION_MS) { + throw new Error("GIF duration is outside the supported range."); + } + return new GifImageSource(source); + } finally { + source.destroy(); + } + } + + public frameIndexAt(timeMs: number): number { + const localTime = ((timeMs % this.duration) + this.duration) % this.duration; + const index = this.frames.findIndex(frame => frame.start <= localTime && frame.end > localTime); + return index < 0 ? this.frames.length - 1 : index; + } + + public destroy(): void { + for (const frame of this.frames) frame.texture.destroy(true); + this.frames.length = 0; + } +} diff --git a/src/core/player-reconciler.ts b/src/core/player-reconciler.ts index 82c41de0..7d70fa4b 100644 --- a/src/core/player-reconciler.ts +++ b/src/core/player-reconciler.ts @@ -39,6 +39,8 @@ export class PlayerReconciler { /** In-flight player load promises, so off-playback captures (captureFrame) can await asset readiness. */ private readonly inFlightLoads = new Set>(); + private readonly playerLoads = new WeakMap>(); + private isInitialReconcile = false; constructor(private readonly edit: Edit) { this.edit.getInternalEvents().on(InternalEvent.Resolved, this.onResolved); @@ -59,9 +61,14 @@ export class PlayerReconciler { * @returns Promise that resolves when all players are loaded */ public async reconcileInitial(resolved: ResolvedEdit): Promise { - const result = this.reconcile(resolved); - await Promise.all(result.pendingLoads); - return result; + this.isInitialReconcile = true; + try { + const result = this.reconcile(resolved); + await Promise.all(result.pendingLoads); + return result; + } finally { + this.isInitialReconcile = false; + } } /** @@ -76,6 +83,14 @@ export class PlayerReconciler { } } + public async whenPlayerSettled(player: Player): Promise { + for (;;) { + const pending = this.playerLoads.get(player); + if (!pending) return; + await Promise.allSettled([pending]); + } + } + /** * Reconcile Players to match the ResolvedEdit. * @@ -130,7 +145,7 @@ export class PlayerReconciler { } } else if (this.enableCreation) { // Create new Player - this.createPlayer(clip, clipId, trackIndex, clipIndex); + pendingLoads.push(this.createPlayer(clip, clipId, trackIndex, clipIndex)); result.created.push(clipId); } } @@ -247,9 +262,7 @@ export class PlayerReconciler { }); }); - // Track the load so off-playback captures can await asset readiness (see whenSettled()). - this.inFlightLoads.add(loadPromise); - return loadPromise.finally(() => this.inFlightLoads.delete(loadPromise)); + return this.trackPlayerLoad(player, loadPromise); } /** @@ -372,7 +385,7 @@ export class PlayerReconciler { } if (needsReload && player.reloadAsset) { - player + const loadPromise = player .reloadAsset() .then(() => { player.reconfigureAfterRestore(); @@ -380,11 +393,31 @@ export class PlayerReconciler { .catch(error => { console.error("Failed to reload asset:", error); }); + this.trackPlayerLoad(player, loadPromise); } else { player.reconfigureAfterRestore(); } } + private trackPlayerLoad(player: Player, loadPromise: Promise): Promise { + this.inFlightLoads.add(loadPromise); + this.playerLoads.set(player, loadPromise); + + const cleanup = (): void => { + if (this.playerLoads.get(player) !== loadPromise) return; + this.playerLoads.delete(player); + if (this.isInitialReconcile || player.getTimingIntent().length !== "auto") return; + + queueMicrotask(() => { + const currentPlayer = player.clipId ? this.edit.getPlayerByClipId(player.clipId) : null; + if (currentPlayer !== player) return; + this.edit.resolveClipAutoLength(player).catch(error => console.error("Failed to resolve auto clip timing:", error)); + }); + }; + loadPromise.then(cleanup, cleanup); + return loadPromise.finally(() => this.inFlightLoads.delete(loadPromise)); + } + /** * Sync PIXI track containers to match resolved track count. * Creates new containers for added tracks, removes empty containers for deleted tracks. diff --git a/src/core/resolver.ts b/src/core/resolver.ts index 1df2e621..1c4017af 100644 --- a/src/core/resolver.ts +++ b/src/core/resolver.ts @@ -18,12 +18,14 @@ import type { EditDocument } from "./edit-document"; import type { MergeFieldService } from "./merge/merge-field-service"; import type { Clip, ResolvedClip, ResolvedEdit, ResolvedTrack } from "./schemas"; +import { resolveAutoLength } from "./timing/resolver"; import { type Seconds, sec, isAliasReference, parseAliasName } from "./timing/types"; // ─── Types ──────────────────────────────────────────────────────────────────── export interface ResolveContext { mergeFields: MergeFieldService; + mediaDurationByClipId?: ReadonlyMap; } /** @@ -81,7 +83,7 @@ function resolveMergeFieldsInClip(clip: InternalClip, mergeFields: MergeFieldSer return num !== null ? num : mergeFields.resolve(value); } if (Array.isArray(value)) { - return value.map((item) => processValue(item, key)); + return value.map(item => processValue(item, key)); } if (value !== null && typeof value === "object") { const result: Record = {}; @@ -246,7 +248,12 @@ function topologicalSort(dependencies: Map>, allClipIds: str * This is called after topological sorting ensures dependencies are resolved first. * Note: Merge fields should be resolved via resolveMergeFieldsInClip() BEFORE calling this. */ -function resolveClipWithAliases(clip: InternalClip, previousClipEnd: Seconds, resolvedAliases: Map): PartialResolvedClip { +function resolveClipWithAliases( + clip: InternalClip, + previousClipEnd: Seconds, + resolvedAliases: Map, + intrinsicDuration: Seconds | null = null +): PartialResolvedClip { // Resolve start let start: Seconds; if (clip.start === "auto") { @@ -269,9 +276,7 @@ function resolveClipWithAliases(clip: InternalClip, previousClipEnd: Seconds, re length = sec(1); // Temporary placeholder pendingEndLength = true; } else if (clip.length === "auto") { - // Use intrinsic duration if available, else fallback - // Note: For now use fallback; intrinsic duration will be provided by Players - length = sec(3); + length = resolveAutoLength(clip.asset, intrinsicDuration); } else if (isAliasReference(clip.length)) { const aliasName = parseAliasName(clip.length); const aliasValue = resolvedAliases.get(aliasName); @@ -341,6 +346,9 @@ export interface SingleClipContext extends ResolveContext { * Resolved alias values for alias reference resolution. */ resolvedAliases?: Map; + + /** Intrinsic media duration in seconds, or null when unavailable. */ + intrinsicDuration?: Seconds | null; } /** @@ -379,7 +387,7 @@ export function resolveClip(document: EditDocument, clipId: string, context: Sin // 3. Resolve the single clip using alias-aware logic const resolvedAliases = context.resolvedAliases ?? new Map(); - const resolvedClip = resolveClipWithAliases(processedClip, context.previousClipEnd, resolvedAliases); + const resolvedClip = resolveClipWithAliases(processedClip, context.previousClipEnd, resolvedAliases, context.intrinsicDuration); // 4. Handle "end" length (second pass for this single clip) if (resolvedClip.pendingEndLength && context.cachedTimelineEnd !== undefined) { @@ -453,7 +461,8 @@ export function resolve(document: EditDocument, context: ResolveContext): Resolv const previousClipEnd = previousClipEndByTrack.get(trackIndex) ?? sec(0); // Resolve the clip with alias support - const resolvedClip = resolveClipWithAliases(processedClip, previousClipEnd, resolvedAliases); + const intrinsicDuration = processedClip.id ? (context.mediaDurationByClipId?.get(processedClip.id) ?? null) : null; + const resolvedClip = resolveClipWithAliases(processedClip, previousClipEnd, resolvedAliases, intrinsicDuration); // Store in map by position key resolvedClipsByPosition.set(`${trackIndex}-${clipIndex}`, resolvedClip); diff --git a/src/core/timing-manager.ts b/src/core/timing-manager.ts index cb3a4c25..f6bf0b4f 100644 --- a/src/core/timing-manager.ts +++ b/src/core/timing-manager.ts @@ -4,7 +4,7 @@ import type { Player } from "@canvas/players/player"; import { EditEvent } from "@core/events/edit-events"; -import { calculateTimelineEnd, resolveAutoLength, resolveAutoStart, resolveEndLength } from "@core/timing/resolver"; +import { calculateTimelineEnd, resolveAutoStart, resolveEndLength } from "@core/timing/resolver"; import { type Seconds, isAliasReference, sec } from "@core/timing/types"; import type { Edit } from "./edit-session"; @@ -53,18 +53,13 @@ export class TimingManager { const resolvedClip = resolvedTrack?.clips[clipIdx]; if (resolvedClip) { - const intent = player.getTimingIntent(); - // Use resolved values from the resolver const resolvedStart = resolvedClip.start; - let resolvedLength = resolvedClip.length; - - // Special handling for "auto" length - requires async asset loading - if (intent.length === "auto") { - resolvedLength = await resolveAutoLength(player.clipConfiguration.asset); - } + const resolvedLength = resolvedClip.length; + const changed = player.getStart() !== resolvedStart || player.getLength() !== resolvedLength; player.setResolvedTiming({ start: resolvedStart, length: resolvedLength }); + if (changed) player.reconfigureAfterRestore(); // Sync resolved edit cache so timeline UI sees actual timing resolvedClip.start = resolvedStart; diff --git a/src/core/timing/resolver.ts b/src/core/timing/resolver.ts index 1dd9067c..3ff056bd 100644 --- a/src/core/timing/resolver.ts +++ b/src/core/timing/resolver.ts @@ -8,8 +8,6 @@ import type { Asset } from "@schemas"; import { type ResolutionContext, type ResolvedTiming, type Seconds, type TimingIntent, isAliasReference, sec } from "./types"; -const DEFAULT_AUTO_LENGTH_FALLBACK = sec(1); - const DEFAULT_AUTO_LENGTH_SEC = sec(3); export function resolveTimingIntent(intent: TimingIntent, context: Readonly): ResolvedTiming { @@ -31,7 +29,7 @@ export function resolveTimingIntent(intent: TimingIntent, context: Readonly { - return new Promise(resolve => { - const video = document.createElement("video"); - video.preload = "metadata"; - video.crossOrigin = "anonymous"; - video.onloadedmetadata = (): void => resolve(video.duration); - video.onerror = (): void => resolve(null); - video.src = src; - }); -} - -export async function resolveAutoLength(asset: Asset): Promise { - const assetWithSrc = asset as { type: string; src?: string; trim?: number }; - - if (["video", "audio", "luma"].includes(assetWithSrc.type) && assetWithSrc.src) { - const duration = await probeMediaDuration(assetWithSrc.src); - if (duration !== null && !Number.isNaN(duration)) { - const trim = assetWithSrc.trim ?? 0; - return sec(duration - trim); - } - } - - return DEFAULT_AUTO_LENGTH_SEC; +export function resolveAutoLength(asset: Asset, intrinsicDuration: Seconds | null = null): Seconds { + if (intrinsicDuration === null || !Number.isFinite(intrinsicDuration) || intrinsicDuration <= 0) return DEFAULT_AUTO_LENGTH_SEC; + const trim = "trim" in asset && typeof asset.trim === "number" && Number.isFinite(asset.trim) ? asset.trim : 0; + return sec(Math.max(0, intrinsicDuration - trim)); } export function resolveAutoStart(trackIndex: number, clipIndex: number, tracks: Player[][]): Seconds { diff --git a/tests/asset-loader.test.ts b/tests/asset-loader.test.ts index 21b3082b..b8ea22be 100644 --- a/tests/asset-loader.test.ts +++ b/tests/asset-loader.test.ts @@ -8,8 +8,15 @@ */ import { AssetLoader } from "@loaders/asset-loader"; +import { GifImageSource } from "@loaders/gif-image-source"; import * as pixi from "pixi.js"; +jest.mock("@loaders/gif-image-source", () => ({ + GifImageSource: { fetch: jest.fn() } +})); + +const mockGifFetch = GifImageSource.fetch as jest.Mock; + // Mock pixi.js VideoSource and Texture jest.mock("pixi.js", () => ({ VideoSource: jest.fn().mockImplementation(({ resource }) => ({ @@ -57,6 +64,46 @@ describe("AssetLoader", () => { expect(loader.loadTracker.registry[url]).toEqual({ progress: 1, status: "failed" }); expect(pixiMock.Assets.unload).toHaveBeenCalledWith(url); }); + + it("unloads an unreferenced asset after its pending load settles", async () => { + const loader = new AssetLoader(); + const url = "https://example.com/pending.png"; + let resolveAsset!: (asset: object) => void; + pixiMock.Assets.load.mockReturnValueOnce( + new Promise(resolve => { + resolveAsset = resolve; + }) + ); + + const load = loader.load(url, { src: url }); + loader.release(url); + pixiMock.Assets.cache.has.mockReturnValue(true); + resolveAsset({}); + await load; + await Promise.resolve(); + + expect(pixiMock.Assets.unload).toHaveBeenCalledWith(url); + }); + }); + + describe("loadGif", () => { + it("shares one decoded source until the final clip releases it", async () => { + const source = { destroy: jest.fn() }; + mockGifFetch.mockResolvedValue(source); + const loader = new AssetLoader(); + const src = "https://example.com/animation.gif"; + + const [first, second] = await Promise.all([loader.loadGif(src, `${src}?x-cors=1`), loader.loadGif(src, `${src}?x-cors=1`)]); + + expect(first).toBe(source); + expect(second).toBe(source); + expect(mockGifFetch).toHaveBeenCalledTimes(1); + loader.release(src); + expect(source.destroy).not.toHaveBeenCalled(); + loader.release(src); + await Promise.resolve(); + expect(source.destroy).toHaveBeenCalledTimes(1); + }); }); describe("loadVideoUnique", () => { diff --git a/tests/edit-clip-operations.test.ts b/tests/edit-clip-operations.test.ts index f93ae50e..32d82ad3 100644 --- a/tests/edit-clip-operations.test.ts +++ b/tests/edit-clip-operations.test.ts @@ -117,6 +117,7 @@ jest.mock("@loaders/asset-loader", () => ({ getProgress: jest.fn().mockReturnValue(100), incrementRef: jest.fn(), decrementRef: jest.fn().mockReturnValue(true), + release: jest.fn(), loadTracker: { on: jest.fn(), off: jest.fn() } })) })); @@ -202,6 +203,7 @@ const createMockPlayer = (edit: Edit, config: ResolvedClip, type: PlayerType) => length: config.length }; }, + getMediaDuration: () => null, getResolvedTiming: () => ({ ...resolvedTiming }), setResolvedTiming: jest.fn((timing: { start: number; length: number }) => { resolvedTiming = { ...timing }; diff --git a/tests/edit-commands.test.ts b/tests/edit-commands.test.ts index e2977443..9fa51dad 100644 --- a/tests/edit-commands.test.ts +++ b/tests/edit-commands.test.ts @@ -117,6 +117,7 @@ jest.mock("@loaders/asset-loader", () => ({ getProgress: jest.fn().mockReturnValue(100), incrementRef: jest.fn(), decrementRef: jest.fn().mockReturnValue(true), + release: jest.fn(), loadTracker: { on: jest.fn(), off: jest.fn() diff --git a/tests/edit-load.test.ts b/tests/edit-load.test.ts index 234f3d15..27104cd2 100644 --- a/tests/edit-load.test.ts +++ b/tests/edit-load.test.ts @@ -115,6 +115,7 @@ const mockAssetLoader = { getProgress: jest.fn().mockReturnValue(100), incrementRef: jest.fn(), decrementRef: jest.fn().mockReturnValue(true), + release: jest.fn(), loadTracker: { on: jest.fn(), off: jest.fn() } }; @@ -164,6 +165,7 @@ const createMockPlayerContainer = () => { // Track player instances by type for assertions const createdPlayers: Map = new Map(); +let mockMediaDuration: number | null = null; // Mock player factory - create functional mock players const createMockPlayer = (edit: Edit, config: ResolvedClip, type: PlayerType) => { @@ -209,6 +211,7 @@ const createMockPlayer = (edit: Edit, config: ResolvedClip, type: PlayerType) => length: config.length }; }, + getMediaDuration: () => mockMediaDuration, getResolvedTiming: () => ({ ...resolvedTiming }), setResolvedTiming: jest.fn((timing: { start: number; length: number }) => { resolvedTiming = { ...timing }; @@ -361,6 +364,7 @@ describe("Edit loadEdit()", () => { beforeEach(async () => { // Reset player creation tracking createdPlayers.clear(); + mockMediaDuration = null; // Reset all mocks jest.clearAllMocks(); @@ -626,6 +630,36 @@ describe("Edit loadEdit()", () => { expect(player?.getLength()).toBe(3); }); + it("re-resolves loaded media duration through auto starts and aliases", async () => { + mockMediaDuration = 2.4; + const mediaEdit = new Edit({ + timeline: { + tracks: [ + { + clips: [ + { alias: "media", asset: { type: "image", src: "https://example.com/animation.gif" }, start: 0, length: "auto" }, + { asset: { type: "luma", src: "https://example.com/matte.mp4" }, start: 0, length: 3 }, + { asset: { type: "image", src: "https://example.com/image.jpg" }, start: "auto", length: 1 } + ] + }, + { clips: [{ asset: { type: "image", src: "https://example.com/alias.jpg" }, start: 0, length: "alias://media" }] } + ] + }, + output: { size: { width: 1920, height: 1080 }, format: "mp4" } + }); + + await mediaEdit.load(); + expect(mediaEdit.getPlayerClip(0, 0)?.getLength()).toBe(2.4); + expect(mediaEdit.getPlayerClip(0, 1)?.getLength()).toBe(2.4); + expect(mediaEdit.getPlayerClip(0, 2)?.getStart()).toBe(2.4); + expect(mediaEdit.getPlayerClip(1, 0)?.getLength()).toBe(2.4); + + mockMediaDuration = 1.2; + await mediaEdit.resolveClipAutoLength(mediaEdit.getPlayerClip(0, 0)!); + expect(mediaEdit.getResolvedEdit().timeline.tracks[1].clips[0].length).toBe(1.2); + mediaEdit.dispose(); + }); + it("sets totalDuration to max clip end time", async () => { const editConfig = createMinimalEdit([ { diff --git a/tests/edit-merge-fields.test.ts b/tests/edit-merge-fields.test.ts index a2dbf19d..fff8e2fc 100644 --- a/tests/edit-merge-fields.test.ts +++ b/tests/edit-merge-fields.test.ts @@ -98,6 +98,7 @@ jest.mock("@loaders/asset-loader", () => ({ getProgress: jest.fn().mockReturnValue(100), incrementRef: jest.fn(), decrementRef: jest.fn().mockReturnValue(true), + release: jest.fn(), loadTracker: { on: jest.fn(), off: jest.fn() } })) })); @@ -181,6 +182,7 @@ const createMockPlayer = (edit: ShotstackEdit, config: ResolvedClip, type: Playe length: config.length }; }, + getMediaDuration: () => null, getResolvedTiming: () => ({ ...resolvedTiming }), setResolvedTiming: jest.fn((timing: { start: number; length: number }) => { resolvedTiming = { ...timing }; diff --git a/tests/edit-playback.test.ts b/tests/edit-playback.test.ts index dbebe794..d0836bae 100644 --- a/tests/edit-playback.test.ts +++ b/tests/edit-playback.test.ts @@ -87,6 +87,7 @@ jest.mock("@loaders/asset-loader", () => ({ getProgress: jest.fn().mockReturnValue(100), incrementRef: jest.fn(), decrementRef: jest.fn().mockReturnValue(true), + release: jest.fn(), loadTracker: { on: jest.fn(), off: jest.fn() diff --git a/tests/edit-timing.test.ts b/tests/edit-timing.test.ts index c0a0cf80..a10a7e7c 100644 --- a/tests/edit-timing.test.ts +++ b/tests/edit-timing.test.ts @@ -103,6 +103,7 @@ jest.mock("@loaders/asset-loader", () => ({ getProgress: jest.fn().mockReturnValue(100), incrementRef: jest.fn(), decrementRef: jest.fn().mockReturnValue(true), + release: jest.fn(), loadTracker: { on: jest.fn(), off: jest.fn() } })) })); @@ -188,6 +189,7 @@ const createMockPlayer = (edit: Edit, config: ResolvedClip, type: PlayerType) => length: config.length }; }, + getMediaDuration: () => null, getResolvedTiming: () => ({ ...resolvedTiming }), setResolvedTiming: jest.fn((timing: { start: number; length: number }) => { resolvedTiming = { ...timing }; diff --git a/tests/gif-image-source.test.ts b/tests/gif-image-source.test.ts new file mode 100644 index 00000000..91c72dff --- /dev/null +++ b/tests/gif-image-source.test.ts @@ -0,0 +1,85 @@ +/** @jest-environment jsdom */ + +import { GifImageSource, isGifUrl } from "@loaders/gif-image-source"; +import { GifSource as PixiGifSource } from "pixi.js/gif"; + +jest.mock("pixi.js", () => ({ + CanvasSource: jest.fn(function CanvasSource(this: { resource: HTMLCanvasElement }, { resource }: { resource: HTMLCanvasElement }) { + this.resource = resource; + }), + Texture: jest.fn(function Texture(this: { source: unknown; destroy: jest.Mock }, { source }: { source: unknown }) { + this.source = source; + this.destroy = jest.fn(); + }) +})); + +jest.mock("pixi.js/gif", () => ({ GifSource: { from: jest.fn() } })); + +const pixiGifFrom = PixiGifSource.from as jest.MockedFunction; +const TWO_FRAME_GIF = "R0lGODlhAgACAPAAAP8AAAAAACH/C05FVFNDQVBFMi4wAwEAAAAh+QQAAAAAACwAAAAAAgACAAACAoRRACH5BAAKAAAALAAAAAACAAIAgAAA/wAAAAIChFEAOw=="; + +function decodeBase64(value: string): ArrayBuffer { + return Uint8Array.from(atob(value), character => character.charCodeAt(0)).buffer; +} + +function appendUint16(bytes: number[], value: number): void { + bytes.push(value % 256, Math.floor(value / 256) % 256); +} + +function createGif(width: number, height: number, frameCount: number): ArrayBuffer { + const bytes = [71, 73, 70, 56, 57, 97]; + appendUint16(bytes, width); + appendUint16(bytes, height); + bytes.push(0, 0, 0); + for (let frame = 0; frame < frameCount; frame += 1) { + bytes.push(0x2c, 0, 0, 0, 0); + appendUint16(bytes, width); + appendUint16(bytes, height); + bytes.push(0, 2, 2, 0x44, 0x01, 0); + } + bytes.push(0x3b); + return Uint8Array.from(bytes).buffer; +} + +function createDecodedSource(): PixiGifSource & { destroy: jest.Mock } { + const frames = [ + { start: 0, end: 40, texture: { source: { resource: document.createElement("canvas") } } }, + { start: 40, end: 140, texture: { source: { resource: document.createElement("canvas") } } } + ]; + return { + width: 2, + height: 2, + duration: 140, + frames, + textures: frames.map(frame => frame.texture), + totalFrames: frames.length, + destroy: jest.fn() + } as unknown as PixiGifSource & { destroy: jest.Mock }; +} + +describe("GifImageSource", () => { + beforeEach(() => pixiGifFrom.mockReset()); + + it("classifies GIF paths and data URLs without probing ordinary images", () => { + expect(isGifUrl("https://example.com/animation.GIF?token=1")).toBe(true); + expect(isGifUrl("data:image/gif;base64,R0lGODlh")).toBe(true); + expect(isGifUrl("https://example.com/photo.png")).toBe(false); + }); + + it("uses Pixi decoding and selects frames from looping playhead time", () => { + const decoded = createDecodedSource(); + pixiGifFrom.mockReturnValue(decoded); + + const source = GifImageSource.from(decodeBase64(TWO_FRAME_GIF)); + + expect(source.frameIndexAt(39)).toBe(0); + expect(source.frameIndexAt(40)).toBe(1); + expect(source.frameIndexAt(180)).toBe(1); + expect(decoded.destroy).toHaveBeenCalledTimes(1); + }); + + it("rejects an eager decode that exceeds the browser memory budget", () => { + expect(() => GifImageSource.from(createGif(4096, 4096, 2))).toThrow(/decoded size/); + expect(pixiGifFrom).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/intent-stripping.test.ts b/tests/intent-stripping.test.ts index f20075c1..19eb384b 100644 --- a/tests/intent-stripping.test.ts +++ b/tests/intent-stripping.test.ts @@ -110,6 +110,7 @@ jest.mock("@loaders/asset-loader", () => ({ getProgress: jest.fn().mockReturnValue(100), incrementRef: jest.fn(), decrementRef: jest.fn().mockReturnValue(true), + release: jest.fn(), loadTracker: { on: jest.fn(), off: jest.fn() diff --git a/tests/media-player-fallback.test.ts b/tests/media-player-fallback.test.ts index a0324633..1c516b68 100644 --- a/tests/media-player-fallback.test.ts +++ b/tests/media-player-fallback.test.ts @@ -14,6 +14,11 @@ jest.mock("@canvas/players/placeholder-graphic", () => ({ createPlaceholderGraphic: mockCreatePlaceholderGraphic })); +jest.mock("@core/loaders/gif-image-source", () => ({ + appendCorsQuery: (src: string) => `${src}?x-cors=1`, + isGifUrl: (src: string) => src.endsWith(".gif") +})); + jest.mock("pixi.js", () => { class MockPoint { public x: number; @@ -162,8 +167,10 @@ function createEdit() { isPlaying: false, assetLoader: { load: jest.fn(), + loadGif: jest.fn(), loadVideoUnique: jest.fn(), - rejectAsset: jest.fn() + rejectAsset: jest.fn(), + release: jest.fn() } }; } @@ -192,8 +199,17 @@ function createVideoClip(): ResolvedClip { } as ResolvedClip; } -function createVideoTexture(width: number, height: number) { +function createGifClip(): ResolvedClip { + return { + asset: { type: "image", src: "https://example.com/animation.gif" }, + start: 0, + length: 5 + } as ResolvedClip; +} + +function createVideoTexture(width: number, height: number, duration = 5) { const resource = { + duration, volume: 1, currentTime: 0, pause: jest.fn(), @@ -221,6 +237,60 @@ describe("media player fallbacks", () => { warnSpy.mockRestore(); }); + it("uses the GIF cycle for auto timing and selects frames from the playhead", async () => { + const edit = createEdit(); + const first = new pixi.Texture({ width: 2, height: 2 } as ConstructorParameters[0]); + const second = new pixi.Texture({ width: 2, height: 2 } as ConstructorParameters[0]); + const gifSource = { + duration: 200, + frames: [ + { start: 0, end: 100, texture: first }, + { start: 100, end: 200, texture: second } + ], + frameIndexAt: jest.fn((time: number) => (time < 100 ? 0 : 1)) + }; + edit.assetLoader.loadGif.mockResolvedValue(gifSource); + const player = new ImagePlayer(edit as never, createGifClip()); + + await player.load(); + expect(player.getMediaDuration()).toBe(0.2); + + edit.playbackTime = 0.15; + player.update(0, 0); + const { sprite } = player as unknown as { sprite: { texture: unknown } }; + expect(sprite.texture).toBe(second); + }); + + it("releases replaced and disposed pending GIF loads exactly once", async () => { + const edit = createEdit(); + let resolveFirst!: (source: unknown) => void; + let resolveSecond!: (source: unknown) => void; + edit.assetLoader.loadGif.mockReturnValueOnce( + new Promise(resolve => { + resolveFirst = resolve; + }) + ); + edit.assetLoader.loadGif.mockReturnValueOnce( + new Promise(resolve => { + resolveSecond = resolve; + }) + ); + const player = new ImagePlayer(edit as never, createGifClip()); + const staleLoad = player.load(); + await Promise.resolve(); + + (player.clipConfiguration.asset as { src: string }).src = "https://example.com/replacement.gif"; + const replacementLoad = player.reloadAsset(); + await Promise.resolve(); + edit.assetLoader.release("https://example.com/replacement.gif"); + player.dispose(); + resolveFirst({ duration: 100, frames: [] }); + resolveSecond({ duration: 100, frames: [] }); + await Promise.all([staleLoad, replacementLoad]); + + expect(edit.assetLoader.release.mock.calls).toEqual([["https://example.com/animation.gif"], ["https://example.com/replacement.gif"]]); + }); + it("uses display dimensions for a failed image placeholder", async () => { const edit = createEdit(); edit.assetLoader.load.mockResolvedValueOnce(null); @@ -258,4 +328,25 @@ describe("media player fallbacks", () => { expect(player.getSize()).toEqual({ width: 1280, height: 720 }); expect(Number.isFinite(player.getScale())).toBe(true); }); + + it("ignores a stale video load after a newer source is ready", async () => { + const edit = createEdit(); + let resolveFirst!: (texture: ReturnType) => void; + const firstLoad = new Promise>(resolve => { + resolveFirst = resolve; + }); + const secondTexture = createVideoTexture(1920, 1080, 2); + edit.assetLoader.loadVideoUnique.mockReturnValueOnce(firstLoad).mockResolvedValueOnce(secondTexture); + const player = new VideoPlayer(edit as never, createVideoClip()); + + const staleLoad = player.load(); + await Promise.resolve(); + (player.clipConfiguration.asset as { src: string }).src = "https://example.com/new-video.mp4"; + await player.reloadAsset(); + resolveFirst(createVideoTexture(640, 360, 8)); + await staleLoad; + + expect(player.getMediaDuration()).toBe(2); + expect((player as unknown as { texture: unknown }).texture).toBe(secondTexture); + }); }); diff --git a/tests/resolver.test.ts b/tests/resolver.test.ts index eaaec278..8c5714e4 100644 --- a/tests/resolver.test.ts +++ b/tests/resolver.test.ts @@ -2,6 +2,7 @@ import { EditDocument } from "@core/edit-document"; import { EventEmitter } from "@core/events/event-emitter"; import { MergeFieldService } from "@core/merge/merge-field-service"; import { resolve } from "@core/resolver"; +import { sec } from "@core/timing/types"; import type { Edit } from "@schemas"; function createMergeFieldService(): MergeFieldService { @@ -79,6 +80,32 @@ describe("Resolver", () => { expect(clips[2].start).toBe(5); // After second clip (2 + 3) }); + it("propagates a loaded GIF cycle through auto and alias timing", () => { + const doc = new EditDocument({ + timeline: { + tracks: [ + { + clips: [ + { alias: "gif", asset: { type: "image", src: "https://example.com/a.gif" }, start: 0, length: "auto" }, + { asset: { type: "image", src: "https://example.com/b.jpg" }, start: "auto", length: 1 } + ] + }, + { clips: [{ asset: { type: "image", src: "https://example.com/c.jpg" }, start: 0, length: "alias://gif" }] } + ] + }, + output: { format: "mp4", size: { width: 1920, height: 1080 } } + }); + const gifId = doc.getClipId(0, 0)!; + + const resolved = resolve(doc, { + mergeFields: createMergeFieldService(), + mediaDurationByClipId: new Map([[gifId, sec(2.4)]]) + }); + expect(resolved.timeline.tracks[0].clips[0].length).toBe(2.4); + expect(resolved.timeline.tracks[0].clips[1].start).toBe(2.4); + expect(resolved.timeline.tracks[1].clips[0].length).toBe(2.4); + }); + it("resolves 'end' length to extend to timeline end", () => { const doc = new EditDocument(createEditWithEndLength()); const mergeFields = createMergeFieldService(); diff --git a/vite.shared.ts b/vite.shared.ts index 47fcf33e..d763ba57 100644 --- a/vite.shared.ts +++ b/vite.shared.ts @@ -20,6 +20,7 @@ export const globals: Record = { }; export function external(id: string): boolean { + if (id === "pixi.js/gif") return false; if (id === "pixi.js" || id.startsWith("pixi.js/")) return true; if (id === "pixi-filters" || id.startsWith("pixi-filters/")) return true; if (id.startsWith("@napi-rs/")) return true;