Skip to content

feat: support animated GIF image assets #146

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
dazzatronus wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat: support animated GIF image assets #146

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions src/components/canvas/players/audio-player.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { KeyframeBuilder } from "@animations/keyframe-builder";
import type { Edit } from "@core/edit-session";
import { sec } from "@core/timing/types";
import { type Size } from "@layouts/geometry";
import { AudioLoadParser } from "@loaders/audio-load-parser";
import { type AudioAsset, type ResolvedClip, type Keyframe } from "@schemas";
Expand All @@ -11,6 +12,7 @@ import { Player, PlayerType } from "./player";
export class AudioPlayer extends Player {
private audioResource: howler.Howl | null;
private isPlaying: boolean;
private assetAcquisitions = new Map<number, string>();

private volumeKeyframeBuilder!: KeyframeBuilder;

Expand All @@ -25,24 +27,37 @@ export class AudioPlayer extends Player {
}

public override async load(): Promise<void> {
const revision = this.beginMediaTimingLoad();
await super.load();
if (!this.isMediaTimingLoadCurrent(revision)) return;

const audioClipConfiguration = this.clipConfiguration.asset as AudioAsset;

const identifier = audioClipConfiguration.src;
if (!identifier) {
this.completeMediaTimingLoad(revision, null);
// Prompt-bearing assets route to pending placeholder players — reaching here without a src is invalid data
throw new Error("Audio asset has no src to load.");
}
const loadOptions: pixi.UnresolvedAsset = { src: identifier, parser: AudioLoadParser.Name };
this.assetAcquisitions.set(revision, identifier);
const audioResource = await this.edit.assetLoader.load<howler.Howl>(identifier, loadOptions);
if (!this.isMediaTimingLoadCurrent(revision)) {
if (audioResource) this.releaseAssetAcquisition(revision);
else this.assetAcquisitions.delete(revision);
return;
}

const isValidAudioSource = audioResource instanceof howler.Howl;
if (!isValidAudioSource) {
if (audioResource) this.releaseAssetAcquisition(revision);
else this.assetAcquisitions.delete(revision);
this.completeMediaTimingLoad(revision, null);
throw new Error(`Invalid audio source '${audioClipConfiguration.src}'.`);
}

this.audioResource = audioResource;
this.completeMediaTimingLoad(revision, sec(audioResource.duration()));

// Create volume keyframes after timing is resolved (not in constructor)
const baseVolume = typeof audioClipConfiguration.volume === "number" ? audioClipConfiguration.volume : 1;
Expand Down Expand Up @@ -107,6 +122,8 @@ export class AudioPlayer extends Player {
}

public override dispose(): void {
const { src } = this.clipConfiguration.asset as AudioAsset;
this.releaseAssetAcquisitions(src);
if (this.audioResource) {
this.audioResource.stop();
this.audioResource.unload();
Expand All @@ -118,6 +135,8 @@ export class AudioPlayer extends Player {

/** Reload the audio asset when asset.src changes (e.g., merge field update or loadEdit) */
public override async reloadAsset(): Promise<void> {
const revision = this.beginMediaTimingLoad();
this.releaseAssetAcquisitions();
if (this.audioResource) {
this.audioResource.stop();
this.audioResource.unload();
Expand All @@ -129,19 +148,49 @@ export class AudioPlayer extends Player {
const audioAsset = this.clipConfiguration.asset as AudioAsset;
const { src } = audioAsset;
if (!src) {
this.completeMediaTimingLoad(revision, null);
throw new Error("Audio asset has no src to load.");
}
const loadOptions: pixi.UnresolvedAsset = { src, parser: AudioLoadParser.Name };
this.assetAcquisitions.set(revision, src);
const audioResource = await this.edit.assetLoader.load<howler.Howl>(src, loadOptions);
if (!this.isMediaTimingLoadCurrent(revision)) {
if (audioResource) this.releaseAssetAcquisition(revision);
else this.assetAcquisitions.delete(revision);
return;
}

if (!(audioResource instanceof howler.Howl)) {
if (audioResource) this.releaseAssetAcquisition(revision);
else this.assetAcquisitions.delete(revision);
this.completeMediaTimingLoad(revision, null);
throw new Error(`Invalid audio source '${audioAsset.src}'.`);
}

this.audioResource = audioResource;
this.completeMediaTimingLoad(revision, sec(audioResource.duration()));
this.audioResource.volume(this.getVolume());
}

private releaseAssetAcquisition(revision: number): void {
const identifier = this.assetAcquisitions.get(revision);
if (!identifier) return;
this.assetAcquisitions.delete(revision);
this.edit.assetLoader.release(identifier);
}

private releaseAssetAcquisitions(alreadyReleasedIdentifier?: string): void {
let skipIdentifier = alreadyReleasedIdentifier;
for (const [revision, identifier] of this.assetAcquisitions) {
this.assetAcquisitions.delete(revision);
if (identifier === skipIdentifier) {
skipIdentifier = undefined;
} else {
this.edit.assetLoader.release(identifier);
}
}
}

public override reconfigureAfterRestore(): void {
super.reconfigureAfterRestore();

Expand Down
129 changes: 114 additions & 15 deletions src/components/canvas/players/image-player.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import type { Edit } from "@core/edit-session";
import { appendCorsQuery, type GifImageSource, isGifUrl } from "@core/loaders/gif-image-source";
import { sec } from "@core/timing/types";
import { type Size } from "@layouts/geometry";
import { type ResolvedClip, type ImageAsset } from "@schemas";
import * as pixi from "pixi.js";
Expand All @@ -7,9 +9,14 @@ import { createPlaceholderGraphic } from "./placeholder-graphic";
import { Player, PlayerType } from "./player";

export class ImagePlayer extends Player {
private texture: pixi.Texture<pixi.ImageSource> | null;
private texture: pixi.Texture | null;
private sprite: pixi.Sprite | null;
private placeholder: pixi.Graphics | null;
private gifSource: GifImageSource | null = null;
private gifFrameTextures: pixi.Texture[] = [];
private ownedTextureWrappers: pixi.Texture[] = [];
private currentGifFrame = -1;
private assetAcquisitions = new Map<number, string>();

constructor(edit: Edit, clipConfiguration: ResolvedClip) {
super(edit, clipConfiguration, PlayerType.Image);
Expand All @@ -20,11 +27,17 @@ export class ImagePlayer extends Player {
}

public override async load(): Promise<void> {
const revision = this.beginMediaTimingLoad();
await super.load();
if (!this.isMediaTimingLoadCurrent(revision)) return;

try {
await this.loadTexture();
if (!(await this.loadTexture(revision))) return;
this.completeMediaTimingLoad(revision, this.gifSource && this.gifSource.frames.length > 1 ? sec(this.gifSource.duration / 1000) : null);
this.configureKeyframes();
} catch {
if (!this.isMediaTimingLoadCurrent(revision)) return;
this.completeMediaTimingLoad(revision, null);
this.createFallbackGraphic();
}
}
Expand All @@ -40,9 +53,12 @@ export class ImagePlayer extends Player {

public override update(deltaTime: number, elapsed: number): void {
super.update(deltaTime, elapsed);
this.updateGifFrame();
}

public override dispose(): void {
const { src } = this.clipConfiguration.asset as ImageAsset;
this.releaseAssetAcquisitions(src);
this.disposeTexture();
this.clearPlaceholder();
super.dispose();
Expand Down Expand Up @@ -71,35 +87,68 @@ export class ImagePlayer extends Player {
return this.placeholder ? this.getDisplaySize() : { width: 0, height: 0 };
}

public override async prepareStaticRender(): Promise<void> {
this.updateGifFrame();
}

/** Reload the image asset when asset.src changes (e.g., merge field update) */
public override async reloadAsset(): Promise<void> {
const revision = this.beginMediaTimingLoad();
this.disposeTexture();
this.clearPlaceholder();
this.releaseAssetAcquisitions();

try {
await this.loadTexture();
if (!(await this.loadTexture(revision))) return;
this.completeMediaTimingLoad(revision, this.gifSource && this.gifSource.frames.length > 1 ? sec(this.gifSource.duration / 1000) : null);
} catch {
if (!this.isMediaTimingLoadCurrent(revision)) return;
this.completeMediaTimingLoad(revision, null);
this.createFallbackGraphic();
}
}

private async loadTexture(): Promise<void> {
private async loadTexture(revision: number): Promise<boolean> {
const imageAsset = this.clipConfiguration.asset as ImageAsset;
const { src } = imageAsset;
if (!src) {
// Prompt-bearing assets route to pending placeholder players — reaching here without a src is invalid data
throw new Error("Image asset has no src to load.");
}

const corsUrl = `${src}${src.includes("?") ? "&" : "?"}x-cors=1`;
const loadOptions: pixi.UnresolvedAsset = { src: corsUrl, crossorigin: "anonymous", data: {} };
const texture = await this.edit.assetLoader.load<pixi.Texture<pixi.ImageSource>>(corsUrl, loadOptions);
const requestUrl = appendCorsQuery(src);
this.assetAcquisitions.set(revision, src);
if (isGifUrl(src)) {
const source = await this.edit.assetLoader.loadGif(src, requestUrl);
if (!this.isMediaTimingLoadCurrent(revision)) {
if (source) this.releaseAssetAcquisition(revision);
else this.assetAcquisitions.delete(revision);
return false;
}
if (!source) {
this.assetAcquisitions.delete(revision);
throw new Error(`Unable to decode GIF image source '${src}'.`);
}
try {
this.configureGif(source);
return true;
} catch (error) {
this.disposeTexture();
this.releaseAssetAcquisition(revision);
throw error;
}
}

const loadOptions: pixi.UnresolvedAsset = { alias: src, src: requestUrl, crossorigin: "anonymous", data: {} };
const texture = await this.edit.assetLoader.load<pixi.Texture<pixi.ImageSource>>(src, loadOptions);
if (!this.isMediaTimingLoadCurrent(revision)) {
if (texture) this.releaseAssetAcquisition(revision);
else this.assetAcquisitions.delete(revision);
return false;
}
if (!(texture?.source instanceof pixi.ImageSource)) {
if (texture) {
texture.destroy(true);
await this.edit.assetLoader.rejectAsset(corsUrl);
}
this.assetAcquisitions.delete(revision);
if (texture) await this.edit.assetLoader.rejectAsset(src);
throw new Error(`Invalid image source '${src}'.`);
}

Expand All @@ -111,6 +160,32 @@ export class ImagePlayer extends Player {
if (this.clipConfiguration.width && this.clipConfiguration.height) {
this.applyFixedDimensions();
}
return true;
}

private configureGif(source: GifImageSource): void {
this.clearPlaceholder();
this.gifSource = source;
this.gifFrameTextures = source.frames.map(frame => this.createCroppedTexture(frame.texture));
this.texture = this.gifFrameTextures[0] ?? null;
if (!this.texture) throw new Error("GIF contains no renderable frames.");

this.sprite = new pixi.Sprite(this.texture);
this.contentContainer.addChild(this.sprite);
this.currentGifFrame = 0;
if (this.clipConfiguration.width && this.clipConfiguration.height) this.applyFixedDimensions();
this.updateGifFrame();
}

private updateGifFrame(): void {
if (!this.gifSource || !this.sprite || !this.isActive()) return;
const frameIndex = this.gifSource.frameIndexAt(this.getPlaybackTime() * 1000);
if (frameIndex === this.currentGifFrame) return;
const texture = this.gifFrameTextures[frameIndex];
if (!texture) return;
this.sprite.texture = texture;
this.texture = texture;
this.currentGifFrame = frameIndex;
}

private disposeTexture(): void {
Expand All @@ -119,9 +194,31 @@ export class ImagePlayer extends Player {
this.sprite.destroy();
this.sprite = null;
}
// DON'T destroy the texture - it's managed by Assets
// The unloadClipAssets() method handles proper cleanup via Assets.unload()
for (const texture of this.ownedTextureWrappers) texture.destroy(false);
this.ownedTextureWrappers = [];
this.texture = null;
this.gifFrameTextures = [];
this.gifSource = null;
this.currentGifFrame = -1;
}

private releaseAssetAcquisition(revision: number): void {
const identifier = this.assetAcquisitions.get(revision);
if (!identifier) return;
this.assetAcquisitions.delete(revision);
this.edit.assetLoader.release(identifier);
}

private releaseAssetAcquisitions(alreadyReleasedIdentifier?: string): void {
let skipIdentifier = alreadyReleasedIdentifier;
for (const [revision, identifier] of this.assetAcquisitions) {
this.assetAcquisitions.delete(revision);
if (identifier === skipIdentifier) {
skipIdentifier = undefined;
} else {
this.edit.assetLoader.release(identifier);
}
}
}

private clearPlaceholder(): void {
Expand All @@ -136,7 +233,7 @@ export class ImagePlayer extends Player {
return true;
}

private createCroppedTexture(texture: pixi.Texture<pixi.ImageSource>): pixi.Texture<pixi.ImageSource> {
private createCroppedTexture<TSource extends pixi.TextureSource>(texture: pixi.Texture<TSource>): pixi.Texture<TSource> {
const imageAsset = this.clipConfiguration.asset as ImageAsset;

if (!imageAsset.crop) {
Expand All @@ -157,6 +254,8 @@ export class ImagePlayer extends Player {
const height = originalHeight - top - bottom;

const crop = new pixi.Rectangle(x, y, width, height);
return new pixi.Texture({ source: texture.source, frame: crop });
const croppedTexture = new pixi.Texture({ source: texture.source, frame: crop });
this.ownedTextureWrappers.push(croppedTexture);
return croppedTexture;
}
}
4 changes: 4 additions & 0 deletions src/components/canvas/players/luma-player.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import type { Edit } from "@core/edit-session";
import { sec } from "@core/timing/types";
import { type Size } from "@layouts/geometry";
import { type ResolvedClip, type LumaAsset } from "@schemas";
import * as pixi from "pixi.js";
Expand All @@ -21,6 +22,7 @@ export class LumaPlayer extends Player {
}

public override async load(): Promise<void> {
const revision = this.beginMediaTimingLoad();
await super.load();

const lumaAsset = this.clipConfiguration.asset as LumaAsset;
Expand All @@ -31,6 +33,7 @@ export class LumaPlayer extends Player {

const isValidLumaSource = texture?.source instanceof pixi.ImageSource || texture?.source instanceof pixi.VideoSource;
if (!isValidLumaSource) {
this.completeMediaTimingLoad(revision, null);
if (texture) {
texture.destroy(true);
await this.edit.assetLoader.rejectAsset(identifier);
Expand All @@ -45,6 +48,7 @@ export class LumaPlayer extends Player {
}

this.texture = texture;
this.completeMediaTimingLoad(revision, texture.source instanceof pixi.VideoSource ? sec(texture.source.resource.duration) : null);
this.sprite = new pixi.Sprite(this.texture);

this.contentContainer.addChild(this.sprite);
Expand Down
Loading
Loading