import React from "react";
import {
AbsoluteFill,
interpolate,
Sequence,
spring,
useCurrentFrame,
useVideoConfig,
} from "remotion";
import {
characters,
type SpeechEvent,
type TimelineEvent,
} from "./data/pizza-oven-project-01/script";
import {
audioFileForSpeech,
hasAudioForSpeech,
pizzaOvenProject01Scenario,
} from "./data/pizza-oven-project-01/timing";
import {roundedFontFamily} from "./fonts";
import {
activeVQChronologicalScenarioSegmentForFrame,
scheduleVQChronologicalScenario,
VQSpeechOverlay,
VQStageCornerStandee,
VQStillBackground,
VQTimelineAudio,
VQTimelineVideo,
VQWarmGradientBackground,
type VQAudioEvent,
type VQScheduledScenarioSegment,
type VQStageCornerStandeeLayouts,
type VQStageCornerStandeeMode,
type VQStillEvent,
type VQVideoEvent,
vqDefaultStageCornerStandeeLayouts,
} from "./lib/VQRemotionLib";
import {getMouthForSpeechFrame} from "./lipsync/manifest";
const sayoAvatar = characters.sayo.avatar;
const PizzaOvenSpeechOverlay = VQSpeechOverlay<SpeechEvent>;
const pizzaOvenStandeeLayouts = {
...vqDefaultStageCornerStandeeLayouts,
stage: {
...vqDefaultStageCornerStandeeLayouts.stage,
frameWidth: 520,
frameHeight: 720,
right: 330,
bottom: -82,
},
} as const satisfies VQStageCornerStandeeLayouts;
const clampInterpolation = {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
} as const;
type ScheduledTimelineEvent = VQScheduledScenarioSegment<TimelineEvent>;
const videoSequenceDuration = (
scheduledVideo: ScheduledTimelineEvent & Readonly<{event: VQVideoEvent}>,
scheduledEvents: readonly ScheduledTimelineEvent[]
) => {
const clearFrame = scheduledEvents.find(
(scheduledEvent) =>
scheduledEvent.from >= scheduledVideo.from &&
scheduledEvent.event.type === "clearVideo" &&
(scheduledEvent.event.videoId === undefined ||
scheduledEvent.event.videoId === scheduledVideo.event.id)
)?.from;
return clearFrame === undefined
? scheduledVideo.durationInFrames
: Math.min(
scheduledVideo.durationInFrames,
Math.max(0, clearFrame - scheduledVideo.from)
);
};
const audioSequenceDuration = (
scheduledAudio: ScheduledTimelineEvent & Readonly<{event: VQAudioEvent}>,
timelineEndFrame: number
) => {
const hasExplicitDuration =
scheduledAudio.event.durationFrames !== undefined ||
scheduledAudio.event.durationSeconds !== undefined;
return scheduledAudio.event.playback === "loop" && !hasExplicitDuration
? Math.max(0, timelineEndFrame - scheduledAudio.from)
: scheduledAudio.durationInFrames;
};
const SayoStandee: React.FC<
Readonly<{
mode: VQStageCornerStandeeMode;
frame: number;
fps: number;
activeSpeech?: SpeechEvent;
speakingLocalFrame: number;
}>
> = ({mode, frame, fps, activeSpeech, speakingLocalFrame}) => {
const entrance = spring({
frame,
fps,
config: {damping: 18, mass: 0.6},
});
const translateY = interpolate(entrance, [0, 1], [32, 0], clampInterpolation);
const mouth = activeSpeech
? getMouthForSpeechFrame(activeSpeech.id, speakingLocalFrame, fps)
: "rest";
return React.createElement(VQStageCornerStandee, {
mode,
imagePath: sayoAvatar.imagePath,
mouthImageDir: sayoAvatar.mouthImageDir,
mouth,
translateY,
layouts: pizzaOvenStandeeLayouts,
zIndex: 2,
});
};
const TimelineOverlay: React.FC<Readonly<{event: TimelineEvent}>> = ({
event,
}) => {
if (event.type !== "say") {
if (event.type === "video") {
return React.createElement(VQTimelineVideo, {video: event});
}
return event.type === "audio"
? React.createElement(VQTimelineAudio, {audio: event})
: null;
}
const character = characters[event.character];
return React.createElement(PizzaOvenSpeechOverlay, {
speech: event,
speakerName: character.displayName,
accentColor: character.avatar.accentColor,
hasAudio: hasAudioForSpeech,
getAudioPath: audioFileForSpeech,
subtitleOptions: {
fontFamily: roundedFontFamily,
fontSize: 40,
lineHeight: 1.35,
backgroundColor: "rgba(255, 255, 255, 0.9)",
},
containerStyle: {zIndex: 3},
});
};
export const PizzaOvenProject01: React.FC = () => {
const frame = useCurrentFrame();
const {fps} = useVideoConfig();
const scheduledEvents = scheduleVQChronologicalScenario(
pizzaOvenProject01Scenario,
fps
);
const activeSegment = activeVQChronologicalScenarioSegmentForFrame(
scheduledEvents,
frame
);
const isInsideActiveSegment = activeSegment
? frame < activeSegment.from + activeSegment.durationInFrames
: false;
const activeSpeech =
activeSegment && isInsideActiveSegment && activeSegment.event.type === "say"
? activeSegment.event
: undefined;
const activeStill = scheduledEvents.reduce<VQStillEvent | undefined>(
(currentStill, scheduledEvent) => {
if (scheduledEvent.from > frame) {
return currentStill;
}
if (scheduledEvent.event.type === "still") {
return scheduledEvent.event;
}
if (scheduledEvent.event.type === "clearStill") {
return undefined;
}
return currentStill;
},
undefined
);
const activeStandeePosition =
scheduledEvents.reduce<VQStageCornerStandeeMode>(
(currentPosition, scheduledEvent) => {
if (scheduledEvent.from > frame) {
return currentPosition;
}
if (
scheduledEvent.event.type === "standeePosition" &&
scheduledEvent.event.character === "sayo"
) {
return scheduledEvent.event.position;
}
return currentPosition;
},
"stage"
);
const speakingLocalFrame =
activeSegment && activeSpeech ? frame - activeSegment.from : 0;
const timelineEndFrame = scheduledEvents.reduce(
(endFrame, scheduledEvent) =>
Math.max(endFrame, scheduledEvent.from + scheduledEvent.durationInFrames),
0
);
const sequences = scheduledEvents
.map((scheduledEvent) => ({
scheduledEvent,
durationInFrames:
scheduledEvent.event.type === "video"
? videoSequenceDuration(
scheduledEvent as ScheduledTimelineEvent & Readonly<{
event: VQVideoEvent;
}>,
scheduledEvents
)
: scheduledEvent.event.type === "audio"
? audioSequenceDuration(
scheduledEvent as ScheduledTimelineEvent & Readonly<{
event: VQAudioEvent;
}>,
timelineEndFrame
)
: scheduledEvent.durationInFrames,
}))
.filter(({durationInFrames}) => durationInFrames > 0)
.map((scheduledEvent) =>
React.createElement(
Sequence,
{
key: scheduledEvent.scheduledEvent.event.id,
from: scheduledEvent.scheduledEvent.from,
durationInFrames: scheduledEvent.durationInFrames,
premountFor: Math.min(fps, scheduledEvent.scheduledEvent.from),
},
React.createElement(TimelineOverlay, {
event: scheduledEvent.scheduledEvent.event,
})
)
);
return React.createElement(
AbsoluteFill,
{
style: {
display: "flex",
flexDirection: "column",
alignItems: "center",
},
},
React.createElement(VQWarmGradientBackground, null),
React.createElement(VQStillBackground, {still: activeStill}),
React.createElement(SayoStandee, {
mode: activeStandeePosition,
frame,
fps,
activeSpeech,
speakingLocalFrame,
}),
sequences
);
};