import React from "react";
import {
AbsoluteFill,
Sequence,
useCurrentFrame,
useVideoConfig,
} from "remotion";
import {
audioFileForSpeech,
hasAudioForSpeech,
pizzaKilnScenario,
} from "./data/pizza-kiln/timing";
import {
characters,
type SpeechEvent,
type TimelineEvent,
} from "./data/pizza-kiln/script";
import {roundedFontFamily} from "./fonts";
import {
scheduleVQChronologicalScenario,
VQSpeechOverlay,
VQStageCornerStandee,
VQTimelineVideo,
VQWarmGradientBackground,
type VQScheduledScenarioSegment,
vqSpeakingAvatarAnimations,
} from "./lib/VQRemotionLib";
import {getMouthForSpeechFrame} from "./lipsync/manifest";
const pizzaSubtitleOptions = {
fontFamily: roundedFontFamily,
fontSize: 34,
lineHeight: 1.45,
backgroundColor: "rgba(255, 255, 255, 0.9)",
} as const;
const SayoStandee: React.FC<
Readonly<{
mode: "stage" | "corner";
speaking: boolean;
localFrame: number;
fps: number;
speechId?: string;
}>
> = ({mode, speaking, localFrame, fps, speechId}) => {
const {avatar} = characters.sayo;
const speakingAnimationType = avatar.speakingAnimationType ?? "none";
const translateY = speaking
? vqSpeakingAvatarAnimations[speakingAnimationType]({
frame: localFrame,
fps,
focused: true,
hasMultipleCharacters: false,
})
: 0;
const mouth =
speaking && speakingAnimationType === "rhubarbLipSync"
? getMouthForSpeechFrame(speechId, localFrame, fps)
: "rest";
return (
<VQStageCornerStandee
mode={mode}
imagePath={avatar.imagePath}
mouthImageDir={avatar.mouthImageDir}
mouth={mouth}
translateY={translateY}
zIndex={3}
/>
);
};
const SpeechOverlay: React.FC<Readonly<{speech: SpeechEvent}>> = ({speech}) => {
const character = characters[speech.character];
return (
<VQSpeechOverlay
speech={speech}
speakerName={character.displayName}
accentColor={character.avatar.accentColor}
hasAudio={hasAudioForSpeech}
getAudioPath={audioFileForSpeech}
subtitleOptions={pizzaSubtitleOptions}
containerStyle={{zIndex: 4}}
/>
);
};
type ScheduledTimelineEvent = VQScheduledScenarioSegment<TimelineEvent>;
type ScheduledSpeechEvent = ScheduledTimelineEvent & Readonly<{
event: SpeechEvent;
}>;
const isInsideSegment = (segment: ScheduledTimelineEvent, frame: number) =>
frame >= segment.from && frame < segment.from + segment.durationInFrames;
const isActiveSpeechSegment = (
segment: ScheduledTimelineEvent,
frame: number
): segment is ScheduledSpeechEvent =>
segment.event.type === "say" && isInsideSegment(segment, frame);
export const PizzaKilnSayoComposition: React.FC = () => {
const frame = useCurrentFrame();
const {fps} = useVideoConfig();
const scheduledEvents = scheduleVQChronologicalScenario(
pizzaKilnScenario,
fps
);
const activeSpeechSegment = scheduledEvents.find((segment) =>
isActiveSpeechSegment(segment, frame)
);
const isVideoVisible = scheduledEvents.some(
(segment) => segment.event.type === "video" && isInsideSegment(segment, frame)
);
const activeSpeech = activeSpeechSegment?.event;
const speechLocalFrame = activeSpeechSegment
? frame - activeSpeechSegment.from
: 0;
const videoSequences = scheduledEvents.map((scheduledEvent) => {
if (scheduledEvent.event.type !== "video") {
return null;
}
return (
<Sequence
key={scheduledEvent.event.id}
from={scheduledEvent.from}
durationInFrames={scheduledEvent.durationInFrames}
premountFor={Math.min(fps, scheduledEvent.from)}
>
<VQTimelineVideo video={scheduledEvent.event} />
</Sequence>
);
});
const speechSequences = scheduledEvents.map((scheduledEvent) => {
if (scheduledEvent.event.type !== "say") {
return null;
}
return (
<Sequence
key={scheduledEvent.event.id}
from={scheduledEvent.from}
durationInFrames={scheduledEvent.durationInFrames}
premountFor={Math.min(fps, scheduledEvent.from)}
>
<SpeechOverlay speech={scheduledEvent.event} />
</Sequence>
);
});
return (
<AbsoluteFill style={{backgroundColor: "#1a1a1a"}}>
<VQWarmGradientBackground />
{videoSequences}
<SayoStandee
mode={isVideoVisible ? "corner" : "stage"}
speaking={Boolean(activeSpeech)}
localFrame={speechLocalFrame}
fps={fps}
speechId={activeSpeech?.id}
/>
{speechSequences}
</AbsoluteFill>
);
};