import React from "react";
import {
AbsoluteFill,
interpolate,
Sequence,
spring,
useCurrentFrame,
useVideoConfig,
} from "remotion";
import {
characters,
compositionTitle,
initialVisibleCharacters,
timeline,
type CharacterId,
type TimelineEvent,
} from "./data/zundamon-jiron/script";
import {
ZUNDAMON_JIRON_GAP_FRAMES,
durationForTimelineEvent,
audioFileForSpeech,
hasAudioForSpeech,
} from "./data/zundamon-jiron/timing";
import {roundedFontFamily} from "./fonts";
import {
VQCaptionOverlay,
VQCharacterStage,
VQSpeechOverlay,
VQTimelineAudio,
VQWarmGradientBackground,
type VQAudioEvent,
type VQMouthResolver,
} from "./lib/VQRemotionLib";
import {getMouthForSpeechFrame} from "./lipsync/manifest";
type ScheduledTimelineEvent = Readonly<{
event: TimelineEvent;
from: number;
durationInFrames: number;
visibleCharacters: CharacterId[];
focusedCharacter?: CharacterId;
avatarFlipXByCharacter: Partial<Record<CharacterId, boolean>>;
avatarTranslateYByCharacter: Partial<Record<CharacterId, number>>;
}>;
const clampInterpolation = {
extrapolateLeft: "clamp",
extrapolateRight: "clamp",
} as const;
const subtitleOptions = {
fontFamily: roundedFontFamily,
fontSize: 36,
lineHeight: 1.4,
backgroundColor: "rgba(255, 255, 255, 0.88)",
} as const;
const flipXForFacingDirection = (direction: "left" | "right") =>
direction === "left";
const characterForEvent = (event: TimelineEvent) =>
"character" in event ? event.character : undefined;
const doesEventAdvanceTimeline = (event: TimelineEvent) =>
event.type !== "audio" &&
event.type !== "standeeFacingDirection" &&
event.type !== "standeeVerticalOffset";
const audioSequenceDuration = (
scheduledAudio: ScheduledTimelineEvent & Readonly<{event: VQAudioEvent}>,
timelineEndFrame: number
) => {
const hasExplicitDuration =
scheduledAudio.event.durationFrames !== undefined ||
scheduledAudio.event.durationSeconds !== undefined;
return scheduledAudio.event.playback === "loop" && !hasExplicitDuration
? Math.max(0, timelineEndFrame - scheduledAudio.from)
: scheduledAudio.durationInFrames;
};
// 用途: script.ts の timeline をフレーム単位の表示スケジュールへ変換する。
// 使用方法: Composition コンポーネント内で fps を渡して呼び出す。
// オプションや引数詳細: character を持つイベントだけ表示対象とフォーカスに反映する。
const scheduleTimeline = (fps: number): ScheduledTimelineEvent[] => {
let cursor = 0;
const visibleCharacters = new Set<CharacterId>(initialVisibleCharacters);
const avatarFlipXByCharacter: Partial<Record<CharacterId, boolean>> = {};
const avatarTranslateYByCharacter: Partial<Record<CharacterId, number>> = {};
let focusedCharacter: CharacterId | undefined = initialVisibleCharacters[0];
return timeline.map((event, index) => {
const eventCharacter = characterForEvent(event);
if (eventCharacter) {
visibleCharacters.add(eventCharacter);
focusedCharacter = eventCharacter;
}
if (event.type === "standeeFacingDirection") {
avatarFlipXByCharacter[event.character] = flipXForFacingDirection(
event.direction
);
}
if (event.type === "standeeVerticalOffset") {
avatarTranslateYByCharacter[event.character] = event.translateY;
}
const durationInFrames = durationForTimelineEvent(event, fps);
const scheduledEvent = {
event,
from: cursor,
durationInFrames,
visibleCharacters: Array.from(visibleCharacters),
focusedCharacter,
avatarFlipXByCharacter: {...avatarFlipXByCharacter},
avatarTranslateYByCharacter: {...avatarTranslateYByCharacter},
};
if (doesEventAdvanceTimeline(event) && durationInFrames > 0) {
cursor += durationInFrames;
}
if (
doesEventAdvanceTimeline(event) &&
durationInFrames > 0 &&
index < timeline.length - 1
) {
cursor += ZUNDAMON_JIRON_GAP_FRAMES;
}
return scheduledEvent;
});
};
// 用途: 現在フレームに対応するタイムライン区間を取得する。
// 使用方法: Composition の毎フレーム描画で activeSegment を求める。
// オプションや引数詳細: 最後に開始した区間を返すため、ギャップ中も直前の表示状態を維持する。
const activeSegmentForFrame = (
scheduledEvents: ScheduledTimelineEvent[],
frame: number
) => {
let activeSegment = scheduledEvents[0];
for (const scheduledEvent of scheduledEvents) {
if (frame >= scheduledEvent.from) {
activeSegment = scheduledEvent;
} else {
break;
}
}
return activeSegment;
};
const resolveMouth: VQMouthResolver<CharacterId> = ({
speechId,
speakingLocalFrame,
fps,
}) => getMouthForSpeechFrame(speechId, speakingLocalFrame, fps);
const Title: React.FC<Readonly<{progress: number}>> = ({progress}) => {
const opacity = interpolate(progress, [0, 1], [0, 1], clampInterpolation);
const translateY = interpolate(progress, [0, 1], [-30, 0], clampInterpolation);
return (
<div
style={{
fontFamily: roundedFontFamily,
fontSize: 54,
fontWeight: 700,
color: "#1f2a44",
textAlign: "center",
marginTop: 40,
opacity,
transform: `translateY(${translateY}px)`,
textShadow: "0 6px 18px rgba(31, 42, 68, 0.2)",
}}
>
{compositionTitle}
</div>
);
};
const TimelineOverlay: React.FC<Readonly<{event: TimelineEvent}>> = ({
event,
}) => {
if (event.type === "say") {
const character = characters[event.character];
return (
<VQSpeechOverlay
speech={event}
speakerName={character.displayName}
accentColor={character.avatar.accentColor}
hasAudio={hasAudioForSpeech}
getAudioPath={audioFileForSpeech}
subtitleOptions={subtitleOptions}
/>
);
}
if (event.type === "audio") {
return <VQTimelineAudio audio={event} />;
}
if (event.type !== "show") {
return null;
}
return (
<VQCaptionOverlay
text={event.caption}
subtitleOptions={subtitleOptions}
/>
);
};
const keyForEvent = (event: TimelineEvent, index: number) => {
if ("id" in event) {
return event.id;
}
return `${event.type}-${event.character}-${index}`;
};
export const ZundamonJiron: React.FC = () => {
const frame = useCurrentFrame();
const {fps} = useVideoConfig();
const scheduledEvents = scheduleTimeline(fps);
const activeSegment = activeSegmentForFrame(scheduledEvents, frame);
const isInsideActiveSegment =
frame < activeSegment.from + activeSegment.durationInFrames;
const titleProgress = spring({
frame,
fps,
config: {damping: 18, mass: 0.6},
});
const activeSpeech =
isInsideActiveSegment && activeSegment.event.type === "say"
? activeSegment.event
: undefined;
const speakingCharacter = activeSpeech?.character;
const speakingLocalFrame = activeSpeech ? frame - activeSegment.from : 0;
const timelineEndFrame = scheduledEvents.reduce(
(endFrame, scheduledEvent) =>
Math.max(endFrame, scheduledEvent.from + scheduledEvent.durationInFrames),
0
);
const sequences = scheduledEvents.map((scheduledEvent, index) =>
(
scheduledEvent.event.type === "audio"
? audioSequenceDuration(
scheduledEvent as ScheduledTimelineEvent &
Readonly<{event: VQAudioEvent}>,
timelineEndFrame
)
: scheduledEvent.durationInFrames
) > 0 ? (
<Sequence
key={keyForEvent(scheduledEvent.event, index)}
from={scheduledEvent.from}
durationInFrames={
scheduledEvent.event.type === "audio"
? audioSequenceDuration(
scheduledEvent as ScheduledTimelineEvent &
Readonly<{event: VQAudioEvent}>,
timelineEndFrame
)
: scheduledEvent.durationInFrames
}
premountFor={Math.min(fps, scheduledEvent.from)}
>
<TimelineOverlay event={scheduledEvent.event} />
</Sequence>
) : null
);
return (
<AbsoluteFill
style={{
display: "flex",
flexDirection: "column",
alignItems: "center",
}}
>
<VQWarmGradientBackground />
<Title progress={titleProgress} />
<VQCharacterStage
characters={characters}
visibleCharacters={activeSegment.visibleCharacters}
focusedCharacter={
isInsideActiveSegment ? activeSegment.focusedCharacter : undefined
}
speakingCharacter={speakingCharacter}
speakingSpeechId={activeSpeech?.id}
speakingLocalFrame={speakingLocalFrame}
frame={frame}
fps={fps}
resolveMouth={resolveMouth}
fontFamily={roundedFontFamily}
stageStyle={{paddingBottom: 130}}
avatarFlipXByCharacter={activeSegment.avatarFlipXByCharacter}
avatarTranslateYByCharacter={activeSegment.avatarTranslateYByCharacter}
/>
{sequences}
</AbsoluteFill>
);
};