Newer
Older
remotion_docker_devcontainer / voicevox-remotion-template / src / zundamon-jiron.tsx
import React from "react";
import {
  AbsoluteFill,
  interpolate,
  Sequence,
  spring,
  useCurrentFrame,
  useVideoConfig,
} from "remotion";
import {
  characters,
  compositionTitle,
  initialVisibleCharacters,
  timeline,
  type CharacterId,
  type TimelineEvent,
} from "./data/zundamon-jiron/script";
import {
  ZUNDAMON_JIRON_GAP_FRAMES,
  durationForTimelineEvent,
  audioFileForSpeech,
  hasAudioForSpeech,
} from "./data/zundamon-jiron/timing";
import {roundedFontFamily} from "./fonts";
import {
  VQCaptionOverlay,
  VQCharacterStage,
  VQSpeechOverlay,
  VQTimelineAudio,
  VQWarmGradientBackground,
  type VQAudioEvent,
  type VQMouthResolver,
} from "./lib/VQRemotionLib";
import {getMouthForSpeechFrame} from "./lipsync/manifest";

type ScheduledTimelineEvent = Readonly<{
  event: TimelineEvent;
  from: number;
  durationInFrames: number;
  visibleCharacters: CharacterId[];
  focusedCharacter?: CharacterId;
  avatarFlipXByCharacter: Partial<Record<CharacterId, boolean>>;
  avatarTranslateYByCharacter: Partial<Record<CharacterId, number>>;
}>;

const clampInterpolation = {
  extrapolateLeft: "clamp",
  extrapolateRight: "clamp",
} as const;

const subtitleOptions = {
  fontFamily: roundedFontFamily,
  fontSize: 36,
  lineHeight: 1.4,
  backgroundColor: "rgba(255, 255, 255, 0.88)",
} as const;

const flipXForFacingDirection = (direction: "left" | "right") =>
  direction === "left";

const characterForEvent = (event: TimelineEvent) =>
  "character" in event ? event.character : undefined;

const doesEventAdvanceTimeline = (event: TimelineEvent) =>
  event.type !== "audio" &&
  event.type !== "standeeFacingDirection" &&
  event.type !== "standeeVerticalOffset";

const audioSequenceDuration = (
  scheduledAudio: ScheduledTimelineEvent & Readonly<{event: VQAudioEvent}>,
  timelineEndFrame: number
) => {
  const hasExplicitDuration =
    scheduledAudio.event.durationFrames !== undefined ||
    scheduledAudio.event.durationSeconds !== undefined;

  return scheduledAudio.event.playback === "loop" && !hasExplicitDuration
    ? Math.max(0, timelineEndFrame - scheduledAudio.from)
    : scheduledAudio.durationInFrames;
};

// 用途: script.ts の timeline をフレーム単位の表示スケジュールへ変換する。
// 使用方法: Composition コンポーネント内で fps を渡して呼び出す。
// オプションや引数詳細: character を持つイベントだけ表示対象とフォーカスに反映する。
const scheduleTimeline = (fps: number): ScheduledTimelineEvent[] => {
  let cursor = 0;
  const visibleCharacters = new Set<CharacterId>(initialVisibleCharacters);
  const avatarFlipXByCharacter: Partial<Record<CharacterId, boolean>> = {};
  const avatarTranslateYByCharacter: Partial<Record<CharacterId, number>> = {};
  let focusedCharacter: CharacterId | undefined = initialVisibleCharacters[0];

  return timeline.map((event, index) => {
    const eventCharacter = characterForEvent(event);
    if (eventCharacter) {
      visibleCharacters.add(eventCharacter);
      focusedCharacter = eventCharacter;
    }

    if (event.type === "standeeFacingDirection") {
      avatarFlipXByCharacter[event.character] = flipXForFacingDirection(
        event.direction
      );
    }

    if (event.type === "standeeVerticalOffset") {
      avatarTranslateYByCharacter[event.character] = event.translateY;
    }

    const durationInFrames = durationForTimelineEvent(event, fps);
    const scheduledEvent = {
      event,
      from: cursor,
      durationInFrames,
      visibleCharacters: Array.from(visibleCharacters),
      focusedCharacter,
      avatarFlipXByCharacter: {...avatarFlipXByCharacter},
      avatarTranslateYByCharacter: {...avatarTranslateYByCharacter},
    };

    if (doesEventAdvanceTimeline(event) && durationInFrames > 0) {
      cursor += durationInFrames;
    }
    if (
      doesEventAdvanceTimeline(event) &&
      durationInFrames > 0 &&
      index < timeline.length - 1
    ) {
      cursor += ZUNDAMON_JIRON_GAP_FRAMES;
    }

    return scheduledEvent;
  });
};

// 用途: 現在フレームに対応するタイムライン区間を取得する。
// 使用方法: Composition の毎フレーム描画で activeSegment を求める。
// オプションや引数詳細: 最後に開始した区間を返すため、ギャップ中も直前の表示状態を維持する。
const activeSegmentForFrame = (
  scheduledEvents: ScheduledTimelineEvent[],
  frame: number
) => {
  let activeSegment = scheduledEvents[0];

  for (const scheduledEvent of scheduledEvents) {
    if (frame >= scheduledEvent.from) {
      activeSegment = scheduledEvent;
    } else {
      break;
    }
  }

  return activeSegment;
};

const resolveMouth: VQMouthResolver<CharacterId> = ({
  speechId,
  speakingLocalFrame,
  fps,
}) => getMouthForSpeechFrame(speechId, speakingLocalFrame, fps);

const Title: React.FC<Readonly<{progress: number}>> = ({progress}) => {
  const opacity = interpolate(progress, [0, 1], [0, 1], clampInterpolation);
  const translateY = interpolate(progress, [0, 1], [-30, 0], clampInterpolation);

  return (
    <div
      style={{
        fontFamily: roundedFontFamily,
        fontSize: 54,
        fontWeight: 700,
        color: "#1f2a44",
        textAlign: "center",
        marginTop: 40,
        opacity,
        transform: `translateY(${translateY}px)`,
        textShadow: "0 6px 18px rgba(31, 42, 68, 0.2)",
      }}
    >
      {compositionTitle}
    </div>
  );
};

const TimelineOverlay: React.FC<Readonly<{event: TimelineEvent}>> = ({
  event,
}) => {
  if (event.type === "say") {
    const character = characters[event.character];

    return (
      <VQSpeechOverlay
        speech={event}
        speakerName={character.displayName}
        accentColor={character.avatar.accentColor}
        hasAudio={hasAudioForSpeech}
        getAudioPath={audioFileForSpeech}
        subtitleOptions={subtitleOptions}
      />
    );
  }

  if (event.type === "audio") {
    return <VQTimelineAudio audio={event} />;
  }

  if (event.type !== "show") {
    return null;
  }

  return (
    <VQCaptionOverlay
      text={event.caption}
      subtitleOptions={subtitleOptions}
    />
  );
};

const keyForEvent = (event: TimelineEvent, index: number) => {
  if ("id" in event) {
    return event.id;
  }

  return `${event.type}-${event.character}-${index}`;
};

export const ZundamonJiron: React.FC = () => {
  const frame = useCurrentFrame();
  const {fps} = useVideoConfig();
  const scheduledEvents = scheduleTimeline(fps);
  const activeSegment = activeSegmentForFrame(scheduledEvents, frame);
  const isInsideActiveSegment =
    frame < activeSegment.from + activeSegment.durationInFrames;

  const titleProgress = spring({
    frame,
    fps,
    config: {damping: 18, mass: 0.6},
  });
  const activeSpeech =
    isInsideActiveSegment && activeSegment.event.type === "say"
      ? activeSegment.event
      : undefined;
  const speakingCharacter = activeSpeech?.character;
  const speakingLocalFrame = activeSpeech ? frame - activeSegment.from : 0;
  const timelineEndFrame = scheduledEvents.reduce(
    (endFrame, scheduledEvent) =>
      Math.max(endFrame, scheduledEvent.from + scheduledEvent.durationInFrames),
    0
  );

  const sequences = scheduledEvents.map((scheduledEvent, index) =>
    (
      scheduledEvent.event.type === "audio"
        ? audioSequenceDuration(
            scheduledEvent as ScheduledTimelineEvent &
              Readonly<{event: VQAudioEvent}>,
            timelineEndFrame
          )
        : scheduledEvent.durationInFrames
    ) > 0 ? (
      <Sequence
        key={keyForEvent(scheduledEvent.event, index)}
        from={scheduledEvent.from}
        durationInFrames={
          scheduledEvent.event.type === "audio"
            ? audioSequenceDuration(
                scheduledEvent as ScheduledTimelineEvent &
                  Readonly<{event: VQAudioEvent}>,
                timelineEndFrame
              )
            : scheduledEvent.durationInFrames
        }
        premountFor={Math.min(fps, scheduledEvent.from)}
      >
        <TimelineOverlay event={scheduledEvent.event} />
      </Sequence>
    ) : null
  );

  return (
    <AbsoluteFill
      style={{
        display: "flex",
        flexDirection: "column",
        alignItems: "center",
      }}
    >
      <VQWarmGradientBackground />
      <Title progress={titleProgress} />
      <VQCharacterStage
        characters={characters}
        visibleCharacters={activeSegment.visibleCharacters}
        focusedCharacter={
          isInsideActiveSegment ? activeSegment.focusedCharacter : undefined
        }
        speakingCharacter={speakingCharacter}
        speakingSpeechId={activeSpeech?.id}
        speakingLocalFrame={speakingLocalFrame}
        frame={frame}
        fps={fps}
        resolveMouth={resolveMouth}
        fontFamily={roundedFontFamily}
        stageStyle={{paddingBottom: 130}}
        avatarFlipXByCharacter={activeSegment.avatarFlipXByCharacter}
        avatarTranslateYByCharacter={activeSegment.avatarTranslateYByCharacter}
      />
      {sequences}
    </AbsoluteFill>
  );
};