Newer
Older
remotion_docker_devcontainer / voicevox-remotion-template / src / pizzaOvenProject01.ts
import React from "react";
import {
  AbsoluteFill,
  interpolate,
  Sequence,
  spring,
  useCurrentFrame,
  useVideoConfig,
} from "remotion";
import {
  characters,
  type SpeechEvent,
  type TimelineEvent,
} from "./data/pizza-oven-project-01/script";
import {
  audioFileForSpeech,
  hasAudioForSpeech,
  pizzaOvenProject01Scenario,
} from "./data/pizza-oven-project-01/timing";
import {roundedFontFamily} from "./fonts";
import {
  activeVQChronologicalScenarioSegmentForFrame,
  scheduleVQChronologicalScenario,
  VQSpeechOverlay,
  VQStageCornerStandee,
  VQStillBackground,
  VQTimelineAudio,
  VQTimelineVideo,
  VQWarmGradientBackground,
  type VQAudioEvent,
  type VQScheduledScenarioSegment,
  type VQStageCornerStandeeLayouts,
  type VQStageCornerStandeeMode,
  type VQStillEvent,
  type VQVideoEvent,
  vqDefaultStageCornerStandeeLayouts,
} from "./lib/VQRemotionLib";
import {getMouthForSpeechFrame} from "./lipsync/manifest";

const sayoAvatar = characters.sayo.avatar;
const PizzaOvenSpeechOverlay = VQSpeechOverlay<SpeechEvent>;
const pizzaOvenStandeeLayouts = {
  ...vqDefaultStageCornerStandeeLayouts,
  stage: {
    ...vqDefaultStageCornerStandeeLayouts.stage,
    frameWidth: 520,
    frameHeight: 720,
    right: 330,
    bottom: -82,
  },
} as const satisfies VQStageCornerStandeeLayouts;

const clampInterpolation = {
  extrapolateLeft: "clamp",
  extrapolateRight: "clamp",
} as const;

type ScheduledTimelineEvent = VQScheduledScenarioSegment<TimelineEvent>;

const videoSequenceDuration = (
  scheduledVideo: ScheduledTimelineEvent & Readonly<{event: VQVideoEvent}>,
  scheduledEvents: readonly ScheduledTimelineEvent[]
) => {
  const clearFrame = scheduledEvents.find(
    (scheduledEvent) =>
      scheduledEvent.from >= scheduledVideo.from &&
      scheduledEvent.event.type === "clearVideo" &&
      (scheduledEvent.event.videoId === undefined ||
        scheduledEvent.event.videoId === scheduledVideo.event.id)
  )?.from;

  return clearFrame === undefined
    ? scheduledVideo.durationInFrames
    : Math.min(
        scheduledVideo.durationInFrames,
        Math.max(0, clearFrame - scheduledVideo.from)
      );
};

const audioSequenceDuration = (
  scheduledAudio: ScheduledTimelineEvent & Readonly<{event: VQAudioEvent}>,
  timelineEndFrame: number
) => {
  const hasExplicitDuration =
    scheduledAudio.event.durationFrames !== undefined ||
    scheduledAudio.event.durationSeconds !== undefined;

  return scheduledAudio.event.playback === "loop" && !hasExplicitDuration
    ? Math.max(0, timelineEndFrame - scheduledAudio.from)
    : scheduledAudio.durationInFrames;
};

const SayoStandee: React.FC<
  Readonly<{
    mode: VQStageCornerStandeeMode;
    frame: number;
    fps: number;
    activeSpeech?: SpeechEvent;
    speakingLocalFrame: number;
  }>
> = ({mode, frame, fps, activeSpeech, speakingLocalFrame}) => {
  const entrance = spring({
    frame,
    fps,
    config: {damping: 18, mass: 0.6},
  });
  const translateY = interpolate(entrance, [0, 1], [32, 0], clampInterpolation);
  const mouth = activeSpeech
    ? getMouthForSpeechFrame(activeSpeech.id, speakingLocalFrame, fps)
    : "rest";

  return React.createElement(VQStageCornerStandee, {
    mode,
    imagePath: sayoAvatar.imagePath,
    mouthImageDir: sayoAvatar.mouthImageDir,
    mouth,
    translateY,
    layouts: pizzaOvenStandeeLayouts,
    zIndex: 2,
  });
};

const TimelineOverlay: React.FC<Readonly<{event: TimelineEvent}>> = ({
  event,
}) => {
  if (event.type !== "say") {
    if (event.type === "video") {
      return React.createElement(VQTimelineVideo, {video: event});
    }

    return event.type === "audio"
      ? React.createElement(VQTimelineAudio, {audio: event})
      : null;
  }

  const character = characters[event.character];

  return React.createElement(PizzaOvenSpeechOverlay, {
    speech: event,
    speakerName: character.displayName,
    accentColor: character.avatar.accentColor,
    hasAudio: hasAudioForSpeech,
    getAudioPath: audioFileForSpeech,
    subtitleOptions: {
      fontFamily: roundedFontFamily,
      fontSize: 40,
      lineHeight: 1.35,
      backgroundColor: "rgba(255, 255, 255, 0.9)",
    },
    containerStyle: {zIndex: 3},
  });
};

export const PizzaOvenProject01: React.FC = () => {
  const frame = useCurrentFrame();
  const {fps} = useVideoConfig();
  const scheduledEvents = scheduleVQChronologicalScenario(
    pizzaOvenProject01Scenario,
    fps
  );
  const activeSegment = activeVQChronologicalScenarioSegmentForFrame(
    scheduledEvents,
    frame
  );
  const isInsideActiveSegment = activeSegment
    ? frame < activeSegment.from + activeSegment.durationInFrames
    : false;
  const activeSpeech =
    activeSegment && isInsideActiveSegment && activeSegment.event.type === "say"
      ? activeSegment.event
      : undefined;
  const activeStill = scheduledEvents.reduce<VQStillEvent | undefined>(
    (currentStill, scheduledEvent) => {
      if (scheduledEvent.from > frame) {
        return currentStill;
      }

      if (scheduledEvent.event.type === "still") {
        return scheduledEvent.event;
      }

      if (scheduledEvent.event.type === "clearStill") {
        return undefined;
      }

      return currentStill;
    },
    undefined
  );
  const activeStandeePosition =
    scheduledEvents.reduce<VQStageCornerStandeeMode>(
      (currentPosition, scheduledEvent) => {
        if (scheduledEvent.from > frame) {
          return currentPosition;
        }

        if (
          scheduledEvent.event.type === "standeePosition" &&
          scheduledEvent.event.character === "sayo"
        ) {
          return scheduledEvent.event.position;
        }

        return currentPosition;
      },
      "stage"
    );
  const speakingLocalFrame =
    activeSegment && activeSpeech ? frame - activeSegment.from : 0;
  const timelineEndFrame = scheduledEvents.reduce(
    (endFrame, scheduledEvent) =>
      Math.max(endFrame, scheduledEvent.from + scheduledEvent.durationInFrames),
    0
  );
  const sequences = scheduledEvents
    .map((scheduledEvent) => ({
      scheduledEvent,
      durationInFrames:
        scheduledEvent.event.type === "video"
          ? videoSequenceDuration(
              scheduledEvent as ScheduledTimelineEvent & Readonly<{
                event: VQVideoEvent;
              }>,
              scheduledEvents
            )
          : scheduledEvent.event.type === "audio"
            ? audioSequenceDuration(
                scheduledEvent as ScheduledTimelineEvent & Readonly<{
                  event: VQAudioEvent;
                }>,
                timelineEndFrame
              )
          : scheduledEvent.durationInFrames,
    }))
    .filter(({durationInFrames}) => durationInFrames > 0)
    .map((scheduledEvent) =>
      React.createElement(
        Sequence,
        {
          key: scheduledEvent.scheduledEvent.event.id,
          from: scheduledEvent.scheduledEvent.from,
          durationInFrames: scheduledEvent.durationInFrames,
          premountFor: Math.min(fps, scheduledEvent.scheduledEvent.from),
        },
        React.createElement(TimelineOverlay, {
          event: scheduledEvent.scheduledEvent.event,
        })
      )
    );

  return React.createElement(
    AbsoluteFill,
    {
      style: {
        display: "flex",
        flexDirection: "column",
        alignItems: "center",
      },
    },
    React.createElement(VQWarmGradientBackground, null),
    React.createElement(VQStillBackground, {still: activeStill}),
    React.createElement(SayoStandee, {
      mode: activeStandeePosition,
      frame,
      fps,
      activeSpeech,
      speakingLocalFrame,
    }),
    sequences
  );
};