diff --git a/voicevox-remotion-template/src/lib/VQRemotionLib/avatarAnimations.ts b/voicevox-remotion-template/src/lib/VQRemotionLib/avatarAnimations.ts new file mode 100644 index 0000000..28cd78c --- /dev/null +++ b/voicevox-remotion-template/src/lib/VQRemotionLib/avatarAnimations.ts @@ -0,0 +1,34 @@ +import type { + VQAvatarAnimation, + VQAvatarAnimationContext, + VQIdleAvatarAnimationType, + VQSpeakingAvatarAnimationType, +} from "./types"; + +const gentleBobAmplitude = ({focused}: VQAvatarAnimationContext) => + focused ? 10 : 3.5; + +export const vqIdleAvatarAnimations: Record< + VQIdleAvatarAnimationType, + VQAvatarAnimation +> = { + none: () => 0, + gentleBob: (context) => + Math.sin((context.frame / context.fps) * Math.PI * 2) * + gentleBobAmplitude(context), +}; + +export const vqSpeakingAvatarAnimations: Record< + VQSpeakingAvatarAnimationType, + VQAvatarAnimation +> = { + none: () => 0, + rhubarbLipSync: () => 0, + gentleBob: vqIdleAvatarAnimations.gentleBob, + quickHop: ({frame, fps}) => { + const cycleFrames = Math.max(1, Math.round(fps * 0.25)); + const progress = (frame % cycleFrames) / cycleFrames; + + return -Math.sin(progress * Math.PI) * 7; + }, +}; diff --git a/voicevox-remotion-template/src/lib/VQRemotionLib/components/VQCaptionOverlay.tsx b/voicevox-remotion-template/src/lib/VQRemotionLib/components/VQCaptionOverlay.tsx new file mode 100644 index 0000000..ba0bb33 --- /dev/null +++ b/voicevox-remotion-template/src/lib/VQRemotionLib/components/VQCaptionOverlay.tsx @@ -0,0 +1,55 @@ +import React from "react"; +import {spring, useCurrentFrame, useVideoConfig} from "remotion"; +import { + VQSpeechSubtitle, + type VQSpeechSubtitleStyleOptions, +} from "./VQSpeechSubtitle"; + +export type VQCaptionOverlayProps = Readonly<{ + text?: string; + speakerName?: string; + accentColor?: string; + subtitleOptions?: VQSpeechSubtitleStyleOptions; + containerStyle?: React.CSSProperties; +}>; + +const defaultOverlayStyle: React.CSSProperties = { + position: "absolute", + bottom: 40, + left: 0, + right: 0, + display: "flex", + justifyContent: "center", +}; + +export const VQCaptionOverlay: React.FC = ({ + text, + speakerName, + accentColor, + subtitleOptions, + containerStyle, +}) => { + const frame = useCurrentFrame(); + const {fps} = useVideoConfig(); + const subtitleProgress = spring({ + frame, + fps, + config: {damping: 20, mass: 0.7}, + }); + + if (!text) { + return null; + } + + return ( +
+ +
+ ); +}; diff --git a/voicevox-remotion-template/src/lib/VQRemotionLib/components/VQCharacterAvatar.tsx b/voicevox-remotion-template/src/lib/VQRemotionLib/components/VQCharacterAvatar.tsx new file mode 100644 index 0000000..9d25e82 --- /dev/null +++ b/voicevox-remotion-template/src/lib/VQRemotionLib/components/VQCharacterAvatar.tsx @@ -0,0 +1,154 @@ +import React from "react"; +import {vqIdleAvatarAnimations, vqSpeakingAvatarAnimations} from "../avatarAnimations"; +import type { + VQCharacterDefinition, + VQMouthResolver, + VQMouthShape, +} from "../types"; +import {VQLipSyncedStandeeImage} from "./VQLipSyncedStandeeImage"; + +export type VQCharacterAvatarProps = + Readonly<{ + characterId: CharacterId; + character: VQCharacterDefinition; + focused: boolean; + hasMultipleCharacters: boolean; + frame: number; + fps: number; + isSpeaking: boolean; + speakingSpeechId?: string; + speakingLocalFrame: number; + resolveMouth?: VQMouthResolver; + fontFamily?: string; + imageFilter?: string; + }>; + +const mouthForAvatar = ({ + characterId, + character, + isSpeaking, + speakingSpeechId, + speakingLocalFrame, + fps, + resolveMouth, +}: Readonly<{ + characterId: CharacterId; + character: VQCharacterDefinition; + isSpeaking: boolean; + speakingSpeechId?: string; + speakingLocalFrame: number; + fps: number; + resolveMouth?: VQMouthResolver; +}>): VQMouthShape => { + const speakingAnimationType = + character.avatar.speakingAnimationType ?? "gentleBob"; + + if (!isSpeaking || speakingAnimationType !== "rhubarbLipSync") { + return "rest"; + } + + return ( + resolveMouth?.({ + characterId, + character, + speechId: speakingSpeechId, + speakingLocalFrame, + fps, + speakingAnimationType, + }) ?? "rest" + ); +}; + +export const VQCharacterAvatar = ({ + characterId, + character, + focused, + hasMultipleCharacters, + frame, + fps, + isSpeaking, + speakingSpeechId, + speakingLocalFrame, + resolveMouth, + fontFamily = "system-ui, sans-serif", + imageFilter = "drop-shadow(0 18px 40px rgba(31, 42, 68, 0.22))", +}: VQCharacterAvatarProps) => { + const {avatar} = character; + const scale = hasMultipleCharacters ? 0.82 : focused ? 0.94 : 0.9; + const opacity = focused || !hasMultipleCharacters ? 1 : 0.72; + const idleAnimationType = avatar.idleAnimationType ?? "gentleBob"; + const speakingAnimationType = avatar.speakingAnimationType ?? "gentleBob"; + const translateY = isSpeaking + ? vqSpeakingAvatarAnimations[speakingAnimationType]({ + frame, + fps, + focused, + hasMultipleCharacters, + }) + : vqIdleAvatarAnimations[idleAnimationType]({ + frame, + fps, + focused, + hasMultipleCharacters, + }); + const imageTranslateY = avatar.imageLayout?.translateY ?? 0; + const imageScaleX = avatar.imageLayout?.flipX ? -1 : 1; + const imageTransform = `translateY(${imageTranslateY}px) scaleX(${imageScaleX})`; + const nameplatePosition = avatar.nameplatePosition ?? "bottom"; + const showNameplate = nameplatePosition !== "none"; + const mouth = mouthForAvatar({ + characterId, + character, + isSpeaking, + speakingSpeechId, + speakingLocalFrame, + fps, + resolveMouth, + }); + + const nameplate = ( +
+ {character.displayName} +
+ ); + + return ( +
+ {showNameplate && nameplatePosition === "top" ? nameplate : null} + + {showNameplate && nameplatePosition === "bottom" ? nameplate : null} +
+ ); +}; diff --git a/voicevox-remotion-template/src/lib/VQRemotionLib/components/VQCharacterStage.tsx b/voicevox-remotion-template/src/lib/VQRemotionLib/components/VQCharacterStage.tsx new file mode 100644 index 0000000..2d15440 --- /dev/null +++ b/voicevox-remotion-template/src/lib/VQRemotionLib/components/VQCharacterStage.tsx @@ -0,0 +1,69 @@ +import React from "react"; +import type {VQCharacterDefinition, VQMouthResolver} from "../types"; +import {VQCharacterAvatar} from "./VQCharacterAvatar"; + +export type VQCharacterStageProps = + Readonly<{ + characters: Readonly>; + visibleCharacters: readonly CharacterId[]; + focusedCharacter?: CharacterId; + speakingCharacter?: CharacterId; + speakingSpeechId?: string; + speakingLocalFrame: number; + frame: number; + fps: number; + resolveMouth?: VQMouthResolver; + fontFamily?: string; + stageStyle?: React.CSSProperties; + avatarImageFilter?: string; + }>; + +export const VQCharacterStage = ({ + characters, + visibleCharacters, + focusedCharacter, + speakingCharacter, + speakingSpeechId, + speakingLocalFrame, + frame, + fps, + resolveMouth, + fontFamily, + stageStyle, + avatarImageFilter, +}: VQCharacterStageProps) => { + const hasMultipleCharacters = visibleCharacters.length > 1; + + return ( +
+ {visibleCharacters.map((characterId) => ( + + ))} +
+ ); +}; diff --git a/voicevox-remotion-template/src/lib/VQRemotionLib/components/VQLipSyncedStandeeImage.tsx b/voicevox-remotion-template/src/lib/VQRemotionLib/components/VQLipSyncedStandeeImage.tsx new file mode 100644 index 0000000..32d421b --- /dev/null +++ b/voicevox-remotion-template/src/lib/VQRemotionLib/components/VQLipSyncedStandeeImage.tsx @@ -0,0 +1,63 @@ +import React from "react"; +import {Img, staticFile} from "remotion"; +import type {VQMouthShape} from "../types"; + +export type VQLipSyncedStandeeImageProps = Readonly<{ + imagePath: string; + mouthImageDir: string; + mouth: VQMouthShape; + width: number | string; + maxHeight: number | string; + height?: number | string; + transform?: string; + filter?: string; +}>; + +export const VQLipSyncedStandeeImage: React.FC< + VQLipSyncedStandeeImageProps +> = ({ + imagePath, + mouthImageDir, + mouth, + width, + maxHeight, + height, + transform, + filter, +}) => { + return ( +
+ + +
+ ); +}; diff --git a/voicevox-remotion-template/src/lib/VQRemotionLib/components/VQSpeechOverlay.tsx b/voicevox-remotion-template/src/lib/VQRemotionLib/components/VQSpeechOverlay.tsx new file mode 100644 index 0000000..9e4c4f4 --- /dev/null +++ b/voicevox-remotion-template/src/lib/VQRemotionLib/components/VQSpeechOverlay.tsx @@ -0,0 +1,65 @@ +import React from "react"; +import {Audio} from "@remotion/media"; +import {spring, staticFile, useCurrentFrame, useVideoConfig} from "remotion"; +import type {VQSpeechEventLike} from "../types"; +import { + VQSpeechSubtitle, + type VQSpeechSubtitleStyleOptions, +} from "./VQSpeechSubtitle"; + +export type VQSpeechOverlayProps = + Readonly<{ + speech: Speech; + speakerName?: string; + accentColor?: string; + hasAudio?: (speech: Speech) => boolean; + getAudioPath?: (speech: Speech) => string; + subtitleOptions?: VQSpeechSubtitleStyleOptions; + containerStyle?: React.CSSProperties; + }>; + +const defaultOverlayStyle: React.CSSProperties = { + position: "absolute", + bottom: 40, + left: 0, + right: 0, + display: "flex", + justifyContent: "center", +}; + +export const VQSpeechOverlay = ({ + speech, + speakerName, + accentColor, + hasAudio, + getAudioPath, + subtitleOptions, + containerStyle, +}: VQSpeechOverlayProps) => { + const frame = useCurrentFrame(); + const {fps} = useVideoConfig(); + const subtitleProgress = spring({ + frame, + fps, + config: {damping: 20, mass: 0.7}, + }); + const text = speech.subtitle ?? speech.text; + const shouldPlayAudio = hasAudio && getAudioPath && hasAudio(speech); + + return ( + <> + {text ? ( +
+ +
+ ) : null} + {shouldPlayAudio ?