diff --git a/voicevox-remotion-template/src/data/pizza-kiln/script.ts b/voicevox-remotion-template/src/data/pizza-kiln/script.ts index fbe9ced..d594512 100644 --- a/voicevox-remotion-template/src/data/pizza-kiln/script.ts +++ b/voicevox-remotion-template/src/data/pizza-kiln/script.ts @@ -1,7 +1,11 @@ import { defineVQTimeline, say, + video, type VQSpeechEvent, + type VQTimelineEvent, + type VQTimelineInputEvent, + type VQVideoEvent, } from "../../lib/VQRemotionLib"; import type {CharacterId, VoicevoxVoice} from "./characters"; @@ -9,6 +13,16 @@ export type {CharacterDefinition, CharacterId, VoicevoxVoice} from "./characters"; export type SpeechEvent = VQSpeechEvent; +export type VideoEvent = VQVideoEvent; +export type TimelineEvent = VQTimelineEvent; +export type TimelineInputEvent = VQTimelineInputEvent< + CharacterId, + VoicevoxVoice +>; + +export const PIZZA_KILN_BACKGROUND_VIDEO_PATH = + "video/pizza-kiln-background.mp4"; +export const PIZZA_KILN_BACKGROUND_VIDEO_FRAMES = 154; export const timeline = defineVQTimeline([ say( @@ -16,9 +30,16 @@ "sayo", "小夜です。今日は、お手製の耐熱煉瓦ピザ窯を、全体ショットでご紹介します。" ), + video("pizza-kiln-video-001", PIZZA_KILN_BACKGROUND_VIDEO_PATH, { + placement: "blocking", + playback: "once", + durationFrames: PIZZA_KILN_BACKGROUND_VIDEO_FRAMES, + muted: true, + fit: "cover", + }), say( "pizza-kiln-sayo-002", "sayo", "以上、小夜がお届けしました。ピザ窯の雰囲気、伝わっていたらうれしいです。" ), -] satisfies readonly SpeechEvent[]); +] satisfies readonly TimelineInputEvent[]); diff --git a/voicevox-remotion-template/src/data/pizza-kiln/timing.ts b/voicevox-remotion-template/src/data/pizza-kiln/timing.ts index 9ccd362..d798033 100644 --- a/voicevox-remotion-template/src/data/pizza-kiln/timing.ts +++ b/voicevox-remotion-template/src/data/pizza-kiln/timing.ts @@ -1,4 +1,12 @@ -import {timeline, type SpeechEvent} from "./script"; +import { + defineVQChronologicalScenario, + totalVQChronologicalScenarioDurationInFrames, +} from "../../lib/VQRemotionLib/scenario"; +import { + doesVQTimelineEventAdvanceTimeline, + type VQVideoEvent, +} from "../../lib/VQRemotionLib"; +import {timeline, type SpeechEvent, type TimelineEvent} from "./script"; import voicevoxManifest from "./voicevox-manifest.json"; type ManifestEntry = { @@ -18,7 +26,6 @@ export const PIZZA_KILN_FPS = 30; export const PIZZA_KILN_GAP_FRAMES = 6; -export const PIZZA_KILN_VIDEO_FRAMES = 154; export const hasAudioForSpeech = (speech: SpeechEvent) => manifestById.has(speech.id); @@ -40,9 +47,53 @@ return Math.ceil(estimatedSeconds * fps); }; +export const durationForVideo = ( + event: VQVideoEvent, + fps = PIZZA_KILN_FPS +) => { + if (event.durationFrames && Number.isFinite(event.durationFrames)) { + return Math.max(1, Math.ceil(event.durationFrames)); + } + + if (event.durationSeconds && Number.isFinite(event.durationSeconds)) { + return Math.max(1, Math.ceil(event.durationSeconds * fps)); + } + + if (event.trimAfterFrames && Number.isFinite(event.trimAfterFrames)) { + return Math.max( + 1, + Math.ceil(event.trimAfterFrames - (event.trimBeforeFrames ?? 0)) + ); + } + + throw new Error(`Video event "${event.id}" needs durationFrames or durationSeconds.`); +}; + +export const durationForTimelineEvent = ( + event: TimelineEvent, + fps = PIZZA_KILN_FPS +) => { + if (event.type === "say") { + return durationForSpeech(event, fps); + } + + if (event.type === "video") { + return durationForVideo(event, fps); + } + + if (event.durationSeconds && Number.isFinite(event.durationSeconds)) { + return Math.max(1, Math.ceil(event.durationSeconds * fps)); + } + + throw new Error(`Timeline event "${event.type}" needs a duration.`); +}; + +export const pizzaKilnScenario = defineVQChronologicalScenario({ + timeline, + gapFrames: PIZZA_KILN_GAP_FRAMES, + durationForEvent: durationForTimelineEvent, + doesEventAdvanceTimeline: doesVQTimelineEventAdvanceTimeline, +}); + export const totalPizzaKilnDurationInFrames = (fps = PIZZA_KILN_FPS) => - durationForSpeech(timeline[0], fps) + - PIZZA_KILN_GAP_FRAMES + - PIZZA_KILN_VIDEO_FRAMES + - PIZZA_KILN_GAP_FRAMES + - durationForSpeech(timeline[1], fps); + totalVQChronologicalScenarioDurationInFrames(pizzaKilnScenario, fps); diff --git a/voicevox-remotion-template/src/lib/VQRemotionLib/components/VQTimelineVideo.tsx b/voicevox-remotion-template/src/lib/VQRemotionLib/components/VQTimelineVideo.tsx new file mode 100644 index 0000000..de87347 --- /dev/null +++ b/voicevox-remotion-template/src/lib/VQRemotionLib/components/VQTimelineVideo.tsx @@ -0,0 +1,49 @@ +import React from "react"; +import {Video} from "@remotion/media"; +import {AbsoluteFill, staticFile} from "remotion"; +import type {VQVideoEvent} from "../timeline"; + +const externalVideoSourcePattern = /^(https?:|data:|blob:)/; + +// 用途: videoイベントのsrcをRemotionで再生できるURLへ変換する。 +// 使用方法: VQTimelineVideo内でpublic配下の相対パスと外部URLを同じpropsから扱う。 +// オプションや引数詳細: http/https/data/blobはそのまま返し、それ以外はstaticFileのpublic相対パスとして扱う。 +const resolveVideoSource = (src: string) => + externalVideoSourcePattern.test(src) ? src : staticFile(src); + +// 用途: VQVideoEventをタイムライン上の動画レイヤーとして描画する。 +// 使用方法: scheduled segmentのSequence内でvideoイベントを渡して使う。 +// オプションや引数詳細: 表示尺は親Sequenceで制御し、loop/trim/音量/速度/配置はeventから反映する。 +export const VQTimelineVideo: React.FC< + Readonly<{ + video: VQVideoEvent; + }> +> = ({video}) => { + return ( + + + ); +}; diff --git a/voicevox-remotion-template/src/lib/VQRemotionLib/index.ts b/voicevox-remotion-template/src/lib/VQRemotionLib/index.ts index 5fd79e1..2933bf3 100644 --- a/voicevox-remotion-template/src/lib/VQRemotionLib/index.ts +++ b/voicevox-remotion-template/src/lib/VQRemotionLib/index.ts @@ -10,4 +10,5 @@ export * from "./components/VQSpeechSubtitle"; export * from "./components/VQStageCornerStandee"; export * from "./components/VQStillBackground"; +export * from "./components/VQTimelineVideo"; export * from "./components/VQWarmGradientBackground"; diff --git a/voicevox-remotion-template/src/lib/VQRemotionLib/scenario.ts b/voicevox-remotion-template/src/lib/VQRemotionLib/scenario.ts index 096f95a..312e519 100644 --- a/voicevox-remotion-template/src/lib/VQRemotionLib/scenario.ts +++ b/voicevox-remotion-template/src/lib/VQRemotionLib/scenario.ts @@ -2,6 +2,7 @@ timeline: readonly Event[]; gapFrames: number; durationForEvent: (event: Event, fps: number) => number; + doesEventAdvanceTimeline?: (event: Event) => boolean; assetWorkflow?: VQScenarioAssetWorkflow; }>; @@ -46,6 +47,8 @@ return scenario.timeline.map((event, index) => { const durationInFrames = scenario.durationForEvent(event, fps); + const advancesTimeline = + scenario.doesEventAdvanceTimeline?.(event) ?? true; const segment = { event, index, @@ -53,8 +56,11 @@ durationInFrames, }; - cursor += durationInFrames; - if (index < scenario.timeline.length - 1) { + if (advancesTimeline) { + cursor += durationInFrames; + } + + if (advancesTimeline && index < scenario.timeline.length - 1) { cursor += scenario.gapFrames; } @@ -84,9 +90,7 @@ fps: number ) => scheduleVQChronologicalScenario(scenario, fps).reduce( - (total, segment, index) => - total + - segment.durationInFrames + - (index < scenario.timeline.length - 1 ? scenario.gapFrames : 0), + (total, segment) => + Math.max(total, segment.from + segment.durationInFrames), 0 ); diff --git a/voicevox-remotion-template/src/lib/VQRemotionLib/timeline.ts b/voicevox-remotion-template/src/lib/VQRemotionLib/timeline.ts index 8bb34fb..678d979 100644 --- a/voicevox-remotion-template/src/lib/VQRemotionLib/timeline.ts +++ b/voicevox-remotion-template/src/lib/VQRemotionLib/timeline.ts @@ -45,6 +45,31 @@ durationSeconds: number; }>; +export type VQVideoPlacement = "blocking" | "overlay" | "background"; + +export type VQVideoPlayback = "once" | "loop"; + +export type VQVideoEvent = Readonly<{ + type: "video"; + id: string; + src: string; + placement?: VQVideoPlacement; + playback?: VQVideoPlayback; + durationFrames?: number; + durationSeconds?: number; + trimBeforeFrames?: number; + trimAfterFrames?: number; + muted?: boolean; + volume?: number; + playbackRate?: number; + toneFrequency?: number; + loopVolumeCurveBehavior?: "repeat" | "extend"; + fit?: VQStillObjectFit; + objectPosition?: string; + opacity?: number; + zIndex?: number; +}>; + export type VQCustomTimelineEvent< Type extends string, Payload extends object = object @@ -58,7 +83,12 @@ CharacterId extends string = string, Voice = unknown, ExtraEvent extends Readonly<{type: string}> = never -> = VQSpeechEvent | VQStillEvent | VQWaitEvent | ExtraEvent; +> = + | VQSpeechEvent + | VQStillEvent + | VQWaitEvent + | VQVideoEvent + | ExtraEvent; export type VQTimelineInputEvent< CharacterId extends string = string, @@ -69,6 +99,7 @@ | VQStillEvent | VQWaitEvent | VQWaitEventInput + | VQVideoEvent | ExtraEvent; type VQTimelineInputEventBase = Readonly<{type: string}>; @@ -122,6 +153,30 @@ durationSeconds, }); +// 用途: 動画をタイムラインイベントとして定義する。 +// 使用方法: script.ts の timeline 内で video("id", "path.mp4", options) として呼び出す。 +// オプションや引数詳細: placement はタイムライン進行、playback は単発/ループ、duration は表示尺を指定する。 +export const video = ( + id: string, + src: string, + options: Omit = {} +): VQVideoEvent => ({ + type: "video", + id, + src, + ...options, +}); + +// 用途: 共通タイムラインイベントが時間カーソルを進めるかを判定する。 +// 使用方法: scheduleVQChronologicalScenarioのdoesEventAdvanceTimelineに渡して使う。 +// オプションや引数詳細: videoのoverlay/backgroundは現在位置で再生を始め、次イベントを同時に開始できる。 +export const doesVQTimelineEventAdvanceTimeline = ( + event: Readonly<{type: string; placement?: string}> +) => + event.type !== "video" || + event.placement === undefined || + event.placement === "blocking"; + // 用途: waitイベントの連番からタイムライン内IDを作る。 // 使用方法: defineVQTimeline内で未解決waitイベントを正規化するときに呼び出す。 // オプションや引数詳細: index は1始まりのwait連番で、3桁ゼロ埋めにする。 diff --git a/voicevox-remotion-template/src/pizza-kiln-composition.tsx b/voicevox-remotion-template/src/pizza-kiln-composition.tsx index a82da59..49c5e1a 100644 --- a/voicevox-remotion-template/src/pizza-kiln-composition.tsx +++ b/voicevox-remotion-template/src/pizza-kiln-composition.tsx @@ -1,34 +1,32 @@ import React from "react"; -import {Video} from "@remotion/media"; import { AbsoluteFill, Sequence, - staticFile, useCurrentFrame, useVideoConfig, } from "remotion"; import { audioFileForSpeech, - durationForSpeech, hasAudioForSpeech, - PIZZA_KILN_GAP_FRAMES, - PIZZA_KILN_VIDEO_FRAMES, + pizzaKilnScenario, } from "./data/pizza-kiln/timing"; import { characters, - timeline, type SpeechEvent, + type TimelineEvent, } from "./data/pizza-kiln/script"; import {roundedFontFamily} from "./fonts"; import { + scheduleVQChronologicalScenario, VQSpeechOverlay, VQStageCornerStandee, + VQTimelineVideo, VQWarmGradientBackground, + type VQScheduledScenarioSegment, vqSpeakingAvatarAnimations, } from "./lib/VQRemotionLib"; import {getMouthForSpeechFrame} from "./lipsync/manifest"; -const BACKGROUND_VIDEO_PATH = "video/pizza-kiln-background.mp4"; const pizzaSubtitleOptions = { fontFamily: roundedFontFamily, fontSize: 34, @@ -87,40 +85,74 @@ ); }; +type ScheduledTimelineEvent = VQScheduledScenarioSegment; +type ScheduledSpeechEvent = ScheduledTimelineEvent & Readonly<{ + event: SpeechEvent; +}>; + +const isInsideSegment = (segment: ScheduledTimelineEvent, frame: number) => + frame >= segment.from && frame < segment.from + segment.durationInFrames; + +const isActiveSpeechSegment = ( + segment: ScheduledTimelineEvent, + frame: number +): segment is ScheduledSpeechEvent => + segment.event.type === "say" && isInsideSegment(segment, frame); + export const PizzaKilnSayoComposition: React.FC = () => { const frame = useCurrentFrame(); const {fps} = useVideoConfig(); - const introSpeech = timeline[0]; - const outroSpeech = timeline[1]; - const introFrames = durationForSpeech(introSpeech, fps); - const outroFrames = durationForSpeech(outroSpeech, fps); - const videoFrom = introFrames + PIZZA_KILN_GAP_FRAMES; - const outroFrom = videoFrom + PIZZA_KILN_VIDEO_FRAMES + PIZZA_KILN_GAP_FRAMES; - const isVideoVisible = - frame >= videoFrom && frame < videoFrom + PIZZA_KILN_VIDEO_FRAMES; - const isOutro = frame >= outroFrom; - const activeSpeech = - frame < introFrames ? introSpeech : isOutro ? outroSpeech : undefined; - const speechLocalFrame = activeSpeech === outroSpeech ? frame - outroFrom : frame; + const scheduledEvents = scheduleVQChronologicalScenario( + pizzaKilnScenario, + fps + ); + const activeSpeechSegment = scheduledEvents.find((segment) => + isActiveSpeechSegment(segment, frame) + ); + const isVideoVisible = scheduledEvents.some( + (segment) => segment.event.type === "video" && isInsideSegment(segment, frame) + ); + const activeSpeech = activeSpeechSegment?.event; + const speechLocalFrame = activeSpeechSegment + ? frame - activeSpeechSegment.from + : 0; + const videoSequences = scheduledEvents.map((scheduledEvent) => { + if (scheduledEvent.event.type !== "video") { + return null; + } + + return ( + + + + ); + }); + const speechSequences = scheduledEvents.map((scheduledEvent) => { + if (scheduledEvent.event.type !== "say") { + return null; + } + + return ( + + + + ); + }); return ( - - + {videoSequences} - - - - - - + {speechSequences} ); };