Catalog
No preview yet

Source

1250 lines
// News Shots — JSON-first reel composition.
//
// Sibling to ai_news, but the entire timeline is described by ONE
// `shots` JSON array. Every visual beat — text, timing, layout,
// transition, kenBurns, per-shot visual, headline, text style —
// lives inside a shot object. Block-level fields are defaults that
// individual shots override.
//
// Shot shape (all fields optional except text + start/end):
//   {
//     text: string,
//     start: number,            // seconds (preferred) or startFrame
//     end: number,              // seconds (preferred) or endFrame
//     words?: [{word, start, end}],   // per-word timing for karaoke
//     layout?: 'hero' | 'split' | 'fullVisual' | 'insetAvatar',
//     visual?: string,          // image/video URL for this shot
//     transition?: 'cut' | 'flash' | 'zoom' | 'fade',
//     transitionDuration?: number,    // seconds, default 0.33
//     kenBurns?: 'zoomIn' | 'zoomOut' | 'panLeft' | 'panRight' | 'panUp' | 'none',
//     headline?: string | null, // null = hide; string = show
//     textStyle?: { color?, inactiveColor?, size?, font?, position? },
//     avatar?: { scale?, offsetY? },
//   }
//
// Karaoke spotlights the active word inside the shot's full text. No
// 3-word chunking — the shot's text is shown whole. Words missing
// timings auto-distribute evenly across the shot's duration.

import { loadFont as loadInter } from '@remotion/google-fonts/Inter';
import { loadFont as loadRoboto } from '@remotion/google-fonts/Roboto';
import { loadFont } from '@remotion/google-fonts/SpaceGrotesk';
import {
  AbsoluteFill,
  Audio,
  Img,
  interpolate,
  OffthreadVideo,
  Series,
  spring,
  useCurrentFrame,
  useRemotionEnvironment,
  useVideoConfig,
  Video,
} from 'remotion';

// Inlined from news_shots-helpers.ts: the block bundler bundles this .tsx in
// isolation (copied to a temp dir) and cannot resolve sibling imports. The
// helpers file + its unit tests remain the canonical spec — keep in sync.
interface AvatarClip {
  url: string;
  durationSeconds?: number;
}

function parseAvatarClips(raw: unknown): AvatarClip[] {
  let parsed: unknown = raw;
  if (typeof raw === 'string') {
    const s = raw.trim();
    if (/^https?:\/\//i.test(s)) return [];
    try {
      parsed = JSON.parse(s);
    } catch {
      return [];
    }
  }
  if (!Array.isArray(parsed)) return [];
  const out: AvatarClip[] = [];
  for (const item of parsed) {
    if (typeof item === 'string' && /^https?:\/\//i.test(item)) {
      out.push({ url: item });
    } else if (item && typeof item === 'object') {
      const o = item as { url?: unknown; durationSeconds?: unknown };
      if (typeof o.url === 'string' && /^https?:\/\//i.test(o.url)) {
        out.push({
          url: o.url,
          durationSeconds:
            typeof o.durationSeconds === 'number' && o.durationSeconds > 0
              ? o.durationSeconds
              : undefined,
        });
      }
    }
  }
  return out;
}

function planAvatarDurations(
  clips: AvatarClip[],
  totalFrames: number,
  fps: number
): number[] {
  if (clips.length === 0) return [];
  const total = Math.max(1, Math.round(totalFrames));
  const haveExact = clips.every(
    (c) => typeof c.durationSeconds === 'number' && c.durationSeconds > 0
  );

  let durations: number[];
  if (haveExact) {
    durations = clips.map((c) =>
      Math.max(1, Math.round((c.durationSeconds as number) * fps))
    );
  } else {
    const per = Math.max(1, Math.floor(total / clips.length));
    durations = clips.map(() => per);
  }
  const summedBeforeLast = durations.slice(0, -1).reduce((s, d) => s + d, 0);
  durations[durations.length - 1] = Math.max(1, total - summedBeforeLast);
  return durations;
}

const { fontFamily: SPACE_GROTESK } = loadFont();
const { fontFamily: INTER } = loadInter();
const { fontFamily: ROBOTO } = loadRoboto();

/**
 * Use OffthreadVideo for Lambda renders (frame-perfect ffmpeg extraction)
 * and the HTML <Video> element for studio/preview (smooth real-time
 * playback, zero server proxy load). Prevents the dev /proxy route from
 * choking on large upscaled avatar files (>20MB) where per-frame ffmpeg
 * extraction tanks the dev server.
 */
function useVideoComponent(): typeof OffthreadVideo {
  const env = useRemotionEnvironment();
  return env.isRendering ? OffthreadVideo : (Video as typeof OffthreadVideo);
}

const FONT_MAP: Record<string, string> = {
  SpaceGrotesk: SPACE_GROTESK,
  Inter: INTER,
  Roboto: ROBOTO,
};

type Layout = 'hero' | 'split' | 'fullVisual' | 'insetAvatar';
type Transition = 'cut' | 'flash' | 'zoom' | 'fade';
type KenBurns =
  | 'zoomIn'
  | 'zoomOut'
  | 'panLeft'
  | 'panRight'
  | 'panUp'
  | 'none';
type TextPosition = 'top' | 'middle' | 'bottom';

const LAYOUTS: Layout[] = ['hero', 'split', 'fullVisual', 'insetAvatar'];
const TRANSITIONS: Transition[] = ['cut', 'flash', 'zoom', 'fade'];
const KEN_BURNS: KenBurns[] = [
  'zoomIn',
  'zoomOut',
  'panLeft',
  'panRight',
  'panUp',
  'none',
];
const TEXT_POSITIONS: TextPosition[] = ['top', 'middle', 'bottom'];

type ShotWord = {
  word: string;
  startFrame: number;
  endFrame: number;
};

type Shot = {
  startFrame: number;
  endFrame: number;
  text: string;
  words: ShotWord[];
  layout: Layout;
  transition: Transition;
  transitionDuration: number; // frames
  kenBurns: KenBurns;
  visual: string | null;
  visualPrompt: string | undefined; // present iff the shot wants a B-roll image
  headline: string | null | undefined; // undefined = default, null = hide, string = show
  textStyle: {
    color?: string;
    inactiveColor?: string;
    size?: number;
    font?: string;
    position?: TextPosition;
  };
  avatar: {
    scale?: number;
    offsetY?: string;
  };
};

function pick<T extends string>(v: unknown, allowed: T[]): T | undefined {
  return typeof v === 'string' && (allowed as string[]).includes(v)
    ? (v as T)
    : undefined;
}

// Accept seconds (preferred) or frames; both keys checked so authors
// can write naturally either way.
function toFrame(value: unknown, key: string, fps: number): number | null {
  if (typeof value === 'number') {
    // Heuristic: if the key explicitly names "Frame" treat as frame,
    // otherwise treat as seconds. start/end fields default to seconds.
    if (key.toLowerCase().includes('frame')) return Math.round(value);
    return Math.round(value * fps);
  }
  return null;
}

function parseWordsArray(raw: unknown, fps: number): ShotWord[] {
  if (!Array.isArray(raw) || raw.length === 0) return [];
  const out: ShotWord[] = [];
  for (const item of raw as unknown[]) {
    if (!item || typeof item !== 'object') continue;
    const o = item as Record<string, unknown>;
    // Whisper style: { word, start, end } (seconds) OR { text, timestamp: [start, end] }
    const word = String(o.word ?? o.text ?? '').trim();
    if (!word) continue;
    let startFrame =
      toFrame(o.startFrame, 'startFrame', fps) ??
      toFrame(o.start, 'start', fps);
    let endFrame =
      toFrame(o.endFrame, 'endFrame', fps) ?? toFrame(o.end, 'end', fps);
    if (
      (startFrame === null || endFrame === null) &&
      Array.isArray((o as { timestamp?: unknown }).timestamp)
    ) {
      const ts = (o as { timestamp: number[] }).timestamp;
      if (startFrame === null && typeof ts[0] === 'number') {
        startFrame = Math.round(ts[0] * fps);
      }
      if (endFrame === null && typeof ts[1] === 'number') {
        endFrame = Math.round(ts[1] * fps);
      }
    }
    if (startFrame === null || endFrame === null) continue;
    out.push({ word, startFrame, endFrame });
  }
  return out;
}

function parseCaptionSource(raw: unknown, fps: number): ShotWord[] {
  if (!raw) return [];
  if (typeof raw === 'string') {
    try {
      raw = JSON.parse(raw);
    } catch {
      return [];
    }
  }
  // Full whisper object: { text, chunks: [{text, timestamp:[s,e]}] }
  if (raw && typeof raw === 'object' && !Array.isArray(raw)) {
    const r = raw as { chunks?: unknown };
    if (Array.isArray(r.chunks)) return parseWordsArray(r.chunks, fps);
  }
  return parseWordsArray(raw, fps);
}

function distributeByText(
  text: string,
  startFrame: number,
  endFrame: number
): ShotWord[] {
  const tokens = (text ?? '').split(/\s+/).filter(Boolean);
  if (tokens.length === 0) return [];
  const span = Math.max(1, endFrame - startFrame);
  const per = span / tokens.length;
  return tokens.map((word, i) => ({
    word,
    startFrame: startFrame + Math.round(i * per),
    endFrame: startFrame + Math.round((i + 1) * per),
  }));
}

function wordsForShot(
  shotWordsRaw: unknown,
  shotStartFrame: number,
  shotEndFrame: number,
  text: string,
  captionWords: ShotWord[],
  fps: number
): ShotWord[] {
  // 1) Explicit per-shot words win
  const explicit = parseWordsArray(shotWordsRaw, fps);
  if (explicit.length > 0) return explicit;
  // 2) Whisper overlap — any word whose span intersects the shot
  if (captionWords.length > 0) {
    const overlap = captionWords.filter(
      (w) => w.endFrame > shotStartFrame && w.startFrame < shotEndFrame
    );
    if (overlap.length > 0) return overlap;
  }
  // 3) Last-resort even distribution from text
  return distributeByText(text, shotStartFrame, shotEndFrame);
}

// LLMs regularly wrap JSON output in markdown fences (```json ... ```)
// or prepend a sentence ("Here is the JSON:"). Strip both before parse
// so a chatty model doesn't silently kill the whole reel.
function stripMarkdownFences(s: string): string {
  let out = s.trim();
  // Strip opening fence: ```json or ``` or ```JSON etc.
  out = out.replace(/^```(?:json|JSON|js|ts)?\s*\n?/, '');
  // Strip closing fence
  out = out.replace(/\n?```\s*$/, '');
  // If there's still prose before the array, find the first '[' and
  // last ']' and slice between them.
  const firstBracket = out.indexOf('[');
  const lastBracket = out.lastIndexOf(']');
  if (
    firstBracket >= 0 &&
    lastBracket > firstBracket &&
    (firstBracket > 0 || lastBracket < out.length - 1)
  ) {
    out = out.slice(firstBracket, lastBracket + 1);
  }
  return out.trim();
}

function normalizeShots(
  raw: unknown,
  fps: number,
  captionWords: ShotWord[]
): Shot[] {
  if (!raw) return [];
  if (typeof raw === 'string') {
    try {
      raw = JSON.parse(stripMarkdownFences(raw));
    } catch {
      return [];
    }
  }
  if (!Array.isArray(raw)) return [];

  const out: Shot[] = [];
  for (const item of raw as unknown[]) {
    if (!item || typeof item !== 'object') continue;
    const o = item as Record<string, unknown>;
    const startFrame =
      toFrame(o.startFrame, 'startFrame', fps) ??
      toFrame(o.start, 'start', fps);
    const endFrame =
      toFrame(o.endFrame, 'endFrame', fps) ?? toFrame(o.end, 'end', fps);
    if (startFrame === null || endFrame === null) continue;

    const text = typeof o.text === 'string' ? o.text : '';
    const words = wordsForShot(
      o.words,
      startFrame,
      endFrame,
      text,
      captionWords,
      fps
    );

    const transitionDuration =
      typeof o.transitionDuration === 'number'
        ? Math.max(1, Math.round(o.transitionDuration * fps))
        : Math.round(0.33 * fps);

    const textStyleRaw = (o.textStyle ?? {}) as Record<string, unknown>;
    const avatarRaw = (o.avatar ?? {}) as Record<string, unknown>;

    out.push({
      startFrame,
      endFrame,
      text,
      words,
      layout: pick(o.layout, LAYOUTS) ?? 'split',
      transition: pick(o.transition, TRANSITIONS) ?? 'cut',
      transitionDuration,
      kenBurns: pick(o.kenBurns, KEN_BURNS) ?? 'zoomIn',
      visual: typeof o.visual === 'string' && o.visual.length ? o.visual : null,
      visualPrompt:
        typeof o.visualPrompt === 'string' && o.visualPrompt.length > 0
          ? o.visualPrompt
          : undefined,
      headline:
        o.headline === null
          ? null
          : typeof o.headline === 'string'
          ? o.headline
          : undefined,
      textStyle: {
        color:
          typeof textStyleRaw.color === 'string'
            ? textStyleRaw.color
            : undefined,
        inactiveColor:
          typeof textStyleRaw.inactiveColor === 'string'
            ? textStyleRaw.inactiveColor
            : undefined,
        size:
          typeof textStyleRaw.size === 'number' ? textStyleRaw.size : undefined,
        font:
          typeof textStyleRaw.font === 'string' ? textStyleRaw.font : undefined,
        position: pick(textStyleRaw.position, TEXT_POSITIONS),
      },
      avatar: {
        scale:
          typeof avatarRaw.scale === 'number' ? avatarRaw.scale : undefined,
        offsetY:
          typeof avatarRaw.offsetY === 'string' ? avatarRaw.offsetY : undefined,
      },
    });
  }
  // Stable ordering by startFrame so prev-shot lookup is deterministic.
  out.sort((a, b) => a.startFrame - b.startFrame);
  return out;
}

function kenBurnsTransform(type: KenBurns, p: number): string {
  switch (type) {
    case 'zoomOut':
      return `scale(${interpolate(p, [0, 1], [1.08, 1.0])})`;
    case 'panLeft':
      return `scale(1.06) translateX(${interpolate(p, [0, 1], [0, -3])}%)`;
    case 'panRight':
      return `scale(1.06) translateX(${interpolate(p, [0, 1], [0, 3])}%)`;
    case 'panUp':
      return `scale(1.06) translateY(${interpolate(p, [0, 1], [0, -3])}%)`;
    case 'none':
      return 'scale(1)';
    case 'zoomIn':
    default:
      return `scale(${interpolate(p, [0, 1], [1.0, 1.08])})`;
  }
}

// Boxes for each layout. Visual stacks on top of avatar with z-index;
// avatar stays mounted in every layout (even if hidden behind a full
// visual) so OffthreadVideo keeps its audio track playing.
//
// For talking-head avatars (face in top portion of 9:16 frame), the
// split layout needs the avatar positioned at 50–100% with
// object-position: top so the face anchors to the top of the visible
// half — not cropped out by default cover-center behavior.
type VisualBox = {
  top: number;
  height: number;
  opacity: number;
  pipVisible: boolean;
};
type AvatarBox = {
  top: number;
  height: number;
  objectPosition: string; // "center center" | "center top"
};

function visualBoxFor(layout: Layout): VisualBox {
  switch (layout) {
    case 'hero':
      return { top: 0, height: 0, opacity: 0, pipVisible: false };
    case 'fullVisual':
      return { top: 0, height: 100, opacity: 1, pipVisible: false };
    case 'insetAvatar':
      return { top: 0, height: 100, opacity: 1, pipVisible: true };
    case 'split':
    default:
      return { top: 0, height: 50, opacity: 1, pipVisible: false };
  }
}

function avatarBoxFor(layout: Layout): AvatarBox {
  switch (layout) {
    case 'split':
      // Avatar occupies the bottom half; face-anchored to top of box so
      // talking-head framing stays visible.
      return { top: 50, height: 50, objectPosition: 'center top' };
    case 'hero':
    case 'fullVisual':
    case 'insetAvatar':
    default:
      return { top: 0, height: 100, objectPosition: 'center center' };
  }
}

type CaptionRowProps = {
  shot: Shot;
  frame: number;
  fps: number;
  defaults: {
    activeColor: string;
    inactiveColor: string;
    fontFamily: string;
    fontSize: number;
  };
};

function CaptionRow({ shot, frame, fps, defaults }: CaptionRowProps) {
  if (!shot.text || shot.words.length === 0) return null;
  const activeColor = shot.textStyle.color ?? defaults.activeColor;
  const inactiveColor = shot.textStyle.inactiveColor ?? defaults.inactiveColor;
  const fontSize = shot.textStyle.size ?? defaults.fontSize;
  const fontFamily =
    (shot.textStyle.font && FONT_MAP[shot.textStyle.font]) ??
    defaults.fontFamily;
  const position = shot.textStyle.position ?? 'bottom';

  // Phrase + width-aware chunking. Captions break on natural
  // punctuation (.,;:!?—) and also break early when the chunk would
  // exceed a character budget — avoids long words like "Researchers
  // wired flexible electronic" overflowing the 9:16 frame at default
  // font size. Cap is conservative; the renderer also allows wrapping
  // as a final safety net.
  const CHUNK_MAX_WORDS = 4;
  const CHUNK_MAX_CHARS = 22; // soft cap, includes joining spaces
  const chunks: Array<typeof shot.words> = [];
  let bucket: typeof shot.words = [];
  let bucketChars = 0;
  const PHRASE_END = /[.,;:!?—]$/;
  for (let i = 0; i < shot.words.length; i++) {
    const w = shot.words[i];
    const wordLen = (w.word || '').length;
    // Flush before adding if this word would exceed the char budget,
    // but only if the bucket already has content (single overflowing
    // word still goes into its own chunk — better than dropping it).
    if (bucket.length > 0 && bucketChars + wordLen + 1 > CHUNK_MAX_CHARS) {
      chunks.push(bucket);
      bucket = [];
      bucketChars = 0;
    }
    bucket.push(w);
    bucketChars += wordLen + (bucket.length > 1 ? 1 : 0);
    const wordTrimmed = (w.word || '').trim();
    const hitsPunctuation = PHRASE_END.test(wordTrimmed);
    if (hitsPunctuation || bucket.length >= CHUNK_MAX_WORDS) {
      chunks.push(bucket);
      bucket = [];
      bucketChars = 0;
    }
  }
  if (bucket.length > 0) chunks.push(bucket);
  let activeIdx = shot.words.findIndex(
    (w) => frame >= w.startFrame && frame < w.endFrame
  );
  if (activeIdx < 0) {
    // Between words — use the most recent one so caption stays visible.
    for (let i = shot.words.length - 1; i >= 0; i--) {
      if (frame >= shot.words[i].startFrame) {
        activeIdx = i;
        break;
      }
    }
  }
  if (activeIdx < 0) return null;
  // Find which chunk contains the active word (chunks now have variable
  // length due to phrase-aware splitting; can't divide by a constant).
  const activeWord = shot.words[activeIdx];
  const chunkIdx = chunks.findIndex((c) =>
    c.some(
      (w) =>
        w.startFrame === activeWord.startFrame &&
        w.endFrame === activeWord.endFrame
    )
  );
  const activeChunk = chunks[chunkIdx];
  if (!activeChunk) return null;

  const chunkStart = activeChunk[0].startFrame;
  const chunkEnd = activeChunk[activeChunk.length - 1].endFrame;
  const localFrame = frame - chunkStart;
  const chunkDuration = Math.max(1, chunkEnd - chunkStart);

  const scale = spring({
    frame: localFrame,
    fps,
    config: { damping: 14, stiffness: 220, mass: 0.4 },
  });
  const opacity = interpolate(
    localFrame,
    [0, 2, chunkDuration - 2, chunkDuration],
    [0, 1, 1, 0],
    { extrapolateLeft: 'clamp', extrapolateRight: 'clamp' }
  );

  const verticalStyle: React.CSSProperties =
    position === 'top'
      ? { top: '10%' }
      : position === 'middle'
      ? { top: '46%', transform: 'translateY(-50%)' }
      : { bottom: '14%' };

  return (
    <div
      style={{
        position: 'absolute',
        left: 0,
        right: 0,
        display: 'flex',
        justifyContent: 'center',
        alignItems: 'center',
        zIndex: 10,
        pointerEvents: 'none',
        ...verticalStyle,
      }}
    >
      <div
        style={{
          transform: `scale(${scale})`,
          opacity,
          padding: '10px 28px',
          display: 'flex',
          // wrap as a safety net — chunk grouping uses CHUNK_MAX_CHARS
          // to prevent overflow but very long single words (e.g.
          // "Researchers") can still push past the frame; allowing
          // wrap drops them onto a second line instead of clipping.
          flexWrap: 'wrap',
          justifyContent: 'center',
          gap: '0 12px',
          maxWidth: '92%',
          margin: '0 auto',
        }}
      >
        {activeChunk.map((wt, i) => {
          const isCurrent = frame >= wt.startFrame && frame < wt.endFrame;
          return (
            <span
              key={`${chunkStart}-${i}`}
              style={{
                color: isCurrent ? activeColor : inactiveColor,
                fontSize,
                fontWeight: 700,
                fontFamily,
                lineHeight: 1.2,
                textTransform: 'uppercase',
                letterSpacing: -0.5,
                WebkitTextStroke: isCurrent ? '0px' : '1.5px rgba(0,0,0,0.6)',
                paintOrder: 'stroke fill',
                textShadow: isCurrent
                  ? `0 0 20px ${activeColor}99, 0 0 40px ${activeColor}55, 0 2px 8px rgba(0,0,0,0.8)`
                  : '0 2px 8px rgba(0,0,0,0.9)',
                display: 'inline-block',
                whiteSpace: 'nowrap',
              }}
            >
              {wt.word}
            </span>
          );
        })}
      </div>
    </div>
  );
}

function VisualPanel({
  src,
  topPercent,
  heightPercent,
  opacity,
  kenBurns,
  kenBurnsProgress,
}: {
  src: string;
  topPercent: number;
  heightPercent: number;
  opacity: number;
  kenBurns: KenBurns;
  kenBurnsProgress: number;
}) {
  const isUrl = typeof src === 'string' && /^https?:\/\//i.test(src);
  const isVideo = isUrl && /\.(mp4|webm|mov)(\?|$)/i.test(src);
  const transform = isVideo
    ? 'scale(1)'
    : kenBurnsTransform(kenBurns, kenBurnsProgress);
  const VideoTag = useVideoComponent();
  if (heightPercent <= 0.5) return null;
  return (
    <div
      style={{
        position: 'absolute',
        top: `${topPercent}%`,
        left: 0,
        right: 0,
        height: `${heightPercent}%`,
        overflow: 'hidden',
        backgroundColor: '#0a0b0f',
        opacity,
      }}
    >
      {isUrl ? (
        isVideo ? (
          <VideoTag
            src={src}
            muted
            style={{ width: '100%', height: '100%', objectFit: 'cover' }}
          />
        ) : (
          <Img
            src={src}
            style={{
              width: '100%',
              height: '100%',
              objectFit: 'cover',
              transform,
              transformOrigin: 'center center',
            }}
          />
        )
      ) : (
        <div
          style={{
            color: 'rgba(255,255,255,0.25)',
            display: 'flex',
            width: '100%',
            height: '100%',
            alignItems: 'center',
            justifyContent: 'center',
            fontFamily: SPACE_GROTESK,
            fontSize: 22,
            letterSpacing: '0.1em',
            textTransform: 'uppercase',
          }}
        >
          No visual
        </div>
      )}
    </div>
  );
}

function AvatarLayer({
  src,
  scale,
  offsetY,
  muted,
  inset,
  topPercent,
  heightPercent,
  leftPercent,
  widthPercent,
  objectPosition,
}: {
  src: string;
  scale: number;
  offsetY: string;
  muted: boolean;
  inset: boolean;
  topPercent: number;
  heightPercent: number;
  leftPercent: number;
  widthPercent: number;
  objectPosition: string;
}) {
  const isUrl = typeof src === 'string' && /^https?:\/\//i.test(src);
  const VideoTag = useVideoComponent();
  return (
    <div
      style={{
        position: 'absolute',
        top: `${topPercent}%`,
        left: `${leftPercent}%`,
        width: `${widthPercent}%`,
        height: `${heightPercent}%`,
        overflow: 'hidden',
        backgroundColor: '#0a0b0f',
        borderRadius: inset ? 18 : 0,
        boxShadow: inset ? '0 8px 28px rgba(0,0,0,0.55)' : undefined,
        border: inset ? '2px solid rgba(255,255,255,0.15)' : undefined,
      }}
    >
      {isUrl ? (
        <VideoTag
          src={src}
          muted={muted}
          style={{
            width: '100%',
            height: '100%',
            objectFit: 'cover',
            objectPosition,
            // 1.012 base scale crops out the ~4px constant-color
            // borders HeyGen Avatar IV ships at top + bottom of its
            // 1080p output. Combined with author-provided scale.
            transform: `scale(${scale * 1.012}) translateY(${offsetY})`,
            transformOrigin: 'center center',
          }}
        />
      ) : (
        <div
          style={{
            color: 'rgba(255,255,255,0.25)',
            display: 'flex',
            width: '100%',
            height: '100%',
            alignItems: 'center',
            justifyContent: 'center',
            fontFamily: SPACE_GROTESK,
            fontSize: 22,
          }}
        >
          Avatar
        </div>
      )}
    </div>
  );
}

// Native sequencing of 2-8 avatar clips back-to-back across the avatar's
// on-screen time via <Series>. One clip is active per frame (Remotion plays
// only the current Series segment), so Lambda renders cleanly — this is the
// same per-shot-video pattern the reel already uses, no ffmpeg concat needed.
//
// Per-clip length: exact `durationSeconds` when provided (media_splitter
// segments), otherwise an even split of the total avatar window. The last
// clip absorbs any rounding remainder so the clips tile the window exactly.
function AvatarSequence({
  clips,
  totalFrames,
  fps,
  scale,
  offsetY,
  muted,
  topPercent,
  heightPercent,
  leftPercent,
  widthPercent,
  objectPosition,
  inset,
}: {
  clips: AvatarClip[];
  totalFrames: number;
  fps: number;
  scale: number;
  offsetY: string;
  muted: boolean;
  topPercent: number;
  heightPercent: number;
  leftPercent: number;
  widthPercent: number;
  objectPosition: string;
  inset: boolean;
}) {
  const durations = planAvatarDurations(clips, totalFrames, fps);

  return (
    <div
      style={{
        position: 'absolute',
        top: `${topPercent}%`,
        left: `${leftPercent}%`,
        width: `${widthPercent}%`,
        height: `${heightPercent}%`,
        overflow: 'hidden',
        backgroundColor: '#0a0b0f',
        borderRadius: inset ? 18 : 0,
        boxShadow: inset ? '0 8px 28px rgba(0,0,0,0.55)' : undefined,
        border: inset ? '2px solid rgba(255,255,255,0.15)' : undefined,
      }}
    >
      <Series>
        {clips.map((clip, i) => (
          <Series.Sequence
            key={`${clip.url}-${i}`}
            durationInFrames={durations[i]}
          >
            <ClipVideo
              src={clip.url}
              muted={muted}
              scale={scale}
              offsetY={offsetY}
              objectPosition={objectPosition}
            />
          </Series.Sequence>
        ))}
      </Series>
    </div>
  );
}

// The <OffthreadVideo>/<Video> for one avatar clip, with the same crop +
// scale treatment AvatarLayer applies. Extracted so AvatarSequence can reuse
// it inside <Series>.
function ClipVideo({
  src,
  muted,
  scale,
  offsetY,
  objectPosition,
}: {
  src: string;
  muted: boolean;
  scale: number;
  offsetY: string;
  objectPosition: string;
}) {
  const VideoTag = useVideoComponent();
  return (
    <VideoTag
      src={src}
      muted={muted}
      style={{
        width: '100%',
        height: '100%',
        objectFit: 'cover',
        objectPosition,
        transform: `scale(${scale * 1.012}) translateY(${offsetY})`,
        transformOrigin: 'center center',
      }}
    />
  );
}

export type NewsShotsProps = {
  shots: unknown;
  captions: unknown;
  visuals: unknown;
  avatarVideo: string;
  avatarVideos: unknown;
  audioFile: string;
  defaultVisual: string;
  defaultHeadline: string;
  defaultActiveColor: string;
  defaultInactiveColor: string;
  defaultFont: 'SpaceGrotesk' | 'Inter' | 'Roboto';
  defaultFontSize: number;
  defaultLayout: Layout;
  backgroundColor: string;
};

// Generated visuals come in as an array of URLs (or { url } objects)
// from a fan-out image generator. Map them to shots by stepping through
// shots in order and assigning visuals[promptIdx] to each shot that has
// a visualPrompt. Hero shots without prompts skip the lookup → keep
// the fan-out lean (no waste image gens for hidden positions).
function mapShotIndexToVisual(
  shots: Array<{ visualPrompt?: string; visual?: string | null }>,
  visualsRaw: unknown
): Array<string | null> {
  let parsedVisuals: unknown = visualsRaw;
  if (typeof visualsRaw === 'string') {
    try {
      parsedVisuals = JSON.parse(visualsRaw);
    } catch {
      parsedVisuals = [];
    }
  }
  const visuals: string[] = [];
  if (Array.isArray(parsedVisuals)) {
    for (const v of parsedVisuals) {
      if (typeof v === 'string' && v.length > 0) visuals.push(v);
      else if (v && typeof v === 'object') {
        const url =
          (v as { url?: unknown }).url ?? (v as { image?: unknown }).image;
        if (typeof url === 'string' && url.length > 0) visuals.push(url);
      }
    }
  }
  const out: Array<string | null> = [];
  let promptIdx = 0;
  for (const shot of shots) {
    const hasPrompt =
      typeof (shot as { visualPrompt?: unknown }).visualPrompt === 'string' &&
      (shot as { visualPrompt: string }).visualPrompt.length > 0;
    if (hasPrompt) {
      out.push(visuals[promptIdx] ?? null);
      promptIdx += 1;
    } else {
      out.push(null);
    }
  }
  return out;
}

export default function NewsShots({
  shots,
  captions,
  visuals,
  avatarVideo,
  avatarVideos,
  audioFile,
  defaultVisual,
  defaultHeadline = '',
  defaultActiveColor = '#00F0FF',
  defaultInactiveColor = '#FFFFFF',
  defaultFont = 'SpaceGrotesk',
  defaultFontSize = 56,
  defaultLayout = 'split',
  backgroundColor = '#05060a',
}: Partial<NewsShotsProps>) {
  const frame = useCurrentFrame();
  const { fps, durationInFrames } = useVideoConfig();

  // Multi-clip avatar (2-8 sub-15s clips, e.g. Seedance) sequenced natively
  // across the full comp via <Series>. A single clip still uses avatarVideo.
  const avatarClips = parseAvatarClips(avatarVideos);
  const useSequencedAvatar = avatarClips.length >= 2;
  // Single-clip source. An explicit avatarVideo port wins; else the lone
  // parsed clip; else a bare avatarVideos URL string. parseAvatarClips returns
  // [] for a single URL string (it only treats arrays as multi-clip), so a
  // single Seedance clip wired to the ARRAY port (avatarVideos) must be
  // resolved here or the avatar renders EMPTY (avatarVideo is undefined →
  // src=''). Live repro: exec bbdef590 shipped a reel with no avatar.
  const singleAvatarSrc =
    (typeof avatarVideo === 'string' && avatarVideo.trim()) ||
    avatarClips[0]?.url ||
    (typeof avatarVideos === 'string' &&
    /^https?:\/\//i.test(avatarVideos.trim())
      ? avatarVideos.trim()
      : '') ||
    '';

  const captionWords = parseCaptionSource(captions, fps);
  const normalized = normalizeShots(shots, fps, captionWords);
  const generatedVisuals = mapShotIndexToVisual(
    normalized as Array<{ visualPrompt?: string; visual?: string | null }>,
    visuals
  );
  const fallbackVisual =
    typeof defaultVisual === 'string' && /^https?:\/\//i.test(defaultVisual)
      ? defaultVisual
      : '';
  const captionFontFamily = FONT_MAP[defaultFont] ?? SPACE_GROTESK;
  const hasAudio =
    typeof audioFile === 'string' && /^https?:\/\//i.test(audioFile);

  const activeIdx = normalized.findIndex(
    (s) => frame >= s.startFrame && frame < s.endFrame
  );
  const activeShot = activeIdx >= 0 ? normalized[activeIdx] : null;
  const activeGeneratedVisual =
    activeIdx >= 0 ? generatedVisuals[activeIdx] : null;

  // Snappy hard-cuts between layouts. Snag the visual + avatar boxes
  // directly for the active shot — no cross-fade, no lerp. Transition
  // effects (flash / zoom) still play as optional per-shot punch beats.
  //
  // When no shot covers the current frame (audio outlives the shots
  // timeline OR a gap between shots), fall back to `hero` so the
  // avatar fills the frame instead of the empty visual panel rendering
  // its "No visual" placeholder. Keeps the reel coherent across the
  // full audio duration without the user having to extend every shot.
  // Resolve this shot's b-roll FIRST: a shot with no real visual must fall
  // back to `hero` (avatar fills, no panel) instead of rendering the
  // "No visual" placeholder. Fixes reels where the b-roll node under-produces
  // (e.g. 1 image for several visual shots) — those shots now show the avatar.
  const rawVisual =
    activeShot?.visual ?? activeGeneratedVisual ?? fallbackVisual;
  const hasVisual =
    typeof rawVisual === 'string' && /^https?:\/\//i.test(rawVisual);
  const targetLayout: Layout =
    hasVisual && activeShot?.layout ? activeShot.layout : 'hero';
  const box = visualBoxFor(targetLayout);
  const avatarBox = avatarBoxFor(targetLayout);
  // defaultLayout from props is intentionally ignored when there's no
  // active shot — falling back to hero is more graceful than 'split'
  // with an empty image panel. Kept in scope so style controls in the
  // Properties panel still bind, but no behavioural effect here.
  void defaultLayout;

  const localFrame = activeShot ? frame - activeShot.startFrame : 0;
  const segmentDuration = activeShot
    ? Math.max(1, activeShot.endFrame - activeShot.startFrame)
    : 1;
  const transitionFrames = activeShot?.transitionDuration ?? 10;

  // Per-shot visual resolution precedence:
  //   1. shot.visual — explicit URL on the shot (author override)
  //   2. activeGeneratedVisual — image generated for this shot via the
  //      visuals port's fan-out (mapped by visualPrompt presence, not
  //      raw index, so hero shots are skipped naturally)
  //   3. fallbackVisual — wired defaultVisual port
  //   4. '' — block renders the no-visual placeholder
  const visual = hasVisual ? rawVisual : '';
  const kenBurns: KenBurns = activeShot?.kenBurns ?? 'zoomIn';
  const kenBurnsProgress = Math.min(1, localFrame / segmentDuration);
  const transition: Transition = activeShot?.transition ?? 'cut';

  // Headline: shot.headline === null → hide; string → show; undefined →
  // fall back to defaultHeadline.
  const headlineText =
    activeShot?.headline === null
      ? ''
      : activeShot?.headline ?? defaultHeadline;

  const avatarScale = activeShot?.avatar.scale ?? 1.0;
  const avatarOffsetY = activeShot?.avatar.offsetY ?? '0%';

  // Transition effects — applied across the first `transitionFrames`
  // of the active shot, so authors get visual feedback proportional
  // to their declared transitionDuration.
  const t = Math.min(1, localFrame / transitionFrames);
  const flashOpacity =
    transition === 'flash' && activeShot
      ? interpolate(t, [0, 1], [0.45, 0], { extrapolateRight: 'clamp' })
      : 0;
  const fadeOpacity =
    transition === 'fade' && activeShot
      ? interpolate(t, [0, 1], [0, 1], { extrapolateRight: 'clamp' })
      : 1;
  const zoomPunch =
    transition === 'zoom' && activeShot
      ? interpolate(t, [0, 0.4, 1], [1.06, 1.03, 1.0], {
          extrapolateRight: 'clamp',
        })
      : 1.0;

  return (
    <AbsoluteFill style={{ backgroundColor }}>
      <div
        style={{
          width: '100%',
          height: '100%',
          transform: `scale(${zoomPunch})`,
          transformOrigin: 'center center',
          opacity: fadeOpacity,
        }}
      >
        {/* Avatar is always mounted — OffthreadVideo keeps its audio
            track playing across every layout. In split, avatar's box
            is the bottom half so the face (framed in the top of the
            9:16 source) lands in the visible area via
            object-position: center top. Other layouts render the
            avatar full-frame (covered by the visual when applicable).

            Multi-clip avatars (2-8 Seedance clips) sequence natively via
            <Series>. When a master audioFile is wired the avatar is muted to
            avoid doubling the voice (each clip carries its own slice audio);
            without a master track the avatar audio is the voice. */}
        {useSequencedAvatar ? (
          <AvatarSequence
            clips={avatarClips}
            totalFrames={durationInFrames}
            fps={fps}
            topPercent={avatarBox.top}
            heightPercent={avatarBox.height}
            leftPercent={0}
            widthPercent={100}
            scale={avatarScale}
            offsetY={avatarOffsetY}
            muted={hasAudio}
            inset={false}
            objectPosition={avatarBox.objectPosition}
          />
        ) : (
          <AvatarLayer
            src={singleAvatarSrc}
            topPercent={avatarBox.top}
            heightPercent={avatarBox.height}
            leftPercent={0}
            widthPercent={100}
            scale={avatarScale}
            offsetY={avatarOffsetY}
            muted={false}
            inset={false}
            objectPosition={avatarBox.objectPosition}
          />
        )}
        {box.opacity > 0 ? (
          <div style={{ position: 'absolute', inset: 0, zIndex: 2 }}>
            <VisualPanel
              src={visual}
              topPercent={box.top}
              heightPercent={box.height}
              opacity={box.opacity}
              kenBurns={kenBurns}
              kenBurnsProgress={kenBurnsProgress}
            />
          </div>
        ) : null}
        {box.pipVisible ? (
          <div style={{ position: 'absolute', inset: 0, zIndex: 3 }}>
            {useSequencedAvatar ? (
              <AvatarSequence
                clips={avatarClips}
                totalFrames={durationInFrames}
                fps={fps}
                topPercent={62}
                heightPercent={28}
                leftPercent={60}
                widthPercent={36}
                scale={avatarScale}
                offsetY={avatarOffsetY}
                muted={true}
                inset={true}
                objectPosition='center top'
              />
            ) : (
              <AvatarLayer
                src={singleAvatarSrc}
                topPercent={62}
                heightPercent={28}
                leftPercent={60}
                widthPercent={36}
                scale={avatarScale}
                offsetY={avatarOffsetY}
                muted={true}
                inset={true}
                objectPosition='center top'
              />
            )}
          </div>
        ) : null}
        {targetLayout === 'split' ? (
          <div
            style={{
              position: 'absolute',
              top: '50%',
              left: 0,
              right: 0,
              height: 3,
              backgroundColor: '#1e2128',
              zIndex: 5,
            }}
          />
        ) : null}
        {headlineText ? (
          <div
            style={{
              position: 'absolute',
              top: 32,
              left: 0,
              right: 0,
              textAlign: 'center',
              color: '#FFFFFF',
              fontFamily: captionFontFamily,
              fontWeight: 800,
              fontSize: 52,
              textShadow: '0 2px 12px rgba(0,0,0,0.85)',
              padding: '0 48px',
              zIndex: 6,
            }}
          >
            {headlineText}
          </div>
        ) : null}
      </div>
      {flashOpacity > 0 ? (
        <div
          style={{
            position: 'absolute',
            inset: 0,
            backgroundColor: `rgba(255,255,255,${flashOpacity})`,
            pointerEvents: 'none',
            zIndex: 11,
          }}
        />
      ) : null}
      {activeShot ? (
        <CaptionRow
          shot={activeShot}
          frame={frame}
          fps={fps}
          defaults={{
            activeColor: defaultActiveColor,
            inactiveColor: defaultInactiveColor,
            fontFamily: captionFontFamily,
            fontSize: defaultFontSize,
          }}
        />
      ) : null}
      {hasAudio ? <Audio src={audioFile as string} /> : null}
    </AbsoluteFill>
  );
}

Schema (Zod)

// Zod schema for news_shots — the JSON-first sibling of ai_news.
//
// Markers:
//   - $port:<type> — workflow input port (video/image/audio/text/json)
//   - $style:<kind> — direct-manipulation UI control in Properties panel
//
// Every visual decision is per-shot inside the `shots` JSON. The
// block-level style fields are DEFAULTS that shots override.

import { z } from 'zod';

export const Schema = z.object({
  // ─── Workflow ports ─────────────────────────────────────────────────────
  shots: z.any().optional().describe('$port:json Shots'),
  captions: z.any().optional().describe('$port:json Captions'),
  avatarVideo: z.string().optional().describe('$port:video Avatar'),
  // JSON (list-shape) port so a 2-8 clip array survives input normalization
  // instead of collapsing to its first element. Sequenced natively via
  // <Series> across the avatar's on-screen time. Falls back to avatarVideo
  // (single clip) when empty.
  avatarVideos: z
    .any()
    .optional()
    .describe('$port:json Avatar Clips (2-8, auto-sequenced)'),
  audioFile: z.string().optional().describe('$port:audio Audio'),
  defaultVisual: z.string().optional().describe('$port:image Fallback Image'),
  visuals: z.any().optional().describe('$port:json B-roll'),

  // ─── Defaults the shots override ────────────────────────────────────────
  defaultHeadline: z
    .string()
    .default('')
    .describe('Headline shown when shot.headline is not set'),
  defaultActiveColor: z
    .string()
    .default('#00F0FF')
    .describe('$style:color Default active-word color'),
  defaultInactiveColor: z
    .string()
    .default('#FFFFFF')
    .describe('$style:color Default inactive-word color'),
  defaultFont: z
    .enum(['SpaceGrotesk', 'Inter', 'Roboto'])
    .default('SpaceGrotesk')
    .describe('$style:font Default caption font'),
  defaultFontSize: z
    .number()
    .default(56)
    .describe('$style:size Default caption font size'),
  defaultLayout: z
    .enum(['hero', 'split', 'fullVisual', 'insetAvatar'])
    .default('split')
    .describe('$style:select Layout used when shot.layout is unset'),
  backgroundColor: z
    .string()
    .default('#05060a')
    .describe('$style:color Background color'),
});

News Shots (JSON)

news_shots

JSON-driven news reel. One `shots` array describes every beat: text, timing, layout (hero/split/fullVisual/insetAvatar), transition, ken-burns, per-shot visual + headline + text style. Replaces ai_news for new reels.

Workflow Inputs (7)

  • Shots
    shots
    JSON
  • Captions
    captions
    JSON
  • Avatar
    avatarVideo
    VIDEO
  • Avatar Clips (2-8, auto-sequenced)
    avatarVideos
    JSON
  • Audio
    audioFile
    AUDIO
  • Fallback Image
    defaultVisual
    IMAGE
  • B-roll
    visuals
    JSON

Config Fields (7)

  • defaultFont
  • defaultLayout
  • backgroundColor
  • defaultFontSize
  • defaultHeadline
  • defaultActiveColor
  • defaultInactiveColor

Meta

Updated
7/10/2026