Newer
Older
bugtrail / packages / extension / src / background / video.ts
/**
 * Encodes the recorded screencast frames into a silent WebM video. WebCodecs
 * VideoEncoder is not exposed to service workers, so this runs in the
 * offscreen document (hosted by src/offscreen/main.ts). With cursor crop
 * options the output is a square that follows the cursor (like the GIF
 * path); without them the whole frame is downscaled to 800px wide. Returns
 * null when WebCodecs is unavailable (Firefox) so the caller can fall back
 * to the GIF path.
 */
import { Muxer, ArrayBufferTarget } from "webm-muxer";
import { cursorCropRect, type CursorCropOptions } from "./gif";
import type { VideoFrame } from "./gif";

/** VP8 needs even dimensions. */
function even(value: number): number {
  return Math.max(2, Math.floor(value / 2) * 2);
}

export async function buildVideoWebm(frames: VideoFrame[], crop?: CursorCropOptions): Promise<Blob | null> {
  if (!frames.length || typeof VideoEncoder === "undefined") return null;

  // the screencast metadata's device size can exceed the JPEG Chrome actually
  // delivers (frames are scaled to fit maxWidth/maxHeight), so decode the
  // first frame to learn the true pixel space the crop math must work in
  const firstImage = await createImageBitmap(
    new Blob([frames[0].jpeg.buffer as ArrayBuffer], { type: "image/jpeg" })
  );

  // fixed output size for the whole file; the cursor crop outputs the source
  // square at (nearly) native resolution — capped at the requested square size
  let width: number;
  let height: number;
  if (crop && crop.viewport.w > 0 && crop.viewport.h > 0) {
    const scale = firstImage.width / crop.viewport.w;
    const cropDevice = Math.min(crop.size, crop.viewport.w, crop.viewport.h) * scale;
    width = even(Math.min(cropDevice, crop.size));
    height = width;
  } else {
    // fallback: full frame capped at 800px wide
    const scale = Math.min(1, 800 / firstImage.width);
    width = even(firstImage.width * scale);
    height = even(firstImage.height * scale);
  }

  const muxer = new Muxer({
    target: new ArrayBufferTarget(),
    video: { codec: "V_VP8", width, height, frameRate: 15 },
    firstTimestampBehavior: "offset",
  });

  // the encoder reports async failures through its error callback
  const failure: { error: Error | null } = { error: null };
  const encoder = new VideoEncoder({
    output: (chunk, meta) => muxer.addVideoChunk(chunk, meta),
    error: (error) => {
      failure.error = error instanceof Error ? error : new Error(String(error));
    },
  });
  encoder.configure({ codec: "vp8", width, height, bitrate: 1_200_000 });

  const canvas = new OffscreenCanvas(width, height);
  const context = canvas.getContext("2d");
  if (!context) throw new Error("no 2d context");

  try {
    // media timestamps start at the first frame: the muxer requires a
    // zero-based timeline (see firstTimestampBehavior above)
    const baseAt = frames[0].at;
    for (let position = 0; position < frames.length; position++) {
      const frame = frames[position];
      const nextAt = position + 1 < frames.length ? frames[position + 1].at : null;
      const timestampUs = Math.max(frame.at - baseAt, 0) * 1000;
      const durationUs = nextAt != null ? Math.max(nextAt - frame.at, 1) * 1000 : 33_000;

      // the first frame is already decoded (its size sized the output)
      const image = position === 0 ? firstImage : await createImageBitmap(new Blob([frame.jpeg.buffer as ArrayBuffer], { type: "image/jpeg" }));
      try {
        if (crop && crop.viewport.w > 0 && crop.viewport.h > 0) {
          // crop in the JPEG's true pixel space — the metadata's device size
          // can be larger than the delivered (downscaled) frame
          const { sx, sy, s } = cursorCropRect({ ...frame, w: image.width, h: image.height }, crop);
          // letterbox fill guards against rounding gaps on the edge px
          context.fillStyle = "#10121c";
          context.fillRect(0, 0, width, height);
          context.drawImage(image, sx, sy, s, s, 0, 0, width, height);
        } else {
          context.drawImage(image, 0, 0, width, height);
        }
      } finally {
        image.close();
      }
      const videoFrame = new VideoFrame(canvas, { timestamp: timestampUs, duration: durationUs });
      encoder.encode(videoFrame, { keyFrame: position % 30 === 0 });
      videoFrame.close();
      if (failure.error) throw failure.error;
    }
    if (encoder.state === "configured") await encoder.flush();
  } finally {
    encoder.close();
  }

  if (failure.error) throw failure.error;
  muxer.finalize();
  const { buffer } = muxer.target as ArrayBufferTarget;
  return new Blob([buffer], { type: "video/webm" });
}