WebCodecs

 encode:  canvas / camera / pixels
            └─► VideoFrame ─► VideoEncoder ─► EncodedVideoChunk
                                                  └─► muxer ─► .mp4 / .webm
 
 decode:  .mp4 / .webm ─► demuxer ─► EncodedVideoChunk
            └─► VideoDecoder ─► VideoFrame ─► canvas / WebGL / WebGPU

const encoder = new VideoEncoder({
  output(chunk, meta) {
    // meta.decoderConfig arrives with the first keyframe
    // (and after each reconfigure): hand both to a muxer
  },
  error: (e) => console.error("encode failed", e),
});
 
encoder.configure({
  codec: "avc1.42001f", // H.264 Baseline, level 3.1
  width: 1280,
  height: 720,
  bitrate: 2_500_000, // bits per second
  framerate: 30,
  latencyMode: "quality", // "realtime" for live streams
  hardwareAcceleration: "no-preference",
  avc: { format: "avc" }, // "annexb" for raw streams
});
 
encoder.encode(frame, { keyFrame: true }); // then close it
frame.close();
await encoder.flush();
 
declare const frame: VideoFrame;

const decoder = new VideoDecoder({
  output(frame) {
    ctx.drawImage(frame, 0, 0);
    frame.close(); // every frame, every time
  },
  error: (e) => console.error("decode failed", e),
});
 
decoder.configure({
  codec: "avc1.64001f",
  codedWidth: 1920,
  codedHeight: 1080,
  description: avcC, // from the MP4 avcC box / demuxer
  optimizeForLatency: false,
});
 
decoder.decode(
  new EncodedVideoChunk({
    type: "key", // decoding must start on a keyframe
    timestamp: 0, // microseconds
    data: bytes,
  }),
);
await decoder.flush();
 
declare const ctx: CanvasRenderingContext2D;
declare const avcC: Uint8Array;
declare const bytes: Uint8Array;

const audioEncoder = new AudioEncoder({
  output: (chunk, meta) => mux(chunk, meta),
  error: (e) => console.error(e),
});
audioEncoder.configure({
  codec: "opus",
  sampleRate: 48_000,
  numberOfChannels: 2,
  bitrate: 128_000,
});
 
const pcm = new Float32Array(2 * 960); // 20 ms, planar
const data = new AudioData({
  format: "f32-planar", // "s16", "f32", "u8", "*-planar"
  sampleRate: 48_000,
  numberOfChannels: 2,
  numberOfFrames: 960,
  timestamp: 0,
  data: pcm,
});
audioEncoder.encode(data);
data.close();
 
declare function mux(
  c: EncodedAudioChunk, m?: EncodedAudioChunkMetadata,
): void;

// from anything drawable: canvas, OffscreenCanvas,
// ImageBitmap, <video>, <img>, another VideoFrame
const a = new VideoFrame(canvas, { timestamp: 0 });
 
// from raw pixels
const rgba = new Uint8Array(640 * 480 * 4);
const b = new VideoFrame(rgba, {
  format: "RGBA", // I420, NV12, RGBX, BGRA, …
  codedWidth: 640,
  codedHeight: 480,
  timestamp: 33_333, // µs
});
 
// back to bytes (pixel format stays frame.format)
const out = new Uint8Array(b.allocationSize());
await b.copyTo(out);
a.close();
b.close();
 
declare const canvas: HTMLCanvasElement;

const { supported, config } =
  await VideoEncoder.isConfigSupported({
    codec: "av01.0.08M.08",
    width: 1920,
    height: 1080,
    bitrate: 4_000_000,
    framerate: 30,
    hardwareAcceleration: "prefer-hardware",
  });
// resolves { supported: false } for unknown combinations;
// rejects with TypeError only for malformed configs

bun add mediabunny   # npm i mediabunny / pnpm add mediabunny

import {
  ALL_FORMATS, BlobSource, BufferTarget, Input,
  Mp4OutputFormat, Output, Conversion,
} from "mediabunny";
 
// transcode/remux any input file to MP4 in a few lines
export async function toMp4(file: File): Promise<Blob> {
  const input = new Input({
    source: new BlobSource(file),
    formats: ALL_FORMATS,
  });
  const output = new Output({
    format: new Mp4OutputFormat({ fastStart: "in-memory" }),
    target: new BufferTarget(),
  });
  const job = await Conversion.init({ input, output });
  await job.execute();
  return new Blob([output.target.buffer!], {
    type: "video/mp4",
  });
}

import {
  BufferTarget, EncodedPacket, EncodedVideoPacketSource,
  Mp4OutputFormat, Output,
} from "mediabunny";
 
const target = new BufferTarget();
const output = new Output({
  format: new Mp4OutputFormat(),
  target,
});
const source = new EncodedVideoPacketSource("avc");
output.addVideoTrack(source, { frameRate: 30 });
await output.start();
 
let muxing = Promise.resolve(); // keeps packets in order
const encoder = new VideoEncoder({
  output(chunk, meta) {
    const packet = EncodedPacket.fromEncodedChunk(chunk);
    muxing = muxing.then(() => source.add(packet, meta));
  },
  error: (e) => console.error("encode failed", e),
});
encoder.configure({
  codec: "avc1.640028",
  width: 1920,
  height: 1080,
  bitrate: 5e6,
});
// ... encoder.encode(frame) for each frame, close frames
await encoder.flush();
await muxing;
await output.finalize();
const mp4 = new Blob([target.buffer!], {
  type: "video/mp4",
});

const type = "image/gif";
if (await ImageDecoder.isTypeSupported(type)) {
  const res = await fetch("/party.gif");
  const decoder = new ImageDecoder({
    data: res.body!, // stream, ArrayBuffer or TypedArray
    type,
  });
  await decoder.tracks.ready;
  const track = decoder.tracks.selectedTrack;
  const count = track?.frameCount ?? 1; // grows if streaming
  for (let i = 0; i < count; i++) {
    const { image } = await decoder.decode({
      frameIndex: i,
    });
    // image is a VideoFrame: draw it, then close it
    image.close();
  }
  decoder.close();
}

const canvas = document.querySelector("canvas")!;
const offscreen = canvas.transferControlToOffscreen();
const worker = new Worker(
  new URL("./decode.worker.ts", import.meta.url),
  { type: "module" },
);
worker.postMessage({ canvas: offscreen, file }, [offscreen]);
 
declare const file: File;

type Job = { canvas: OffscreenCanvas; file: File };
 
self.onmessage = (e: MessageEvent<Job>) => {
  const ctx = e.data.canvas.getContext("2d");
  if (!ctx) throw new Error("no 2d context");
  const decoder = new VideoDecoder({
    output(frame) {
      ctx.drawImage(frame, 0, 0);
      frame.close();
    },
    error: (err) => self.postMessage({ error: err.message }),
  });
  // demux e.data.file, configure, decode (see recipes)
};

declare class MediaStreamTrackProcessor {
  constructor(init: { track: MediaStreamTrack });
  readonly readable: ReadableStream<VideoFrame>;
}
 
async function encodeTrack(
  track: MediaStreamTrack,
  encoder: VideoEncoder,
) {
  const reader = new MediaStreamTrackProcessor({ track })
    .readable.getReader();
  for (let n = 0; ; n++) {
    const { value: frame, done } = await reader.read();
    if (done) break;
    if (encoder.encodeQueueSize > 2) {
      frame.close(); // drop frames instead of lagging
      continue;
    }
    encoder.encode(frame, { keyFrame: n % 150 === 0 });
    frame.close();
  }
}

import {
  BufferTarget, CanvasSource, Mp4OutputFormat, Output,
  QUALITY_HIGH, WebMOutputFormat,
} from "mediabunny";
export async function renderVideo(
  canvas: HTMLCanvasElement | OffscreenCanvas, n: number,
  draw: (i: number) => void | Promise<void>,
  { fps = 30, webm = false } = {},
): Promise<Blob> {
  const target = new BufferTarget();
  const output = new Output({ target, format: webm
    ? new WebMOutputFormat() : new Mp4OutputFormat() });
  const source = new CanvasSource(canvas, {
    codec: webm ? "vp9" : "avc", quality: QUALITY_HIGH,
  });
  output.addVideoTrack(source, { frameRate: fps });
  await output.start();
  for (let i = 0; i < n; i++) {
    await draw(i);
    await source.add(i / fps, 1 / fps); // seconds, not µs
  }
  await output.finalize();
  const type = webm ? "video/webm" : "video/mp4";
  return new Blob([target.buffer!], { type });
}

import * as mb from "mediabunny";
export async function drawAll(
  file: File, canvas: HTMLCanvasElement,
) {
  const input = new mb.Input({
    source: new mb.BlobSource(file), formats: mb.ALL_FORMATS,
  });
  const track = await input.getPrimaryVideoTrack();
  const config = await track?.getDecoderConfig();
  if (!track || !config) throw new Error("no video track");
  const ctx = canvas.getContext("2d")!;
  const decoder = new VideoDecoder({
    output: (f) => { ctx.drawImage(f, 0, 0); f.close(); },
    error: (e) => console.error(e),
  });
  decoder.configure(config);
  const sink = new mb.EncodedPacketSink(track);
  for await (const packet of sink.packets()) {
    while (decoder.decodeQueueSize > 8)
      await new Promise((r) => (decoder.ondequeue = r));
    decoder.decode(packet.toEncodedVideoChunk());
  }
  await decoder.flush();
  decoder.close();
}

export async function thumbnail(file: File, maxW = 320) {
  const { type } = file;
  let src: VideoFrame | ImageBitmap;
  let dec: ImageDecoder | undefined;
  if ("ImageDecoder" in globalThis &&
      (await ImageDecoder.isTypeSupported(type))) {
    dec = new ImageDecoder({ data: file.stream(), type });
    src = (await dec.decode({ frameIndex: 0 })).image;
  } else {
    src = await createImageBitmap(file); // Safari path
  }
  const [w0, h0] = src instanceof ImageBitmap
    ? [src.width, src.height]
    : [src.displayWidth, src.displayHeight];
  const k = Math.min(1, maxW / w0);
  const w = Math.round(w0 * k), h = Math.round(h0 * k);
  const canvas = new OffscreenCanvas(w, h);
  canvas.getContext("2d")!.drawImage(src, 0, 0, w, h);
  src.close();
  dec?.close();
  return canvas.convertToBlob({ type: "image/jpeg" });
}

const CANDIDATES = {
  av1: "av01.0.08M.08",
  vp9: "vp09.00.40.08",
  h264: "avc1.640028",
  vp8: "vp8",
} as const;
type Name = keyof typeof CANDIDATES;
 
export async function videoEncoders(
  width = 1920, height = 1080,
): Promise<Name[]> {
  if (typeof VideoEncoder === "undefined") return [];
  const names = Object.keys(CANDIDATES) as Name[];
  const ok = await Promise.all(names.map((name) =>
    VideoEncoder.isConfigSupported({
      codec: CANDIDATES[name], width, height,
      bitrate: 5_000_000, framerate: 30,
    }).then((r) => r.supported === true, () => false)));
  return names.filter((_, i) => ok[i]);
}