import init, {
MediaInput,
PreviewOptions,
errorCode,
seekLatencyBudgetMs,
} from "./pkg/zvidlib.js";
import { scrubWalkStart, shouldDrawPreview } from "./scrub.js";
import { SAMPLES, decoderConfig, supportedSamples } from "./samples.js";
const canvas = document.querySelector("#video");
const playButton = document.querySelector("#play");
const rewindButton = document.querySelector("#rewind");
const fastForwardButton = document.querySelector("#fast-forward");
const previousFrameButton = document.querySelector("#previous-frame");
const nextFrameButton = document.querySelector("#next-frame");
const timeline = document.querySelector("#timeline");
const status = document.querySelector("#status");
const fps = document.querySelector("#fps");
const seekLatency = document.querySelector("#seek-latency");
const VERTEX_SHADER = `
attribute vec2 position;
varying vec2 uv;
void main() {
uv = position * 0.5 + 0.5;
gl_Position = vec4(position, 0.0, 1.0);
}
`;
const FRAGMENT_SHADER = `
precision mediump float;
varying vec2 uv;
uniform sampler2D frame;
void main() {
gl_FragColor = texture2D(frame, vec2(uv.x, 1.0 - uv.y));
}
`;
function compileProgram(gl) {
const compile = (type, source) => {
const shader = gl.createShader(type);
gl.shaderSource(shader, source);
gl.compileShader(shader);
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
throw new Error(gl.getShaderInfoLog(shader));
}
return shader;
};
const program = gl.createProgram();
gl.attachShader(program, compile(gl.VERTEX_SHADER, VERTEX_SHADER));
gl.attachShader(program, compile(gl.FRAGMENT_SHADER, FRAGMENT_SHADER));
gl.linkProgram(program);
if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
throw new Error(gl.getProgramInfoLog(program));
}
return program;
}
function syntheticFrame(width, height, phase) {
const pixels = new Uint8Array(width * height * 4);
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
const offset = (y * width + x) * 4;
pixels[offset] = Math.floor((255 * x) / width);
pixels[offset + 1] = Math.floor((255 * y) / height);
pixels[offset + 2] = Math.floor(255 * phase);
pixels[offset + 3] = 255;
}
}
return pixels;
}
async function probeSupport(samples) {
if (!globalThis.VideoDecoder) return [];
const answers = new Map(
await Promise.all(
samples.map(async (sample) => {
try {
const support = await VideoDecoder.isConfigSupported(decoderConfig(sample));
return [sample, support?.supported === true];
} catch {
return [sample, false];
}
}),
),
);
return supportedSamples(samples, (sample) => answers.get(sample));
}
async function openSample(sample, lines) {
const response = await fetch(sample.file);
if (!response.ok) {
lines.push(
`${sample.file}: HTTP ${response.status}. See examples/README.md for how to fetch the sample.`,
);
return null;
}
const input = await MediaInput.open(await response.blob());
return { input, video: input.video(0), audio: input.audio(0) };
}
async function openDecodableSample(lines) {
const supported = await probeSupport(SAMPLES);
lines.push(
supported.length > 0
? `This browser reports a decoder for: ${supported.map((sample) => sample.description).join(", ")}.`
: "This browser reports no decoder for any bundled sample.",
);
for (const sample of supported) {
const opened = await openSample(sample, lines);
if (!opened) continue;
try {
const decodedFrame = await opened.video.get(0n);
lines.push(
`${sample.description}: video.get(0n) decoded a real ${decodedFrame.width}x${decodedFrame.height} frame.`,
);
return { ...opened, sample, decodedFrame };
} catch (error) {
lines.push(`${sample.description}: video.get(0n) rejected with ${errorCode(error)}.`);
opened.input.close();
}
}
const fallback = await openSample(SAMPLES[0], lines);
if (!fallback) return null;
lines.push("No bundled sample decodes here: falling back to a synthetic frame.");
return { ...fallback, sample: SAMPLES[0], decodedFrame: null };
}
async function main() {
await init();
const gl = canvas.getContext("webgl2", { alpha: false });
if (!gl) throw new Error("WebGL 2 is unavailable");
const program = compileProgram(gl);
const positionBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.bufferData(
gl.ARRAY_BUFFER,
new Float32Array([-1, -1, 1, -1, -1, 1, 1, 1]),
gl.STATIC_DRAW,
);
const positionLocation = gl.getAttribLocation(program, "position");
gl.enableVertexAttribArray(positionLocation);
gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0);
const texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
status.textContent = "Choosing a sample this browser can decode…";
const lines = [];
const opened = await openDecodableSample(lines);
if (!opened) {
status.textContent = lines.join("\n");
return;
}
const { input, video, audio, sample, decodedFrame } = opened;
lines.push(
`Opened ${sample.file} (${sample.description}), ${input.byteLength} bytes.`,
`Video stream 0 direction: ${video.direction}`,
`Audio stream 0 direction: ${audio.direction}`,
);
const useRealDecode = decodedFrame !== null;
let audioState = null;
try {
audioState = await prepareAudio(audio);
lines.push(`${audioState.config.codec} audio ready: ${audioState.config.sampleRate} Hz, ${audioState.config.numberOfChannels} channels.`);
} catch (error) {
lines.push(`Audio unavailable (${errorCode(error) ?? error?.name ?? "ERROR"}): playback will be silent.`);
}
const frameStarts = await buildFrameStarts(video);
let randomAccessPoints = [0];
try {
randomAccessPoints = (await video.randomAccessPoints()).map(Number);
if (randomAccessPoints.length === 0) randomAccessPoints = [0];
} catch (error) {
lines.push(`video.randomAccessPoints() rejected with ${errorCode(error)}: scrubbing backwards restarts at frame 0.`);
}
const mediaDurationMs = frameStarts[frameStarts.length - 1] || 1;
const lastFrameIndex = Math.max(0, frameStarts.length - 2);
timeline.max = String(lastFrameIndex);
status.textContent = lines.join("\n");
let playing = false;
let frameIndex = 0;
let stopped = false;
let videoClockStart = 0;
let pausedMediaTimeMs = 0;
function uploadFrame(pixels, width, height) {
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, pixels);
gl.viewport(0, 0, canvas.width, canvas.height);
gl.useProgram(program);
gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
}
const budgetMs = seekLatencyBudgetMs();
let previews = null;
let worstPreviewMs = 0;
if (useRealDecode) {
try {
const framesPerSecond = 1000 / (frameStarts[1] - frameStarts[0] || 1000 / 24);
const previewOptions = new PreviewOptions(framesPerSecond);
previews = await video.previews(previewOptions);
previewOptions.free();
lines.push(
`Seek previews: ${previews.total} positions, ${previews.stride} frames apart, filling in the background.`,
);
pumpPreviews();
} catch (error) {
lines.push(`video.previews() rejected with ${errorCode(error)}: a drag walks the group of pictures instead.`);
}
status.textContent = lines.join("\n");
}
function scheduleIdle(callback) {
if (typeof requestIdleCallback === "function") requestIdleCallback(callback);
else setTimeout(callback, 0);
}
function pumpPreviews() {
if (!previews || previews.complete || stopped) return;
scheduleIdle(async () => {
try {
await previews.step();
} catch (error) {
previews = null;
lines.push(`Preview pass stopped with ${errorCode(error) ?? "ERROR"}.`);
status.textContent = lines.join("\n");
return;
}
pumpPreviews();
});
}
function drawPreview(target) {
if (!previews || !shouldDrawPreview(frameIndex, target)) return;
const started = performance.now();
const preview = previews.nearest(BigInt(target));
if (!preview) return;
const picture = preview.picture;
uploadFrame(picture.pixels, picture.width, picture.height);
picture.free();
preview.free();
const elapsed = performance.now() - started;
worstPreviewMs = Math.max(worstPreviewMs, elapsed);
if (seekLatency) {
seekLatency.textContent =
`seek ${elapsed.toFixed(2)} ms (worst ${worstPreviewMs.toFixed(2)} of ${budgetMs} ms)`;
}
}
async function frameDuration(index) {
try {
return await video.frameDuration(BigInt(index));
} catch {
frameIndex = 0;
return await video.frameDuration(0n);
}
}
async function buildFrameStarts(video) {
const starts = [0];
for (let index = 0; ; index++) {
try {
starts.push(starts[index] + (await video.frameDuration(BigInt(index))));
} catch {
return starts;
}
}
}
function frameForMediaTime(milliseconds) {
let low = 0;
let high = Math.max(0, frameStarts.length - 2);
while (low < high) {
const mid = Math.floor((low + high + 1) / 2);
if (frameStarts[mid] <= milliseconds) low = mid;
else high = mid - 1;
}
return Math.min(low, frameStarts.length - 1);
}
function mediaTimeForFrame(index) {
return frameStarts[Math.min(Math.max(index, 0), lastFrameIndex)] ?? 0;
}
function syncTimeline() {
timeline.value = String(Math.min(frameIndex, lastFrameIndex));
}
function currentMediaTimeMs() {
if (!playing) return pausedMediaTimeMs;
if (audioState?.startedAt !== undefined) {
const elapsedMs = Math.max((audioState.context.currentTime - audioState.startedAt) * 1000, 0);
return ((elapsedMs % mediaDurationMs) + mediaDurationMs) % mediaDurationMs;
}
return (performance.now() - videoClockStart) % mediaDurationMs;
}
async function seekToFrame(index, keepPlaying = playing) {
frameIndex = Math.min(Math.max(index, 0), lastFrameIndex);
pausedMediaTimeMs = mediaTimeForFrame(frameIndex);
if (audioState) stopAudio(audioState);
if (keepPlaying && audioState) await startAudio(audioState, pausedMediaTimeMs);
if (keepPlaying && !audioState) videoClockStart = performance.now() - pausedMediaTimeMs;
await renderFrame(false);
syncTimeline();
}
let pendingSeek = null;
let seeking = false;
async function requestSeek(index, keepPlaying) {
pendingSeek = { index, keepPlaying: keepPlaying ?? playing };
if (seeking) return;
seeking = true;
try {
while (pendingSeek) {
const next = pendingSeek;
pendingSeek = null;
await seekToFrame(next.index, next.keepPlaying);
}
} finally {
seeking = false;
}
}
let scrubTarget = null;
let scrubbing = false;
let scrubAbort = null;
let scrubStep = null;
let scrubPosition = null;
let scrubMsPerFrame = null;
async function scrubTo(index) {
if (!useRealDecode) return requestSeek(index, false);
scrubTarget = Math.min(Math.max(index, 0), lastFrameIndex);
drawPreview(scrubTarget);
if (scrubbing) {
if (scrubStep !== null && scrubStep > scrubTarget) scrubAbort?.abort();
return;
}
scrubbing = true;
scrubPosition = frameIndex;
playing = false;
playButton.textContent = "Play";
if (audioState) stopAudio(audioState);
try {
while (scrubTarget !== null) {
const target = scrubTarget;
const next = scrubWalkStart(scrubPosition, target, scrubMsPerFrame, randomAccessPoints);
scrubAbort = new AbortController();
scrubStep = next;
const startedAt = performance.now();
try {
const frame = await video.get(BigInt(next), scrubAbort.signal);
uploadFrame(frame.pixels, frame.width, frame.height);
const span = scrubPosition === null ? 0 : next - scrubPosition;
if (span > 0) scrubMsPerFrame = (performance.now() - startedAt) / span;
frameIndex = next;
scrubPosition = next;
pausedMediaTimeMs = mediaTimeForFrame(frameIndex);
} catch (error) {
if (errorCode(error) !== "CANCELED") {
scrubTarget = null;
break;
}
scrubPosition = null;
} finally {
scrubStep = null;
}
if (scrubTarget === target && frameIndex === target) scrubTarget = null;
}
} finally {
scrubbing = false;
scrubAbort = null;
scrubStep = null;
syncTimeline();
}
}
async function seekByMilliseconds(deltaMs) {
const targetTime = ((currentMediaTimeMs() + deltaMs) % mediaDurationMs + mediaDurationMs) % mediaDurationMs;
await requestSeek(frameForMediaTime(targetTime));
}
async function stepFrame(delta) {
const from = playing ? frameForMediaTime(currentMediaTimeMs()) : frameIndex;
playing = false;
playButton.textContent = "Play";
await requestSeek(from + delta, false);
}
async function renderFrame(advance = true) {
if (useRealDecode) {
try {
const duration = await frameDuration(frameIndex);
const frame = await video.get(BigInt(frameIndex));
uploadFrame(frame.pixels, frame.width, frame.height);
if (advance) frameIndex += 1;
syncTimeline();
return duration;
} catch {
frameIndex = 0;
const duration = await frameDuration(frameIndex);
const frame = await video.get(0n);
uploadFrame(frame.pixels, frame.width, frame.height);
if (advance) frameIndex = 1;
syncTimeline();
return duration;
}
}
const duration = await frameDuration(frameIndex);
const phase = (frameIndex % 60) / 60;
uploadFrame(syntheticFrame(canvas.width, canvas.height, phase), canvas.width, canvas.height);
if (advance) frameIndex += 1;
syncTimeline();
return duration;
}
let framesSinceFpsUpdate = 0;
let lastFpsUpdate = performance.now();
let lastDisplayedFrame = -1;
function tickFps() {
framesSinceFpsUpdate += 1;
const now = performance.now();
const elapsed = now - lastFpsUpdate;
if (elapsed >= 500) {
fps.textContent = `${((framesSinceFpsUpdate * 1000) / elapsed).toFixed(1)} fps`;
framesSinceFpsUpdate = 0;
lastFpsUpdate = now;
}
}
async function renderLoop() {
while (!stopped) {
await new Promise((resolve) => requestAnimationFrame(resolve));
if (playing) {
frameIndex = frameForMediaTime(currentMediaTimeMs());
const displayedFrame = frameIndex;
await renderFrame();
if (displayedFrame !== lastDisplayedFrame) {
lastDisplayedFrame = displayedFrame;
tickFps();
}
}
}
}
playButton.addEventListener("click", async () => {
playing = !playing;
if (playing) {
videoClockStart = performance.now() - pausedMediaTimeMs;
if (audioState) await startAudio(audioState, pausedMediaTimeMs);
} else if (audioState) {
pausedMediaTimeMs = currentMediaTimeMs();
stopAudio(audioState);
} else {
pausedMediaTimeMs = currentMediaTimeMs();
}
playButton.textContent = playing ? "Pause" : "Play";
});
rewindButton.addEventListener("click", () => seekByMilliseconds(-5000));
fastForwardButton.addEventListener("click", () => seekByMilliseconds(5000));
previousFrameButton.addEventListener("click", () => stepFrame(-1));
nextFrameButton.addEventListener("click", () => stepFrame(1));
timeline.addEventListener("input", () => scrubTo(Number(timeline.value)));
if (decodedFrame) {
uploadFrame(decodedFrame.pixels, decodedFrame.width, decodedFrame.height);
frameIndex = 1;
} else {
await renderFrame();
}
renderLoop();
window.addEventListener(
"pagehide",
() => {
stopped = true;
input.close();
},
{ once: true },
);
}
async function prepareAudio(audio) {
const config = await audio.decoderConfig();
const sampleCount = await audio.sampleCount();
const context = new AudioContext({ sampleRate: config.sampleRate });
return {
audio,
config,
sampleCount,
context,
buffer: undefined,
sources: [],
};
}
async function decodeAudioBuffer(state) {
if (state.buffer) return state.buffer;
const pcm = await state.audio.getRange(0n, state.sampleCount);
const channels = pcm.channels;
const samples = pcm.samples;
const frames = Math.max(samples.length / channels, 1);
const buffer = state.context.createBuffer(channels, frames, pcm.sampleRate);
for (let channel = 0; channel < channels; channel++) {
const plane = new Float32Array(frames);
for (let frame = 0; frame < samples.length / channels; frame++) {
plane[frame] = samples[frame * channels + channel];
}
buffer.copyToChannel(plane, channel);
}
state.buffer = buffer;
return buffer;
}
async function startAudio(state, offsetMs = 0) {
stopAudio(state);
const buffer = await decodeAudioBuffer(state);
await state.context.resume();
const offsetSeconds = Math.min(Math.max(offsetMs / 1000, 0), Math.max(buffer.duration - 0.001, 0));
const startAt = state.context.currentTime + 0.05;
const source = state.context.createBufferSource();
source.buffer = buffer;
source.loop = true;
source.connect(state.context.destination);
source.start(startAt, offsetSeconds);
state.sources.push(source);
state.startedAt = startAt - offsetSeconds;
}
function stopAudio(state) {
for (const source of state.sources.splice(0)) {
try {
source.stop();
} catch {}
}
state.startedAt = undefined;
}
main().catch((error) => {
status.textContent = `Error: ${error?.message ?? error}`;
console.error(error);
});