export const audioFormats = ["audio/webm;codecs=opus", "audio/ogg;codecs=opus", "audio/mpeg"];
function object(value) {
if (!value || typeof value !== "object" || Array.isArray(value))
throw new Error("Invalid realtime media object");
return value;
}
function positive(value, maximum = Number.MAX_SAFE_INTEGER) {
if (typeof value !== "number" || !Number.isSafeInteger(value) || value <= 0 || value > maximum)
throw new Error("Invalid audio metadata");
return value;
}
function identifier(value) {
if (typeof value !== "string" || !value.trim())
throw new Error("Audio identity is required");
return value;
}
export class RealtimeMedia {
audioFormats;
constructor(audioFormats = ["audio/ogg;codecs=opus", "audio/mpeg", "audio/webm;codecs=opus"]) {
if (!Array.isArray(audioFormats) || audioFormats.length < 1 || audioFormats.length > 8)
throw new Error("Provide one to eight audio MIME types");
this.audioFormats = Object.freeze(audioFormats.map(identifier));
}
toJSON() { return { version: "1.0", audio_formats: [...this.audioFormats] }; }
validateNegotiation(capabilities) {
const media = object(capabilities.media);
if (media.version !== "1.0" || media.delivery !== "http" || !this.audioFormats.includes(String(object(media.audio_format).mime_type))) {
throw new Error("The runtime did not negotiate a supported realtime media format");
}
}
}
export class RealtimeAudio {
streamId;
responseId;
uri;
mimeType;
codec;
sampleRate;
channels;
expiresAt;
maxBytes;
constructor(value) {
const data = object(value), format = object(data.format);
if (data.version !== "1.0")
throw new Error("Unsupported realtime media version");
this.streamId = identifier(data.stream_id);
this.responseId = identifier(data.response_id);
this.uri = identifier(data.uri);
const uri = new URL(this.uri);
const loopback = uri.hostname === "localhost" || uri.hostname === "[::1]" || /^127\.(?:\d{1,3}\.){2}\d{1,3}$/.test(uri.hostname);
if (uri.username || uri.password || uri.hash || /^https?:\/\/[^/?#]*@/.test(this.uri) || !(uri.protocol === "https:" || uri.protocol === "http:" && loopback))
throw new Error("Audio URL must use HTTPS or loopback HTTP");
this.mimeType = identifier(format.mime_type);
this.codec = identifier(format.codec);
this.sampleRate = positive(format.sample_rate);
this.channels = positive(format.channels);
const expected = this.mimeType === "audio/mpeg" ? ["mp3", 24000] : ["audio/ogg;codecs=opus", "audio/webm;codecs=opus"].includes(this.mimeType) ? ["opus", 48000] : [];
if (this.codec !== expected[0] || this.sampleRate !== expected[1] || this.channels !== 1)
throw new Error("Audio format metadata is inconsistent");
this.expiresAt = positive(data.expires_at);
this.maxBytes = positive(data.max_bytes, 2 * 1024 * 1024);
}
toJSON() {
return { version: "1.0", stream_id: this.streamId, response_id: this.responseId, uri: this.uri,
format: { mime_type: this.mimeType, codec: this.codec, sample_rate: this.sampleRate, channels: this.channels },
expires_at: this.expiresAt, max_bytes: this.maxBytes };
}
content() {
if (new URL(this.uri).protocol !== "https:")
throw new Error("B2A media references require HTTPS");
return { type: "audio", mime_type: this.mimeType, uri: this.uri, representation_id: this.responseId };
}
async *chunks(signal) {
if (this.expiresAt * 1000 <= Date.now())
throw new Error("The audio stream has expired");
const controller = new AbortController();
const abort = () => controller.abort(signal?.reason);
if (signal?.aborted)
abort();
else
signal?.addEventListener("abort", abort, { once: true });
let reader;
let timer;
try {
timer = setTimeout(() => controller.abort(), 30_000);
const response = await fetch(this.uri, { headers: { Accept: this.mimeType }, redirect: "error", credentials: "omit", signal: controller.signal });
clearTimeout(timer);
if (!response.ok || !response.body)
throw new Error("Audio stream is unavailable");
if (response.headers.get("content-type")?.split(";", 1)[0] !== this.mimeType.split(";", 1)[0])
throw new Error("Unexpected audio MIME type");
reader = response.body.getReader();
let received = 0;
for (;;) {
timer = setTimeout(() => controller.abort(), 30_000);
const { done, value } = await reader.read();
clearTimeout(timer);
if (done)
break;
received += value.length;
if (received > this.maxBytes)
throw new Error("Audio exceeds its resource limit");
yield value;
}
}
finally {
clearTimeout(timer);
signal?.removeEventListener("abort", abort);
await reader?.cancel().catch(() => { });
controller.abort();
}
}
async writeTo(destination, signal) {
let size = 0;
for await (const chunk of this.chunks(signal)) {
await destination.write(chunk);
size += chunk.length;
}
return size;
}
}
export class AudioPlayer extends EventTarget {
element;
media;
pending = new Map();
queue = [];
active;
preparation;
retry;
blocked = false;
constructor(element) {
super();
if (!element && typeof Audio === "undefined")
throw new Error("Provide a media output on this platform; headless clients can consume RealtimeAudio.chunks()");
this.element = element ?? new Audio();
this.element.crossOrigin = "anonymous";
const streaming = audioFormats.filter(format => typeof MediaSource !== "undefined" && MediaSource.isTypeSupported(format));
const native = ["audio/mpeg", "audio/ogg;codecs=opus", "audio/webm;codecs=opus"].filter(format => this.element.canPlayType(format));
this.media = new RealtimeMedia(streaming.length ? streaming : native);
}
get needsUserGesture() { return this.blocked; }
prepare() {
if (this.active)
return this.resume();
if (this.preparation)
return this.preparation.promise;
const bytes = new Uint8Array(844), view = new DataView(bytes.buffer);
const text = (at, value) => { for (let i = 0; i < value.length; i++)
bytes[at + i] = value.charCodeAt(i); };
text(0, "RIFF");
view.setUint32(4, 836, true);
text(8, "WAVEfmt ");
view.setUint32(16, 16, true);
view.setUint16(20, 1, true);
view.setUint16(22, 1, true);
view.setUint32(24, 8000, true);
view.setUint32(28, 16000, true);
view.setUint16(32, 2, true);
view.setUint16(34, 16, true);
text(36, "data");
view.setUint32(40, 800, true);
const uri = URL.createObjectURL(new Blob([bytes], { type: "audio/wav" }));
let resolve, reject;
const promise = new Promise((yes, no) => { resolve = yes; reject = no; });
const operation = { promise, cancel: (error) => finish(error) };
const finish = (error) => {
if (this.preparation !== operation)
return;
this.preparation = undefined;
this.element.pause();
URL.revokeObjectURL(uri);
if (error)
reject(error);
else
resolve();
this.advance();
};
this.preparation = operation;
this.element.src = uri;
try {
void this.element.play().then(() => finish(), error => finish(error));
}
catch (error) {
finish(error);
}
return promise;
}
resume() { return this.retry?.() ?? Promise.resolve(); }
play(audio) {
const previous = this.pending.get(audio.streamId);
if (previous)
return previous;
if (this.pending.size >= 1024)
return Promise.reject(new Error("Create a new player for the next conversation"));
const promise = new Promise((resolve, reject) => this.queue.push({ audio, resolve, reject }));
this.pending.set(audio.streamId, promise);
this.advance();
return promise;
}
advance() {
if (this.preparation || this.active || !this.queue.length)
return;
const next = this.queue.shift();
this.active = next;
const controller = new AbortController();
let objectUrl;
let source;
const cleanup = () => {
controller.abort();
source?.removeEventListener("sourceopen", sourceOpen);
if (objectUrl) {
URL.revokeObjectURL(objectUrl);
objectUrl = undefined;
}
this.element.removeEventListener("ended", ended);
this.element.removeEventListener("error", mediaFailed);
this.element.removeEventListener("playing", playing);
if (this.active === next) {
this.active = undefined;
this.retry = undefined;
this.blocked = false;
}
};
const sourceOpen = () => {
const load = async () => {
const buffer = source.addSourceBuffer(next.audio.mimeType);
for await (const chunk of next.audio.chunks(controller.signal)) {
await new Promise((resolve, reject) => {
const detach = () => { buffer.removeEventListener("updateend", done); buffer.removeEventListener("error", error); controller.signal.removeEventListener("abort", error); };
const done = () => { detach(); resolve(); };
const error = () => { detach(); reject(new Error("Audio delivery stopped")); };
if (controller.signal.aborted) {
error();
return;
}
buffer.addEventListener("updateend", done, { once: true });
buffer.addEventListener("error", error, { once: true });
controller.signal.addEventListener("abort", error, { once: true });
try {
buffer.appendBuffer(chunk.slice().buffer);
}
catch (failure) {
detach();
reject(failure);
}
});
}
if (source?.readyState === "open" && !controller.signal.aborted)
source.endOfStream();
};
void load().catch(error => { if (this.active === next)
failed(error); });
};
const ended = () => { next.resolve(); this.advance(); };
const failed = (error) => { next.reject(error); this.advance(); };
const mediaFailed = () => failed(new DOMException(this.element.error?.message || "Audio playback failed", "MediaPlaybackError"));
const playing = () => { if (this.active === next) {
this.blocked = false;
this.dispatchEvent(new Event("playing"));
} };
const resolve = next.resolve, reject = next.reject;
next.resolve = () => { cleanup(); resolve(); };
next.reject = error => { cleanup(); reject(error); };
if (next.audio.expiresAt * 1000 <= Date.now()) {
failed(new Error("The audio stream has expired"));
return;
}
if (!this.media.audioFormats.includes(next.audio.mimeType)) {
failed(new DOMException("Unsupported audio format", "NotSupportedError"));
return;
}
this.element.addEventListener("ended", ended, { once: true });
this.element.addEventListener("error", mediaFailed, { once: true });
this.element.addEventListener("playing", playing);
if (typeof MediaSource !== "undefined" && MediaSource.isTypeSupported(next.audio.mimeType)) {
source = new MediaSource();
source.addEventListener("sourceopen", sourceOpen, { once: true });
objectUrl = URL.createObjectURL(source);
this.element.src = objectUrl;
}
else {
this.element.src = next.audio.uri;
}
this.retry = () => {
let started;
try {
started = this.element.play();
}
catch (error) {
started = Promise.reject(error);
}
return started.catch(error => {
if (this.active !== next)
return;
if (error?.name === "NotAllowedError") {
this.blocked = true;
this.dispatchEvent(new Event("playbackblocked"));
}
else {
failed(error);
}
throw error;
});
};
void this.retry().catch(() => { });
}
stop() {
const error = new DOMException("Audio playback stopped", "AbortError");
for (const item of this.queue.splice(0))
item.reject(error);
this.preparation?.cancel(error);
this.active?.reject(error);
this.active = undefined;
this.retry = undefined;
this.blocked = false;
this.element.pause();
this.element.removeAttribute("src");
this.element.load();
}
}
export class RealtimeWebRTC extends EventTarget {
audio;
peer;
channel;
input;
closed = false;
pending = [];
listening = false;
connected;
failed;
ready = new Promise((resolve, reject) => { this.connected = resolve; this.failed = reject; });
constructor(audio = document.createElement('audio')) {
super();
this.audio = audio;
audio.autoplay = true;
this.ready.catch(() => { });
}
get readyState() { return this.closed ? 3 : this.channel?.readyState === 'open' ? 1 : 0; }
addEventListener(type, listener, options) {
super.addEventListener(type, listener, options);
if (type === 'message' && !this.listening) {
this.listening = true;
queueMicrotask(() => { for (const event of this.pending.splice(0))
this.dispatchEvent(event); });
}
}
static async connect(endpoint, options = {}) {
const url = new URL(endpoint, globalThis.location?.href);
if (url.username || url.password || url.hash || (url.protocol !== 'https:' && !(url.protocol === 'http:' && ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname))))
throw new Error('WebRTC signaling requires HTTPS');
const peer = new RealtimeWebRTC(options.audio);
try {
const offer = await peer.offer(options.iceServers, options.microphone);
const response = await fetch(url, { method: 'POST', headers: { ...options.headers, 'Content-Type': 'application/json' }, body: JSON.stringify(offer), redirect: 'error', signal: AbortSignal.timeout(30000) });
if (!response.ok)
throw new Error('WebRTC signaling was rejected');
const reader = response.body.getReader();
const chunks = [];
let size = 0;
try {
while (true) {
const { done, value } = await reader.read();
if (done)
break;
size += value.length;
if (size > 131072)
throw new Error('WebRTC answer exceeds limit');
chunks.push(value);
}
}
finally {
await reader.cancel();
}
const body = new Uint8Array(size);
let position = 0;
for (const chunk of chunks) {
body.set(chunk, position);
position += chunk.length;
}
await peer.answer(JSON.parse(new TextDecoder().decode(body)));
await Promise.race([peer.ready, new Promise((_, reject) => { const timer = setTimeout(() => reject(new Error('WebRTC connection timed out')), 30000); peer.ready.finally(() => clearTimeout(timer)).catch(() => { }); })]);
return peer;
}
catch (error) {
peer.close();
throw error;
}
}
async offer(iceServers = [], microphone = false) {
if (this.peer || this.closed)
throw new Error('Peer already used');
const peer = this.peer = new RTCPeerConnection({ iceServers });
peer.ontrack = event => {
if (event.track.kind !== 'audio')
return;
this.audio.srcObject = new MediaStream([event.track]);
this.audio.play().catch(error => this.dispatchEvent(new CustomEvent('playbackblocked', { detail: error })));
};
peer.onconnectionstatechange = () => {
if (peer.connectionState === 'failed') {
this.dispatchEvent(new Event('error'));
this.close();
}
};
if (microphone) {
this.input = await navigator.mediaDevices.getUserMedia({ audio: true });
for (const track of this.input.getTracks())
peer.addTrack(track, this.input);
}
else
peer.addTransceiver('audio', { direction: 'recvonly' });
const channel = this.channel = peer.createDataChannel('marona', { ordered: true });
channel.onopen = () => { this.connected(); this.dispatchEvent(new Event('open')); };
channel.onclose = () => this.close();
channel.onmessage = event => {
if (typeof event.data !== 'string' || new TextEncoder().encode(event.data).length > 262144) {
this.dispatchEvent(new Event('error'));
this.close();
return;
}
if (this.listening)
this.dispatchEvent(new MessageEvent('message', { data: event.data }));
else if (this.pending.length < 256)
this.pending.push(new MessageEvent('message', { data: event.data }));
else
this.close();
};
const gathered = new Promise((resolve, reject) => {
const timer = setTimeout(() => reject(new Error('ICE gathering timed out')), 15000);
peer.addEventListener('icegatheringstatechange', () => { if (peer.iceGatheringState === 'complete') {
clearTimeout(timer);
resolve();
} });
});
await peer.setLocalDescription(await peer.createOffer());
await gathered;
return { type: 'offer', sdp: peer.localDescription.sdp };
}
async answer(value) {
if (value.type !== 'answer' || typeof value.sdp !== 'string' || value.sdp.length > 65536 || !value.sdp.includes('a=fingerprint:'))
throw new Error('Invalid WebRTC answer');
await this.peer.setRemoteDescription(value);
}
send(event) {
if (this.readyState !== 1 || new TextEncoder().encode(event).length > 262144 || this.channel.bufferedAmount > 1048576)
throw new Error('WebRTC connection is unavailable or backpressured');
this.channel.send(event);
}
async resume() { await this.audio.play(); }
close() {
if (this.closed)
return;
this.closed = true;
this.failed(new Error('WebRTC connection closed'));
this.input?.getTracks().forEach(track => track.stop());
this.channel?.close();
this.peer?.close();
this.audio.pause();
this.audio.srcObject = null;
this.pending = [];
this.dispatchEvent(new Event('close'));
}
}