use std::{
collections::HashMap,
time::{Duration, SystemTime, UNIX_EPOCH},
};
use base64::{
alphabet,
engine::{DecodePaddingMode, GeneralPurpose, GeneralPurposeConfig},
Engine,
};
use tokio::time::Instant;
pub const ENDPOINT_OVERRIDE_ENV: &str = "LK_TELEMETRY_ENDPOINT";
const MAX_TOKEN_LIFETIME: Duration = Duration::from_secs(366 * 24 * 60 * 60);
const CLOUD_SUFFIX: &str = ".livekit.cloud";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum Signal {
Logs,
Traces,
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct Target {
pub project: Option<String>,
pub logs: String,
pub traces: String,
pub token: Option<String>,
}
impl Target {
pub fn url(&self, signal: Signal) -> &str {
match signal {
Signal::Logs => &self.logs,
Signal::Traces => &self.traces,
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct Token {
raw: String,
grant: bool,
expires: Option<Expiry>,
id: u64,
}
impl From<&str> for Token {
fn from(raw: &str) -> Self {
let claims = raw
.split('.')
.nth(1)
.and_then(|payload| {
const URL_SAFE: GeneralPurpose = GeneralPurpose::new(
&alphabet::URL_SAFE,
GeneralPurposeConfig::new()
.with_decode_padding_mode(DecodePaddingMode::Indifferent),
);
URL_SAFE.decode(payload).ok()
})
.and_then(|json| serde_json::from_slice::<serde_json::Value>(&json).ok());
let grant = claims.as_ref().is_some_and(|c| {
let observability = &c["observability"];
observability["write"].as_bool() == Some(true)
|| observability["clientWrite"].as_bool() == Some(true)
});
let expires = claims.as_ref().and_then(|c| c.get("exp")).map(|exp| {
let now = unix_now_secs();
let wall_secs = exp.as_f64().filter(|exp| exp.is_finite()).unwrap_or(0.0);
let left = if wall_secs > now {
Duration::from_secs_f64((wall_secs - now).min(MAX_TOKEN_LIFETIME.as_secs_f64()))
} else {
Duration::ZERO
};
let monotonic = Instant::now().checked_add(left).unwrap_or_else(Instant::now);
Expiry { monotonic, wall_secs }
});
let id = {
use std::hash::{Hash, Hasher};
let mut hasher = std::collections::hash_map::DefaultHasher::new();
raw.hash(&mut hasher);
hasher.finish()
};
Self { raw: raw.to_owned(), grant, expires, id }
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
struct Expiry {
monotonic: Instant,
wall_secs: f64,
}
impl Expiry {
fn passed(&self) -> bool {
Instant::now() >= self.monotonic || unix_now_secs() >= self.wall_secs
}
}
fn unix_now_secs() -> f64 {
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs_f64();
#[cfg(test)]
let now = now + tests::WALL_OFFSET_SECS.with(|offset| offset.get());
now
}
impl Token {
fn expired(&self) -> bool {
self.expires.is_some_and(|at| at.passed())
}
fn usable(&self, refused: &Refusals) -> bool {
self.grant && !self.expired() && !refused.contains(self)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum Dead {
NotCloud,
NoGrant,
NotFound,
Disabled,
}
#[derive(Debug, Default)]
struct Project {
latest: Option<Token>,
consented: bool,
dead: Option<Dead>,
}
fn offer(slot: &mut Option<Token>, new: &Token, refused: &Refusals) {
if refused.contains(new) {
return;
}
let keep = slot.as_ref().is_some_and(|t| t.raw == new.raw || (!new.grant && t.usable(refused)));
if !keep {
*slot = Some(new.clone());
}
}
#[derive(Debug, Default)]
struct Refusals(HashMap<u64, Option<Expiry>>);
impl Refusals {
fn contains(&self, token: &Token) -> bool {
self.0.contains_key(&token.id)
}
fn add(&mut self, token: &Token) -> bool {
self.prune();
self.0.insert(token.id, token.expires).is_none()
}
fn prune(&mut self) {
let now = Instant::now();
self.0.retain(|_, expires| expires.is_none_or(|at| now < at.monotonic));
}
}
pub(crate) const PROCESS_OWNER: &str = "process";
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum Route {
Send(Target),
Wait,
Drop,
}
enum Resolved<'a> {
Override(&'a Target),
Send { host: &'a str, token: &'a Token },
Wait,
Drop,
}
#[derive(Debug, Default)]
pub(crate) struct Destinations {
projects: HashMap<String, Project>,
compacted_dead: bool,
sessions: HashMap<(String, String), Token>,
rooms: HashMap<String, String>,
refused: Refusals,
latest: Option<String>,
endpoint_override: Option<Target>,
}
impl Destinations {
pub fn new(endpoint_override: Option<&str>) -> Self {
Self {
endpoint_override: endpoint_override
.filter(|e| !e.is_empty())
.and_then(override_target),
..Self::default()
}
}
pub fn first_project(&self, owner: &str) -> Option<String> {
self.rooms.get(owner).cloned()
}
pub fn has_override(&self) -> bool {
self.endpoint_override.is_some()
}
pub fn set(&mut self, url: &str, token: &str, owner: &str) -> Option<String> {
let Server { key: host, cloud } = Server::parse(url)?;
if self.endpoint_override.is_some() {
return Some(host);
}
self.rooms.entry(owner.to_owned()).or_insert_with(|| host.clone());
let project = self.projects.entry(host.clone()).or_default();
if project.dead.is_none() && !cloud {
project.dead = Some(Dead::NotCloud);
log::warn!("{host} is not LiveKit Cloud: client telemetry stays on this device");
}
if matches!(project.dead, Some(Dead::NotCloud | Dead::Disabled)) {
return Some(host);
}
let token = Token::from(token);
if project.dead == Some(Dead::NotFound) {
log::debug!("new token for {host}: trying its ingest again");
project.dead = None;
}
if token.grant {
project.consented = true;
if project.dead == Some(Dead::NoGrant) {
project.dead = None;
}
} else if !project.consented && project.dead.is_none() {
project.dead = Some(Dead::NoGrant);
log::info!("the token for {host} has no observability grant: telemetry stays local");
}
offer(&mut project.latest, &token, &self.refused);
let key = (host.clone(), owner.to_owned());
let mut own = self.sessions.remove(&key);
offer(&mut own, &token, &self.refused);
self.sessions.extend(own.map(|own| (key, own)));
self.latest = Some(host.clone());
Some(host)
}
pub fn route(&self, host: Option<&str>, owner: Option<&str>) -> Route {
match self.resolve(host, owner) {
Resolved::Override(target) => Route::Send(target.clone()),
Resolved::Send { host, token } => Route::Send(Target {
project: Some(host.to_owned()),
logs: ingest_url(host, Signal::Logs),
traces: ingest_url(host, Signal::Traces),
token: Some(token.raw.clone()),
}),
Resolved::Wait => Route::Wait,
Resolved::Drop => Route::Drop,
}
}
fn resolve<'a>(&'a self, host: Option<&'a str>, owner: Option<&'a str>) -> Resolved<'a> {
if let Some(target) = &self.endpoint_override {
return Resolved::Override(target);
}
let room = owner.filter(|o| *o != PROCESS_OWNER);
let host = match (host, room) {
(Some(host), _) => host,
(None, Some(room)) => match self.rooms.get(room) {
Some(first) => first.as_str(),
None => return Resolved::Wait,
},
(None, None) => match self.fallback() {
Some(host) => host,
None if self.all_dead() => return Resolved::Drop,
None => return Resolved::Wait,
},
};
let Some(project) = self.projects.get(host) else { return Resolved::Wait };
if project.dead.is_some() {
return Resolved::Drop;
}
let token = match room {
Some(room) if self.rooms.contains_key(room) => {
self.sessions.get(&(host.to_owned(), room.to_owned()))
}
_ => project.latest.as_ref(),
};
match token {
Some(token) if token.usable(&self.refused) => Resolved::Send { host, token },
_ => Resolved::Wait,
}
}
fn fallback(&self) -> Option<&str> {
let alive = |host: &&String| self.projects.get(*host).is_some_and(|p| p.dead.is_none());
self.latest
.as_ref()
.filter(alive)
.or_else(|| self.projects.keys().find(alive))
.map(String::as_str)
}
fn all_dead(&self) -> bool {
(self.compacted_dead || !self.projects.is_empty())
&& self.projects.values().all(|p| p.dead.is_some())
}
pub fn alive(&self, host: Option<&str>, owner: Option<&str>) -> bool {
!matches!(self.resolve(host, owner), Resolved::Drop)
}
pub fn refuse(&mut self, token: &str) {
if self.refused.add(&Token::from(token)) {
log::warn!("the collector refused a token; waiting for a new one");
}
}
pub fn retain_owners(
&mut self,
live: &HashMap<String, Option<String>>,
backlog: &std::collections::HashSet<(Option<String>, String)>,
) {
let backlog: std::collections::HashSet<(Option<String>, String)> = backlog
.iter()
.map(|(host, owner)| match (host, self.rooms.get(owner)) {
(None, Some(first)) if owner != PROCESS_OWNER => {
(Some(first.clone()), owner.clone())
}
_ => (host.clone(), owner.clone()),
})
.collect();
let backed =
|host: &str, owner: &str| backlog.contains(&(Some(host.to_owned()), owner.to_owned()));
self.sessions.retain(|(host, owner), _| backed(host, owner) || live.contains_key(owner));
self.rooms
.retain(|owner, _| live.contains_key(owner) || backlog.iter().any(|(_, o)| o == owner));
let unrouted = backlog.iter().any(|(host, _)| host.is_none());
let latest = self.latest.clone();
let held: std::collections::HashSet<String> = self
.sessions
.keys()
.map(|(host, _)| host)
.chain(self.rooms.values())
.cloned()
.collect();
self.projects.retain(|host, project| {
let needed = live.values().any(|route| route.as_deref() == Some(host))
|| backlog.iter().any(|(h, _)| h.as_deref() == Some(host))
|| (unrouted && latest.as_deref() == Some(host))
|| held.contains(host);
if !needed {
project.latest = None;
}
if needed {
return true;
}
match project.dead {
Some(Dead::Disabled) => true,
Some(Dead::NotFound) if project.consented => true,
Some(_) => {
self.compacted_dead = true;
false
}
None => false,
}
});
if self.latest.as_ref().is_some_and(|host| !self.projects.contains_key(host)) {
self.latest = None;
}
self.projects.shrink_to(self.projects.len().saturating_mul(2));
self.sessions.shrink_to(self.sessions.len().saturating_mul(2));
self.rooms.shrink_to(self.rooms.len().saturating_mul(2));
}
#[cfg(test)]
pub fn credentials(&self) -> usize {
self.sessions.len() + self.projects.values().filter(|p| p.latest.is_some()).count()
}
pub fn revive(&mut self, host: &str) -> bool {
match self.projects.get_mut(host) {
Some(project) if project.dead == Some(Dead::NotFound) => {
project.dead = None;
true
}
_ => false,
}
}
pub fn kill(&mut self, host: &str, why: Dead) {
let project = self.projects.entry(host.to_owned()).or_default();
if project.dead.is_none() {
match why {
Dead::NotFound => log::warn!("{host} has no client telemetry ingest; going silent"),
Dead::Disabled => log::warn!("{host} disabled data recording; going silent"),
Dead::NotCloud | Dead::NoGrant => {}
}
project.dead = Some(why);
}
}
}
struct Server {
key: String,
cloud: bool,
}
impl Server {
fn parse(url: &str) -> Option<Self> {
let url = url::Url::parse(url).ok()?;
let host = url.host_str()?;
let host = host.strip_suffix('.').unwrap_or(host).to_ascii_lowercase();
if host.is_empty() {
return None;
}
let key = match url.port() {
Some(port) => format!("{host}:{port}"),
None => host.clone(),
};
let cloud = matches!(url.scheme(), "wss" | "https")
&& matches!(url.host(), Some(url::Host::Domain(_)))
&& host.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'.')
&& host.strip_suffix(CLOUD_SUFFIX).is_some_and(|label| !label.is_empty())
&& url.port().is_none()
&& url.username().is_empty()
&& url.password().is_none();
let key = if cloud { key } else { format!("{}+{}", url.scheme(), file_safe(&key)) };
Some(Self { key, cloud })
}
}
fn file_safe(text: &str) -> String {
let mut safe = String::with_capacity(text.len());
for byte in text.bytes() {
match byte {
b'a'..=b'z' | b'0'..=b'9' | b'.' | b'-' | b'_' => safe.push(byte as char),
_ => safe.push_str(&format!("%{byte:02X}")),
}
}
safe
}
fn ingest_url(host: &str, signal: Signal) -> String {
let signal = match signal {
Signal::Logs => "logs",
Signal::Traces => "traces",
};
format!("https://{host}/observability/client/{signal}/otlp/v0")
}
fn override_target(endpoint: &str) -> Option<Target> {
let endpoint = endpoint.trim_end_matches('/');
let url = url::Url::parse(endpoint).ok().filter(|u| matches!(u.scheme(), "http" | "https"));
let Some(mut url) = url else {
log::warn!("{ENDPOINT_OVERRIDE_ENV} is not an HTTP URL: ignoring it");
return None;
};
let traces = url.path_segments().and_then(|segments| {
let mut segments: Vec<String> = segments.map(str::to_owned).collect();
let logs = segments.iter().rposition(|s| s == "logs")?;
segments[logs] = "traces".to_owned();
Some(segments.join("/"))
});
let (logs, traces) = match traces {
Some(path) => {
url.set_path(&path);
(endpoint.to_owned(), url.to_string())
}
None => (format!("{endpoint}/v1/logs"), format!("{endpoint}/v1/traces")),
};
Some(Target { project: None, logs, traces, token: None })
}
#[cfg(test)]
pub(crate) mod tests {
use super::*;
pub(crate) fn jwt(claims: serde_json::Value) -> String {
let engine = base64::engine::general_purpose::URL_SAFE_NO_PAD;
format!(
"{}.{}.sig",
engine.encode(br#"{"alg":"HS256"}"#),
engine.encode(claims.to_string())
)
}
pub(crate) fn granted(ttl_secs: u64) -> String {
let exp = SystemTime::now().duration_since(UNIX_EPOCH).expect("clock").as_secs() + ttl_secs;
jwt(serde_json::json!({ "exp": exp, "observability": { "write": true } }))
}
pub(crate) fn grantless(ttl_secs: u64) -> String {
let exp = SystemTime::now().duration_since(UNIX_EPOCH).expect("clock").as_secs() + ttl_secs;
jwt(serde_json::json!({ "exp": exp, "video": { "roomJoin": true } }))
}
fn send(route: Route) -> Target {
match route {
Route::Send(target) => target,
other => panic!("expected a target, got {other:?}"),
}
}
#[test]
fn only_livekit_cloud_hosts_get_an_ingest_url() {
let mut d = Destinations::default();
let token = granted(600);
let host = d.set("wss://Proj.LiveKit.Cloud/rtc?access_token=x#f", &token, "s");
assert_eq!(host.as_deref(), Some("proj.livekit.cloud"));
let target = send(d.route(host.as_deref(), Some("s")));
assert_eq!(target.logs, "https://proj.livekit.cloud/observability/client/logs/otlp/v0");
assert_eq!(target.traces, "https://proj.livekit.cloud/observability/client/traces/otlp/v0");
assert_eq!(target.token.as_deref(), Some(token.as_str()));
let staging = d.set("https://p.staging.livekit.cloud", &token, "s");
assert!(matches!(d.route(staging.as_deref(), None), Route::Send(_)), "staging is Cloud");
for url in
["ws://localhost:7880", "wss://livekit.example.com", "wss://livekit.cloud.evil.io"]
{
let host = d.set(url, &token, "s");
assert_eq!(d.route(host.as_deref(), None), Route::Drop, "{url}: no ingest");
}
assert_eq!(d.set("nonsense", &token, "s"), None);
let empty = d.set("wss:///rtc", &token, "s"); assert_eq!(d.route(empty.as_deref(), None), Route::Drop);
}
#[test]
fn look_alike_server_urls_never_get_the_token() {
let token = granted(600);
for url in [
"wss://evil.example\\.livekit.cloud",
"wss://evil.example\\x.livekit.cloud/rtc",
"wss://user:pass@p.livekit.cloud",
"wss://evil.example@p.livekit.cloud",
"wss://p.livekit.cloud:8443",
"ws://p.livekit.cloud",
"http://p.livekit.cloud",
"wss://livekit.cloud",
"wss://.livekit.cloud",
"wss://p.livekit.cloud.evil.io",
"wss://p.livekit.cloud%2eevil.io",
"wss://evil.io#.livekit.cloud",
"wss://evil.io?.livekit.cloud",
"wss://[::1]",
"wss://1.2.3.4",
"wss://p.livekit.cloud\u{0}.evil",
] {
let mut d = Destinations::default();
let host = d.set(url, &token, "s");
let route = d.route(host.as_deref(), Some("s"));
assert!(!matches!(route, Route::Send(_)), "{url} → {host:?} must not get the token");
}
let mut d = Destinations::default();
let host = d.set("WSS://P.LiveKit.Cloud:443/rtc?x=1#y", &token, "s");
let target = send(d.route(host.as_deref(), Some("s")));
assert_eq!(target.logs, "https://p.livekit.cloud/observability/client/logs/otlp/v0");
}
#[test]
fn a_trailing_dot_names_the_same_cloud_project() {
let mut d = Destinations::default();
let token = granted(600);
let host = d.set("wss://p.livekit.cloud./rtc", &token, "s");
assert_eq!(host.as_deref(), Some("p.livekit.cloud"));
let target = send(d.route(host.as_deref(), Some("s")));
assert_eq!(target.logs, "https://p.livekit.cloud/observability/client/logs/otlp/v0");
assert_eq!(d.set("wss://p.livekit.cloud/rtc", &token, "s"), host, "one key");
for url in ["wss://livekit.cloud.", "wss://.livekit.cloud.", "wss://p.livekit.cloud.."] {
let mut d = Destinations::default();
let host = d.set(url, &token, "s");
assert_ne!(host.as_deref(), Some("p.livekit.cloud"), "{url}");
assert!(!matches!(d.route(host.as_deref(), Some("s")), Route::Send(_)), "{url}");
}
}
#[test]
fn a_token_without_the_grant_is_not_consent() {
let mut d = Destinations::default();
let host = d.set("wss://p.livekit.cloud", &grantless(600), "s");
assert_eq!(d.route(host.as_deref(), Some("s")), Route::Drop, "never opted in");
assert_eq!(d.route(None, None), Route::Drop, "and no project takes process data");
let host = d.set("wss://p.livekit.cloud", &granted(600), "s");
assert!(matches!(d.route(host.as_deref(), Some("s")), Route::Send(_)), "a grant opts in");
}
#[tokio::test(start_paused = true)]
async fn a_refresh_that_drops_the_grant_keeps_the_granted_token_until_it_expires() {
let mut d = Destinations::default();
let join = granted(60);
let host = d.set("wss://p.livekit.cloud", &join, "s");
d.set("wss://p.livekit.cloud", &grantless(600), "s");
assert_eq!(send(d.route(host.as_deref(), Some("s"))).token, Some(join), "still granted");
tokio::time::advance(Duration::from_secs(61)).await;
assert_eq!(d.route(host.as_deref(), Some("s")), Route::Wait, "expired: hold, never drop");
let refreshed = granted(600);
d.set("wss://p.livekit.cloud", &refreshed, "s");
assert_eq!(send(d.route(host.as_deref(), Some("s"))).token, Some(refreshed));
}
#[test]
fn a_refused_token_is_never_sent_again() {
let mut d = Destinations::default();
let first = granted(600);
let host = d.set("wss://p.livekit.cloud", &first, "s").expect("host");
d.refuse(&first);
assert_eq!(d.route(Some(&host), Some("s")), Route::Wait);
d.set("wss://p.livekit.cloud", &first, "s");
assert_eq!(d.route(Some(&host), Some("s")), Route::Wait, "handing it over again: still no");
d.set("wss://p.livekit.cloud", &granted(900), "s");
assert!(matches!(d.route(Some(&host), Some("s")), Route::Send(_)));
}
thread_local! {
pub(super) static WALL_OFFSET_SECS: std::cell::Cell<f64> = const { std::cell::Cell::new(0.0) };
}
#[test]
fn a_refused_token_stays_refused_across_a_wall_clock_jump() {
let mut d = Destinations::default();
let token = granted(600);
let host = d.set("wss://p.livekit.cloud", &token, "s").expect("host");
assert!(matches!(d.route(Some(&host), Some("s")), Route::Send(_)));
let live: HashMap<String, Option<String>> = [("s".to_owned(), Some(host.clone()))].into();
WALL_OFFSET_SECS.with(|offset| offset.set(1_200.0));
d.refuse(&token);
d.retain_owners(&live, &Default::default());
WALL_OFFSET_SECS.with(|offset| offset.set(0.0));
assert_eq!(d.route(Some(&host), Some("s")), Route::Wait, "refused, whatever the clock did");
let mut d = Destinations::default();
let host = d.set("wss://p.livekit.cloud", &token, "s").expect("host");
d.refuse(&token);
d.retain_owners(&live, &Default::default());
assert_eq!(d.route(Some(&host), Some("s")), Route::Wait);
}
#[tokio::test(start_paused = true)]
async fn a_refusal_outlives_the_room_that_held_the_token_across_a_wall_clock_jump() {
let mut d = Destinations::default();
let token = granted(600);
let host = d.set("wss://p.livekit.cloud", &token, "s").expect("host");
WALL_OFFSET_SECS.with(|offset| offset.set(1_200.0));
d.refuse(&token);
d.retain_owners(&Default::default(), &Default::default());
WALL_OFFSET_SECS.with(|offset| offset.set(0.0));
d.set("wss://p.livekit.cloud", &token, "t");
assert_eq!(d.route(Some(&host), Some("t")), Route::Wait, "the same token stays refused");
assert!(d.refused.contains(&Token::from(token.as_str())));
let mut d = Destinations::default();
d.set("wss://p.livekit.cloud", &token, "s");
WALL_OFFSET_SECS.with(|offset| offset.set(1_200.0));
d.refuse(&token);
WALL_OFFSET_SECS.with(|offset| offset.set(0.0));
d.retain_owners(&Default::default(), &Default::default());
assert!(d.refused.contains(&Token::from(token.as_str())), "kept until the deadline");
tokio::time::advance(Duration::from_secs(700)).await;
d.retain_owners(&Default::default(), &Default::default());
assert!(d.refused.contains(&Token::from(token.as_str())), "cleanup forgets nothing");
d.refuse(&granted(60));
assert!(!d.refused.contains(&Token::from(token.as_str())), "the next refusal does");
}
#[tokio::test(start_paused = true)]
async fn a_refused_token_held_by_a_live_room_stays_refused_past_its_monotonic_deadline() {
let mut d = Destinations::default();
let token = granted(600);
let host = d.set("wss://p.livekit.cloud", &token, "s").expect("host");
d.refuse(&token);
let live: HashMap<String, Option<String>> = [("s".to_owned(), Some(host.clone()))].into();
tokio::time::advance(Duration::from_secs(601)).await;
d.retain_owners(&live, &Default::default());
d.set("wss://p.livekit.cloud", &token, "t");
assert_eq!(d.route(Some(&host), Some("t")), Route::Wait, "the same token stays refused");
}
#[tokio::test(start_paused = true)]
async fn a_refused_token_stays_refused_after_its_holder_retired_and_its_deadline_passed() {
let mut d = Destinations::default();
let token = granted(600);
let host = d.set("wss://p.livekit.cloud", &token, "s").expect("host");
d.refuse(&token);
d.retain_owners(&Default::default(), &Default::default());
tokio::time::advance(Duration::from_secs(601)).await;
d.retain_owners(&Default::default(), &Default::default());
d.set("wss://p.livekit.cloud", &token, "t");
assert_eq!(d.route(Some(&host), Some("t")), Route::Wait, "the same token stays refused");
d.refuse(&granted(60));
assert!(!d.refused.contains(&Token::from(token.as_str())), "forgotten at the next refusal");
}
#[test]
fn rooms_never_borrow_each_others_tokens() {
let mut d = Destinations::default();
let (a, b, c) = (granted(600), granted(700), granted(800));
let host_a = d.set("wss://a.livekit.cloud", &a, "room-a");
let host_b = d.set("wss://b.livekit.cloud", &b, "room-b");
d.set("wss://a.livekit.cloud", &c, "room-c"); let target = send(d.route(host_a.as_deref(), Some("room-a")));
assert!(target.logs.starts_with("https://a.") && target.token == Some(a.clone()));
let target = send(d.route(host_a.as_deref(), Some("room-c")));
assert_eq!(target.token, Some(c.clone()), "same project, its own token");
let target = send(d.route(host_b.as_deref(), Some("room-b")));
assert!(target.logs.starts_with("https://b.") && target.token == Some(b));
d.refuse(&a);
assert_eq!(d.route(host_a.as_deref(), Some("room-a")), Route::Wait, "no borrowing c");
let previous_launch = send(d.route(host_a.as_deref(), Some("gone")));
assert_eq!(previous_launch.token, Some(c), "a previous launch's batch: project's latest");
d.kill("b.livekit.cloud", Dead::Disabled);
assert!(send(d.route(None, None)).logs.starts_with("https://a."), "process: a live one");
}
#[test]
fn ownership_survives_a_room_changing_projects() {
let mut d = Destinations::default();
let (a, b) = (granted(600), granted(700));
let host_a = d.set("wss://a.livekit.cloud", &a, "room").expect("a");
d.set("wss://b.livekit.cloud", &b, "room");
let target = send(d.route(Some(&host_a), Some("room")));
assert!(target.logs.starts_with("https://a.") && target.token == Some(a));
d.set("wss://c.livekit.cloud", &granted(800), "other");
assert_eq!(d.route(None, Some("unconnected")), Route::Wait, "no borrowing c");
let early = send(d.route(None, Some("room")));
assert!(early.logs.starts_with("https://a."), "a Room's first project");
assert!(matches!(d.route(None, Some(PROCESS_OWNER)), Route::Send(_)));
}
#[test]
fn refusals_stick_and_credentials_follow_live_rooms() {
let mut d = Destinations::default();
let tokens: Vec<String> = (0..40).map(|n| granted(600 + n)).collect();
for (n, token) in tokens.iter().enumerate() {
d.set("wss://p.livekit.cloud", token, &format!("room-{n}"));
d.refuse(token);
}
for (n, token) in tokens.iter().enumerate() {
let room = format!("room-{n}");
d.set("wss://p.livekit.cloud", token, &room); assert_eq!(d.route(Some("p.livekit.cloud"), Some(&room)), Route::Wait, "{n}");
d.set("wss://p.livekit.cloud", token, &format!("new-{n}")); assert_eq!(d.route(Some("p.livekit.cloud"), Some(&format!("new-{n}"))), Route::Wait);
assert_eq!(d.route(None, Some(PROCESS_OWNER)), Route::Wait, "{n}: process route");
assert_eq!(d.route(Some("p.livekit.cloud"), Some("gone")), Route::Wait, "{n}: restart");
}
let (a, b) = (granted(900), granted(901));
d.set("wss://q.livekit.cloud", &a, "ra");
d.refuse(&a);
d.set("wss://q.livekit.cloud", &b, "rb");
d.set("wss://q.livekit.cloud", &a, "ra");
let process = send(d.route(Some("q.livekit.cloud"), Some("gone")));
assert_eq!(process.token, Some(b), "the project copy is b, never the refused a");
for n in 0..50 {
d.set(&format!("wss://c{n}.livekit.cloud"), &granted(1000 + n), &format!("churn-{n}"));
}
let live: HashMap<String, Option<String>> =
[("rb".to_owned(), Some("q.livekit.cloud".to_owned()))].into();
d.retain_owners(&live, &Default::default());
assert_eq!(d.credentials(), 2, "rb's token and q's project copy, nothing else");
let backlog = [(Some("q.livekit.cloud".to_owned()), "ra".to_owned())].into();
d.set("wss://q.livekit.cloud", &granted(902), "ra");
d.retain_owners(&Default::default(), &backlog);
assert_eq!(d.credentials(), 2, "a gone Room's backlog keeps its credential and q's copy");
}
#[test]
fn not_found_recovers_with_the_next_token_disabled_does_not() {
let mut d = Destinations::default();
let host = d.set("wss://p.livekit.cloud", &granted(600), "s").expect("host");
d.kill(&host, Dead::NotFound);
assert_eq!(d.route(Some(&host), Some("s")), Route::Drop);
d.set("wss://p.livekit.cloud", &granted(700), "s");
assert!(matches!(d.route(Some(&host), Some("s")), Route::Send(_)), "a new connect retries");
d.kill(&host, Dead::Disabled);
d.set("wss://p.livekit.cloud", &granted(800), "s");
assert_eq!(d.route(Some(&host), Some("s")), Route::Drop, "the owner's switch stands");
}
#[test]
fn the_override_takes_everything_without_a_token() {
let d = Destinations::new(Some("http://localhost:4318/"));
let target = send(d.route(None, None));
assert_eq!(target.logs, "http://localhost:4318/v1/logs");
assert_eq!(target.traces, "http://localhost:4318/v1/traces");
assert_eq!(target.token, None);
let full = Destinations::new(Some("https://h/observability/client/logs/otlp/v0"));
assert_eq!(
send(full.route(Some("ignored"), None)).traces,
"https://h/observability/client/traces/otlp/v0"
);
let mut local = Destinations::new(Some("http://localhost:4318"));
let host = local.set("ws://localhost:7880", "not-a-jwt", "s");
assert!(matches!(local.route(host.as_deref(), Some("s")), Route::Send(_)));
}
#[test]
fn an_override_that_is_not_an_http_url_is_ignored() {
crate::test_log::capture();
for endpoint in ["not a url", "localhost:4318", "ftp://collector/v1/logs", "/v1/logs"] {
let d = Destinations::new(Some(endpoint));
assert!(!d.has_override(), "{endpoint}");
assert_eq!(d.route(None, None), Route::Wait, "{endpoint}: no collector, no override");
let lines = crate::test_log::take();
assert!(
lines.iter().any(|l| l.contains(ENDPOINT_OVERRIDE_ENV)),
"{endpoint}: {lines:?}"
);
}
}
#[test]
fn the_override_swaps_only_a_whole_logs_segment() {
let targets = |endpoint| {
let target = send(Destinations::new(Some(endpoint)).route(None, None));
(target.logs, target.traces)
};
assert_eq!(
targets("http://logs-collector:4318"),
(
"http://logs-collector:4318/v1/logs".to_owned(),
"http://logs-collector:4318/v1/traces".to_owned()
)
);
assert_eq!(
targets("http://collector:4318/catalogs"),
(
"http://collector:4318/catalogs/v1/logs".to_owned(),
"http://collector:4318/catalogs/v1/traces".to_owned()
)
);
}
#[test]
fn non_cloud_urls_never_share_a_routing_key_with_cloud() {
let cloud = "p.livekit.cloud";
let mut d = Destinations::default();
d.set("wss://p.livekit.cloud", &grantless(600), "a");
let plain = d.set("ws://p.livekit.cloud", &granted(600), "b");
assert_ne!(plain.as_deref(), Some(cloud));
assert_eq!(d.route(Some(cloud), Some("b")), Route::Drop, "still never opted in");
assert_eq!(d.route(plain.as_deref(), Some("b")), Route::Drop, "not Cloud");
let mut d = Destinations::default();
d.set("wss://p.livekit.cloud", &granted(600), "a");
d.kill(cloud, Dead::NotFound);
d.set("ws://p.livekit.cloud", &granted(700), "b");
assert_eq!(d.route(Some(cloud), Some("a")), Route::Drop, "only a Cloud connect revives");
for first in ["ws://p.livekit.cloud", "wss://user@p.livekit.cloud"] {
let mut d = Destinations::default();
d.set(first, &granted(600), "a");
let host = d.set("wss://p.livekit.cloud", &granted(600), "b");
assert!(matches!(d.route(host.as_deref(), Some("b")), Route::Send(_)), "{first}");
}
}
#[test]
fn secure_then_plaintext_keeps_cloud_credentials_isolated() {
let mut d = Destinations::default();
let secure_token = granted(600);
let cloud = d.set("wss://p.livekit.cloud", &secure_token, "secure").unwrap();
for (n, scheme) in ["ws", "http"].into_iter().enumerate() {
let owner = format!("plain-{n}");
let plain = d.set(&format!("{scheme}://p.livekit.cloud"), &granted(700), &owner);
assert_ne!(plain.as_deref(), Some(cloud.as_str()));
assert_eq!(d.route(plain.as_deref(), Some(&owner)), Route::Drop);
assert_eq!(d.route(None, Some(&owner)), Route::Drop);
assert_eq!(
send(d.route(Some(&cloud), Some("secure"))).token.as_deref(),
Some(secure_token.as_str())
);
assert_eq!(send(d.route(None, None)).token.as_deref(), Some(secure_token.as_str()));
}
let control = granted(800);
d.set("https://p.livekit.cloud", &control, "second-secure");
assert_eq!(send(d.route(Some(&cloud), Some("second-secure"))).token, Some(control));
assert_eq!(send(d.route(Some(&cloud), Some("secure"))).token, Some(secure_token));
}
#[test]
fn a_room_without_a_project_waits_for_its_first_one() {
let mut d = Destinations::default();
d.set("ws://localhost:7880", &granted(600), "a");
assert!(!d.alive(None, None), "process data: only dead projects so far");
assert!(d.alive(None, Some("b")), "a Room without a server waits");
}
#[test]
fn the_project_table_follows_the_live_sessions() {
let mut d = Destinations::default();
for n in 0..1000 {
d.set(&format!("wss://p{n}.livekit.cloud"), &granted(600), &format!("room-{n}"));
}
d.kill("p0.livekit.cloud", Dead::Disabled);
d.retain_owners(&Default::default(), &Default::default());
assert_eq!(d.projects.len(), 1, "only the owner's switch stays");
let route = d.route(Some("p0.livekit.cloud"), Some("room-0"));
assert_eq!(route, Route::Drop, "and keeps the project silent");
}
#[test]
fn routing_keys_are_file_names_on_every_platform() {
let refused = |c: char| c.is_control() || r#"<>:"/\|?*"#.contains(c);
let urls = [
"ws://localhost:7880",
"wss://[::1]:7880",
"wss://a*b\"c.example",
"wss://x*y.livekit.cloud",
"https://p.livekit.cloud:8443",
"wss://p.livekit.cloud",
];
let mut d = Destinations::default();
let keys: Vec<String> =
urls.iter().map(|url| d.set(url, &granted(600), "s").expect(url)).collect();
assert!(keys.iter().all(|key| !key.contains(refused)), "{keys:?}");
let distinct: std::collections::HashSet<&String> = keys.iter().collect();
assert_eq!(distinct.len(), keys.len(), "distinct servers keep distinct keys: {keys:?}");
assert_eq!(keys[5], "p.livekit.cloud", "a Cloud project is keyed by its host");
}
#[test]
fn early_records_belong_to_a_first_project_that_is_not_cloud() {
let mut d = Destinations::default();
let token = granted(600);
let first = d.set("ws://localhost:7880", &token, "room").expect("a host");
d.set("wss://later.livekit.cloud", &token, "room");
assert_eq!(d.first_project("room"), Some(first));
assert_eq!(d.route(None, Some("room")), Route::Drop, "dropped with that project");
}
#[test]
fn early_records_belong_to_a_first_project_that_is_disabled() {
let mut d = Destinations::default();
let token = granted(600);
d.set("wss://disabled.livekit.cloud", &token, "other");
d.kill("disabled.livekit.cloud", Dead::Disabled);
d.set("wss://disabled.livekit.cloud", &token, "room");
d.set("wss://later.livekit.cloud", &token, "room");
assert_eq!(d.first_project("room").as_deref(), Some("disabled.livekit.cloud"));
assert_eq!(d.route(None, Some("room")), Route::Drop, "dropped with that project");
}
#[test]
fn a_dead_project_outlives_its_room() {
for (url, token) in
[("ws://localhost:7880", granted(600)), ("wss://p.livekit.cloud", grantless(600))]
{
let mut d = Destinations::default();
d.set(url, &token, "room");
d.retain_owners(&Default::default(), &Default::default());
assert_eq!(d.route(None, Some(PROCESS_OWNER)), Route::Drop, "{url}");
assert!(!d.alive(None, Some(PROCESS_OWNER)), "{url}: process data is not collected");
}
}
#[test]
fn dead_project_churn_is_compacted_without_reviving_process_capture() {
let mut d = Destinations::default();
for n in 0..1000 {
d.set(&format!("ws://host-{n}.example"), &granted(600), "room");
d.set(&format!("wss://p{n}.livekit.cloud"), &grantless(600), "room");
d.retain_owners(&Default::default(), &Default::default());
}
assert!(d.projects.is_empty(), "unowned recomputable dead projects must compact");
assert_eq!(d.projects.capacity(), 0);
assert!(d.rooms.is_empty() && d.sessions.is_empty());
assert_eq!((d.rooms.capacity(), d.sessions.capacity()), (0, 0));
assert_eq!(d.credentials(), 0);
assert!(d.latest.is_none(), "the last routing key must be released too");
assert_eq!(d.route(None, None), Route::Drop);
assert_eq!(d.route(None, Some("new-room")), Route::Wait);
let token = granted(700);
let host = d.set("wss://p0.livekit.cloud", &token, "new-room");
assert_eq!(send(d.route(host.as_deref(), Some("new-room"))).token, Some(token));
d.kill("p0.livekit.cloud", Dead::Disabled);
assert_eq!(d.route(None, None), Route::Drop, "previously alive is now dead");
d.retain_owners(&Default::default(), &Default::default());
assert_eq!(d.route(None, None), Route::Drop);
d.set("wss://p0.livekit.cloud", &granted(800), "late");
assert_eq!(d.route(None, Some("late")), Route::Drop, "Disabled stays authoritative");
}
#[test]
fn compaction_preserves_needed_dead_routes_and_late_grantless_reconnects() {
let mut d = Destinations::default();
let host = d.set("wss://p.livekit.cloud", &granted(600), "room").unwrap();
d.kill(&host, Dead::NotFound);
d.retain_owners(&Default::default(), &Default::default());
d.set("wss://p.livekit.cloud", &grantless(700), "late");
assert_eq!(d.route(Some(&host), Some("late")), Route::Wait, "consent is remembered");
let dead = d.set("wss://no.livekit.cloud", &grantless(600), "queued").unwrap();
let backlog = [(None, "queued".to_owned())].into();
d.retain_owners(&Default::default(), &backlog);
assert_eq!(d.first_project("queued"), Some(dead));
assert_eq!(d.route(None, Some("queued")), Route::Drop);
d.set("wss://alive.livekit.cloud", &granted(600), "alive");
assert!(matches!(d.route(None, None), Route::Send(_)));
d.retain_owners(&Default::default(), &Default::default());
assert_eq!(d.route(None, None), Route::Drop);
}
#[tokio::test(start_paused = true)]
async fn untrusted_expiry_claims_fail_closed() {
let now = SystemTime::now().duration_since(UNIX_EPOCH).expect("clock").as_secs_f64();
let token = |exp: serde_json::Value| {
jwt(serde_json::json!({ "exp": exp, "observability": { "write": true } }))
};
for exp in [
serde_json::json!(-1),
serde_json::json!(0),
serde_json::json!(now - 0.5),
serde_json::json!(-1e308),
serde_json::json!("tomorrow"),
serde_json::json!(true),
serde_json::json!(null),
serde_json::json!({}),
] {
let mut d = Destinations::default();
let host = d.set("wss://p.livekit.cloud", &token(exp.clone()), "s");
assert_eq!(d.route(host.as_deref(), Some("s")), Route::Wait, "exp {exp}: expired");
}
let mut d = Destinations::default();
let host = d.set("wss://p.livekit.cloud", &token(serde_json::json!(now + 60.9)), "s");
assert!(matches!(d.route(host.as_deref(), Some("s")), Route::Send(_)), "fractional future");
let host = d.set("wss://p.livekit.cloud", &token(serde_json::json!(1e300)), "s");
assert!(matches!(d.route(host.as_deref(), Some("s")), Route::Send(_)), "extreme: clamped");
}
#[test]
fn expiry_is_checked_on_the_wall_clock_too() {
let mut token = Token::from(granted(600).as_str());
assert!(!token.expired());
token.expires.as_mut().expect("exp").wall_secs = unix_now_secs() - 1.0;
assert!(token.expired(), "monotonic deadline ahead, wall-clock expiry passed");
let mut d = Destinations::default();
d.set("wss://p.livekit.cloud", &token.raw, "s");
d.sessions
.values_mut()
.chain(d.projects.values_mut().filter_map(|p| p.latest.as_mut()))
.for_each(|t| t.expires.as_mut().expect("exp").wall_secs = unix_now_secs() - 1.0);
assert_eq!(d.route(Some("p.livekit.cloud"), Some("s")), Route::Wait, "a hard hold");
}
#[test]
fn tokens_are_read_not_verified() {
assert!(Token::from(granted(60).as_str()).grant);
let client_write = jwt(serde_json::json!({ "observability": { "clientWrite": true } }));
let parsed = Token::from(client_write.as_str());
assert!(parsed.grant && parsed.expires.is_none(), "no exp: no known expiry");
assert!(!Token::from("garbage").grant);
assert!(!Token::from("a.!!!.c").grant);
}
}