1use std::{
16 collections::{hash_map::Entry, HashMap, HashSet},
17 sync::Arc,
18 time::Duration,
19};
20
21use tokio::time::Instant;
22
23use crate::{
24 event::now_unix_nanos, scope::ScopeState, store::QueuedEvent, AttributeValue, TelemetryEvent,
25};
26
27#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))]
29#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
30pub enum TrackKind {
31 Audio,
33 Video,
35}
36
37#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))]
39#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
40pub enum StreamDirection {
41 Inbound,
43 Outbound,
45}
46
47#[cfg_attr(feature = "uniffi", derive(uniffi::Record))]
57#[derive(Debug, Clone, PartialEq)]
58pub struct RtcStatsSample {
59 pub track_id: String,
61 pub kind: TrackKind,
63 pub direction: StreamDirection,
65 #[cfg_attr(feature = "uniffi", uniffi(default))]
67 pub codec: Option<String>,
68 #[cfg_attr(feature = "uniffi", uniffi(default))]
71 pub bytes: Option<u64>,
72 #[cfg_attr(feature = "uniffi", uniffi(default))]
74 pub packets: Option<u64>,
75 #[cfg_attr(feature = "uniffi", uniffi(default))]
77 pub packets_lost: Option<u64>,
78 #[cfg_attr(feature = "uniffi", uniffi(default))]
82 pub freeze_count: Option<u64>,
83 #[cfg_attr(feature = "uniffi", uniffi(default))]
85 pub freezes_duration_ms: Option<u64>,
86 #[cfg_attr(feature = "uniffi", uniffi(default))]
88 pub concealed_samples: Option<u64>,
89 #[cfg_attr(feature = "uniffi", uniffi(default))]
92 pub concealment_events: Option<u64>,
93 #[cfg_attr(feature = "uniffi", uniffi(default))]
96 pub jitter_buffer_delay_ms: Option<u64>,
97 #[cfg_attr(feature = "uniffi", uniffi(default))]
99 pub jitter_buffer_emitted_count: Option<u64>,
100 #[cfg_attr(feature = "uniffi", uniffi(default))]
102 pub quality_limitation_bandwidth_ms: Option<u64>,
103 #[cfg_attr(feature = "uniffi", uniffi(default))]
106 pub quality_limitation_cpu_ms: Option<u64>,
107 #[cfg_attr(feature = "uniffi", uniffi(default))]
109 pub quality_limitation_other_ms: Option<u64>,
110 #[cfg_attr(feature = "uniffi", uniffi(default))]
113 pub pause_count: Option<u64>,
114 #[cfg_attr(feature = "uniffi", uniffi(default))]
116 pub pauses_duration_ms: Option<u64>,
117 #[cfg_attr(feature = "uniffi", uniffi(default))]
120 pub silent_concealed_samples: Option<u64>,
121 #[cfg_attr(feature = "uniffi", uniffi(default))]
123 pub interruption_count: Option<u64>,
124 #[cfg_attr(feature = "uniffi", uniffi(default))]
126 pub interruptions_duration_ms: Option<u64>,
127 #[cfg_attr(feature = "uniffi", uniffi(default))]
130 pub jitter_ms: Option<f64>,
131 #[cfg_attr(feature = "uniffi", uniffi(default))]
134 pub rtt_ms: Option<f64>,
135 #[cfg_attr(feature = "uniffi", uniffi(default))]
137 pub frames_per_second: Option<f64>,
138 #[cfg_attr(feature = "uniffi", uniffi(default))]
140 pub audio_level: Option<f64>,
141 #[cfg_attr(feature = "uniffi", uniffi(default))]
143 pub timestamp_ns: Option<u64>,
144 #[cfg_attr(feature = "uniffi", uniffi(default))]
148 pub layer: Option<String>,
149}
150
151#[cfg_attr(feature = "uniffi", derive(uniffi::Record))]
155#[derive(Debug, Clone, PartialEq)]
156pub struct RtcStat {
157 pub kind: String,
159 pub id: String,
161 pub members: HashMap<String, AttributeValue>,
163}
164
165fn num(stat: &RtcStat, key: &str) -> Option<f64> {
167 match stat.members.get(key)? {
168 AttributeValue::Int(i) => Some(*i as f64),
169 AttributeValue::Double(d) => Some(*d),
170 AttributeValue::Str(s) => s.parse().ok(),
171 AttributeValue::Bool(_) => None,
172 }
173 .filter(|v: &f64| v.is_finite())
174}
175
176fn count(stat: &RtcStat, key: &str) -> Option<u64> {
178 num(stat, key).map(|v| v.max(0.0) as u64)
179}
180
181fn ms(stat: &RtcStat, key: &str) -> Option<u64> {
183 num(stat, key).map(|s| (s.max(0.0) * 1000.0) as u64)
184}
185
186fn ms_f64(stat: &RtcStat, key: &str) -> Option<f64> {
188 num(stat, key).map(|s| s * 1000.0).filter(|ms| ms.is_finite())
189}
190
191fn text(stat: &RtcStat, key: &str) -> Option<String> {
193 match stat.members.get(key)? {
194 AttributeValue::Str(s) => Some(s.clone()),
195 _ => None,
196 }
197}
198
199pub(crate) fn samples_from_report(
204 track_id: &str,
205 kind: TrackKind,
206 direction: StreamDirection,
207 report: &[RtcStat],
208 timestamp_ns: Option<u64>,
209) -> Vec<RtcStatsSample> {
210 let timestamp_ns = Some(timestamp_ns.unwrap_or_else(now_unix_nanos));
212 let index = ReportIndex::new(report);
213 index
214 .rtp
215 .iter()
216 .filter(|(_, d)| *d == direction)
217 .map(|(stat, direction)| index.sample(stat, track_id, kind, *direction, timestamp_ns))
218 .collect()
219}
220
221pub(crate) fn samples_from_peer_report(
227 report: &[RtcStat],
228 tracks: &HashMap<String, String>,
229 timestamp_ns: Option<u64>,
230) -> Vec<RtcStatsSample> {
231 let timestamp_ns = Some(timestamp_ns.unwrap_or_else(now_unix_nanos));
232 let index = ReportIndex::new(report);
233 index
234 .rtp
235 .iter()
236 .filter_map(|(stat, direction)| {
237 let track_id = match direction {
238 StreamDirection::Inbound => text(stat, "trackIdentifier"),
239 StreamDirection::Outbound => text(stat, "mediaSourceId")
240 .and_then(|id| index.by_id.get(id.as_str()).copied())
241 .filter(|source| source.kind == "media-source")
242 .and_then(|source| text(source, "trackIdentifier")),
243 }?;
244 let sid = tracks.get(&track_id)?;
245 let kind = match text(stat, "kind").or_else(|| text(stat, "mediaType"))?.as_str() {
246 "audio" => TrackKind::Audio,
247 "video" => TrackKind::Video,
248 _ => return None,
249 };
250 Some(index.sample(stat, sid, kind, *direction, timestamp_ns))
251 })
252 .collect()
253}
254
255struct ReportIndex<'a> {
259 by_id: HashMap<&'a str, &'a RtcStat>,
260 remote_by_local: HashMap<String, &'a RtcStat>,
261 pair_rtt_ms: Option<f64>,
262 rtp: Vec<(&'a RtcStat, StreamDirection)>,
263}
264
265impl<'a> ReportIndex<'a> {
266 fn new(report: &'a [RtcStat]) -> Self {
267 let mut index = Self {
268 by_id: HashMap::with_capacity(report.len()),
269 remote_by_local: HashMap::new(),
270 pair_rtt_ms: None,
271 rtp: Vec::new(),
272 };
273 let mut selected = None;
275 let mut ambiguous = false;
276 let mut fallback = Vec::new();
277 let flag =
278 |stat: &RtcStat, key| matches!(stat.members.get(key), Some(AttributeValue::Bool(true)));
279 for stat in report {
280 index.by_id.insert(stat.id.as_str(), stat);
281 match stat.kind.as_str() {
282 "outbound-rtp" => index.rtp.push((stat, StreamDirection::Outbound)),
283 "inbound-rtp" => index.rtp.push((stat, StreamDirection::Inbound)),
284 "remote-inbound-rtp" => {
285 if let Some(local) = text(stat, "localId") {
286 index.remote_by_local.insert(local, stat);
287 }
288 }
289 "transport" => {
290 if let Some(id) = text(stat, "selectedCandidatePairId") {
291 ambiguous |= selected.as_ref().is_some_and(|selected| selected != &id);
292 selected = Some(id);
293 }
294 }
295 "candidate-pair" => {
296 let succeeded = text(stat, "state").as_deref() == Some("succeeded");
297 if flag(stat, "selected") || (flag(stat, "nominated") && succeeded) {
298 fallback.push(stat);
299 }
300 }
301 _ => {}
302 }
303 }
304 let pair = if ambiguous {
305 None
306 } else if let Some(id) = selected {
307 index.by_id.get(id.as_str()).copied().filter(|pair| pair.kind == "candidate-pair")
308 } else if fallback.len() == 1 {
309 fallback.first().copied()
310 } else {
311 None
312 };
313 index.pair_rtt_ms = pair.and_then(|pair| ms_f64(pair, "currentRoundTripTime"));
314 index
315 }
316
317 fn sample(
319 &self,
320 stat: &RtcStat,
321 track_id: &str,
322 kind: TrackKind,
323 direction: StreamDirection,
324 timestamp_ns: Option<u64>,
325 ) -> RtcStatsSample {
326 let mut sample = RtcStatsSample::new(track_id, kind, direction);
327 sample.codec = text(stat, "codecId")
328 .and_then(|id| self.by_id.get(id.as_str()).copied())
329 .filter(|codec| codec.kind == "codec")
330 .and_then(|codec| text(codec, "mimeType"));
331 sample.timestamp_ns = timestamp_ns;
332 match direction {
333 StreamDirection::Outbound => {
334 sample.layer = Some(text(stat, "rid").unwrap_or_else(|| stat.id.clone()));
335 sample.bytes = count(stat, "bytesSent");
336 sample.packets = count(stat, "packetsSent");
337 sample.frames_per_second = num(stat, "framesPerSecond");
338 sample.quality_limitation_bandwidth_ms =
339 ms(stat, "qualityLimitationDurations.bandwidth");
340 sample.quality_limitation_cpu_ms = ms(stat, "qualityLimitationDurations.cpu");
341 sample.quality_limitation_other_ms = ms(stat, "qualityLimitationDurations.other");
342 sample.rtt_ms = self
343 .remote_by_local
344 .get(&stat.id)
345 .and_then(|remote| ms_f64(remote, "roundTripTime"));
346 }
347 StreamDirection::Inbound => {
348 sample.bytes = count(stat, "bytesReceived");
349 sample.packets = count(stat, "packetsReceived");
350 sample.packets_lost = count(stat, "packetsLost");
351 sample.freeze_count = count(stat, "freezeCount");
352 sample.freezes_duration_ms = ms(stat, "totalFreezesDuration");
353 sample.pause_count = count(stat, "pauseCount");
354 sample.pauses_duration_ms = ms(stat, "totalPausesDuration");
355 sample.concealed_samples = count(stat, "concealedSamples");
356 sample.silent_concealed_samples = count(stat, "silentConcealedSamples");
357 sample.concealment_events = count(stat, "concealmentEvents");
358 sample.interruption_count = count(stat, "interruptionCount");
359 sample.interruptions_duration_ms = ms(stat, "totalInterruptionDuration");
360 sample.jitter_buffer_delay_ms = ms(stat, "jitterBufferDelay");
361 sample.jitter_buffer_emitted_count = count(stat, "jitterBufferEmittedCount");
362 sample.jitter_ms = ms_f64(stat, "jitter");
363 sample.frames_per_second = num(stat, "framesPerSecond");
364 sample.audio_level = num(stat, "audioLevel");
365 sample.rtt_ms = match text(stat, "transportId") {
366 Some(id) => self
367 .by_id
368 .get(id.as_str())
369 .filter(|transport| transport.kind == "transport")
370 .and_then(|transport| text(transport, "selectedCandidatePairId"))
371 .and_then(|id| self.by_id.get(id.as_str()))
372 .filter(|pair| pair.kind == "candidate-pair")
373 .and_then(|pair| ms_f64(pair, "currentRoundTripTime"))
374 .or(self.pair_rtt_ms),
375 None => self.pair_rtt_ms,
376 };
377 }
378 }
379 sample
380 }
381}
382
383fn larger<T: PartialOrd>(mine: &mut Option<T>, theirs: Option<T>) {
385 if let Some(theirs) = theirs {
386 if mine.as_ref().is_none_or(|mine| theirs > *mine) {
387 *mine = Some(theirs);
388 }
389 }
390}
391
392impl RtcStatsSample {
393 fn sanitize_gauges(&mut self) {
395 for value in [
396 &mut self.jitter_ms,
397 &mut self.rtt_ms,
398 &mut self.frames_per_second,
399 &mut self.audio_level,
400 ] {
401 *value = value.filter(|v| v.is_finite());
402 }
403 }
404
405 fn merge(&mut self, other: RtcStatsSample) {
409 larger(&mut self.codec, other.codec);
410 larger(&mut self.bytes, other.bytes);
411 larger(&mut self.packets, other.packets);
412 larger(&mut self.packets_lost, other.packets_lost);
413 larger(&mut self.freeze_count, other.freeze_count);
414 larger(&mut self.freezes_duration_ms, other.freezes_duration_ms);
415 larger(&mut self.concealed_samples, other.concealed_samples);
416 larger(&mut self.concealment_events, other.concealment_events);
417 larger(&mut self.jitter_buffer_delay_ms, other.jitter_buffer_delay_ms);
418 larger(&mut self.jitter_buffer_emitted_count, other.jitter_buffer_emitted_count);
419 larger(&mut self.quality_limitation_bandwidth_ms, other.quality_limitation_bandwidth_ms);
420 larger(&mut self.quality_limitation_cpu_ms, other.quality_limitation_cpu_ms);
421 larger(&mut self.quality_limitation_other_ms, other.quality_limitation_other_ms);
422 larger(&mut self.pause_count, other.pause_count);
423 larger(&mut self.pauses_duration_ms, other.pauses_duration_ms);
424 larger(&mut self.silent_concealed_samples, other.silent_concealed_samples);
425 larger(&mut self.interruption_count, other.interruption_count);
426 larger(&mut self.interruptions_duration_ms, other.interruptions_duration_ms);
427 larger(&mut self.jitter_ms, other.jitter_ms);
428 larger(&mut self.rtt_ms, other.rtt_ms);
429 larger(&mut self.frames_per_second, other.frames_per_second);
430 larger(&mut self.audio_level, other.audio_level);
431 }
432
433 pub fn new(track_id: impl Into<String>, kind: TrackKind, direction: StreamDirection) -> Self {
435 Self {
436 track_id: track_id.into(),
437 kind,
438 direction,
439 codec: None,
440 bytes: None,
441 packets: None,
442 packets_lost: None,
443 freeze_count: None,
444 freezes_duration_ms: None,
445 concealed_samples: None,
446 concealment_events: None,
447 jitter_buffer_delay_ms: None,
448 jitter_buffer_emitted_count: None,
449 quality_limitation_bandwidth_ms: None,
450 quality_limitation_cpu_ms: None,
451 quality_limitation_other_ms: None,
452 pause_count: None,
453 pauses_duration_ms: None,
454 silent_concealed_samples: None,
455 interruption_count: None,
456 interruptions_duration_ms: None,
457 jitter_ms: None,
458 rtt_ms: None,
459 frames_per_second: None,
460 audio_level: None,
461 timestamp_ns: None,
462 layer: None,
463 }
464 }
465}
466
467#[derive(Debug, Default, Clone, Copy)]
469struct Gauge {
470 min: f64,
471 max: f64,
472 sum: f64,
473 n: u32,
474}
475
476impl Gauge {
477 fn add(&mut self, value: Option<f64>) {
478 let Some(value) = value else { return };
479 if self.n == 0 {
480 self.min = value;
481 self.max = value;
482 } else {
483 self.min = self.min.min(value);
484 self.max = self.max.max(value);
485 }
486 self.sum += value;
487 self.n += 1;
488 }
489
490 fn avg(&self) -> Option<f64> {
492 (self.n > 0).then(|| self.sum / self.n as f64)
493 }
494
495 fn attach(&self, event: TelemetryEvent, key: &str) -> TelemetryEvent {
496 let Some(avg) = self.avg() else { return event };
497 event
498 .with_attribute(format!("{key}.min"), self.min)
499 .with_attribute(format!("{key}.max"), self.max)
500 .with_attribute(format!("{key}.avg"), avg)
501 }
502}
503
504struct Window {
506 session: Arc<ScopeState>,
510 start_ns: u64,
511 samples: u32,
513 first: RtcStatsSample,
515 last: RtcStatsSample,
517 jitter: Gauge,
518 rtt: Gauge,
519 fps: Gauge,
520 audio_level: Gauge,
521}
522
523impl Window {
524 fn open(start_ns: u64, first: RtcStatsSample, session: Arc<ScopeState>) -> Self {
525 let mut window = Self {
526 session,
527 start_ns,
528 samples: 0,
529 first: first.clone(),
530 last: first,
531 jitter: Gauge::default(),
532 rtt: Gauge::default(),
533 fps: Gauge::default(),
534 audio_level: Gauge::default(),
535 };
536 window.commit();
537 window
538 }
539
540 fn summary(&self) -> String {
544 let mut parts = Vec::new();
545 let delta = |a: Option<u64>, b: Option<u64>| Some(a?.saturating_sub(b?));
546 let interval_ms =
547 delta(self.last.timestamp_ns, self.first.timestamp_ns).map(|ns| ns / 1_000_000);
548 if let (Some(bytes), Some(ms)) =
549 (delta(self.last.bytes, self.first.bytes), interval_ms.filter(|ms| *ms > 0))
550 {
551 parts.push(format!("{} kbps", u128::from(bytes) * 8 / u128::from(ms)));
552 }
553 if let (Some(lost), Some(packets)) = (
554 delta(self.last.packets_lost, self.first.packets_lost),
555 delta(self.last.packets, self.first.packets),
556 ) {
557 let total = u128::from(lost) + u128::from(packets);
558 if total > 0 {
559 parts.push(format!("loss {:.1}%", lost as f64 * 100.0 / total as f64));
560 }
561 }
562 if let Some(rtt) = self.rtt.avg() {
563 parts.push(format!("rtt {rtt:.0} ms"));
564 }
565 if let Some(fps) = self.fps.avg() {
566 parts.push(format!("{fps:.0} fps"));
567 }
568 if let Some(freezes) =
569 delta(self.last.freeze_count, self.first.freeze_count).filter(|f| *f > 0)
570 {
571 parts.push(format!("{freezes} freezes"));
572 }
573 format!(
574 "{} {}: {}",
575 self.last.kind.as_str(),
576 self.last.direction.as_str(),
577 if parts.is_empty() { "no data".to_owned() } else { parts.join(", ") }
578 )
579 }
580
581 fn into_queued(self, end_ns: u64) -> QueuedEvent {
583 let session = Arc::clone(&self.session);
584 QueuedEvent::new(self.into_event(end_ns), session)
585 }
586
587 fn add(&mut self, reading: RtcStatsSample) {
589 self.last = reading;
590 self.commit();
591 }
592
593 fn commit(&mut self) {
595 self.samples += 1;
596 self.jitter.add(self.last.jitter_ms);
597 self.rtt.add(self.last.rtt_ms);
598 self.fps.add(self.last.frames_per_second);
599 self.audio_level.add(self.last.audio_level);
600 }
601
602 fn into_event(self, end_ns: u64) -> TelemetryEvent {
606 let window_ms = end_ns.saturating_sub(self.start_ns) / 1_000_000;
607 let body = self.summary();
608 let last = self.last;
609 let mut event = TelemetryEvent::new("lk.rtc.stats.sample")
610 .with_body(body)
611 .with_attribute("lk.track.sid", last.track_id)
612 .with_attribute("lk.track.kind", last.kind.as_str())
613 .with_attribute("lk.track.direction", last.direction.as_str())
614 .with_attribute("lk.rtc.window_ms", window_ms as i64)
615 .with_attribute("lk.rtc.samples", self.samples as i64);
616 if let Some(codec) = last.codec {
617 event = event.with_attribute("lk.rtc.codec", codec);
618 }
619 let counters = [
620 ("lk.rtc.bytes", last.bytes),
621 ("lk.rtc.packets", last.packets),
622 ("lk.rtc.packets_lost", last.packets_lost),
623 ("lk.rtc.freeze_count", last.freeze_count),
624 ("lk.rtc.freezes_duration_ms", last.freezes_duration_ms),
625 ("lk.rtc.concealed_samples", last.concealed_samples),
626 ("lk.rtc.concealment_events", last.concealment_events),
627 ("lk.rtc.jitter_buffer_delay_ms", last.jitter_buffer_delay_ms),
628 ("lk.rtc.jitter_buffer_emitted_count", last.jitter_buffer_emitted_count),
629 ("lk.rtc.quality_limitation.bandwidth_ms", last.quality_limitation_bandwidth_ms),
630 ("lk.rtc.quality_limitation.cpu_ms", last.quality_limitation_cpu_ms),
631 ("lk.rtc.quality_limitation.other_ms", last.quality_limitation_other_ms),
632 ("lk.rtc.pause_count", last.pause_count),
633 ("lk.rtc.pauses_duration_ms", last.pauses_duration_ms),
634 ("lk.rtc.silent_concealed_samples", last.silent_concealed_samples),
635 ("lk.rtc.interruption_count", last.interruption_count),
636 ("lk.rtc.interruptions_duration_ms", last.interruptions_duration_ms),
637 ];
638 for (key, value) in counters {
639 if let Some(value) = value {
640 event = event.with_attribute(key, i64::try_from(value).unwrap_or(i64::MAX));
641 }
642 }
643 event = self.jitter.attach(event, "lk.rtc.jitter_ms");
644 event = self.rtt.attach(event, "lk.rtc.rtt_ms");
645 event = self.fps.attach(event, "lk.rtc.fps");
646 event = self.audio_level.attach(event, "lk.rtc.audio_level");
647 event.timestamp_ns = Some(end_ns);
648 event
649 }
650}
651
652#[derive(Default)]
659pub(crate) struct StatsWindows {
660 windows: HashMap<TrackKey, Window>,
661 limitation: HashMap<TrackKey, HashMap<Option<String>, u64>>,
665 cpu_limited_until: Option<Instant>,
666 tracks: HashMap<TrackKey, Track>,
668 seen: HashSet<TrackKey>,
671 revoked: Arc<std::sync::atomic::AtomicBool>,
673}
674
675type TrackKey = ([u8; 16], String, StreamDirection);
678
679fn track_key(session: &ScopeState, sample: &RtcStatsSample) -> TrackKey {
680 (session.trace_id, sample.track_id.clone(), sample.direction)
681}
682
683#[derive(Default)]
685struct Track {
686 poll: Option<Poll>,
689 previous: HashMap<String, StreamCounts>,
692 total: StreamCounts,
694}
695
696struct Poll {
698 session: Arc<ScopeState>,
699 timestamp_ns: u64,
700 readings: HashMap<Option<String>, RtcStatsSample>,
701}
702
703#[derive(Debug, Clone, Copy, Default)]
705struct StreamCounts {
706 bytes: Option<u64>,
707 packets: Option<u64>,
708}
709
710impl Track {
711 fn reading(
718 &mut self,
719 readings: HashMap<Option<String>, RtcStatsSample>,
720 ) -> Option<RtcStatsSample> {
721 let named = readings
723 .values()
724 .any(|r| r.direction == StreamDirection::Outbound && r.layer.is_some());
725 let mut streams: HashMap<String, StreamCounts> = HashMap::new();
726 let mut reading: Option<RtcStatsSample> = None;
727 for sample in readings.into_values() {
728 if named {
729 let stream = streams.entry(sample.layer.clone().unwrap_or_default()).or_default();
730 larger(&mut stream.bytes, sample.bytes);
731 larger(&mut stream.packets, sample.packets);
732 }
733 match &mut reading {
734 Some(reading) => reading.merge(sample),
735 None => reading = Some(sample),
736 }
737 }
738 let mut reading = reading?;
739 reading.layer = None;
740 if named {
741 let mut grown = StreamCounts::default();
742 for (id, now) in &mut streams {
743 let before = self.previous.get(id).copied().unwrap_or_default();
744 grow(&mut grown.bytes, now.bytes, before.bytes);
745 grow(&mut grown.packets, now.packets, before.packets);
746 now.bytes = now.bytes.or(before.bytes);
747 now.packets = now.packets.or(before.packets);
748 }
749 self.previous = streams;
750 add(&mut self.total.bytes, grown.bytes);
751 add(&mut self.total.packets, grown.packets);
752 reading.bytes = self.total.bytes;
753 reading.packets = self.total.packets;
754 }
755 Some(reading)
756 }
757}
758
759fn grow(grown: &mut Option<u64>, now: Option<u64>, before: Option<u64>) {
763 let Some(now) = now else { return };
764 let growth = before.map_or(0, |before| now.saturating_sub(before));
765 *grown = Some(grown.unwrap_or(0).saturating_add(growth));
766}
767
768fn add(total: &mut Option<u64>, grown: Option<u64>) {
770 if let Some(grown) = grown {
771 *total = Some(total.unwrap_or(0).saturating_add(grown));
772 }
773}
774
775const CPU_LIMITED_HOLD: Duration = Duration::from_secs(60);
780
781impl StatsWindows {
782 pub fn with_consent(revoked: Arc<std::sync::atomic::AtomicBool>) -> Self {
784 Self { revoked, ..Self::default() }
785 }
786
787 pub fn record_batch_in(
790 &mut self,
791 samples: impl IntoIterator<Item = RtcStatsSample>,
792 session: &Arc<ScopeState>,
793 ) {
794 let timestamp_ns = now_unix_nanos();
795 for mut sample in samples {
796 sample.timestamp_ns.get_or_insert(timestamp_ns);
797 self.record_in(sample, session);
798 }
799 }
800
801 pub fn record_in(&mut self, mut sample: RtcStatsSample, session: &Arc<ScopeState>) {
805 if self.revoked.load(std::sync::atomic::Ordering::SeqCst) {
806 return;
807 }
808 sample.sanitize_gauges();
809 let key = track_key(session, &sample);
810 self.seen.insert(key.clone());
811 let timestamp_ns = *sample.timestamp_ns.get_or_insert_with(now_unix_nanos);
812 let track = self.tracks.entry(key.clone()).or_default();
813 if track.poll.as_ref().is_some_and(|poll| poll.timestamp_ns != timestamp_ns) {
814 self.end_poll(&key);
815 }
816 self.check_limitation(&key, &sample);
817 let track = self.tracks.entry(key).or_default();
818 let poll = track.poll.get_or_insert_with(|| Poll {
819 session: session.clone(),
820 timestamp_ns,
821 readings: HashMap::new(),
822 });
823 match poll.readings.entry(sample.layer.clone()) {
824 Entry::Occupied(mut entry) => entry.get_mut().merge(sample),
825 Entry::Vacant(entry) => {
826 entry.insert(sample);
827 }
828 }
829 }
830
831 fn end_poll(&mut self, key: &TrackKey) {
834 let Some(track) = self.tracks.get_mut(key) else { return };
835 let Some(Poll { session, timestamp_ns, readings }) = track.poll.take() else { return };
836 let cpu: Vec<(Option<String>, u64)> = readings
837 .iter()
838 .filter(|(_, sample)| sample.direction == StreamDirection::Outbound)
839 .filter_map(|(layer, sample)| Some((layer.clone(), sample.quality_limitation_cpu_ms?)))
840 .collect();
841 let Some(reading) = track.reading(readings) else { return };
842 self.set_limitation_baseline(key, cpu);
843 match self.windows.get_mut(key) {
844 Some(window) => window.add(reading),
845 None => {
846 self.windows.insert(key.clone(), Window::open(timestamp_ns, reading, session));
847 }
848 }
849 }
850
851 fn end_polls(&mut self, ended: impl Fn(&TrackKey) -> bool) {
853 let keys: Vec<TrackKey> = self
854 .tracks
855 .iter()
856 .filter(|(key, track)| track.poll.is_some() && ended(key))
857 .map(|(key, _)| key.clone())
858 .collect();
859 for key in keys {
860 self.end_poll(&key);
861 }
862 }
863
864 pub fn close(&mut self) -> Vec<QueuedEvent> {
869 self.end_polls(|_| true);
870 let seen = std::mem::take(&mut self.seen);
871 self.tracks.retain(|key, _| seen.contains(key));
872 self.limitation.retain(|key, _| seen.contains(key));
873 let end = now_unix_nanos();
874 self.windows.drain().map(|(_, window)| window.into_queued(end)).collect()
875 }
876
877 pub fn retire(&mut self, session: &ScopeState, track_id: Option<&str>) -> Vec<QueuedEvent> {
880 let gone = |(trace_id, sid, _): &TrackKey| {
881 *trace_id == session.trace_id && track_id.is_none_or(|t| t == sid)
882 };
883 self.end_polls(gone);
884 self.tracks.retain(|key, _| !gone(key));
885 self.limitation.retain(|key, _| !gone(key));
886 self.seen.retain(|key| !gone(key));
887 let keys: Vec<TrackKey> = self.windows.keys().filter(|k| gone(k)).cloned().collect();
888 let end = now_unix_nanos();
889 keys.into_iter()
890 .filter_map(|key| self.windows.remove(&key))
891 .map(|window| window.into_queued(end))
892 .collect()
893 }
894
895 pub fn split(&mut self, session: &ScopeState) -> Vec<QueuedEvent> {
898 self.end_polls(|(trace_id, ..)| *trace_id == session.trace_id);
899 let keys: Vec<TrackKey> = self
900 .windows
901 .keys()
902 .filter(|(trace_id, ..)| *trace_id == session.trace_id)
903 .cloned()
904 .collect();
905 let end = now_unix_nanos();
906 keys.into_iter()
907 .filter_map(|key| self.windows.remove(&key))
908 .map(|window| window.into_queued(end))
909 .collect()
910 }
911
912 #[cfg(test)]
915 pub fn tracked(&mut self) -> (usize, usize, usize) {
916 self.end_polls(|_| true);
917 (self.windows.len(), self.tracks.len(), self.limitation.len())
918 }
919
920 #[cfg(test)]
922 pub fn record(&mut self, sample: RtcStatsSample) {
923 self.record_in(sample, &test_session());
924 }
925
926 #[cfg(test)]
928 pub fn close_events(&mut self) -> Vec<TelemetryEvent> {
929 self.close().into_iter().map(|q| q.event).collect()
930 }
931
932 pub fn cpu_limited(&self) -> bool {
935 self.cpu_limited_until.is_some_and(|t| Instant::now() < t)
936 }
937
938 fn check_limitation(&mut self, key: &TrackKey, sample: &RtcStatsSample) {
942 if sample.direction != StreamDirection::Outbound {
943 return;
944 }
945 let previous = self.limitation.get(key).and_then(|streams| streams.get(&sample.layer));
946 let (Some(current), Some(&previous)) = (sample.quality_limitation_cpu_ms, previous) else {
947 return;
948 };
949 if current > previous {
950 self.cpu_limited_until = Some(Instant::now() + CPU_LIMITED_HOLD);
951 }
952 }
953
954 fn set_limitation_baseline(&mut self, key: &TrackKey, cpu: Vec<(Option<String>, u64)>) {
957 if cpu.is_empty() {
958 return;
959 }
960 self.limitation.entry(key.clone()).or_default().extend(cpu);
961 }
962
963 pub fn clear(&mut self) -> u64 {
966 let first_polls = self
967 .tracks
968 .iter()
969 .filter(|(key, track)| track.poll.is_some() && !self.windows.contains_key(*key))
970 .count();
971 let open = (self.windows.len() + first_polls) as u64;
972 *self = Self::with_consent(self.revoked.clone());
973 open
974 }
975}
976
977#[cfg(test)]
979fn test_session() -> Arc<ScopeState> {
980 ScopeState::with_trace_id([7; 16])
981}
982
983impl TrackKind {
984 fn as_str(self) -> &'static str {
985 match self {
986 TrackKind::Audio => "audio",
987 TrackKind::Video => "video",
988 }
989 }
990}
991
992impl StreamDirection {
993 fn as_str(self) -> &'static str {
994 match self {
995 StreamDirection::Inbound => "inbound",
996 StreamDirection::Outbound => "outbound",
997 }
998 }
999}
1000
1001#[cfg(test)]
1002mod tests {
1003 use super::*;
1004
1005 #[test]
1008 fn extreme_counters_have_safe_summaries_and_nonnegative_encoding() {
1009 let reading = |count, second: u64| RtcStatsSample {
1010 bytes: Some(count),
1011 packets: Some(count),
1012 packets_lost: Some(count),
1013 freeze_count: Some(count),
1014 timestamp_ns: Some(second * 1_000_000_000),
1015 ..RtcStatsSample::new("TR_1", TrackKind::Video, StreamDirection::Inbound)
1016 };
1017 let mut window = Window::open(0, reading(0, 1), test_session());
1018 window.add(reading(u64::MAX, 2));
1019 let event = window.into_event(2_000_000_000);
1020 let body = event.body.as_deref().unwrap();
1021 assert!(body.contains("147573952589676412 kbps"), "{body}");
1022 assert!(body.contains("loss 50.0%"), "{body}");
1023 for key in ["lk.rtc.bytes", "lk.rtc.packets", "lk.rtc.packets_lost", "lk.rtc.freeze_count"]
1024 {
1025 assert_eq!(attr(&event, key), Some(AttributeValue::Int(i64::MAX)));
1026 }
1027 }
1028
1029 #[test]
1030 fn extreme_packet_loss_has_a_safe_denominator() {
1031 let sample = |count| RtcStatsSample {
1032 packets: Some(count),
1033 packets_lost: Some(count),
1034 ..RtcStatsSample::new("TR_1", TrackKind::Audio, StreamDirection::Inbound)
1035 };
1036 let mut window = Window::open(0, sample(0), test_session());
1037 window.add(sample(u64::MAX));
1038 assert!(window.summary().contains("loss 50.0%"));
1039 }
1040
1041 #[test]
1042 fn extreme_counter_encoding_saturates() {
1043 let sample = RtcStatsSample {
1044 bytes: Some(u64::MAX),
1045 ..RtcStatsSample::new("TR_1", TrackKind::Audio, StreamDirection::Inbound)
1046 };
1047 let event = Window::open(0, sample, test_session()).into_event(1);
1048 assert_eq!(attr(&event, "lk.rtc.bytes"), Some(AttributeValue::Int(i64::MAX)));
1049 }
1050
1051 #[test]
1052 fn extreme_layer_growth_and_running_totals_saturate() {
1053 for layers in [vec!["h"], vec!["h", "f"]] {
1054 let mut windows = StatsWindows::default();
1055 for (second, bytes) in [(1, 0), (2, u64::MAX), (3, 0), (4, u64::MAX)] {
1056 for id in &layers {
1057 windows.record(stream(id, Some(bytes), second));
1058 }
1059 }
1060 assert_eq!(totals(&mut windows), both(i64::MAX));
1061 }
1062 }
1063
1064 #[test]
1065 fn typed_batches_preserve_explicit_timestamps() {
1066 let mut windows = StatsWindows::default();
1067 let session = test_session();
1068 for second in [1, 2] {
1069 windows.record_batch_in(
1070 [
1071 stream("h", Some(second * 1_000), second),
1072 stream("f", Some(second * 4_000), second),
1073 ],
1074 &session,
1075 );
1076 let poll = windows.tracks.values().next().unwrap().poll.as_ref().unwrap();
1077 assert_eq!(poll.timestamp_ns, second * 1_000_000_000);
1078 }
1079 assert_eq!(totals(&mut windows), both(5_000));
1080 }
1081
1082 #[test]
1083 fn unstamped_layer_batches_preserve_all_byte_growth() {
1084 let mut windows = StatsWindows::default();
1085 let session = test_session();
1086 for bytes in [[1_000, 4_000], [5_000, 12_000]] {
1087 let samples = ["h", "f"].into_iter().zip(bytes).map(|(id, bytes)| {
1088 std::thread::sleep(Duration::from_millis(1));
1090 RtcStatsSample { timestamp_ns: None, ..stream(id, Some(bytes), 1) }
1091 });
1092 windows.record_batch_in(samples, &session);
1093 }
1094 assert_eq!(totals(&mut windows), both(12_000));
1095 }
1096
1097 #[test]
1098 fn duplicate_poll_submissions_keep_one_reading_per_stream() {
1099 let mut windows = StatsWindows::default();
1100 for id in ["h", "f"] {
1101 windows.record(stream(id, Some(0), 1));
1102 }
1103 for bytes in 0..10_000 {
1104 windows.record(stream("h", Some(bytes), 2));
1105 }
1106 windows.record(stream("h", None, 2));
1107 windows.record(stream("f", Some(20_000), 2));
1108 let track = windows.tracks.values().next().unwrap();
1109 assert_eq!(
1110 track.poll.as_ref().unwrap().readings.len(),
1111 2,
1112 "storage follows unique streams"
1113 );
1114 assert_eq!(
1115 totals(&mut windows),
1116 both(29_999),
1117 "duplicates keep the largest known counters"
1118 );
1119 }
1120
1121 #[test]
1126 fn cpu_starvation_on_a_small_layer_is_not_hidden_by_a_larger_sibling() {
1127 let layer = |id: &str, cpu_ms: Option<u64>, second: u64| RtcStatsSample {
1128 quality_limitation_cpu_ms: cpu_ms,
1129 layer: Some(id.into()),
1130 timestamp_ns: Some(second * 1_000_000_000),
1131 ..RtcStatsSample::new("TR_1", TrackKind::Video, StreamDirection::Outbound)
1132 };
1133 for reversed in [false, true] {
1134 let poll = |windows: &mut StatsWindows, h: Option<u64>, f: Option<u64>, s: u64| {
1135 let mut samples = vec![layer("h", h, s), layer("f", f, s)];
1136 if reversed {
1137 samples.reverse();
1138 }
1139 for sample in samples {
1140 windows.record(sample);
1141 }
1142 };
1143 let mut windows = StatsWindows::default();
1144 poll(&mut windows, Some(1_000), Some(100), 1);
1145 poll(&mut windows, Some(1_000), Some(100), 2);
1146 assert!(!windows.cpu_limited(), "reversed {reversed}: nothing grew");
1147 poll(&mut windows, Some(1_000), Some(200), 3);
1149 assert!(windows.cpu_limited(), "reversed {reversed}: the small layer was starved");
1150 let mut windows = StatsWindows::default();
1152 poll(&mut windows, Some(1_000), Some(100), 1);
1153 poll(&mut windows, None, None, 2);
1154 poll(&mut windows, Some(1_000), Some(150), 3);
1155 assert!(windows.cpu_limited(), "reversed {reversed}: baseline survived a silent poll");
1156 }
1157 }
1158
1159 #[test]
1160 fn simulcast_layers_fold_into_one_track_series() {
1161 let mut windows = StatsWindows::default();
1162 let layer = |id: &str, bytes: u64, fps: f64, second: u64| RtcStatsSample {
1163 bytes: Some(bytes),
1164 frames_per_second: Some(fps),
1165 layer: Some(id.into()),
1166 timestamp_ns: Some(second * 1_000_000_000),
1167 ..RtcStatsSample::new("TR_1", TrackKind::Video, StreamDirection::Outbound)
1168 };
1169 windows.record(layer("h", 1_000, 30.0, 1));
1170 windows.record(layer("f", 4_000, 30.0, 1));
1171 windows.record(layer("h", 3_000, 30.0, 2));
1173 windows.record(layer("f", 4_000, 0.0, 2));
1174 let events = windows.close_events();
1175 assert_eq!(events.len(), 1, "one window per track, not per layer");
1176 assert_eq!(attr(&events[0], "lk.rtc.bytes"), Some(AttributeValue::Int(2_000)), "h grew");
1177 let body = events[0].body.clone().unwrap_or_default();
1178 assert!(body.contains("16 kbps"), "a stalled layer is not a stalled track: {body}");
1179 }
1180
1181 #[tokio::test(start_paused = true)]
1182 async fn cpu_limitation_counter_drives_cadence_pressure() {
1183 let mut windows = StatsWindows::default();
1184 let reading = |bandwidth_ms, cpu_ms| RtcStatsSample {
1185 quality_limitation_bandwidth_ms: Some(bandwidth_ms),
1186 quality_limitation_cpu_ms: Some(cpu_ms),
1187 ..RtcStatsSample::new("TR_1", TrackKind::Video, StreamDirection::Outbound)
1188 };
1189 windows.record(reading(0, 0));
1190 assert!(!windows.cpu_limited(), "first reading: no delta");
1191 windows.record(reading(5_000, 0));
1192 assert!(!windows.cpu_limited(), "bandwidth limitation is not pressure");
1193 windows.record(reading(5_000, 250));
1194 assert!(windows.cpu_limited());
1195 tokio::time::advance(Duration::from_secs(59)).await;
1196 assert!(windows.cpu_limited(), "cpu hold is sticky");
1197 tokio::time::advance(Duration::from_secs(2)).await;
1198 assert!(!windows.cpu_limited(), "and expires");
1199 let mut inbound = RtcStatsSample::new("TR_2", TrackKind::Audio, StreamDirection::Inbound);
1200 inbound.quality_limitation_cpu_ms = Some(9_999);
1201 windows.record(inbound.clone());
1202 windows.record(inbound);
1203 assert!(!windows.cpu_limited(), "inbound counters are ignored");
1204 }
1205
1206 #[tokio::test(start_paused = true)]
1207 async fn a_reading_without_the_cpu_counter_keeps_the_baseline() {
1208 let mut windows = StatsWindows::default();
1209 let reading = |cpu_ms| RtcStatsSample {
1210 quality_limitation_cpu_ms: cpu_ms,
1211 ..RtcStatsSample::new("TR_1", TrackKind::Video, StreamDirection::Outbound)
1212 };
1213 for cpu_ms in [Some(5_000), None, Some(5_000)] {
1214 windows.record(reading(cpu_ms));
1215 }
1216 assert!(!windows.cpu_limited(), "the counter did not grow");
1217 }
1218
1219 #[test]
1220 fn a_split_window_keeps_its_tracks_state_through_the_next_close() {
1221 let mut windows = StatsWindows::default();
1222 let session = test_session();
1223 windows.record_in(
1224 RtcStatsSample {
1225 bytes: Some(1_000),
1226 quality_limitation_cpu_ms: Some(0),
1227 layer: Some("f".into()),
1228 ..RtcStatsSample::new("TR_1", TrackKind::Video, StreamDirection::Outbound)
1229 },
1230 &session,
1231 );
1232 windows.split(&session);
1233 windows.close();
1234 assert_eq!(windows.tracked(), (0, 1, 1), "the track is live: it read since the last close");
1235 windows.close();
1236 assert_eq!(windows.tracked(), (0, 0, 0), "a whole window without a reading retires it");
1237 }
1238
1239 fn stat(kind: &str, id: &str, members: &[(&str, AttributeValue)]) -> RtcStat {
1240 RtcStat {
1241 kind: kind.into(),
1242 id: id.into(),
1243 members: members.iter().map(|(k, v)| (k.to_string(), v.clone())).collect(),
1244 }
1245 }
1246
1247 #[test]
1248 fn inbound_rtt_follows_each_streams_transport() {
1249 let transport = |id: &str, pair: &str| {
1250 stat("transport", id, &[("selectedCandidatePairId", AttributeValue::Str(pair.into()))])
1251 };
1252 let pair = |id: &str, rtt| {
1253 stat(
1254 "candidate-pair",
1255 id,
1256 &[
1257 ("currentRoundTripTime", AttributeValue::Double(rtt)),
1258 ("selected", AttributeValue::Bool(true)),
1259 ],
1260 )
1261 };
1262 let mut report = vec![
1263 stat("inbound-rtp", "IN_A", &[("transportId", AttributeValue::Str("T_A".into()))]),
1264 stat("inbound-rtp", "IN_B", &[("transportId", AttributeValue::Str("T_B".into()))]),
1265 stat("inbound-rtp", "UNKNOWN", &[]),
1266 transport("T_A", "A"),
1267 transport("T_B", "B"),
1268 pair("A", 0.01),
1269 pair("B", 0.2),
1270 ];
1271 for reverse in [false, true] {
1272 if reverse {
1273 report.reverse();
1274 }
1275 let index = ReportIndex::new(&report);
1276 for (id, expected) in [("IN_A", Some(10.0)), ("IN_B", Some(200.0)), ("UNKNOWN", None)] {
1277 let sample = index.sample(
1278 index.by_id[id],
1279 "TR_1",
1280 TrackKind::Audio,
1281 StreamDirection::Inbound,
1282 None,
1283 );
1284 assert_eq!(sample.rtt_ms, expected, "{id}");
1285 }
1286 }
1287 report.retain(|s| s.kind != "transport");
1288 let index = ReportIndex::new(&report);
1289 assert_eq!(
1290 index.pair_rtt_ms, None,
1291 "two selected pairs are ambiguous without transports too"
1292 );
1293 }
1294
1295 #[test]
1296 fn inbound_rtt_falls_back_when_transport_selection_is_missing() {
1297 let inbound =
1298 stat("inbound-rtp", "IN", &[("transportId", AttributeValue::Str("T".into()))]);
1299 let pair = stat(
1300 "candidate-pair",
1301 "PAIR",
1302 &[
1303 ("nominated", AttributeValue::Bool(true)),
1304 ("state", AttributeValue::Str("succeeded".into())),
1305 ("currentRoundTripTime", AttributeValue::Double(0.04)),
1306 ],
1307 );
1308 let reports =
1309 [vec![inbound.clone(), pair.clone()], vec![inbound, pair, stat("transport", "T", &[])]];
1310 let rtts: Vec<_> = reports
1311 .iter()
1312 .map(|report| {
1313 samples_from_report(
1314 "TR_1",
1315 TrackKind::Audio,
1316 StreamDirection::Inbound,
1317 report,
1318 None,
1319 )[0]
1320 .rtt_ms
1321 })
1322 .collect();
1323 assert_eq!(rtts, vec![Some(40.0), Some(40.0)], "missing transport and missing selection");
1324 }
1325
1326 #[test]
1327 fn inbound_rtt_comes_from_the_selected_candidate_pair() {
1328 let inbound = stat("inbound-rtp", "IN", &[("bytesReceived", AttributeValue::Int(1))]);
1329 let pair = |id: &str, rtt: f64, nominated: bool| {
1330 stat(
1331 "candidate-pair",
1332 id,
1333 &[
1334 ("nominated", AttributeValue::Bool(nominated)),
1335 ("state", AttributeValue::Str("succeeded".into())),
1336 ("currentRoundTripTime", AttributeValue::Double(rtt)),
1337 ],
1338 )
1339 };
1340 let rtt = |report: Vec<RtcStat>| {
1341 samples_from_report("TR_1", TrackKind::Audio, StreamDirection::Inbound, &report, None)
1342 [0]
1343 .rtt_ms
1344 };
1345 let fallback = vec![inbound.clone(), pair("A", 0.3, false), pair("B", 0.04, true)];
1347 assert_eq!(rtt(fallback), Some(40.0), "a stale succeeded pair listed first is skipped");
1348 let transport =
1350 stat("transport", "T", &[("selectedCandidatePairId", AttributeValue::Str("B".into()))]);
1351 let named = vec![inbound, pair("A", 0.3, true), pair("B", 0.04, false), transport];
1352 assert_eq!(rtt(named.clone()), Some(40.0));
1353 let mut shared = named;
1354 for id in ["T2", "T3"] {
1355 shared.push(stat(
1356 "transport",
1357 id,
1358 &[("selectedCandidatePairId", AttributeValue::Str("B".into()))],
1359 ));
1360 }
1361 assert_eq!(rtt(shared.clone()), Some(40.0), "several transports can select the same pair");
1362 shared.push(stat(
1363 "transport",
1364 "T4",
1365 &[("selectedCandidatePairId", AttributeValue::Str("A".into()))],
1366 ));
1367 shared.push(stat(
1368 "transport",
1369 "T5",
1370 &[("selectedCandidatePairId", AttributeValue::Str("B".into()))],
1371 ));
1372 assert_eq!(rtt(shared), None, "repeating a pair cannot clear ambiguity");
1373 }
1374
1375 #[test]
1376 fn typed_non_finite_gauges_do_not_hide_valid_layers() {
1377 for invalid in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] {
1378 for reverse in [false, true] {
1379 let mut windows = StatsWindows::default();
1380 let gauge = |value| RtcStatsSample {
1381 jitter_ms: Some(value),
1382 rtt_ms: Some(value),
1383 frames_per_second: Some(value),
1384 audio_level: Some(value),
1385 ..stream("h", None, 1)
1386 };
1387 let mut samples = [gauge(invalid), gauge(30.0)];
1388 samples[1].layer = Some("f".into());
1389 if reverse {
1390 samples.reverse();
1391 }
1392 for sample in samples {
1393 windows.record(sample);
1394 }
1395 let event = windows.close_events().pop().unwrap();
1396 for key in ["lk.rtc.jitter_ms", "lk.rtc.rtt_ms", "lk.rtc.fps", "lk.rtc.audio_level"]
1397 {
1398 for suffix in ["min", "max", "avg"] {
1399 assert_eq!(
1400 attr(&event, &format!("{key}.{suffix}")),
1401 Some(AttributeValue::Double(30.0))
1402 );
1403 }
1404 }
1405 }
1406 }
1407 }
1408
1409 #[test]
1410 fn milliseconds_conversion_rejects_overflow() {
1411 let report = [
1412 stat("inbound-rtp", "IN", &[("jitter", AttributeValue::Double(f64::MAX))]),
1413 stat("outbound-rtp", "OUT", &[]),
1414 stat(
1415 "remote-inbound-rtp",
1416 "REMOTE",
1417 &[
1418 ("localId", AttributeValue::Str("OUT".into())),
1419 ("roundTripTime", AttributeValue::Double(f64::MAX)),
1420 ],
1421 ),
1422 stat(
1423 "candidate-pair",
1424 "PAIR",
1425 &[
1426 ("selected", AttributeValue::Bool(true)),
1427 ("currentRoundTripTime", AttributeValue::Double(f64::MAX)),
1428 ],
1429 ),
1430 ];
1431 for direction in [StreamDirection::Inbound, StreamDirection::Outbound] {
1432 let samples = samples_from_report("TR_1", TrackKind::Audio, direction, &report, None);
1433 assert_eq!(samples[0].rtt_ms, None);
1434 assert_eq!(samples[0].jitter_ms, None);
1435 }
1436 }
1437
1438 #[test]
1439 fn non_finite_numbers_are_no_reading() {
1440 let report = [stat(
1441 "inbound-rtp",
1442 "IN",
1443 &[
1444 ("jitter", AttributeValue::Str("NaN".into())),
1445 ("bytesReceived", AttributeValue::Double(f64::INFINITY)),
1446 ],
1447 )];
1448 let sample =
1449 &samples_from_report("TR_1", TrackKind::Audio, StreamDirection::Inbound, &report, None)
1450 [0];
1451 assert_eq!((sample.jitter_ms, sample.bytes), (None, None));
1452 }
1453
1454 fn attr(event: &TelemetryEvent, key: &str) -> Option<AttributeValue> {
1455 event.attributes.iter().find(|a| a.key == key).map(|a| a.value.clone())
1456 }
1457
1458 #[test]
1459 fn window_keeps_last_counter_and_summarises_gauges() {
1460 let mut windows = StatsWindows::default();
1461 for (bytes, jitter) in [(100, 1.0), (200, 3.0), (300, 2.0)] {
1462 let mut sample =
1463 RtcStatsSample::new("TR_1", TrackKind::Audio, StreamDirection::Inbound);
1464 sample.bytes = Some(bytes);
1465 sample.jitter_ms = Some(jitter);
1466 windows.record(sample);
1467 }
1468 let mut other = RtcStatsSample::new("TR_1", TrackKind::Audio, StreamDirection::Outbound);
1469 other.packets = Some(7);
1470 windows.record(other);
1471
1472 let mut events = windows.close_events();
1473 assert!(windows.close_events().is_empty(), "closing again yields nothing");
1474 events.sort_by_key(|e| attr(e, "lk.track.direction").map(|v| format!("{v:?}")));
1475 assert_eq!(events.len(), 2, "one event per track and direction");
1476 let inbound = &events[0];
1477 assert_eq!(inbound.name, "lk.rtc.stats.sample");
1478 assert_eq!(
1479 attr(inbound, "lk.track.direction"),
1480 Some(AttributeValue::Str("inbound".into()))
1481 );
1482 assert_eq!(
1483 attr(inbound, "lk.rtc.bytes"),
1484 Some(AttributeValue::Int(300)),
1485 "cumulative: last value"
1486 );
1487 assert_eq!(attr(inbound, "lk.rtc.samples"), Some(AttributeValue::Int(3)));
1488 assert_eq!(attr(inbound, "lk.rtc.jitter_ms.min"), Some(AttributeValue::Double(1.0)));
1489 assert_eq!(attr(inbound, "lk.rtc.jitter_ms.max"), Some(AttributeValue::Double(3.0)));
1490 assert_eq!(attr(inbound, "lk.rtc.jitter_ms.avg"), Some(AttributeValue::Double(2.0)));
1491 assert_eq!(attr(inbound, "lk.rtc.rtt_ms.avg"), None, "absent gauges are omitted");
1492 assert_eq!(attr(&events[1], "lk.rtc.packets"), Some(AttributeValue::Int(7)));
1493 }
1494
1495 #[test]
1496 fn a_report_maps_to_samples_with_codec_rtt_layers_and_milliseconds() {
1497 let stat = |kind: &str, id: &str, members: &[(&str, AttributeValue)]| RtcStat {
1498 kind: kind.into(),
1499 id: id.into(),
1500 members: members.iter().map(|(k, v)| (k.to_string(), v.clone())).collect(),
1501 };
1502 let s = |v: &str| AttributeValue::Str(v.into());
1503 let report = vec![
1504 stat("codec", "C1", &[("mimeType", s("video/VP8"))]),
1505 stat(
1506 "outbound-rtp",
1507 "OUT_f",
1508 &[
1509 ("rid", s("f")),
1510 ("codecId", s("C1")),
1511 ("bytesSent", AttributeValue::Int(7_000)),
1512 ("packetsSent", s("70")),
1513 ("framesPerSecond", AttributeValue::Double(29.5)),
1514 ("qualityLimitationDurations.cpu", AttributeValue::Double(1.25)),
1515 ],
1516 ),
1517 stat(
1518 "outbound-rtp",
1519 "OUT_h",
1520 &[("rid", s("h")), ("bytesSent", AttributeValue::Int(500))],
1521 ),
1522 stat(
1523 "remote-inbound-rtp",
1524 "RI_f",
1525 &[("localId", s("OUT_f")), ("roundTripTime", AttributeValue::Double(0.042))],
1526 ),
1527 ];
1528 let out = samples_from_report(
1529 "TR_1",
1530 TrackKind::Video,
1531 StreamDirection::Outbound,
1532 &report,
1533 Some(5),
1534 );
1535 assert_eq!(out.len(), 2);
1536 let f = out.iter().find(|x| x.layer.as_deref() == Some("f")).expect("layer f");
1537 assert_eq!(f.codec.as_deref(), Some("video/VP8"));
1538 assert_eq!((f.bytes, f.packets, f.frames_per_second), (Some(7_000), Some(70), Some(29.5)));
1539 assert_eq!(f.quality_limitation_cpu_ms, Some(1_250));
1540 assert_eq!(f.rtt_ms, Some(42.0));
1541 assert_eq!(f.timestamp_ns, Some(5));
1542 assert_eq!(
1543 out.iter().find(|x| x.layer.as_deref() == Some("h")).and_then(|x| x.rtt_ms),
1544 None
1545 );
1546
1547 let report = vec![
1548 stat(
1549 "candidate-pair",
1550 "CP",
1551 &[
1552 ("nominated", AttributeValue::Bool(true)),
1553 ("state", AttributeValue::Str("succeeded".into())),
1554 ("currentRoundTripTime", AttributeValue::Double(0.1)),
1555 ],
1556 ),
1557 stat(
1558 "inbound-rtp",
1559 "IN",
1560 &[
1561 ("bytesReceived", AttributeValue::Int(12)),
1562 ("packetsLost", AttributeValue::Int(-3)),
1563 ("jitter", AttributeValue::Double(0.02)),
1564 ("totalFreezesDuration", AttributeValue::Double(2.5)),
1565 ("jitterBufferDelay", AttributeValue::Double(3.0)),
1566 ("audioLevel", AttributeValue::Double(0.5)),
1567 ],
1568 ),
1569 ];
1570 let out =
1571 samples_from_report("TR_2", TrackKind::Audio, StreamDirection::Inbound, &report, None);
1572 assert_eq!(out.len(), 1);
1573 let x = &out[0];
1574 assert_eq!((x.bytes, x.packets_lost, x.rtt_ms), (Some(12), Some(0), Some(100.0)));
1575 assert_eq!(
1576 (x.jitter_ms, x.freezes_duration_ms, x.jitter_buffer_delay_ms),
1577 (Some(20.0), Some(2_500), Some(3_000))
1578 );
1579 assert_eq!(x.layer, None);
1580 }
1581
1582 #[test]
1585 fn the_same_track_in_two_sessions_never_merges() {
1586 let mut windows = StatsWindows::default();
1587 let (a, b) = (ScopeState::new(), ScopeState::new());
1588 let reading = |bytes| RtcStatsSample {
1589 bytes: Some(bytes),
1590 ..RtcStatsSample::new("TR_shared", TrackKind::Audio, StreamDirection::Inbound)
1591 };
1592 windows.record_in(reading(100), &a);
1593 windows.record_in(reading(900), &b);
1594 windows.record_in(reading(200), &a);
1595 let closed = windows.close();
1596 assert_eq!(closed.len(), 2, "one window per session");
1597 for (session, bytes) in [(&a, 200), (&b, 900)] {
1598 let window = closed.iter().find(|q| q.session == *session).expect("own window");
1599 assert_eq!(attr(&window.event, "lk.rtc.bytes"), Some(AttributeValue::Int(bytes)));
1600 let id = Some(AttributeValue::Str(session.hex()));
1601 assert_eq!(attr(&window.event, "session.id"), id, "decorated at capture");
1602 }
1603 }
1604
1605 #[test]
1608 fn per_track_state_is_retired_with_the_track() {
1609 let mut windows = StatsWindows::default();
1610 let (a, b) = (ScopeState::new(), ScopeState::new());
1611 let outbound = |sid: &str| RtcStatsSample {
1612 bytes: Some(1),
1613 quality_limitation_cpu_ms: Some(0),
1614 layer: Some("f".into()),
1615 ..RtcStatsSample::new(sid, TrackKind::Video, StreamDirection::Outbound)
1616 };
1617 for sid in ["TR_1", "TR_2"] {
1618 windows.record_in(outbound(sid), &a);
1619 }
1620 windows.record_in(outbound("TR_3"), &b);
1621 assert_eq!(windows.tracked(), (3, 3, 3));
1622
1623 let last = windows.retire(&a, Some("TR_1"));
1624 assert_eq!(last.len(), 1, "the unpublished track's partial window ships now");
1625 assert_eq!(windows.tracked(), (2, 2, 2));
1626
1627 let last = windows.retire(&a, None);
1628 assert_eq!(last.len(), 1, "the session's last window ships at disconnect");
1629 assert_eq!(windows.tracked(), (1, 1, 1), "the other session keeps its track");
1630
1631 windows.close();
1632 assert_eq!(windows.tracked(), (0, 1, 1), "counters survive an ordinary window…");
1633 windows.close();
1634 assert_eq!(windows.tracked(), (0, 0, 0), "…not a whole window without a reading");
1635 }
1636
1637 #[test]
1638 fn a_peer_connection_report_maps_to_its_tracks() {
1639 let stat = |kind: &str, id: &str, members: &[(&str, &str)]| RtcStat {
1640 kind: kind.into(),
1641 id: id.into(),
1642 members: members
1643 .iter()
1644 .map(|(k, v)| (k.to_string(), AttributeValue::Str(v.to_string())))
1645 .collect(),
1646 };
1647 let report = vec![
1648 stat("media-source", "MS1", &[("trackIdentifier", "local-cam")]),
1649 stat(
1650 "outbound-rtp",
1651 "O1",
1652 &[("kind", "video"), ("mediaSourceId", "MS1"), ("rid", "f"), ("bytesSent", "10")],
1653 ),
1654 stat(
1655 "outbound-rtp",
1656 "O2",
1657 &[("kind", "video"), ("mediaSourceId", "MS1"), ("rid", "h"), ("bytesSent", "5")],
1658 ),
1659 stat(
1660 "inbound-rtp",
1661 "I1",
1662 &[("kind", "audio"), ("trackIdentifier", "TR_remote"), ("bytesReceived", "7")],
1663 ),
1664 stat(
1665 "inbound-rtp",
1666 "I2",
1667 &[("kind", "audio"), ("trackIdentifier", "unknown"), ("bytesReceived", "1")],
1668 ),
1669 ];
1670 let tracks = HashMap::from([
1671 ("local-cam".to_owned(), "TR_cam".to_owned()),
1672 ("TR_remote".to_owned(), "TR_remote".to_owned()),
1673 ]);
1674 let out = samples_from_peer_report(&report, &tracks, None);
1675 assert_eq!(out.len(), 3, "two camera layers and the known remote track");
1676 assert!(out
1677 .iter()
1678 .filter(|s| s.track_id == "TR_cam")
1679 .all(|s| { s.kind == TrackKind::Video && s.direction == StreamDirection::Outbound }));
1680 let remote = out.iter().find(|s| s.track_id == "TR_remote").expect("remote");
1681 assert_eq!((remote.kind, remote.bytes), (TrackKind::Audio, Some(7)));
1682 }
1683
1684 #[test]
1687 fn the_bitrate_spans_the_readings_not_the_window() {
1688 let mut first = RtcStatsSample::new("TR_1", TrackKind::Video, StreamDirection::Outbound);
1689 first.bytes = Some(0);
1690 first.timestamp_ns = Some(0);
1691 let mut last = first.clone();
1692 last.bytes = Some(300_000);
1693 last.timestamp_ns = Some(30_000_000_000);
1694 let mut window = Window::open(0, first.clone(), test_session());
1695 window.add(last);
1696 let body = window.into_event(60_000_000_000).body.unwrap_or_default();
1697 assert!(body.contains("80 kbps"), "{body}");
1698 let alone = Window::open(0, first, test_session()).into_event(60_000_000_000);
1699 let body = alone.body.unwrap_or_default();
1700 assert!(!body.contains("kbps"), "one reading has no bitrate: {body}");
1701 }
1702
1703 #[test]
1706 fn the_order_of_a_reports_layers_does_not_change_the_gauges() {
1707 let gauges = |order: [(&str, f64); 2]| {
1708 let mut windows = StatsWindows::default();
1709 for (layer, fps) in order {
1710 let mut sample =
1711 RtcStatsSample::new("TR_1", TrackKind::Video, StreamDirection::Outbound);
1712 sample.layer = Some(layer.into());
1713 sample.frames_per_second = Some(fps);
1714 sample.timestamp_ns = Some(1);
1715 windows.record(sample);
1716 }
1717 let event = windows.close_events().pop().expect("a window");
1718 let fps: Vec<_> =
1719 event.attributes.into_iter().filter(|a| a.key.starts_with("lk.rtc.fps")).collect();
1720 (fps, event.body)
1721 };
1722 let (h_first, body) = gauges([("h", 10.0), ("f", 30.0)]);
1723 assert_eq!(h_first, gauges([("f", 30.0), ("h", 10.0)]).0);
1724 assert!(body.unwrap_or_default().contains("30 fps"));
1725 }
1726
1727 #[test]
1730 fn a_polls_reading_keeps_what_any_layer_reports_in_either_order() {
1731 let layer = |rid: &str| {
1732 stat("outbound-rtp", &format!("OUT_{rid}"), &[("rid", AttributeValue::Str(rid.into()))])
1733 };
1734 let rtt = |rid: &str, secs: f64| {
1735 stat(
1736 "remote-inbound-rtp",
1737 &format!("RI_{rid}"),
1738 &[
1739 ("localId", AttributeValue::Str(format!("OUT_{rid}"))),
1740 ("roundTripTime", AttributeValue::Double(secs)),
1741 ],
1742 )
1743 };
1744 let rtt_avg = |report: Vec<RtcStat>| {
1745 let mut windows = StatsWindows::default();
1746 let outbound = StreamDirection::Outbound;
1747 for sample in samples_from_report("TR_1", TrackKind::Video, outbound, &report, Some(1))
1748 {
1749 windows.record(sample);
1750 }
1751 attr(&windows.close_events().pop().expect("a window"), "lk.rtc.rtt_ms.avg")
1752 };
1753 let (f, h, q) = (layer("f"), layer("h"), layer("q"));
1754 for report in [vec![f.clone(), h.clone()], vec![h, f.clone()]] {
1755 let report = [report, vec![rtt("f", 0.042)]].concat();
1756 assert_eq!(rtt_avg(report), Some(AttributeValue::Double(42.0)), "only `f` has one");
1757 }
1758 for report in [vec![f.clone(), q.clone()], vec![q, f]] {
1759 let report = [report, vec![rtt("f", 0.042), rtt("q", 0.05)]].concat();
1760 assert_eq!(rtt_avg(report), Some(AttributeValue::Double(50.0)), "the larger one");
1761 }
1762 }
1763
1764 fn stream(id: &str, bytes: Option<u64>, second: u64) -> RtcStatsSample {
1766 RtcStatsSample {
1767 layer: Some(id.into()),
1768 bytes,
1769 packets: bytes,
1770 timestamp_ns: Some(second * 1_000_000_000),
1771 ..RtcStatsSample::new("TR_1", TrackKind::Video, StreamDirection::Outbound)
1772 }
1773 }
1774
1775 fn totals(windows: &mut StatsWindows) -> (Option<AttributeValue>, Option<AttributeValue>) {
1777 let event = windows.close_events().pop().expect("a window");
1778 (attr(&event, "lk.rtc.bytes"), attr(&event, "lk.rtc.packets"))
1779 }
1780
1781 fn both(n: i64) -> (Option<AttributeValue>, Option<AttributeValue>) {
1782 (Some(AttributeValue::Int(n)), Some(AttributeValue::Int(n)))
1783 }
1784
1785 #[test]
1788 fn a_stream_that_reports_again_adds_only_its_growth() {
1789 let mut windows = StatsWindows::default();
1790 for id in 0..9 {
1791 windows.record(stream(&format!("layer-{id}"), Some(1), id + 1));
1792 }
1793 assert_eq!(totals(&mut windows), both(0), "first polls only set starting points");
1794 windows.record(stream("layer-0", Some(1), 10));
1795 assert_eq!(totals(&mut windows), both(0), "no stream sent another byte");
1796 windows.record(stream("layer-0", Some(3), 11));
1797 assert_eq!(totals(&mut windows), both(2), "what it sent since counts");
1798 }
1799
1800 #[test]
1803 fn an_absent_counter_never_becomes_a_zero_starting_point() {
1804 let mut windows = StatsWindows::default();
1805 for id in 0..9 {
1806 windows.record(stream(&format!("layer-{id}"), Some(1), id + 1));
1807 }
1808 windows.close_events();
1809 windows.record(stream("layer-0", None, 10));
1810 windows.close_events();
1811 windows.record(stream("layer-0", Some(1), 11));
1812 assert_eq!(totals(&mut windows), both(0), "no stream sent anything more");
1813 windows.record(stream("layer-0", None, 12));
1814 windows.record(stream("layer-0", Some(4), 13));
1815 assert_eq!(totals(&mut windows), both(3), "growth across a poll without the counter");
1816 }
1817
1818 #[test]
1820 fn a_polls_totals_do_not_depend_on_its_order() {
1821 let totals_for = |reverse: bool| {
1822 let mut windows = StatsWindows::default();
1823 let mut ids: Vec<u64> = (1..=10).collect();
1824 if reverse {
1825 ids.reverse();
1826 }
1827 for second in [1, 2] {
1828 for &id in &ids {
1829 windows.record(stream(&format!("layer-{id}"), Some(id * second), second));
1830 }
1831 }
1832 totals(&mut windows)
1833 };
1834 assert_eq!(totals_for(false), totals_for(true), "same polls in reverse order");
1835 assert_eq!(totals_for(false), both(55), "each stream grew by its own count");
1836 }
1837
1838 #[test]
1840 fn every_order_of_a_poll_gives_the_same_window() {
1841 let poll = |second: u64| {
1842 [("f", 3_000, 30.0, 0.05), ("h", 2_000, 15.0, 0.04), ("q", 1_000, 7.0, 0.06)].map(
1843 |(id, bytes, fps, rtt)| RtcStatsSample {
1844 frames_per_second: Some(fps / second as f64),
1845 rtt_ms: Some(rtt * 1_000.0),
1846 ..stream(id, Some(bytes * second), second)
1847 },
1848 )
1849 };
1850 let orders = [[0, 1, 2], [0, 2, 1], [1, 0, 2], [1, 2, 0], [2, 0, 1], [2, 1, 0]];
1851 let windows: Vec<_> = orders
1852 .iter()
1853 .map(|order| {
1854 let mut windows = StatsWindows::default();
1855 for second in [1, 2] {
1856 let readings = poll(second);
1857 for &i in order {
1858 windows.record(readings[i].clone());
1859 }
1860 }
1861 let event = windows.close_events().pop().expect("a window");
1862 let mut attributes: Vec<_> = event
1863 .attributes
1864 .into_iter()
1865 .filter(|a| a.key.starts_with("lk.rtc.") && a.key != "lk.rtc.window_ms")
1867 .map(|a| format!("{}={:?}", a.key, a.value))
1868 .collect();
1869 attributes.sort();
1870 (attributes, event.body)
1871 })
1872 .collect();
1873 assert!(windows.iter().all(|w| *w == windows[0]), "{windows:#?}");
1874 }
1875
1876 #[test]
1878 fn a_falling_frame_rate_comes_from_the_current_poll_only() {
1879 let reading = |layer: &str, fps, timestamp_ns| RtcStatsSample {
1880 layer: Some(layer.into()),
1881 frames_per_second: Some(fps),
1882 timestamp_ns: Some(timestamp_ns),
1883 ..RtcStatsSample::new("TR_1", TrackKind::Video, StreamDirection::Outbound)
1884 };
1885 for order in [[("h", 5.0), ("f", 10.0)], [("f", 10.0), ("h", 5.0)]] {
1886 let mut windows = StatsWindows::default();
1887 windows.record(reading("f", 30.0, 1));
1888 windows.record(reading("h", 10.0, 1));
1889 windows.close_events();
1890 for (layer, fps) in order {
1891 windows.record(reading(layer, fps, 2));
1892 }
1893 let event = windows.close_events().pop().expect("a window");
1894 assert_eq!(attr(&event, "lk.rtc.fps.avg"), Some(AttributeValue::Double(10.0)));
1895 }
1896 }
1897
1898 #[test]
1900 fn a_track_keeps_one_reports_streams() {
1901 let mut windows = StatsWindows::default();
1902 for id in 0..1000 {
1903 windows.record(stream(&format!("outbound-rtp-{id}"), Some(1), id + 1));
1904 windows.close_events();
1905 }
1906 let kept = windows.tracks.values().next().expect("the track").previous.len();
1907 assert_eq!(kept, 1, "the previous poll's one stream");
1908 }
1909
1910 #[test]
1912 fn clear_forgets_every_stream() {
1913 let mut windows = StatsWindows::default();
1914 for _ in 0..2 {
1915 for id in 0..9 {
1916 windows.record(stream(&format!("layer-{id}"), Some(1), id + 1));
1917 }
1918 windows.clear();
1919 assert_eq!(windows.tracked(), (0, 0, 0));
1920 }
1921 }
1922
1923 #[test]
1926 fn retire_and_clear_let_the_session_go() {
1927 let mut windows = StatsWindows::default();
1928 let inbound =
1929 |sid: &str| RtcStatsSample::new(sid, TrackKind::Audio, StreamDirection::Inbound);
1930 let scope = ScopeState::new();
1931 let session = Arc::downgrade(&scope);
1932 windows.record_in(inbound("TR_1"), &scope);
1933 let queued = windows.retire(&scope, None);
1934 drop(scope);
1935 assert!(session.upgrade().is_some(), "the last window holds it");
1936 drop(queued);
1937 assert!(session.upgrade().is_none(), "gone with the window");
1938 let scope = ScopeState::new();
1939 let session = Arc::downgrade(&scope);
1940 windows.record_in(inbound("TR_2"), &scope);
1941 drop(scope);
1942 windows.clear();
1943 assert!(session.upgrade().is_none(), "nothing kept after clear");
1944 }
1945
1946 #[test]
1950 fn only_a_layered_outbound_track_keeps_a_running_total() {
1951 let totals_for = |direction| {
1952 let mut windows = StatsWindows::default();
1953 [123, 200].map(|bytes| {
1954 windows
1955 .record(RtcStatsSample { direction, ..stream("ssrc-1", Some(bytes), bytes) });
1956 totals(&mut windows)
1957 })
1958 };
1959 assert_eq!(totals_for(StreamDirection::Inbound), [both(123), both(200)], "as reported");
1960 assert_eq!(totals_for(StreamDirection::Outbound), [both(0), both(77)], "growth");
1961 }
1962
1963 #[test]
1966 fn clear_counts_every_window_and_first_poll_once() {
1967 let mut windows = StatsWindows::default();
1968 windows.record(stream("f", Some(100), 1));
1969 assert_eq!(windows.clear(), 1, "a pending first poll is one record");
1970 windows.record(stream("f", Some(100), 1));
1972 windows.record(stream("f", Some(200), 2));
1973 windows.record(RtcStatsSample { track_id: "TR_2".into(), ..stream("f", Some(100), 1) });
1974 assert_eq!(windows.clear(), 2, "one per track, its pending poll included");
1975 }
1976}