Skip to main content

moqros_cli/
bridges.rs

1//! Running the bridges of one node over one relay connection.
2
3use std::collections::{HashMap, HashSet};
4use std::sync::Arc;
5use std::time::{Duration, Instant};
6
7use anyhow::{Context, bail, ensure};
8use futures::future::BoxFuture;
9use futures::stream::FuturesUnordered;
10use futures::{FutureExt, StreamExt};
11use moqros::ros::{
12	BitrateControl, Bridge, BridgeEnd, BridgeStats, MoqToRos, QosProfile, ROS_COMPRESSED_IMAGE_TYPE, r2r,
13};
14use tokio::sync::watch;
15
16use crate::args::Format;
17use crate::diagnostics::Diagnostics;
18use crate::runtime::{SharedNode, Spinner, shutdown_signal, spawn_spinner};
19use crate::spec::{Discovery, PubTopic, SubBroadcast, check_video_type};
20
21/// What a bridge node does: ROS topics to publish to the relay, and broadcasts to republish
22/// on ROS.
23pub struct Bridges {
24	pub topics: Vec<PubTopic>,
25	pub broadcasts: Vec<SubBroadcast>,
26	/// For video topics.
27	pub video: moqros::ImageBroadcastSettings,
28	pub qos: QosProfile,
29	/// How often to publish the bridges' statistics on `/diagnostics`, if at all.
30	pub diagnostics_period: Option<Duration>,
31	/// Broadcasts to find by prefix and republish too.
32	pub discovery: Option<Discovery>,
33}
34
35/// The relay connection of a bridge node: a publisher for its topics, a subscriber for its
36/// broadcasts, or both over one connection.
37pub struct Relay {
38	publisher: Option<moqros::Publisher>,
39	subscriber: Option<moqros::Subscriber>,
40	session: moqros::Session,
41}
42
43impl Relay {
44	/// For a node that only has topics.
45	pub fn publisher(publisher: moqros::Publisher) -> Self {
46		Self {
47			session: publisher.session().clone(),
48			publisher: Some(publisher),
49			subscriber: None,
50		}
51	}
52
53	/// For a node that only has broadcasts.
54	pub fn subscriber(subscriber: moqros::Subscriber) -> Self {
55		Self {
56			session: subscriber.session().clone(),
57			publisher: None,
58			subscriber: Some(subscriber),
59		}
60	}
61
62	/// Both directions over the one connection [`moqros::connect_both`] returns.
63	pub fn both(publisher: moqros::Publisher, subscriber: moqros::Subscriber) -> Self {
64		Self {
65			session: publisher.session().clone(),
66			publisher: Some(publisher),
67			subscriber: Some(subscriber),
68		}
69	}
70}
71
72/// Run `bridges` until the relay connection gives up, the ROS spinner dies, or SIGINT or SIGTERM
73/// arrives (which closes the connection first, so the relay ends this node's broadcasts and
74/// subscriptions right away).
75///
76/// A topic without a type starts when it appears in the ROS graph. Broadcasts are waited for all
77/// at once, each starting when it's announced; one whose topic can't be created (e.g. a message
78/// type without bindings) is skipped. With `discovery`, every broadcast announced under its
79/// prefixes is republished too, once.
80///
81/// A bridge that stops or fails is restarted on its own, after a delay that grows while it
82/// keeps failing (up to 30 seconds), so one broken topic or broadcast doesn't take the others
83/// down.
84pub async fn run_bridges(node: r2r::Node, relay: Relay, bridges: Bridges) -> anyhow::Result<()> {
85	let mut supervisor = Supervisor::new(node, relay, bridges)?;
86	// Created once, so a signal arriving while the loop body runs isn't missed.
87	let shutdown = shutdown_signal();
88	tokio::pin!(shutdown);
89
90	loop {
91		supervisor.start_ready_topics()?;
92
93		tokio::select! {
94			Some((ready, broadcast)) = supervisor.subscribing.next() => supervisor.on_subscribed(ready, broadcast)?,
95			Some(name) = next_announcement(&mut supervisor.announcements) => supervisor.on_announcement(&name),
96			Some((name, value)) = next_parameter(&mut supervisor.parameters) => supervisor.on_parameter(&name, &value),
97			Some(stopped) = supervisor.pub_bridges.next() => supervisor.on_topic_stopped(stopped),
98			Some(stopped) = supervisor.sub_bridges.next() => supervisor.on_broadcast_stopped(stopped),
99			_ = tokio::time::sleep(Duration::from_millis(500)), if !supervisor.pending.is_empty() => {}
100			Some(diagnostics) = next_report(&mut supervisor.diagnostics) => {
101				if let Err(err) = diagnostics.publish() {
102					tracing::warn!(%err, "publishing diagnostics failed");
103				}
104			}
105			// E.g. the relay refused the session: nothing would ever be announced.
106			res = supervisor.relay.session.closed() => return res.context("relay connection closed"),
107			err = supervisor.spinner.failed() => return Err(err),
108			res = &mut shutdown => {
109				tracing::info!(signal = res?, "shutting down");
110				if let Err(err) = supervisor.relay.session.close().await {
111					tracing::warn!(%err, "the relay connection didn't close cleanly");
112				}
113				return Ok(());
114			}
115		}
116	}
117}
118
119/// A topic bridge that stopped.
120type TopicStopped = (PubTopic, BridgeEnd);
121/// A broadcast bridge that stopped.
122type BroadcastStopped = (SubBroadcast, BridgeEnd);
123
124/// The state of [`run_bridges`]: what runs, what waits to start, and what was taken.
125struct Supervisor {
126	node: Arc<SharedNode>,
127	relay: Relay,
128	/// For video topics; its bitrate follows the `bitrate` parameter.
129	video: moqros::ImageBroadcastSettings,
130	qos: QosProfile,
131	discovery: Option<Discovery>,
132	spinner: Spinner,
133	/// Topics that haven't started (again) yet.
134	pending: PendingTopics,
135	restarts: Restarts,
136	/// Broadcasts waited for, each until it's announced.
137	subscribing: FuturesUnordered<BoxFuture<'static, (MoqToRos, SubBroadcast)>>,
138	pub_bridges: FuturesUnordered<BoxFuture<'static, TopicStopped>>,
139	sub_bridges: FuturesUnordered<BoxFuture<'static, BroadcastStopped>>,
140	/// Broadcasts announced on the relay, with discovery.
141	announcements: Option<moqros::Announcements>,
142	/// Changes to the node's parameters, with video topics.
143	parameters: Option<ParameterChanges>,
144	/// The bitrate control of each running video topic.
145	bitrates: HashMap<String, BitrateControl>,
146	diagnostics: Option<(Diagnostics, tokio::time::Interval)>,
147	/// Broadcasts and topics taken, so a found broadcast doesn't take one again.
148	known_broadcasts: HashSet<String>,
149	used_topics: HashSet<String>,
150}
151
152impl Supervisor {
153	fn new(node: r2r::Node, relay: Relay, bridges: Bridges) -> anyhow::Result<Self> {
154		let Bridges {
155			topics,
156			broadcasts,
157			video,
158			qos,
159			diagnostics_period,
160			discovery,
161		} = bridges;
162		ensure!(
163			topics.is_empty() || relay.publisher.is_some(),
164			"topics to publish need a publisher"
165		);
166		ensure!(
167			(broadcasts.is_empty() && discovery.is_none()) || relay.subscriber.is_some(),
168			"broadcasts to republish need a subscriber"
169		);
170		let known_broadcasts = broadcasts
171			.iter()
172			.map(|b| moqros::normalize_broadcast_name(&b.broadcast).to_string())
173			.collect();
174		let used_topics = broadcasts
175			.iter()
176			.map(|b| b.topic.clone())
177			.chain(topics.iter().map(|t| t.topic.clone()))
178			.collect();
179		let announcements = relay
180			.subscriber
181			.as_ref()
182			.filter(|_| discovery.is_some())
183			.map(|subscriber| subscriber.announced(""));
184
185		let node = SharedNode::new(node);
186		// Only the ROS -> MoQ side subscribes to anything; a node that only publishes has
187		// nothing to spin for.
188		let spinner = match topics.is_empty() {
189			true => Spinner::idle(),
190			false => spawn_spinner(node.clone()),
191		};
192		// Parameter changes need the spinner, which only runs with topics; only video topics take any.
193		let parameters = match topics.iter().any(|topic| topic.format == Format::Video) {
194			true => Some(node.with(watch_parameters)?),
195			false => None,
196		};
197		let diagnostics = diagnostics_period.and_then(|period| match node.with(Diagnostics::new) {
198			Ok(diagnostics) => Some((diagnostics, tokio::time::interval(period))),
199			Err(err) => {
200				tracing::warn!(%err, "not publishing diagnostics; build with diagnostic_msgs in IDL_PACKAGE_FILTER");
201				None
202			}
203		});
204
205		let mut supervisor = Self {
206			node,
207			relay,
208			video,
209			qos,
210			discovery,
211			spinner,
212			pending: PendingTopics::new(topics),
213			restarts: Restarts::default(),
214			subscribing: FuturesUnordered::new(),
215			pub_bridges: FuturesUnordered::new(),
216			sub_bridges: FuturesUnordered::new(),
217			announcements,
218			parameters,
219			bitrates: HashMap::new(),
220			diagnostics,
221			known_broadcasts,
222			used_topics,
223		};
224		// A topic's type depends on what its broadcast carries, so wait for each broadcast
225		// (all at once) before creating its publisher.
226		for broadcast in broadcasts {
227			supervisor.subscribe(broadcast, Duration::ZERO);
228		}
229		Ok(supervisor)
230	}
231
232	/// Start the topics that are due and whose type is known.
233	fn start_ready_topics(&mut self) -> anyhow::Result<()> {
234		let Some(publisher) = &self.relay.publisher else {
235			return Ok(());
236		};
237		let started = self
238			.pending
239			.start_ready(&self.node, publisher, &self.video, &self.qos, &mut self.restarts)?;
240		for (topic, bridge) in started {
241			let stats = bridge.watch_stats();
242			self.report_running(&topic.topic, &format!("ROS -> MoQ {}", topic.format), &stats);
243			if let Some(control) = bridge.bitrate_control() {
244				self.bitrates.insert(topic.topic.clone(), control);
245			}
246			self.pub_bridges.push(async move { (topic, bridge.await) }.boxed());
247		}
248		Ok(())
249	}
250
251	/// Start republishing a broadcast that was announced.
252	///
253	/// One whose topic can't be created (e.g. a message type this build has no bindings for)
254	/// is skipped, so the other broadcasts and topics can go on; the node fails only once
255	/// nothing is left to bridge.
256	fn on_subscribed(&mut self, ready: MoqToRos, broadcast: SubBroadcast) -> anyhow::Result<()> {
257		match self
258			.node
259			.with(|node| spawn_sub_broadcast(ready, node, &broadcast, self.qos.clone()))
260		{
261			Ok(bridge) => {
262				self.report_running(&broadcast.broadcast, "MoQ -> ROS", &bridge.watch_stats());
263				self.sub_bridges.push(async move { (broadcast, bridge.await) }.boxed());
264			}
265			Err(err) => {
266				tracing::error!(%err, broadcast = broadcast.broadcast, topic = broadcast.topic, "skipping broadcast");
267				if self.is_idle() {
268					bail!("nothing left to bridge: no broadcast could be republished");
269				}
270			}
271		}
272		Ok(())
273	}
274
275	/// Republish the announced broadcast `name` if discovery covers it and it's new.
276	fn on_announcement(&mut self, name: &str) {
277		let Some(discovery) = &self.discovery else {
278			return;
279		};
280		if !discovery.covers(name)
281			|| !self
282				.known_broadcasts
283				.insert(moqros::normalize_broadcast_name(name).to_string())
284		{
285			return;
286		}
287		match discovery.broadcast(name) {
288			Ok(broadcast) if self.used_topics.insert(broadcast.topic.clone()) => {
289				tracing::info!(broadcast = name, topic = broadcast.topic, "found a broadcast");
290				self.subscribe(broadcast, Duration::ZERO);
291			}
292			Ok(broadcast) => tracing::warn!(
293				broadcast = name,
294				topic = broadcast.topic,
295				"skipping a found broadcast: its topic is taken"
296			),
297			Err(err) => tracing::warn!(broadcast = name, %err, "skipping a found broadcast"),
298		}
299	}
300
301	/// Apply a change to the node's parameter `name`.
302	fn on_parameter(&mut self, name: &str, value: &r2r::ParameterValue) {
303		match runtime_bitrate(name, value) {
304			Some(Ok(bitrate)) => {
305				tracing::info!(bitrate, "changing the video bitrate");
306				self.video.encoder.bitrate = bitrate;
307				for control in self.bitrates.values() {
308					control.set(bitrate);
309				}
310			}
311			Some(Err(err)) => tracing::warn!(%err, "ignoring the new bitrate"),
312			None => tracing::warn!(
313				parameter = name,
314				"only `bitrate` changes at runtime; restart the node to apply this parameter"
315			),
316		}
317	}
318
319	fn on_topic_stopped(&mut self, (topic, end): TopicStopped) {
320		let delay = self.note_stopped("topic", &topic.topic, &end);
321		tracing::info!(topic = topic.topic, ?delay, "restarting the topic bridge");
322		self.pending.restart(topic, delay);
323	}
324
325	fn on_broadcast_stopped(&mut self, (broadcast, end): BroadcastStopped) {
326		let delay = self.note_stopped("broadcast", &broadcast.broadcast, &end);
327		self.node.with(|node| end.teardown.run(node));
328		tracing::info!(
329			broadcast = broadcast.broadcast,
330			?delay,
331			"restarting the broadcast bridge"
332		);
333		self.subscribe(broadcast, delay);
334	}
335
336	/// Log and report that the bridge of `kind` `name` stopped, returning how long to wait
337	/// before restarting it.
338	fn note_stopped(&mut self, kind: &str, name: &str, end: &BridgeEnd) -> Duration {
339		log_stopped(kind, name, end);
340		if let Some((diagnostics, _)) = &mut self.diagnostics {
341			diagnostics.restarting(name);
342		}
343		self.restarts.next(&format!("{kind} {name}"), end.stats.sent > 0)
344	}
345
346	/// Report the bridge `name` (`what` it bridges) as running, with its counters.
347	fn report_running(&mut self, name: &str, what: &str, stats: &watch::Receiver<BridgeStats>) {
348		if let Some((diagnostics, _)) = &mut self.diagnostics {
349			diagnostics.running(name, what, stats.clone());
350		}
351	}
352
353	/// Wait `delay`, then subscribe to `broadcast`.
354	fn subscribe(&mut self, broadcast: SubBroadcast, delay: Duration) {
355		if let Some(subscriber) = self.relay.subscriber.clone() {
356			self.subscribing
357				.push(subscribe_after(subscriber, broadcast, delay).boxed());
358		}
359	}
360
361	/// Whether nothing runs or waits to run, nor ever will.
362	fn is_idle(&self) -> bool {
363		self.discovery.is_none()
364			&& self.subscribing.is_empty()
365			&& self.sub_bridges.is_empty()
366			&& self.pub_bridges.is_empty()
367			&& self.pending.is_empty()
368	}
369}
370
371/// Changes to the node's ROS parameters, through its `set_parameters` service.
372type ParameterChanges = std::pin::Pin<Box<dyn futures::Stream<Item = (String, r2r::ParameterValue)> + Send>>;
373
374/// Serve the node's parameter services, returning the changes they make.
375fn watch_parameters(node: &mut r2r::Node) -> anyhow::Result<ParameterChanges> {
376	let (serve, changes) = node
377		.make_parameter_handler()
378		.context("failed to serve ROS parameters")?;
379	tokio::spawn(serve);
380	Ok(Box::pin(changes))
381}
382
383/// The next broadcast announced. Never resolves without discovery.
384async fn next_announcement(announcements: &mut Option<moqros::Announcements>) -> Option<String> {
385	match announcements {
386		Some(announcements) => announcements.next().await,
387		None => std::future::pending().await,
388	}
389}
390
391/// The next parameter change. Never resolves without parameter services.
392async fn next_parameter(parameters: &mut Option<ParameterChanges>) -> Option<(String, r2r::ParameterValue)> {
393	match parameters {
394		Some(changes) => changes.next().await,
395		None => std::future::pending().await,
396	}
397}
398
399/// The new bitrate if parameter `name` is the bitrate, or why `value` isn't one; `None` for other
400/// parameters.
401fn runtime_bitrate(name: &str, value: &r2r::ParameterValue) -> Option<Result<u32, String>> {
402	(name == "bitrate").then(|| match value {
403		r2r::ParameterValue::Integer(bitrate) => u32::try_from(*bitrate)
404			.ok()
405			.filter(|&bitrate| bitrate > 0)
406			.ok_or_else(|| format!("{bitrate} is not a bitrate in bit/s")),
407		other => Err(format!("{other:?} is not an integer")),
408	})
409}
410
411/// The diagnostics, once their next report is due. Never resolves without diagnostics.
412async fn next_report(diagnostics: &mut Option<(Diagnostics, tokio::time::Interval)>) -> Option<&mut Diagnostics> {
413	match diagnostics {
414		Some((diagnostics, interval)) => {
415			interval.tick().await;
416			Some(diagnostics)
417		}
418		None => std::future::pending().await,
419	}
420}
421
422/// Wait `delay`, then subscribe to `broadcast` (retrying until it's announced).
423async fn subscribe_after(
424	subscriber: moqros::Subscriber,
425	broadcast: SubBroadcast,
426	delay: Duration,
427) -> (MoqToRos, SubBroadcast) {
428	tokio::time::sleep(delay).await;
429	(
430		MoqToRos::subscribe_with(subscriber, &broadcast.broadcast, broadcast.subscribe_options()).await,
431		broadcast,
432	)
433}
434
435/// Log why the bridge of `kind` `name` stopped.
436fn log_stopped(kind: &str, name: &str, end: &BridgeEnd) {
437	let stats = &end.stats;
438	match &end.result {
439		Ok(()) => tracing::warn!(kind, name, ?stats, "bridge stopped"),
440		Err(err @ moqros::Error::BridgePanicked(_)) => tracing::error!(kind, name, ?stats, %err, "bridge panicked"),
441		Err(err) => tracing::warn!(kind, name, ?stats, %err, "bridge failed"),
442	}
443}
444
445/// How long to wait before restarting each bridge that stopped: one second at first, doubling
446/// up to [`Restarts::MAX`] while it keeps stopping without getting a message through.
447#[derive(Debug, Default)]
448struct Restarts {
449	delays: HashMap<String, Duration>,
450}
451
452impl Restarts {
453	const MIN: Duration = Duration::from_secs(1);
454	const MAX: Duration = Duration::from_secs(30);
455
456	/// The delay before restarting `bridge`, which stopped after sending a message if
457	/// `progressed`.
458	fn next(&mut self, bridge: &str, progressed: bool) -> Duration {
459		let delay = self.delays.entry(bridge.to_string()).or_default();
460		*delay = match progressed || delay.is_zero() {
461			true => Self::MIN,
462			false => (*delay * 2).min(Self::MAX),
463		};
464		*delay
465	}
466}
467
468/// A topic to publish that hasn't started (again) yet.
469struct Pending {
470	topic: PubTopic,
471	/// Not started before this.
472	due: Instant,
473	/// Whether it ran before, so a failure to start is retried instead of fatal.
474	restart: bool,
475}
476
477/// Topics to publish that haven't started yet, because their type isn't known or they wait
478/// to be restarted.
479struct PendingTopics {
480	topics: Vec<Pending>,
481	/// Whether the topics still waiting for their type were logged.
482	logged: bool,
483}
484
485impl PendingTopics {
486	fn new(topics: Vec<PubTopic>) -> Self {
487		let now = Instant::now();
488		let topics = topics
489			.into_iter()
490			.map(|topic| Pending {
491				topic,
492				due: now,
493				restart: false,
494			})
495			.collect();
496		Self { topics, logged: false }
497	}
498
499	fn is_empty(&self) -> bool {
500		self.topics.is_empty()
501	}
502
503	/// Start `topic` again in `delay`.
504	fn restart(&mut self, topic: PubTopic, delay: Duration) {
505		self.topics.push(Pending {
506			topic,
507			due: Instant::now() + delay,
508			restart: true,
509		});
510	}
511
512	/// Start every topic that is due and whose type is known by now, returning each one with
513	/// its bridge. Data topics without a `type` wait until something publishes them.
514	///
515	/// A topic that fails to start fails the node the first time (it's misconfigured), and is
516	/// retried later on a restart.
517	fn start_ready(
518		&mut self,
519		node: &SharedNode,
520		publisher: &moqros::Publisher,
521		video: &moqros::ImageBroadcastSettings,
522		qos: &QosProfile,
523		restarts: &mut Restarts,
524	) -> anyhow::Result<Vec<(PubTopic, Bridge)>> {
525		let now = Instant::now();
526		let mut started = Vec::new();
527		let mut waiting = Vec::new();
528		for mut pending in std::mem::take(&mut self.topics) {
529			if pending.due > now {
530				self.topics.push(pending);
531				continue;
532			}
533			match node.with(|node| start_pub_topic(node, publisher, &pending.topic, video, qos.clone())) {
534				Ok(Some(bridge)) => started.push((pending.topic, bridge)),
535				Ok(None) => {
536					waiting.push(pending.topic.topic.clone());
537					self.topics.push(pending);
538				}
539				Err(err) if pending.restart => {
540					let delay = restarts.next(&format!("topic {}", pending.topic.topic), false);
541					tracing::warn!(topic = pending.topic.topic, %err, ?delay, "restarting the topic bridge failed");
542					pending.due = now + delay;
543					self.topics.push(pending);
544				}
545				Err(err) => return Err(err),
546			}
547		}
548		if !waiting.is_empty() && !self.logged {
549			tracing::info!(topics = ?waiting, "waiting for topics to appear to learn their types");
550			self.logged = true;
551		}
552		Ok(started)
553	}
554}
555
556/// Start bridging `topic` from ROS to MoQ once its type is known: given, or looked up in the ROS
557/// graph. Returns `None` while the topic hasn't appeared there yet.
558fn start_pub_topic(
559	node: &mut r2r::Node,
560	publisher: &moqros::Publisher,
561	topic: &PubTopic,
562	video: &moqros::ImageBroadcastSettings,
563	qos: QosProfile,
564) -> anyhow::Result<Option<Bridge>> {
565	let ros_type = match &topic.ros_type {
566		Some(ros_type) => ros_type.clone(),
567		None => match moqros::ros::topic_type(node, &topic.topic)? {
568			Some(ros_type) => ros_type,
569			None => return Ok(None),
570		},
571	};
572	let source = moqros::ros::RosSource::new(topic.topic.clone(), qos);
573	let bridge = match topic.format.data() {
574		None => {
575			check_video_type(&topic.topic, &ros_type)?;
576			let images = publisher.create_image_broadcast(&topic.broadcast, video.clone())?;
577			match ros_type.as_str() {
578				ROS_COMPRESSED_IMAGE_TYPE => moqros::ros::spawn_ros_to_moq_compressed(node, source, images)?,
579				_ => moqros::ros::spawn_ros_to_moq(node, source, images)?,
580			}
581		}
582		Some(format) => {
583			let settings = moqros::DataSettings::new(format)
584				.with_mode(topic.mode)
585				.with_ros_type(ros_type);
586			let data = publisher.create_data_broadcast(&topic.broadcast, settings)?;
587			moqros::ros::spawn_ros_to_moq_data(node, source, data)?
588		}
589	};
590	tracing::info!(
591		topic = topic.topic,
592		format = ?topic.format,
593		broadcast = topic.broadcast,
594		"bridging ROS -> MoQ"
595	);
596	Ok(Some(bridge))
597}
598
599/// Create the ROS publisher for a subscribed broadcast and start republishing it.
600fn spawn_sub_broadcast(
601	ready: moqros::ros::MoqToRos,
602	node: &mut r2r::Node,
603	broadcast: &SubBroadcast,
604	qos: QosProfile,
605) -> moqros::Result<Bridge> {
606	let bridge = ready.spawn(node, broadcast.target(qos))?;
607	tracing::info!(
608		broadcast = broadcast.broadcast,
609		topic = broadcast.topic,
610		"bridging MoQ -> ROS"
611	);
612	Ok(bridge)
613}
614
615#[cfg(test)]
616mod tests {
617	use super::*;
618
619	#[test]
620	fn reads_runtime_bitrates() {
621		use r2r::ParameterValue::{Double, Integer};
622		assert_eq!(runtime_bitrate("bitrate", &Integer(500_000)), Some(Ok(500_000)));
623		assert!(matches!(runtime_bitrate("bitrate", &Integer(0)), Some(Err(_))));
624		assert!(matches!(runtime_bitrate("bitrate", &Integer(-1)), Some(Err(_))));
625		assert!(matches!(runtime_bitrate("bitrate", &Integer(1 << 40)), Some(Err(_))));
626		assert!(matches!(runtime_bitrate("bitrate", &Double(5e5)), Some(Err(_))));
627		assert_eq!(runtime_bitrate("fps", &Integer(10)), None);
628	}
629
630	#[test]
631	fn restart_delays_back_off_and_reset() {
632		let mut restarts = Restarts::default();
633		let delays: Vec<u64> = (0..7).map(|_| restarts.next("topic /a", false).as_secs()).collect();
634		assert_eq!(delays, [1, 2, 4, 8, 16, 30, 30]);
635		// Each bridge has its own delay.
636		assert_eq!(restarts.next("topic /b", false), Restarts::MIN);
637		// A bridge that got messages through starts over.
638		assert_eq!(restarts.next("topic /a", true), Restarts::MIN);
639		assert_eq!(restarts.next("topic /a", false), Duration::from_secs(2));
640	}
641}