1use crate::{
2 process::Process,
3 trust::{unix_now, TrustChain},
4 RigError,
5};
6use serde_json::json;
7use std::{
8 collections::BTreeSet,
9 path::PathBuf,
10 process::{Command, Stdio},
11 time::{Duration, Instant},
12};
13use tokio::{
14 io::{AsyncReadExt, AsyncWriteExt},
15 net::TcpStream,
16};
17
18pub const LOOPBACK: &str = "127.0.0.1";
19
20pub struct PreloadedAccount {
22 pub public_key: String,
23 pub signing_keys: Vec<String>,
24}
25
26pub struct BusOptions {
27 pub binary: Option<PathBuf>,
29 pub directory: Option<PathBuf>,
31 pub trust: Option<TrustChain>,
32 pub preload: Vec<PreloadedAccount>,
33 pub startup_timeout: Duration,
34}
35
36impl Default for BusOptions {
37 fn default() -> Self {
38 Self {
39 binary: None,
40 directory: None,
41 trust: None,
42 preload: vec![],
43 startup_timeout: Duration::from_secs(20),
44 }
45 }
46}
47
48pub struct BusServer {
49 child: Process,
51 _temporary: Option<tempfile::TempDir>,
52 monitor_port: u16,
54 pub dir: PathBuf,
55 pub url: String,
56 pub port: u16,
57 pub listen_host: String,
58 pub log: PathBuf,
59 pub operator_jwt: PathBuf,
60 pub system_account: String,
61 pub trust: TrustChain,
62 pub version: String,
63}
64
65impl BusServer {
66 pub async fn start(options: BusOptions) -> Result<Self, RigError> {
67 let (bin, version) = match options.binary {
68 Some(path) => crate::binaries::checked_nats(path)?,
69 None => crate::nats_server_bin()?,
70 };
71 let temporary = if options.directory.is_none() {
72 Some(tempfile::tempdir()?)
73 } else {
74 None
75 };
76 let dir = options
77 .directory
78 .unwrap_or_else(|| temporary.as_ref().unwrap().path().to_path_buf());
79 std::fs::create_dir_all(dir.join("jwt"))?;
80 let dir = dir.canonicalize()?;
81 let trust = options.trust.unwrap_or_else(TrustChain::generate);
82 let operator_jwt = dir.join("operator.jwt");
83 std::fs::write(&operator_jwt, trust.operator_jwt())?;
84 let ports_dir = dir.join("ports");
91 if ports_dir.exists() {
92 std::fs::remove_dir_all(&ports_dir)?;
93 }
94 std::fs::create_dir_all(&ports_dir)?;
95 let log = dir.join("server.log");
96 let stderr_path = dir.join("server.stderr");
97 let mut preload = format!(
98 "{}: {}\n",
99 trust.system_account,
100 json!(trust.system_account_jwt())
101 );
102 for account in options.preload {
103 let jwt = trust.account_jwt(&account.public_key, &account.signing_keys);
104 preload.push_str(&format!("{}: {}\n", account.public_key, json!(jwt)));
105 }
106 let conf = format!(
109 "listen: \"127.0.0.1:-1\"\n\
110 max_control_line: 65536\ndebug: true\n\
111 http: \"127.0.0.1:-1\"\n\
112 ports_file_dir: {ports}\n\
113 log_file: {log}\njetstream {{ store_dir: {js} }}\n\
114 operator: {operator}\nsystem_account: {system}\n\
115 resolver {{ type: full, dir: {jwt}, allow_delete: false, interval: \"2m\" }}\n\
116 resolver_preload {{\n{preload}}}\n",
117 ports = json!(ports_dir),
118 log = json!(log),
119 js = json!(dir.join("js")),
120 operator = json!(operator_jwt),
121 system = json!(trust.system_account),
122 jwt = json!(dir.join("jwt")),
123 );
124 let conf_path = dir.join("server.conf");
125 std::fs::write(&conf_path, conf)?;
126 let child = Process(
127 Command::new(bin)
128 .arg("-c")
129 .arg(conf_path)
130 .env("XDG_DATA_HOME", dir.join("data"))
131 .env("XDG_RUNTIME_DIR", dir.join("run"))
132 .env("XDG_CONFIG_HOME", dir.join("config"))
133 .stdout(Stdio::null())
134 .stderr(std::fs::File::create(&stderr_path)?)
137 .spawn()?,
138 );
139 let mut child = child;
140 let deadline = Instant::now() + options.startup_timeout;
141 let ports = match wait_ports_file(&mut child, &ports_dir, deadline).await {
142 Ok(ports) => ports,
143 Err(error) => {
144 return Err(match child.0.try_wait()? {
147 Some(status) => RigError::Invalid(format!(
148 "nats-server exited {status} before writing its ports file: {}",
149 startup_output(&stderr_path, &log)
150 )),
151 None => RigError::Timeout {
152 waiting_for: format!(
153 "nats-server ports file in {} ({error})",
154 ports_dir.display()
155 ),
156 details: startup_output(&stderr_path, &log),
157 },
158 });
159 }
160 };
161 let port = ports.client;
162 let mut server = Self {
163 child,
164 _temporary: temporary,
165 monitor_port: ports.monitor,
166 dir,
167 url: format!("nats://127.0.0.1:{port}"),
168 port,
169 listen_host: LOOPBACK.to_string(),
170 log,
171 operator_jwt,
172 system_account: trust.system_account.clone(),
173 trust,
174 version,
175 };
176 loop {
177 if let Some(status) = server.child.0.try_wait()? {
178 return Err(RigError::Invalid(format!(
179 "nats-server exited {status}: {}",
180 startup_output(&stderr_path, &server.log)
181 )));
182 }
183 if tokio::time::timeout(Duration::from_millis(500), healthz(server.monitor_port))
184 .await
185 .unwrap_or(false)
186 {
187 break;
188 }
189 if Instant::now() >= deadline {
190 return Err(RigError::Timeout {
191 waiting_for: "NATS /healthz".into(),
192 details: startup_output(&stderr_path, &server.log),
193 });
194 }
195 tokio::time::sleep(Duration::from_millis(50)).await;
196 }
197 Ok(server)
198 }
199
200 pub fn pid(&self) -> u32 {
201 self.child.0.id()
202 }
203
204 pub fn ckbus_env(&self) -> Vec<(String, String)> {
206 vec![
207 ("CKBUS_NATS_URL".into(), self.url.clone()),
208 (
209 "CKBUS_OPERATOR_JWT".into(),
210 self.operator_jwt.display().to_string(),
211 ),
212 ("CKBUS_SYSTEM_ACCOUNT".into(), self.system_account.clone()),
213 ]
214 }
215
216 pub fn stored_accounts(&self) -> BTreeSet<String> {
217 std::fs::read_dir(self.dir.join("jwt"))
218 .into_iter()
219 .flatten()
220 .filter_map(Result::ok)
221 .filter_map(|entry| {
222 entry
223 .file_name()
224 .to_str()
225 .and_then(|name| name.strip_suffix(".jwt"))
226 .map(str::to_string)
227 })
228 .collect()
229 }
230 pub fn log_text(&self) -> String {
231 std::fs::read_to_string(&self.log).unwrap_or_default()
232 }
233 pub async fn stop(self) {
234 drop(self);
235 }
236}
237
238#[derive(Debug, Clone, Copy, PartialEq, Eq)]
240struct BoundPorts {
241 client: u16,
242 monitor: u16,
243}
244
245async fn wait_ports_file(
250 child: &mut Process,
251 ports_dir: &std::path::Path,
252 deadline: Instant,
253) -> Result<BoundPorts, String> {
254 let suffix = format!("_{}.ports", child.0.id());
255 let mut last = "no ports file written".to_string();
256 loop {
257 let found = std::fs::read_dir(ports_dir)
258 .into_iter()
259 .flatten()
260 .filter_map(Result::ok)
261 .map(|entry| entry.path())
262 .find(|path| {
263 path.file_name()
264 .and_then(|name| name.to_str())
265 .is_some_and(|name| name.ends_with(&suffix))
266 });
267 if let Some(path) = found {
268 match std::fs::read_to_string(&path)
269 .map_err(|error| error.to_string())
270 .and_then(|text| parse_ports(&text))
271 {
272 Ok(ports) => return Ok(ports),
273 Err(error) => last = format!("{}: {error}", path.display()),
274 }
275 }
276 if Instant::now() >= deadline || matches!(child.0.try_wait(), Ok(Some(_))) {
279 return Err(last);
280 }
281 tokio::time::sleep(Duration::from_millis(20)).await;
282 }
283}
284
285fn parse_ports(text: &str) -> Result<BoundPorts, String> {
288 let value: serde_json::Value =
289 serde_json::from_str(text).map_err(|error| format!("not JSON yet: {error}"))?;
290 let port = |key: &str| -> Result<u16, String> {
291 value[key]
292 .as_array()
293 .and_then(|urls| urls.first())
294 .and_then(|url| url.as_str())
295 .and_then(|url| url.rsplit(':').next())
296 .and_then(|port| port.trim_end_matches('/').parse().ok())
297 .ok_or_else(|| format!("no {key} port in {text:?}"))
298 };
299 Ok(BoundPorts {
300 client: port("nats")?,
301 monitor: port("monitoring")?,
302 })
303}
304
305fn startup_output(stderr: &std::path::Path, log: &std::path::Path) -> String {
307 fn tail(path: &std::path::Path) -> String {
308 let text = std::fs::read_to_string(path).unwrap_or_default();
309 let lines: Vec<&str> = text.lines().collect();
310 lines[lines.len().saturating_sub(40)..].join("\n")
311 }
312 format!(
313 "stderr tail:\n{}\nserver.log tail:\n{}",
314 tail(stderr),
315 tail(log)
316 )
317}
318
319async fn healthz(port: u16) -> bool {
320 let Ok(mut stream) = TcpStream::connect((LOOPBACK, port)).await else {
321 return false;
322 };
323 if stream
324 .write_all(b"GET /healthz HTTP/1.0\r\nHost: 127.0.0.1\r\n\r\n")
325 .await
326 .is_err()
327 {
328 return false;
329 }
330 let mut buf = Vec::new();
331 stream.read_to_end(&mut buf).await.is_ok() && String::from_utf8_lossy(&buf).contains("\"ok\"")
332}
333
334impl TrustChain {
335 pub(crate) fn account_jwt(&self, public_key: &str, signing_keys: &[String]) -> String {
336 crate::trust::encode_jwt(
337 &self.operator_root,
338 public_key,
339 "fixture-account",
340 unix_now() - 120,
341 json!({"type": "account", "version": 2,
342 "limits": {"subs": -1, "data": -1, "payload": -1, "imports": -1, "exports": -1,
343 "wildcards": true, "conn": -1, "leaf": -1},
344 "signing_keys": signing_keys, "default_permissions": {"pub": {}, "sub": {}}}),
345 )
346 }
347}
348
349#[cfg(test)]
350mod tests {
351 use super::*;
352
353 async fn varz_client_port(monitor_port: u16) -> Option<u64> {
356 let mut stream = TcpStream::connect((LOOPBACK, monitor_port)).await.ok()?;
357 stream
358 .write_all(b"GET /varz HTTP/1.0\r\nHost: 127.0.0.1\r\n\r\n")
359 .await
360 .ok()?;
361 let mut buf = Vec::new();
362 stream.read_to_end(&mut buf).await.ok()?;
363 let text = String::from_utf8_lossy(&buf);
364 let (_, body) = text.split_once("\r\n\r\n")?;
365 serde_json::from_str::<serde_json::Value>(body).ok()?["port"].as_u64()
366 }
367
368 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
373 async fn eight_concurrent_brokers_bind_distinct_ports_and_all_answer_healthz() {
374 if let Err(error) = crate::nats_server_bin() {
375 crate::skip_or_fail(error);
376 return;
377 }
378 let started =
379 futures_util::future::join_all((0..8).map(|_| BusServer::start(BusOptions::default())))
380 .await;
381 let servers: Vec<BusServer> = started
382 .into_iter()
383 .enumerate()
384 .map(|(index, result)| {
385 result.unwrap_or_else(|error| panic!("broker {index} did not start: {error}"))
386 })
387 .collect();
388 let mut seen = BTreeSet::new();
389 for (index, server) in servers.iter().enumerate() {
390 for (kind, port) in [("client", server.port), ("monitor", server.monitor_port)] {
391 assert!(
392 seen.insert(port),
393 "broker {index}'s {kind} port {port} is already used by another broker"
394 );
395 }
396 }
397 assert_eq!(seen.len(), 16, "8 client and 8 monitor ports, all distinct");
398 for (index, server) in servers.iter().enumerate() {
399 assert!(
400 healthz(server.monitor_port).await,
401 "broker {index} /healthz on {} did not answer ok: {}",
402 server.monitor_port,
403 server.log_text()
404 );
405 assert_eq!(
406 varz_client_port(server.monitor_port).await,
407 Some(u64::from(server.port)),
408 "broker {index}'s monitor and client ports belong to different servers"
409 );
410 let conf = std::fs::read_to_string(server.dir.join("server.conf")).unwrap();
412 for line in ["listen: \"127.0.0.1:-1\"\n", "http: \"127.0.0.1:-1\"\n"] {
413 assert!(
414 conf.contains(line),
415 "broker {index} conf lacks {line:?}: {conf}"
416 );
417 }
418 }
419 }
420
421 #[test]
422 fn ports_file_parsing_reads_both_ports_and_rejects_partial_writes() {
423 assert_eq!(
424 parse_ports(
425 r#"{"nats":["nats://127.0.0.1:41441"],"monitoring":["http://127.0.0.1:36891"]}"#
426 ),
427 Ok(BoundPorts {
428 client: 41441,
429 monitor: 36891
430 })
431 );
432 assert!(parse_ports(r#"{"nats":["nats://127.0.0.1:41441"],"mon"#).is_err());
433 assert!(parse_ports(r#"{"nats":["nats://127.0.0.1:41441"]}"#).is_err());
434 assert!(parse_ports("").is_err());
435 }
436}