Skip to main content

cortexkit_bus_testkit/
bus.rs

1use crate::{
2    process::Process,
3    trust::{unix_now, TrustChain},
4    RigError,
5};
6use serde_json::json;
7use std::{
8    collections::BTreeSet,
9    path::PathBuf,
10    process::{Command, Stdio},
11    time::{Duration, Instant},
12};
13use tokio::{
14    io::{AsyncReadExt, AsyncWriteExt},
15    net::TcpStream,
16};
17
18pub const LOOPBACK: &str = "127.0.0.1";
19
20/// An additional account for tests that exercise user JWTs without ck-bus bootstrap.
21pub struct PreloadedAccount {
22    pub public_key: String,
23    pub signing_keys: Vec<String>,
24}
25
26pub struct BusOptions {
27    /// Optional test-build path; otherwise use CK_NATS_SERVER_BIN, then PATH.
28    pub binary: Option<PathBuf>,
29    /// Optional externally owned fixture directory; otherwise a temporary tree.
30    pub directory: Option<PathBuf>,
31    pub trust: Option<TrustChain>,
32    pub preload: Vec<PreloadedAccount>,
33    pub startup_timeout: Duration,
34}
35
36impl Default for BusOptions {
37    fn default() -> Self {
38        Self {
39            binary: None,
40            directory: None,
41            trust: None,
42            preload: vec![],
43            startup_timeout: Duration::from_secs(20),
44        }
45    }
46}
47
48pub struct BusServer {
49    // Drop processes before their files.
50    child: Process,
51    _temporary: Option<tempfile::TempDir>,
52    /// The HTTP monitoring port nats-server bound, read from its ports file.
53    monitor_port: u16,
54    pub dir: PathBuf,
55    pub url: String,
56    pub port: u16,
57    pub listen_host: String,
58    pub log: PathBuf,
59    pub operator_jwt: PathBuf,
60    pub system_account: String,
61    pub trust: TrustChain,
62    pub version: String,
63}
64
65impl BusServer {
66    pub async fn start(options: BusOptions) -> Result<Self, RigError> {
67        let (bin, version) = match options.binary {
68            Some(path) => crate::binaries::checked_nats(path)?,
69            None => crate::nats_server_bin()?,
70        };
71        let temporary = if options.directory.is_none() {
72            Some(tempfile::tempdir()?)
73        } else {
74            None
75        };
76        let dir = options
77            .directory
78            .unwrap_or_else(|| temporary.as_ref().unwrap().path().to_path_buf());
79        std::fs::create_dir_all(dir.join("jwt"))?;
80        let dir = dir.canonicalize()?;
81        let trust = options.trust.unwrap_or_else(TrustChain::generate);
82        let operator_jwt = dir.join("operator.jwt");
83        std::fs::write(&operator_jwt, trust.operator_jwt())?;
84        // nats-server binds every listener on a kernel-chosen port (`:-1`) and
85        // reports the result in a ports file. Choosing a free port here and handing
86        // the number over would leave a window in which another broker started at
87        // the same moment can be given that port too, and the loser exits with
88        // "address already in use". The directory is emptied first so a file left by
89        // a killed earlier server in a reused fixture tree is never read.
90        let ports_dir = dir.join("ports");
91        if ports_dir.exists() {
92            std::fs::remove_dir_all(&ports_dir)?;
93        }
94        std::fs::create_dir_all(&ports_dir)?;
95        let log = dir.join("server.log");
96        let stderr_path = dir.join("server.stderr");
97        let mut preload = format!(
98            "{}: {}\n",
99            trust.system_account,
100            json!(trust.system_account_jwt())
101        );
102        for account in options.preload {
103            let jwt = trust.account_jwt(&account.public_key, &account.signing_keys);
104            preload.push_str(&format!("{}: {}\n", account.public_key, json!(jwt)));
105        }
106        // A participant's JWT can exceed NATS's default 4 KiB CONNECT limit.
107        // JSON string quoting also escapes fixture paths containing quotes or slashes.
108        let conf = format!(
109            "listen: \"127.0.0.1:-1\"\n\
110             max_control_line: 65536\ndebug: true\n\
111             http: \"127.0.0.1:-1\"\n\
112             ports_file_dir: {ports}\n\
113             log_file: {log}\njetstream {{ store_dir: {js} }}\n\
114             operator: {operator}\nsystem_account: {system}\n\
115             resolver {{ type: full, dir: {jwt}, allow_delete: false, interval: \"2m\" }}\n\
116             resolver_preload {{\n{preload}}}\n",
117            ports = json!(ports_dir),
118            log = json!(log),
119            js = json!(dir.join("js")),
120            operator = json!(operator_jwt),
121            system = json!(trust.system_account),
122            jwt = json!(dir.join("jwt")),
123        );
124        let conf_path = dir.join("server.conf");
125        std::fs::write(&conf_path, conf)?;
126        let child = Process(
127            Command::new(bin)
128                .arg("-c")
129                .arg(conf_path)
130                .env("XDG_DATA_HOME", dir.join("data"))
131                .env("XDG_RUNTIME_DIR", dir.join("run"))
132                .env("XDG_CONFIG_HOME", dir.join("config"))
133                .stdout(Stdio::null())
134                // Errors raised before the log file is open (a refused config, a
135                // failed listener) only reach stderr.
136                .stderr(std::fs::File::create(&stderr_path)?)
137                .spawn()?,
138        );
139        let mut child = child;
140        let deadline = Instant::now() + options.startup_timeout;
141        let ports = match wait_ports_file(&mut child, &ports_dir, deadline).await {
142            Ok(ports) => ports,
143            Err(error) => {
144                // `try_wait` keeps a status it already collected, so an exit seen
145                // inside the wait is reported here.
146                return Err(match child.0.try_wait()? {
147                    Some(status) => RigError::Invalid(format!(
148                        "nats-server exited {status} before writing its ports file: {}",
149                        startup_output(&stderr_path, &log)
150                    )),
151                    None => RigError::Timeout {
152                        waiting_for: format!(
153                            "nats-server ports file in {} ({error})",
154                            ports_dir.display()
155                        ),
156                        details: startup_output(&stderr_path, &log),
157                    },
158                });
159            }
160        };
161        let port = ports.client;
162        let mut server = Self {
163            child,
164            _temporary: temporary,
165            monitor_port: ports.monitor,
166            dir,
167            url: format!("nats://127.0.0.1:{port}"),
168            port,
169            listen_host: LOOPBACK.to_string(),
170            log,
171            operator_jwt,
172            system_account: trust.system_account.clone(),
173            trust,
174            version,
175        };
176        loop {
177            if let Some(status) = server.child.0.try_wait()? {
178                return Err(RigError::Invalid(format!(
179                    "nats-server exited {status}: {}",
180                    startup_output(&stderr_path, &server.log)
181                )));
182            }
183            if tokio::time::timeout(Duration::from_millis(500), healthz(server.monitor_port))
184                .await
185                .unwrap_or(false)
186            {
187                break;
188            }
189            if Instant::now() >= deadline {
190                return Err(RigError::Timeout {
191                    waiting_for: "NATS /healthz".into(),
192                    details: startup_output(&stderr_path, &server.log),
193                });
194            }
195            tokio::time::sleep(Duration::from_millis(50)).await;
196        }
197        Ok(server)
198    }
199
200    pub fn pid(&self) -> u32 {
201        self.child.0.id()
202    }
203
204    /// Bootstrap inputs for the supervised ck-bus module.
205    pub fn ckbus_env(&self) -> Vec<(String, String)> {
206        vec![
207            ("CKBUS_NATS_URL".into(), self.url.clone()),
208            (
209                "CKBUS_OPERATOR_JWT".into(),
210                self.operator_jwt.display().to_string(),
211            ),
212            ("CKBUS_SYSTEM_ACCOUNT".into(), self.system_account.clone()),
213        ]
214    }
215
216    pub fn stored_accounts(&self) -> BTreeSet<String> {
217        std::fs::read_dir(self.dir.join("jwt"))
218            .into_iter()
219            .flatten()
220            .filter_map(Result::ok)
221            .filter_map(|entry| {
222                entry
223                    .file_name()
224                    .to_str()
225                    .and_then(|name| name.strip_suffix(".jwt"))
226                    .map(str::to_string)
227            })
228            .collect()
229    }
230    pub fn log_text(&self) -> String {
231        std::fs::read_to_string(&self.log).unwrap_or_default()
232    }
233    pub async fn stop(self) {
234        drop(self);
235    }
236}
237
238/// The client and monitoring ports a running nats-server reported.
239#[derive(Debug, Clone, Copy, PartialEq, Eq)]
240struct BoundPorts {
241    client: u16,
242    monitor: u16,
243}
244
245/// Waits until `child` has written `<executable>_<pid>.ports` under `ports_dir` and
246/// returns the ports it names. nats-server writes the file once every listener is
247/// bound. The file is written in place, so an unparsable read is a file still
248/// being written and is read again; the `Err` is the last reason it was not usable.
249async fn wait_ports_file(
250    child: &mut Process,
251    ports_dir: &std::path::Path,
252    deadline: Instant,
253) -> Result<BoundPorts, String> {
254    let suffix = format!("_{}.ports", child.0.id());
255    let mut last = "no ports file written".to_string();
256    loop {
257        let found = std::fs::read_dir(ports_dir)
258            .into_iter()
259            .flatten()
260            .filter_map(Result::ok)
261            .map(|entry| entry.path())
262            .find(|path| {
263                path.file_name()
264                    .and_then(|name| name.to_str())
265                    .is_some_and(|name| name.ends_with(&suffix))
266            });
267        if let Some(path) = found {
268            match std::fs::read_to_string(&path)
269                .map_err(|error| error.to_string())
270                .and_then(|text| parse_ports(&text))
271            {
272                Ok(ports) => return Ok(ports),
273                Err(error) => last = format!("{}: {error}", path.display()),
274            }
275        }
276        // A server that already exited will never write the file; the caller
277        // reports its exit status.
278        if Instant::now() >= deadline || matches!(child.0.try_wait(), Ok(Some(_))) {
279            return Err(last);
280        }
281        tokio::time::sleep(Duration::from_millis(20)).await;
282    }
283}
284
285/// Parses nats-server's ports file: `{"nats": ["nats://host:port"], "monitoring":
286/// ["http://host:port"], ...}`.
287fn parse_ports(text: &str) -> Result<BoundPorts, String> {
288    let value: serde_json::Value =
289        serde_json::from_str(text).map_err(|error| format!("not JSON yet: {error}"))?;
290    let port = |key: &str| -> Result<u16, String> {
291        value[key]
292            .as_array()
293            .and_then(|urls| urls.first())
294            .and_then(|url| url.as_str())
295            .and_then(|url| url.rsplit(':').next())
296            .and_then(|port| port.trim_end_matches('/').parse().ok())
297            .ok_or_else(|| format!("no {key} port in {text:?}"))
298    };
299    Ok(BoundPorts {
300        client: port("nats")?,
301        monitor: port("monitoring")?,
302    })
303}
304
305/// The last lines of the server's stderr and log file, for a startup failure.
306fn startup_output(stderr: &std::path::Path, log: &std::path::Path) -> String {
307    fn tail(path: &std::path::Path) -> String {
308        let text = std::fs::read_to_string(path).unwrap_or_default();
309        let lines: Vec<&str> = text.lines().collect();
310        lines[lines.len().saturating_sub(40)..].join("\n")
311    }
312    format!(
313        "stderr tail:\n{}\nserver.log tail:\n{}",
314        tail(stderr),
315        tail(log)
316    )
317}
318
319async fn healthz(port: u16) -> bool {
320    let Ok(mut stream) = TcpStream::connect((LOOPBACK, port)).await else {
321        return false;
322    };
323    if stream
324        .write_all(b"GET /healthz HTTP/1.0\r\nHost: 127.0.0.1\r\n\r\n")
325        .await
326        .is_err()
327    {
328        return false;
329    }
330    let mut buf = Vec::new();
331    stream.read_to_end(&mut buf).await.is_ok() && String::from_utf8_lossy(&buf).contains("\"ok\"")
332}
333
334impl TrustChain {
335    pub(crate) fn account_jwt(&self, public_key: &str, signing_keys: &[String]) -> String {
336        crate::trust::encode_jwt(
337            &self.operator_root,
338            public_key,
339            "fixture-account",
340            unix_now() - 120,
341            json!({"type": "account", "version": 2,
342                "limits": {"subs": -1, "data": -1, "payload": -1, "imports": -1, "exports": -1,
343                    "wildcards": true, "conn": -1, "leaf": -1},
344                "signing_keys": signing_keys, "default_permissions": {"pub": {}, "sub": {}}}),
345        )
346    }
347}
348
349#[cfg(test)]
350mod tests {
351    use super::*;
352
353    /// Requests `/varz` on `monitor_port` and returns the client port it reports,
354    /// confirming that the monitoring and client listeners belong to one server.
355    async fn varz_client_port(monitor_port: u16) -> Option<u64> {
356        let mut stream = TcpStream::connect((LOOPBACK, monitor_port)).await.ok()?;
357        stream
358            .write_all(b"GET /varz HTTP/1.0\r\nHost: 127.0.0.1\r\n\r\n")
359            .await
360            .ok()?;
361        let mut buf = Vec::new();
362        stream.read_to_end(&mut buf).await.ok()?;
363        let text = String::from_utf8_lossy(&buf);
364        let (_, body) = text.split_once("\r\n\r\n")?;
365        serde_json::from_str::<serde_json::Value>(body).ok()?["port"].as_u64()
366    }
367
368    /// Brokers started at the same moment in one process must each get ports nobody
369    /// else holds. A kit that picks a free port and releases it before nats-server
370    /// binds it can hand one port to two brokers; the loser exits with "address
371    /// already in use" and its caller times out on `/healthz`.
372    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
373    async fn eight_concurrent_brokers_bind_distinct_ports_and_all_answer_healthz() {
374        if let Err(error) = crate::nats_server_bin() {
375            crate::skip_or_fail(error);
376            return;
377        }
378        let started =
379            futures_util::future::join_all((0..8).map(|_| BusServer::start(BusOptions::default())))
380                .await;
381        let servers: Vec<BusServer> = started
382            .into_iter()
383            .enumerate()
384            .map(|(index, result)| {
385                result.unwrap_or_else(|error| panic!("broker {index} did not start: {error}"))
386            })
387            .collect();
388        let mut seen = BTreeSet::new();
389        for (index, server) in servers.iter().enumerate() {
390            for (kind, port) in [("client", server.port), ("monitor", server.monitor_port)] {
391                assert!(
392                    seen.insert(port),
393                    "broker {index}'s {kind} port {port} is already used by another broker"
394                );
395            }
396        }
397        assert_eq!(seen.len(), 16, "8 client and 8 monitor ports, all distinct");
398        for (index, server) in servers.iter().enumerate() {
399            assert!(
400                healthz(server.monitor_port).await,
401                "broker {index} /healthz on {} did not answer ok: {}",
402                server.monitor_port,
403                server.log_text()
404            );
405            assert_eq!(
406                varz_client_port(server.monitor_port).await,
407                Some(u64::from(server.port)),
408                "broker {index}'s monitor and client ports belong to different servers"
409            );
410            // The kit asked nats-server to choose; it never chose a number itself.
411            let conf = std::fs::read_to_string(server.dir.join("server.conf")).unwrap();
412            for line in ["listen: \"127.0.0.1:-1\"\n", "http: \"127.0.0.1:-1\"\n"] {
413                assert!(
414                    conf.contains(line),
415                    "broker {index} conf lacks {line:?}: {conf}"
416                );
417            }
418        }
419    }
420
421    #[test]
422    fn ports_file_parsing_reads_both_ports_and_rejects_partial_writes() {
423        assert_eq!(
424            parse_ports(
425                r#"{"nats":["nats://127.0.0.1:41441"],"monitoring":["http://127.0.0.1:36891"]}"#
426            ),
427            Ok(BoundPorts {
428                client: 41441,
429                monitor: 36891
430            })
431        );
432        assert!(parse_ports(r#"{"nats":["nats://127.0.0.1:41441"],"mon"#).is_err());
433        assert!(parse_ports(r#"{"nats":["nats://127.0.0.1:41441"]}"#).is_err());
434        assert!(parse_ports("").is_err());
435    }
436}