radicle_cli_test/
lib.rs

1use std::borrow::Cow;
2use std::collections::HashMap;
3use std::path::{Path, PathBuf};
4use std::str::FromStr;
5use std::sync;
6use std::{env, fs, io, mem};
7
8use snapbox::cmd::{Command, OutputAssert};
9use snapbox::{Assert, Redactions};
10use thiserror::Error;
11
12const CARGO_TARGET_DIR_DIRNAME: &str = "target";
13
14const CARGO_PROFILE: &str = "debug";
15
16/// Used to ensure the build task is only run once.
17static BUILD: sync::Once = sync::Once::new();
18
19#[derive(Error, Debug)]
20pub enum Error {
21    #[error("parsing failed")]
22    Parse,
23    #[error("invalid file path: {0:?}")]
24    InvalidFilePath(String),
25    #[error("unknown home {0:?}")]
26    UnknownHome(String),
27    #[error("test file not found: {0:?}")]
28    TestNotFound(PathBuf),
29    #[error("i/o: {0}")]
30    Io(#[from] io::Error),
31    #[error("snapbox: {0}")]
32    Snapbox(#[from] snapbox::assert::Error),
33}
34
35#[derive(Debug, PartialEq, Eq)]
36enum ExitStatus {
37    Success,
38    Failure,
39}
40
41/// A test which may contain multiple assertions.
42#[derive(Debug, Default, PartialEq, Eq)]
43pub struct Test {
44    /// Human-readable context around the test. Functions as documentation.
45    context: Vec<String>,
46    /// Test assertions to run.
47    assertions: Vec<Assertion>,
48    /// Whether to check stderr's output instead of stdout.
49    stderr: bool,
50    /// Whether to expect an error status code.
51    fail: bool,
52    /// Home directory under which to run this test.
53    home: Option<String>,
54    /// Local env vars to use just for this test.
55    env: HashMap<String, String>,
56}
57
58/// An assertion is a command to run with an expected output.
59#[derive(Debug, PartialEq, Eq)]
60pub struct Assertion {
61    /// The test file that contains this assertion.
62    path: PathBuf,
63    /// Name of command to run, eg. `git`.
64    command: String,
65    /// Command arguments, eg. `["push"]`.
66    args: Vec<String>,
67    /// Expected output (stdout or stderr).
68    expected: String,
69    /// Expected exit status.
70    exit: ExitStatus,
71    /// Line number in the test file where this assertion is defined.
72    line: usize,
73}
74
75#[derive(Debug, Default, PartialEq, Eq, Clone)]
76pub struct Home {
77    name: Option<String>,
78    path: PathBuf,
79    envs: HashMap<String, String>,
80}
81
82#[derive(Debug)]
83pub struct TestRun {
84    home: Home,
85    env: HashMap<String, String>,
86}
87
88impl TestRun {
89    fn cd(&mut self, path: PathBuf) {
90        self.home.path = path;
91    }
92
93    fn envs(&self) -> impl Iterator<Item = (String, String)> + '_ {
94        self.home
95            .envs
96            .iter()
97            .chain(self.env.iter())
98            .map(|(k, v)| (k.to_owned(), v.to_owned()))
99            .chain(Some((
100                "PWD".to_owned(),
101                self.home.path.to_string_lossy().to_string(),
102            )))
103    }
104
105    fn path(&self) -> PathBuf {
106        self.home.path.clone()
107    }
108}
109
110#[derive(Debug)]
111pub struct TestRunner<'a> {
112    cwd: Option<PathBuf>,
113    homes: HashMap<String, Home>,
114    formula: &'a TestFormula,
115}
116
117impl<'a> TestRunner<'a> {
118    fn new(formula: &'a TestFormula) -> Self {
119        Self {
120            cwd: None,
121            homes: formula.homes.clone(),
122            formula,
123        }
124    }
125
126    fn run(&mut self, test: &'a Test) -> TestRun {
127        let mut env = self.formula.env.clone();
128        env.extend(test.env.clone());
129
130        if let Some(ref h) = test.home {
131            if let Some(home) = self.homes.get(h) {
132                env.insert("USER".to_owned(), h.to_owned());
133                return TestRun {
134                    home: home.clone(),
135                    env,
136                };
137            } else {
138                panic!("TestRunner::test: home `~{h}` does not exist");
139            }
140        }
141        TestRun {
142            home: Home {
143                name: None,
144                path: self.cwd.clone().unwrap_or_else(|| self.formula.cwd.clone()),
145                envs: HashMap::new(),
146            },
147            env,
148        }
149    }
150
151    fn finish(&mut self, run: TestRun) {
152        if let Some(name) = &run.home.name {
153            self.homes.insert(name.clone(), run.home);
154        } else {
155            self.cwd = Some(run.home.path);
156        }
157    }
158}
159
160#[derive(Debug, Default, PartialEq, Eq)]
161pub struct TestFormula {
162    /// Current working directory to run the test in.
163    cwd: PathBuf,
164    /// User homes.
165    homes: HashMap<String, Home>,
166    /// Environment to pass to the test.
167    env: HashMap<String, String>,
168    /// Tests to run.
169    tests: Vec<Test>,
170    /// Output substitutions.
171    subs: Redactions,
172}
173
174impl TestFormula {
175    pub fn new(cwd: PathBuf) -> Self {
176        Self {
177            cwd: cwd.clone(),
178            env: HashMap::new(),
179            homes: HashMap::new(),
180            tests: Vec::new(),
181            subs: Redactions::new(),
182        }
183    }
184
185    pub fn build(&mut self, binaries: &[(&str, &str)]) -> &mut Self {
186        // We don't need to re-build every time the `build` function is called. Once is enough.
187        BUILD.call_once(|| {
188            use escargot::format::Message;
189            use radicle_log::env_level;
190            use radicle_log::test::Logger;
191            use radicle_term::Paint;
192
193            Paint::force(true);
194
195            let level = env_level().unwrap_or(log::Level::Debug);
196            let logger = Box::new(Logger::new(level));
197
198            log::set_boxed_logger(logger).expect("no other logger should have been set already");
199            log::set_max_level(level.to_level_filter());
200
201            for (package, binary) in binaries {
202                log::debug!(target: "test", "Building binaries for package `{package}`..");
203
204                let results = escargot::CargoBuild::new()
205                    .package(package)
206                    .bin(binary)
207                    .manifest_path(cargo_manifest_dir().join("Cargo.toml"))
208                    .target_dir(cargo_target_dir())
209                    .exec()
210                    .unwrap();
211
212                for result in results {
213                    match result {
214                        Ok(msg) => {
215                            if let Ok(Message::CompilerArtifact(a)) = msg.decode()
216                                && let Some(e) = a.executable
217                            {
218                                log::debug!(target: "test", "Built {}", e.display());
219                            }
220                        }
221                        Err(e) => {
222                            log::error!(target: "test", "Error building package `{package}`: {e}");
223                        }
224                    }
225                }
226            }
227        });
228        self
229    }
230
231    pub fn env(&mut self, key: impl ToString, val: impl ToString) -> &mut Self {
232        self.env.insert(key.to_string(), val.to_string());
233        self
234    }
235
236    pub fn home(
237        &mut self,
238        user: impl ToString,
239        path: impl AsRef<Path>,
240        envs: impl IntoIterator<Item = (impl ToString, impl ToString)>,
241    ) -> &mut Self {
242        self.homes.insert(
243            user.to_string(),
244            Home {
245                name: Some(user.to_string()),
246                path: path.as_ref().to_path_buf(),
247                envs: envs
248                    .into_iter()
249                    .map(|(k, v)| (k.to_string(), v.to_string()))
250                    .collect(),
251            },
252        );
253        self
254    }
255
256    pub fn envs<K: ToString, V: ToString>(
257        &mut self,
258        envs: impl IntoIterator<Item = (K, V)>,
259    ) -> &mut Self {
260        for (k, v) in envs {
261            self.env.insert(k.to_string(), v.to_string());
262        }
263        self
264    }
265
266    pub fn file(&mut self, path: impl AsRef<Path>) -> Result<&mut Self, Error> {
267        let path = path.as_ref();
268        let contents = match fs::read(path) {
269            Ok(bytes) => bytes,
270            Err(err) if err.kind() == io::ErrorKind::NotFound => {
271                return Err(Error::TestNotFound(path.to_path_buf()));
272            }
273            Err(err) => return Err(err.into()),
274        };
275        self.read(path, io::Cursor::new(contents))
276    }
277
278    pub fn read(&mut self, path: &Path, r: impl io::BufRead) -> Result<&mut Self, Error> {
279        let mut test = Test::default();
280        let mut fenced = false; // Whether we're inside a fenced code block.
281        let mut file: Option<(PathBuf, String)> = None; // Path and content of file created by this test block.
282
283        for (row, line) in r.lines().enumerate() {
284            let line = line?;
285
286            if line.starts_with("```") {
287                if fenced {
288                    if let Some((ref path, ref mut content)) = file.take() {
289                        // Write file.
290                        let path = self.cwd.join(path);
291
292                        if let Some(dir) = path.parent() {
293                            log::debug!(target: "test", "Creating directory {}..", dir.display());
294                            fs::create_dir_all(dir)?;
295                        }
296                        log::debug!(target: "test", "Writing {} bytes to {}..", content.len(), path.display());
297                        fs::write(path, content)?;
298                    } else {
299                        // End existing code block.
300                        self.tests.push(mem::take(&mut test));
301                    }
302                } else {
303                    for token in line.split_whitespace() {
304                        if let Some(home) = token.strip_prefix('~') {
305                            test.home = Some(home.to_owned());
306                        } else if let Some((key, val)) = token.split_once('=') {
307                            test.env.insert(key.to_owned(), val.to_owned());
308                        } else if token.contains("stderr") {
309                            test.stderr = true;
310                        } else if token.contains("fail") {
311                            test.fail = true;
312                        } else if let Some(path) = token.strip_prefix("./") {
313                            file = Some((
314                                PathBuf::from_str(path)
315                                    .map_err(|_| Error::InvalidFilePath(token.to_owned()))?,
316                                String::new(),
317                            ));
318                        }
319                    }
320                }
321                fenced = !fenced;
322
323                continue;
324            }
325
326            if fenced {
327                if let Some((_, ref mut content)) = file {
328                    content.push_str(line.as_str());
329                    content.push('\n');
330                } else if let Some(line) = line.strip_prefix('$') {
331                    let line = line.trim();
332
333                    #[cfg(unix)]
334                    let parts = shlex::split(line).ok_or(Error::Parse)?;
335
336                    #[cfg(windows)]
337                    let parts = winsplit::split(line);
338
339                    let (cmd, args) = parts.split_first().ok_or(Error::Parse)?;
340
341                    test.assertions.push(Assertion {
342                        path: path.to_path_buf(),
343                        command: cmd.to_owned(),
344                        args: args.to_owned(),
345                        expected: String::new(),
346                        exit: if test.fail {
347                            ExitStatus::Failure
348                        } else {
349                            ExitStatus::Success
350                        },
351                        line: row + 1,
352                    });
353                } else if let Some(a) = test.assertions.last_mut() {
354                    a.expected.push_str(line.as_str());
355                    a.expected.push('\n');
356                } else {
357                    return Err(Error::Parse);
358                }
359            } else {
360                test.context.push(line);
361            }
362        }
363        Ok(self)
364    }
365
366    #[allow(dead_code)]
367    pub fn substitute(
368        &mut self,
369        value: &'static str,
370        other: impl Into<Cow<'static, str>>,
371    ) -> Result<&mut Self, Error> {
372        self.subs.insert(value, other.into())?;
373        Ok(self)
374    }
375
376    /// Convert instances of '[..   ]' to '[..]' where the number of ' 's are arbitrary.
377    ///
378    /// Supporting these bracket types help support using the '[..]' pattern while preserving
379    /// spaces important for text alignment.
380    fn map_spaced_brackets(s: &str) -> String {
381        let mut ret = String::new();
382        let mut pos = 0;
383
384        for c in s.chars() {
385            match (c, pos) {
386                ('[', 0) => pos += 1,
387                (' ', 1) => continue,
388                ('.', 1) => pos += 1,
389                ('.', 2) => pos += 1,
390                ('.', 3) => continue,
391                (' ', 3) => continue,
392                (']', 3) => pos = 0,
393                (_, _) => pos = 0,
394            }
395            ret.push(c);
396        }
397
398        ret
399    }
400
401    pub fn run(&mut self) -> Result<bool, io::Error> {
402        let assert = Assert::new()
403            .normalize_paths(false)
404            .redact_with(self.subs.clone());
405        let mut runner = TestRunner::new(self);
406
407        fs::create_dir_all(&self.cwd)?;
408
409        // For each code block.
410        for test in &self.tests {
411            let mut run = runner.run(test);
412
413            // For each command.
414            for (i, assertion) in test.assertions.iter().enumerate() {
415                let location = assertion
416                    .path
417                    .file_name()
418                    .map(|f| f.to_string_lossy().to_string())
419                    .map(|f| f.strip_suffix(".md").unwrap_or(&f).to_owned())
420                    .map(|f| f + ":" + assertion.line.to_string().as_str())
421                    .unwrap_or(String::from("<none>"));
422
423                if assertion.command == "cd" {
424                    let arg = assertion.args.first().unwrap();
425                    let dir: PathBuf = arg.into();
426                    let dir = run.path().join(dir);
427
428                    // TODO: Add support for `..` and `/`
429                    // TODO: Error if more than one args are given.
430
431                    log::debug!(target: "test", "{location}: `cd {}`..", dir.display());
432
433                    if !dir.exists() {
434                        return Err(io::Error::new(
435                            io::ErrorKind::NotFound,
436                            format!("cd: '{}' does not exist", dir.display()),
437                        ));
438                    }
439                    run.cd(dir);
440
441                    continue;
442                }
443
444                // Expand environment variables.
445                let mut args = assertion.args.clone();
446                for arg in &mut args {
447                    for (k, v) in run.envs() {
448                        *arg = arg.replace(format!("${k}").as_str(), &v);
449                    }
450                }
451
452                if !run.path().exists() {
453                    log::warn!(target: "test", "{location}: Directory {} does not exist. Creating..", run.path().display());
454                    fs::create_dir_all(run.path())?;
455                }
456
457                let jj_envs = if assertion.command == "jj" {
458                    vec![
459                        ("JJ_RANDOMNESS_SEED", i.to_string()),
460                        ("JJ_TIMESTAMP", "2001-02-03T04:05:06+07:00".to_string()),
461                        ("JJ_OP_TIMESTAMP", "2001-02-03T04:05:06+07:00".to_string()),
462                    ]
463                } else {
464                    vec![]
465                };
466
467                let bins = std::env::join_paths(bins(self.cwd.clone())).unwrap();
468
469                let command = Command::new(assertion.command.clone())
470                    .env_clear()
471                    .env("PATH", &bins)
472                    .env("RUST_BACKTRACE", "1")
473                    .envs(jj_envs)
474                    .envs(run.envs())
475                    .current_dir(run.path())
476                    .args(args.clone())
477                    .with_assert(assert.clone());
478
479                log::debug!(target: "test", "{location}: `{} {}` @ {}", assertion.command, args.join(" "), run.path().display());
480                log::trace!(target: "test", "{location}: {}", run.envs().map(|(k, v)| format!("{}={}", k, v)).collect::<Vec<_>>().join(", "));
481                log::logger().flush();
482
483                // Even though it would be possible to use `Command::assert` to directly obtain
484                // `OutputAssert`, we use `Command::output` to be able to handle `io::ErrorKind::NotFound`
485                // separately and provide a more helpful error message in that case.
486                match command.output() {
487                    Ok(output) => {
488                        let assert = OutputAssert::new(output).with_assert(assert.clone());
489                        let expected = Self::map_spaced_brackets(&assertion.expected);
490
491                        let expected = {
492                            #[cfg(windows)]
493                            const EXE: &str = ".exe";
494
495                            #[cfg(unix)]
496                            const EXE: &str = "";
497
498                            expected.replace("[EXE]", EXE)
499                        };
500
501                        let matches = if test.stderr {
502                            assert.stderr_eq(&expected)
503                        } else {
504                            assert.stdout_eq(&expected)
505                        };
506                        match assertion.exit {
507                            ExitStatus::Success => {
508                                matches.success();
509                            }
510                            ExitStatus::Failure => {
511                                matches.failure();
512                            }
513                        }
514                    }
515                    Err(err) => {
516                        if err.kind() == io::ErrorKind::NotFound {
517                            log::error!(target: "test", "{location}: Command `{}` does not exist..", assertion.command);
518                        }
519                        return Err(io::Error::new(
520                            err.kind(),
521                            format!("{location}: {err}: `{}`", assertion.command),
522                        ));
523                    }
524                }
525            }
526            runner.finish(run);
527        }
528        Ok(true)
529    }
530}
531
532fn cargo_manifest_dir() -> PathBuf {
533    env::var("CARGO_MANIFEST_DIR").map(PathBuf::from).unwrap()
534}
535
536fn cargo_target_dir() -> PathBuf {
537    env::var("CARGO_TARGET_DIR")
538        .map(PathBuf::from)
539        .unwrap_or(cargo_manifest_dir().join(CARGO_TARGET_DIR_DIRNAME))
540}
541
542/// Get the list of binary paths to use as `$PATH` for the tests,
543/// starting with the current working directory.
544fn bins(cwd: PathBuf) -> Vec<PathBuf> {
545    let mut bins: Vec<PathBuf> = Vec::new();
546
547    // Add current working directory to `$PATH`,
548    // this makes it more convenient to execute scripts during testing.
549    bins.push(cwd);
550
551    bins.push(cargo_target_dir().join(CARGO_PROFILE));
552
553    // Add the "real" `$PATH`.
554    if let Ok(path) = env::var("PATH") {
555        bins.extend(env::split_paths(&path));
556    }
557
558    #[cfg(windows)]
559    {
560        // Radicle CLI tests rely on various Unix coreutils
561        // (such as `ls` and `touch`) being available.
562        // On Windows, it is very likely that we can find them in the
563        // following location.
564        // Note that adding this path to the end of `$PATH` causes
565        // no harm, even if the directory does not exist.
566        bins.push(PathBuf::from(r#"C:\Program Files\Git\usr\bin"#));
567    }
568
569    bins
570}
571
572#[cfg(test)]
573mod tests {
574    use super::*;
575
576    use pretty_assertions::assert_eq;
577
578    #[test]
579    fn test_parse() {
580        let input = r#"
581Let's try to track @dave and @sean:
582``` RAD_HINT=true
583$ rad track @dave
584Tracking relationship established for @dave.
585Nothing to do.
586
587$ rad track @sean
588Tracking relationship established for @sean.
589Nothing to do.
590```
591Super, now let's move on to the next step.
592``` ~alice (stderr)
593$ rad sync
594```
595"#
596        .trim()
597        .as_bytes()
598        .to_owned();
599
600        let cwd = PathBuf::from("radicle-cli-test");
601
602        let mut actual = TestFormula::new(cwd.clone());
603        let path = Path::new("test.md").to_path_buf();
604        actual
605            .read(path.as_path(), io::BufReader::new(io::Cursor::new(input)))
606            .unwrap();
607
608        let expected = TestFormula {
609            homes: HashMap::new(),
610            cwd: cwd.clone(),
611            env: HashMap::new(),
612            subs: Redactions::new(),
613            tests: vec![
614                Test {
615                    context: vec![String::from("Let's try to track @dave and @sean:")],
616                    home: None,
617                    assertions: vec![
618                        Assertion {
619                            line: 3,
620                            path: path.clone(),
621                            command: String::from("rad"),
622                            args: vec![String::from("track"), String::from("@dave")],
623                            expected: String::from(
624                                "Tracking relationship established for @dave.\nNothing to do.\n\n",
625                            ),
626                            exit: ExitStatus::Success,
627                        },
628                        Assertion {
629                            line: 7,
630                            path: path.clone(),
631                            command: String::from("rad"),
632                            args: vec![String::from("track"), String::from("@sean")],
633                            expected: String::from(
634                                "Tracking relationship established for @sean.\nNothing to do.\n",
635                            ),
636                            exit: ExitStatus::Success,
637                        },
638                    ],
639                    fail: false,
640                    stderr: false,
641                    env: vec![("RAD_HINT".to_owned(), "true".to_owned())]
642                        .into_iter()
643                        .collect(),
644                },
645                Test {
646                    context: vec![String::from("Super, now let's move on to the next step.")],
647                    home: Some("alice".to_owned()),
648                    assertions: vec![Assertion {
649                        line: 13,
650                        path: path.clone(),
651                        command: String::from("rad"),
652                        args: vec![String::from("sync")],
653                        expected: String::new(),
654                        exit: ExitStatus::Success,
655                    }],
656                    fail: false,
657                    stderr: true,
658                    env: HashMap::default(),
659                },
660            ],
661        };
662
663        assert_eq!(actual, expected);
664    }
665
666    #[test]
667    fn test_run() {
668        let input = r#"
669Running a simple command such as `head`:
670```
671$ head -n 2 Cargo.toml
672[package]
673name = "radicle-cli-test"
674```
675"#
676        .trim()
677        .as_bytes()
678        .to_owned();
679
680        let mut formula = TestFormula::new(PathBuf::from_str(env!("CARGO_MANIFEST_DIR")).unwrap());
681        formula
682            .read(
683                Path::new("test.md"),
684                io::BufReader::new(io::Cursor::new(input)),
685            )
686            .unwrap();
687        formula.run().unwrap();
688    }
689
690    #[test]
691    fn test_example_spaced_brackets() {
692        let input = r#"
693Running a simple command such as `head`:
694```
695$ echo "    hello"
696[..]hello
697$ echo "    hello"
698[..  ]hello
699$ echo "    hello"
700[  ..]hello
701$ echo "[bug, good-first-issue]"
702[bug, good-first-issue]
703$ echo "[bug, good-first-issue]"
704[bug, [  ..    ]-issue]
705$ echo "[bug, good-first-issue]"
706[bug, [  ...   ]-issue]
707```
708"#
709        .trim()
710        .as_bytes()
711        .to_owned();
712
713        let mut formula = TestFormula::new(PathBuf::from_str(env!("CARGO_MANIFEST_DIR")).unwrap());
714        formula
715            .read(
716                Path::new("test.md"),
717                io::BufReader::new(io::Cursor::new(input)),
718            )
719            .unwrap();
720        formula.run().unwrap();
721    }
722}
radicle_cli_test/lib.rs

radicle_cli_test/
lib.rs
