radicle_cli_test/
lib.rs

1#![allow(clippy::collapsible_else_if)]
2use std::borrow::Cow;
3use std::collections::HashMap;
4use std::path::{Path, PathBuf};
5use std::str::FromStr;
6use std::sync;
7use std::{env, ffi, fs, io, mem};
8
9use snapbox::cmd::{Command, OutputAssert};
10use snapbox::{Assert, Substitutions};
11use thiserror::Error;
12
13/// Used to ensure the build task is only run once.
14static BUILD: sync::Once = sync::Once::new();
15
16#[derive(Error, Debug)]
17pub enum Error {
18    #[error("parsing failed")]
19    Parse,
20    #[error("invalid file path: {0:?}")]
21    InvalidFilePath(String),
22    #[error("unknown home {0:?}")]
23    UnknownHome(String),
24    #[error("test file not found: {0:?}")]
25    TestNotFound(PathBuf),
26    #[error("i/o: {0}")]
27    Io(#[from] io::Error),
28    #[error("snapbox: {0}")]
29    Snapbox(#[from] snapbox::Error),
30}
31
32#[derive(Debug, PartialEq, Eq)]
33enum ExitStatus {
34    Success,
35    Failure,
36}
37
38/// A test which may contain multiple assertions.
39#[derive(Debug, Default, PartialEq, Eq)]
40pub struct Test {
41    /// Human-readable context around the test. Functions as documentation.
42    context: Vec<String>,
43    /// Test assertions to run.
44    assertions: Vec<Assertion>,
45    /// Whether to check stderr's output instead of stdout.
46    stderr: bool,
47    /// Whether to expect an error status code.
48    fail: bool,
49    /// Home directory under which to run this test.
50    home: Option<String>,
51    /// Local env vars to use just for this test.
52    env: HashMap<String, String>,
53}
54
55/// An assertion is a command to run with an expected output.
56#[derive(Debug, PartialEq, Eq)]
57pub struct Assertion {
58    /// The test file that contains this assertion.
59    path: PathBuf,
60    /// Name of command to run, eg. `git`.
61    command: String,
62    /// Command arguments, eg. `["push"]`.
63    args: Vec<String>,
64    /// Expected output (stdout or stderr).
65    expected: String,
66    /// Expected exit status.
67    exit: ExitStatus,
68}
69
70#[derive(Debug, Default, PartialEq, Eq, Clone)]
71pub struct Home {
72    name: Option<String>,
73    path: PathBuf,
74    envs: HashMap<String, String>,
75}
76
77#[derive(Debug)]
78pub struct TestRun {
79    home: Home,
80    env: HashMap<String, String>,
81}
82
83impl TestRun {
84    fn cd(&mut self, path: PathBuf) {
85        self.home.path = path;
86    }
87
88    fn envs(&self) -> impl Iterator<Item = (String, String)> + '_ {
89        self.home
90            .envs
91            .iter()
92            .chain(self.env.iter())
93            .map(|(k, v)| (k.to_owned(), v.to_owned()))
94            .chain(Some((
95                "PWD".to_owned(),
96                self.home.path.to_string_lossy().to_string(),
97            )))
98    }
99
100    fn path(&self) -> PathBuf {
101        self.home.path.clone()
102    }
103}
104
105#[derive(Debug)]
106pub struct TestRunner<'a> {
107    cwd: Option<PathBuf>,
108    homes: HashMap<String, Home>,
109    formula: &'a TestFormula,
110}
111
112impl<'a> TestRunner<'a> {
113    fn new(formula: &'a TestFormula) -> Self {
114        Self {
115            cwd: None,
116            homes: formula.homes.clone(),
117            formula,
118        }
119    }
120
121    fn run(&mut self, test: &'a Test) -> TestRun {
122        let mut env = self.formula.env.clone();
123        env.extend(test.env.clone());
124
125        if let Some(ref h) = test.home {
126            if let Some(home) = self.homes.get(h) {
127                return TestRun {
128                    home: home.clone(),
129                    env,
130                };
131            } else {
132                panic!("TestRunner::test: home `~{h}` does not exist");
133            }
134        }
135        TestRun {
136            home: Home {
137                name: None,
138                path: self.cwd.clone().unwrap_or_else(|| self.formula.cwd.clone()),
139                envs: HashMap::new(),
140            },
141            env,
142        }
143    }
144
145    fn finish(&mut self, run: TestRun) {
146        if let Some(name) = &run.home.name {
147            self.homes.insert(name.clone(), run.home);
148        } else {
149            self.cwd = Some(run.home.path);
150        }
151    }
152}
153
154#[derive(Debug, Default, PartialEq, Eq)]
155pub struct TestFormula {
156    /// Current working directory to run the test in.
157    cwd: PathBuf,
158    /// User homes.
159    homes: HashMap<String, Home>,
160    /// Environment to pass to the test.
161    env: HashMap<String, String>,
162    /// Tests to run.
163    tests: Vec<Test>,
164    /// Output substitutions.
165    subs: Substitutions,
166    /// Binaries path.
167    bins: Vec<PathBuf>,
168}
169
170impl TestFormula {
171    pub fn new(cwd: PathBuf) -> Self {
172        Self {
173            cwd: cwd.clone(),
174            env: HashMap::new(),
175            homes: HashMap::new(),
176            tests: Vec::new(),
177            subs: Substitutions::new(),
178            bins: env::var("PATH")
179                .map(|env_path| {
180                    let mut bins: Vec<PathBuf> = env_path.split(':').map(PathBuf::from).collect();
181                    // Add current working directory to `$PATH`,
182                    // this makes it more convenient to execute scripts during testing.
183                    bins.push(cwd);
184                    bins
185                })
186                .unwrap_or_default(),
187        }
188    }
189
190    pub fn build(&mut self, binaries: &[(&str, &str)]) -> &mut Self {
191        let manifest = env::var("CARGO_MANIFEST_DIR").expect(
192            "TestFormula::build: cannot build binaries: variable `CARGO_MANIFEST_DIR` is not set",
193        );
194        let profile = if cfg!(debug_assertions) {
195            "debug"
196        } else {
197            "release"
198        };
199        let target_dir = env::var("CARGO_TARGET_DIR").unwrap_or("target".to_string());
200        let manifest = Path::new(manifest.as_str());
201        let bins = manifest.join(&target_dir).join(profile);
202
203        // Add the target dir to the beginning of the list we will use as `PATH`.
204        self.bins.insert(0, bins);
205
206        // We don't need to re-build everytime the `build` function is called. Once is enough.
207        BUILD.call_once(|| {
208            use escargot::format::Message;
209            use radicle::logger::env_level;
210            use radicle::logger::test as logger;
211
212            logger::init(env_level().unwrap_or(log::Level::Debug));
213
214            for (package, binary) in binaries {
215                log::debug!(target: "test", "Building binaries for package `{package}`..");
216
217                let results = escargot::CargoBuild::new()
218                    .package(package)
219                    .bin(binary)
220                    .manifest_path(manifest.join("Cargo.toml"))
221                    .target_dir(&target_dir)
222                    .exec()
223                    .unwrap();
224
225                for result in results {
226                    match result {
227                        Ok(msg) => {
228                            if let Ok(Message::CompilerArtifact(a)) = msg.decode() {
229                                if let Some(e) = a.executable {
230                                    log::debug!(target: "test", "Built {}", e.display());
231                                }
232                            }
233                        }
234                        Err(e) => {
235                            log::error!(target: "test", "Error building package `{package}`: {e}");
236                        }
237                    }
238                }
239            }
240        });
241        self
242    }
243
244    pub fn env(&mut self, key: impl ToString, val: impl ToString) -> &mut Self {
245        self.env.insert(key.to_string(), val.to_string());
246        self
247    }
248
249    pub fn home(
250        &mut self,
251        user: impl ToString,
252        path: impl AsRef<Path>,
253        envs: impl IntoIterator<Item = (impl ToString, impl ToString)>,
254    ) -> &mut Self {
255        self.homes.insert(
256            user.to_string(),
257            Home {
258                name: Some(user.to_string()),
259                path: path.as_ref().to_path_buf(),
260                envs: envs
261                    .into_iter()
262                    .map(|(k, v)| (k.to_string(), v.to_string()))
263                    .collect(),
264            },
265        );
266        self
267    }
268
269    pub fn envs<K: ToString, V: ToString>(
270        &mut self,
271        envs: impl IntoIterator<Item = (K, V)>,
272    ) -> &mut Self {
273        for (k, v) in envs {
274            self.env.insert(k.to_string(), v.to_string());
275        }
276        self
277    }
278
279    pub fn file(&mut self, path: impl AsRef<Path>) -> Result<&mut Self, Error> {
280        let path = path.as_ref();
281        let contents = match fs::read(path) {
282            Ok(bytes) => bytes,
283            Err(err) if err.kind() == io::ErrorKind::NotFound => {
284                return Err(Error::TestNotFound(path.to_path_buf()));
285            }
286            Err(err) => return Err(err.into()),
287        };
288        self.read(path, io::Cursor::new(contents))
289    }
290
291    pub fn read(&mut self, path: &Path, r: impl io::BufRead) -> Result<&mut Self, Error> {
292        let mut test = Test::default();
293        let mut fenced = false; // Whether we're inside a fenced code block.
294        let mut file: Option<(PathBuf, String)> = None; // Path and content of file created by this test block.
295
296        for line in r.lines() {
297            let line = line?;
298
299            if line.starts_with("```") {
300                if fenced {
301                    if let Some((ref path, ref mut content)) = file.take() {
302                        // Write file.
303                        let path = self.cwd.join(path);
304
305                        if let Some(dir) = path.parent() {
306                            log::debug!(target: "test", "Creating directory {}..", dir.display());
307                            fs::create_dir_all(dir)?;
308                        }
309                        log::debug!(target: "test", "Writing {} bytes to {}..", content.len(), path.display());
310                        fs::write(path, content)?;
311                    } else {
312                        // End existing code block.
313                        self.tests.push(mem::take(&mut test));
314                    }
315                } else {
316                    for token in line.split_whitespace() {
317                        if let Some(home) = token.strip_prefix('~') {
318                            test.home = Some(home.to_owned());
319                        } else if let Some((key, val)) = token.split_once('=') {
320                            test.env.insert(key.to_owned(), val.to_owned());
321                        } else if token.contains("stderr") {
322                            test.stderr = true;
323                        } else if token.contains("fail") {
324                            test.fail = true;
325                        } else if let Some(path) = token.strip_prefix("./") {
326                            file = Some((
327                                PathBuf::from_str(path)
328                                    .map_err(|_| Error::InvalidFilePath(token.to_owned()))?,
329                                String::new(),
330                            ));
331                        }
332                    }
333                }
334                fenced = !fenced;
335
336                continue;
337            }
338
339            if fenced {
340                if let Some((_, ref mut content)) = file {
341                    content.push_str(line.as_str());
342                    content.push('\n');
343                } else if let Some(line) = line.strip_prefix('$') {
344                    let line = line.trim();
345                    let parts = shlex::split(line).ok_or(Error::Parse)?;
346                    let (cmd, args) = parts.split_first().ok_or(Error::Parse)?;
347
348                    test.assertions.push(Assertion {
349                        path: path.to_path_buf(),
350                        command: cmd.to_owned(),
351                        args: args.to_owned(),
352                        expected: String::new(),
353                        exit: if test.fail {
354                            ExitStatus::Failure
355                        } else {
356                            ExitStatus::Success
357                        },
358                    });
359                } else if let Some(a) = test.assertions.last_mut() {
360                    a.expected.push_str(line.as_str());
361                    a.expected.push('\n');
362                } else {
363                    return Err(Error::Parse);
364                }
365            } else {
366                test.context.push(line);
367            }
368        }
369        Ok(self)
370    }
371
372    #[allow(dead_code)]
373    pub fn substitute(
374        &mut self,
375        value: &'static str,
376        other: impl Into<Cow<'static, str>>,
377    ) -> Result<&mut Self, Error> {
378        self.subs.insert(value, other)?;
379        Ok(self)
380    }
381
382    /// Convert instances of '[..   ]' to '[..]' where the number of ' 's are arbitrary.
383    ///
384    /// Supporting these bracket types help support using the '[..]' pattern while preserving
385    /// spaces important for text alignment.
386    fn map_spaced_brackets(s: &str) -> String {
387        let mut ret = String::new();
388        let mut pos = 0;
389
390        for c in s.chars() {
391            match (c, pos) {
392                ('[', 0) => pos += 1,
393                (' ', 1) => continue,
394                ('.', 1) => pos += 1,
395                ('.', 2) => pos += 1,
396                ('.', 3) => continue,
397                (' ', 3) => continue,
398                (']', 3) => pos = 0,
399                (_, _) => pos = 0,
400            }
401            ret.push(c);
402        }
403
404        ret
405    }
406
407    pub fn run(&mut self) -> Result<bool, io::Error> {
408        let assert = Assert::new().substitutions(self.subs.clone());
409        let mut runner = TestRunner::new(self);
410
411        fs::create_dir_all(&self.cwd)?;
412        log::debug!(target: "test", "Using PATH {:?}", self.bins);
413
414        // For each code block.
415        for test in &self.tests {
416            let mut run = runner.run(test);
417
418            // For each command.
419            for assertion in &test.assertions {
420                // Expand environment variables.
421                let mut args = assertion.args.clone();
422                for arg in &mut args {
423                    for (k, v) in run.envs() {
424                        *arg = arg.replace(format!("${k}").as_str(), &v);
425                    }
426                }
427                let path = assertion
428                    .path
429                    .file_name()
430                    .map(|f| f.to_string_lossy().to_string())
431                    .unwrap_or(String::from("<none>"));
432                let cmd = if assertion.command == "rad" {
433                    snapbox::cmd::cargo_bin("rad")
434                } else if assertion.command == "cd" {
435                    let arg = assertion.args.first().unwrap();
436                    let dir: PathBuf = arg.into();
437                    let dir = run.path().join(dir);
438
439                    // TODO: Add support for `..` and `/`
440                    // TODO: Error if more than one args are given.
441
442                    log::debug!(target: "test", "{path}: Running `cd {}`..", dir.display());
443
444                    if !dir.exists() {
445                        return Err(io::Error::new(
446                            io::ErrorKind::NotFound,
447                            format!("cd: '{}' does not exist", dir.display()),
448                        ));
449                    }
450                    run.cd(dir);
451
452                    continue;
453                } else {
454                    PathBuf::from(&assertion.command)
455                };
456                log::debug!(target: "test", "{path}: Running `{}` with {:?} in `{}`..", cmd.display(), assertion.args, run.path().display());
457
458                if !run.path().exists() {
459                    log::warn!(target: "test", "{path}: Directory {} does not exist. Creating..", run.path().display());
460                    fs::create_dir_all(run.path())?;
461                }
462
463                let bins = self
464                    .bins
465                    .iter()
466                    .map(|p| p.as_os_str())
467                    .collect::<Vec<_>>()
468                    .join(ffi::OsStr::new(":"));
469                let result = Command::new(cmd.clone())
470                    .env_clear()
471                    .env("PATH", &bins)
472                    .env("RUST_BACKTRACE", "1")
473                    .envs(run.envs())
474                    .current_dir(run.path())
475                    .args(args)
476                    .with_assert(assert.clone())
477                    .output();
478
479                match result {
480                    Ok(output) => {
481                        let assert = OutputAssert::new(output).with_assert(assert.clone());
482                        let expected = Self::map_spaced_brackets(&assertion.expected);
483
484                        let matches = if test.stderr {
485                            assert.stderr_matches(&expected)
486                        } else {
487                            assert.stdout_matches(&expected)
488                        };
489                        match assertion.exit {
490                            ExitStatus::Success => {
491                                matches.success();
492                            }
493                            ExitStatus::Failure => {
494                                matches.failure();
495                            }
496                        }
497                    }
498                    Err(err) => {
499                        if err.kind() == io::ErrorKind::NotFound {
500                            log::error!(target: "test", "{path}: Command `{}` does not exist..", cmd.display());
501                        }
502                        return Err(io::Error::new(
503                            err.kind(),
504                            format!("{path}: {err}: `{}`", cmd.display()),
505                        ));
506                    }
507                }
508            }
509            runner.finish(run);
510        }
511        Ok(true)
512    }
513}
514
515#[cfg(test)]
516mod tests {
517    use super::*;
518
519    use pretty_assertions::assert_eq;
520
521    #[test]
522    fn test_parse() {
523        let input = r#"
524Let's try to track @dave and @sean:
525``` RAD_HINT=true
526$ rad track @dave
527Tracking relationship established for @dave.
528Nothing to do.
529
530$ rad track @sean
531Tracking relationship established for @sean.
532Nothing to do.
533```
534Super, now let's move on to the next step.
535``` ~alice (stderr)
536$ rad sync
537```
538"#
539        .trim()
540        .as_bytes()
541        .to_owned();
542
543        let cwd = PathBuf::from("radicle-cli-test");
544
545        let mut actual = TestFormula::new(cwd.clone());
546        let path = Path::new("test.md").to_path_buf();
547        actual
548            .read(path.as_path(), io::BufReader::new(io::Cursor::new(input)))
549            .unwrap();
550
551        let expected = TestFormula {
552            homes: HashMap::new(),
553            cwd: cwd.clone(),
554            env: HashMap::new(),
555            subs: Substitutions::new(),
556            bins: {
557                let mut bins: Vec<_> = env::var("PATH")
558                    .unwrap_or_default()
559                    .split(':')
560                    .map(PathBuf::from)
561                    .collect();
562                bins.push(cwd);
563                bins
564            },
565            tests: vec![
566                Test {
567                    context: vec![String::from("Let's try to track @dave and @sean:")],
568                    home: None,
569                    assertions: vec![
570                        Assertion {
571                            path: path.clone(),
572                            command: String::from("rad"),
573                            args: vec![String::from("track"), String::from("@dave")],
574                            expected: String::from(
575                                "Tracking relationship established for @dave.\nNothing to do.\n\n",
576                            ),
577                            exit: ExitStatus::Success,
578                        },
579                        Assertion {
580                            path: path.clone(),
581                            command: String::from("rad"),
582                            args: vec![String::from("track"), String::from("@sean")],
583                            expected: String::from(
584                                "Tracking relationship established for @sean.\nNothing to do.\n",
585                            ),
586                            exit: ExitStatus::Success,
587                        },
588                    ],
589                    fail: false,
590                    stderr: false,
591                    env: vec![("RAD_HINT".to_owned(), "true".to_owned())]
592                        .into_iter()
593                        .collect(),
594                },
595                Test {
596                    context: vec![String::from("Super, now let's move on to the next step.")],
597                    home: Some("alice".to_owned()),
598                    assertions: vec![Assertion {
599                        path: path.clone(),
600                        command: String::from("rad"),
601                        args: vec![String::from("sync")],
602                        expected: String::new(),
603                        exit: ExitStatus::Success,
604                    }],
605                    fail: false,
606                    stderr: true,
607                    env: HashMap::default(),
608                },
609            ],
610        };
611
612        assert_eq!(actual, expected);
613    }
614
615    #[test]
616    fn test_run() {
617        let input = r#"
618Running a simple command such as `head`:
619```
620$ head -n 2 Cargo.toml
621[package]
622name = "radicle-cli-test"
623```
624"#
625        .trim()
626        .as_bytes()
627        .to_owned();
628
629        let mut formula = TestFormula::new(PathBuf::from_str(env!("CARGO_MANIFEST_DIR")).unwrap());
630        formula
631            .read(
632                Path::new("test.md"),
633                io::BufReader::new(io::Cursor::new(input)),
634            )
635            .unwrap();
636        formula.run().unwrap();
637    }
638
639    #[test]
640    fn test_example_spaced_brackets() {
641        let input = r#"
642Running a simple command such as `head`:
643```
644$ echo "    hello"
645[..]hello
646$ echo "    hello"
647[..  ]hello
648$ echo "    hello"
649[  ..]hello
650$ echo "[bug, good-first-issue]"
651[bug, good-first-issue]
652$ echo "[bug, good-first-issue]"
653[bug, [  ..    ]-issue]
654$ echo "[bug, good-first-issue]"
655[bug, [  ...   ]-issue]
656```
657"#
658        .trim()
659        .as_bytes()
660        .to_owned();
661
662        let mut formula = TestFormula::new(PathBuf::from_str(env!("CARGO_MANIFEST_DIR")).unwrap());
663        formula
664            .read(
665                Path::new("test.md"),
666                io::BufReader::new(io::Cursor::new(input)),
667            )
668            .unwrap();
669        formula.run().unwrap();
670    }
671}
radicle_cli_test/lib.rs

radicle_cli_test/
lib.rs
