cargo_referendum/
lib.rs

1use fasthash::sea;
2use regex::Regex;
3use std::collections::BTreeSet;
4use std::collections::HashMap;
5use std::fmt::Debug;
6use std::process::Command;
7use std::str;
8use string_builder::Builder;
9
10#[derive(thiserror::Error, Debug)]
11pub enum ReferendumError {
12    #[error("Test run failed to execute")]
13    TestRunFailure(),
14    #[error("Test run extraction failure")]
15    TestResultExtractionFailure(),
16    #[error("Tests not found failure")]
17    TestNotFound(),
18}
19
20pub type Result<T> = std::result::Result<T, ReferendumError>;
21
22fn run_tests(toolkit: &str) -> Result<String> {
23    let output = Command::new("rustup")
24        .arg("run")
25        .arg(toolkit)
26        .arg("cargo")
27        .arg("test")
28        .arg("--")
29        .arg("--test-threads=1")
30        .arg("--show-output")
31        .output()
32        .expect("Error running command");
33
34    match &output.status.success() {
35        false => {
36            //want to add information about the error here too
37            return Err(ReferendumError::TestRunFailure());
38        }
39        true => (),
40    };
41
42    match str::from_utf8(&output.stdout) {
43        Ok(v) => Ok(v.to_string()),
44        Err(e) => panic!("Invalid UTF-8 sequence: {}", e),
45    }
46}
47
48fn parse_test_output(output: &str) -> Vec<String> {
49    let lines: Vec<String> = output.split('\n').map(|x| x.to_string()).collect();
50
51    lines
52}
53
54fn get_test_names(lines: &[String]) -> BTreeSet<String> {
55    let re = Regex::new("test [a-zA-z_0-9]+::[a-zA-Z_0-9]+").unwrap();
56    let lines_string = lines.join(" ");
57    let names: Vec<String> = re
58        .find_iter(&lines_string)
59        .filter_map(|digits| digits.as_str().parse().ok())
60        .collect();
61    names
62        .iter()
63        .map(|name| name.replacen("test ", "", 1))
64        .collect()
65}
66
67fn get_test_result(test_name: &str, lines: &[String]) -> Result<bool> {
68    let re = Regex::new("[a-zA-z_0-9]+::[a-zA-Z_0-9]+ ... (ok|FAILED)").unwrap();
69    let re_should_panic =
70        Regex::new("[a-zA-z_0-9]+::[a-zA-Z_0-9]+ - should panic ... (ok|FAILED)").unwrap();
71    let lines_string = lines.join(" ");
72
73    let mut result_lines: Vec<String> = re
74        .find_iter(&lines_string)
75        .filter_map(|digits| digits.as_str().parse().ok())
76        .collect();
77
78    let panic_result_lines: Vec<String> = re_should_panic
79        .find_iter(&lines_string)
80        .filter_map(|digits| digits.as_str().parse().ok())
81        .collect();
82
83    result_lines.extend(panic_result_lines);
84
85    for item in result_lines.iter() {
86        if item.contains(test_name) {
87            return Ok(item.contains("ok"));
88        }
89    }
90    Err(ReferendumError::TestResultExtractionFailure())
91}
92
93fn generate_output_map(lines: &str) -> HashMap<String, String> {
94    let re = Regex::new(r"---- [a-zA-Z_0-9]+::[a-zA-Z_0-9]+ stdout ----").unwrap();
95    let test_name_re = Regex::new(r"[a-zA-Z_0-9]+::[a-zA-Z_0-9]+").unwrap();
96    let mut map = HashMap::new();
97
98    let lines = parse_test_output(lines);
99    for (mut i, line) in lines.iter().enumerate() {
100        if re.is_match(line) {
101            let start = i;
102            //this must match if re matches
103            let test = test_name_re.find(line).unwrap();
104            let name = line[test.start()..test.end()].to_string();
105            while i < lines.len() - 1 && !lines[i].is_empty() {
106                i += 1;
107            }
108            let output = lines[start + 1..i].join("\n").clone();
109            map.insert(name, output);
110        }
111    }
112    map
113}
114
115fn get_consensus_hash(tests: &[Test]) -> Option<u64> {
116    let mut map: HashMap<u64, u8> = HashMap::new();
117    for test in tests {
118        let count = map.entry(test.hash).or_insert(0);
119        *count += 1;
120    }
121
122    let max_hash = map.iter().max_by(|a, b| a.1.cmp(b.1)).unwrap();
123
124    match max_hash.1 {
125        1 => None,
126        _ => Some(*max_hash.0),
127    }
128}
129
130pub fn vote(tests: Vec<Test>) -> Result<VoteResult> {
131    let mut test_map: HashMap<String, Vec<Test>> = HashMap::new();
132    for test in tests {
133        let entry = test_map.entry(test.name.clone()).or_insert_with(Vec::new);
134        entry.push(test);
135    }
136
137    let mut matches: Vec<Test> = Vec::new();
138    let mut non_matches: Vec<Test> = Vec::new();
139    let mut no_consensus: Vec<Test> = Vec::new();
140
141    for (_name, test_list) in test_map.iter() {
142        let consensus = match get_consensus_hash(test_list) {
143            Some(hash) => hash,
144            None => {
145                for test in test_list {
146                    no_consensus.push(test.clone());
147                }
148                continue;
149            }
150        };
151
152        for test in test_list {
153            match test.hash == consensus {
154                true => matches.push(test.clone()),
155                false => non_matches.push(test.clone()),
156            }
157        }
158    }
159
160    if matches.is_empty() && non_matches.is_empty() && no_consensus.is_empty() {
161        return Err(ReferendumError::TestNotFound());
162    }
163
164    Ok(VoteResult {
165        matches,
166        non_matches,
167        no_consensus,
168    })
169}
170
171pub fn get_tests(toolkits: Vec<&str>) -> Result<Vec<Test>> {
172    let mut tests: Vec<Test> = Vec::new();
173    //should I add something here to check that if a toolkit is installed in rustup
174    for kit in toolkits {
175        let run = &run_tests(kit)?;
176        let output_map = generate_output_map(run);
177        let lines = parse_test_output(run);
178        let unique_test_names = get_test_names(&lines);
179        for test in unique_test_names.iter() {
180            let test_output = match output_map.get(test) {
181                Some(v) => v.to_string(),
182                None => "".to_string(),
183            };
184            let test_result = get_test_result(test, &lines)?;
185            let output_obj = Test {
186                name: test.clone(),
187                toolkit: kit.to_string(),
188                result: test_result,
189                output: test_output.clone(),
190                hash: sea::hash64((test_result.to_string() + &test_output).as_bytes()),
191            };
192            tests.push(output_obj);
193        }
194    }
195    Ok(tests)
196}
197
198fn generate_test_result_output(name: &str, result: bool, toolkit: Option<&str>) -> String {
199    let mut builder = Builder::default();
200    builder.append("test ");
201    builder.append(name.to_string());
202    match toolkit {
203        Some(kit) => {
204            builder.append(" @ ");
205            builder.append(kit);
206        }
207        None => builder.append(" "),
208    }
209    builder.append(" ... ");
210    if result {
211        builder.append("ok");
212    } else {
213        builder.append("FAILED");
214    }
215    builder.string().unwrap()
216}
217
218fn generate_test_output_output(name: &str, output: &str, toolkit: Option<&str>) -> String {
219    let mut builder = Builder::default();
220    builder.append("\n\t---- test ");
221    builder.append(name);
222    if let Some(kit) = toolkit {
223        builder.append(" @ ");
224        builder.append(kit);
225    }
226
227    builder.append(" stdout ----\n");
228    builder.append("\t");
229    builder.append(output);
230    builder.append("\n");
231
232    builder.string().unwrap()
233}
234
235pub fn generate_consensus_map(consensus_votes: &[Test]) -> HashMap<String, Consensus> {
236    let mut consensus_map: HashMap<String, Consensus> = HashMap::new();
237    for matched_vote in consensus_votes.iter() {
238        if !consensus_map.contains_key(&matched_vote.name.to_string()) {
239            let consensus = Consensus {
240                name: matched_vote.name.clone(),
241                result: matched_vote.result,
242                output: matched_vote.output.clone(),
243            };
244            consensus_map.insert(consensus.name.to_string(), consensus);
245        }
246    }
247    consensus_map
248}
249
250pub fn get_consensus_results(consensus_map: &HashMap<String, Consensus>) -> String {
251    let mut builder = Builder::default();
252    builder.append("Consensus Test Results...\n");
253    for (name, vote) in consensus_map.iter() {
254        builder.append(generate_test_result_output(
255            name,
256            vote.result,
257            Some("consensus"),
258        ));
259        if !vote.output.is_empty() {
260            builder.append(generate_test_output_output(
261                name,
262                &vote.output,
263                Some("consensus"),
264            ));
265        }
266        builder.append("\n");
267    }
268
269    builder.string().unwrap()
270}
271
272pub fn get_dissenting_results(
273    dissenting_votes: Vec<Test>,
274    consensus_map: &HashMap<String, Consensus>,
275) -> String {
276    let mut builder = Builder::default();
277    builder.append("Dissenting Test Results...\n");
278    for dissenting_vote in dissenting_votes.iter() {
279        let consensus = consensus_map
280            .get(&dissenting_vote.name)
281            .expect("Non-matched test vote does not have corresponding consensus");
282
283        builder.append(generate_test_result_output(
284            &consensus.name,
285            consensus.result,
286            Some("consensus"),
287        ));
288        builder.append("\n");
289
290        builder.append(generate_test_result_output(
291            &dissenting_vote.name,
292            dissenting_vote.result,
293            Some(&dissenting_vote.toolkit),
294        ));
295
296        builder.append(generate_test_output_output(
297            &consensus.name,
298            &consensus.output,
299            Some("consensus"),
300        ));
301        builder.append(generate_test_output_output(
302            &dissenting_vote.name,
303            &dissenting_vote.output,
304            Some(&dissenting_vote.toolkit),
305        ));
306
307        builder.append("\n");
308    }
309    builder.string().unwrap()
310}
311
312pub fn get_no_consensus_results(no_consensus_votes: Vec<Test>) -> String {
313    let mut builder = Builder::default();
314    builder.append("No Consensus Results...\n");
315    for vote in no_consensus_votes.iter() {
316        builder.append(generate_test_result_output(
317            &vote.name,
318            vote.result,
319            Some(&vote.toolkit),
320        ));
321        builder.append(generate_test_output_output(
322            &vote.name,
323            &vote.output,
324            Some(&vote.toolkit),
325        ));
326        builder.append("\n");
327    }
328    builder.string().unwrap()
329}
330
331#[derive(Debug, Clone)]
332pub struct Test {
333    pub name: String,
334    pub toolkit: String,
335    pub result: bool,
336    pub output: String,
337    pub hash: u64,
338}
339
340#[derive(Debug)]
341pub struct Consensus {
342    pub name: String,
343    pub result: bool,
344    pub output: String,
345}
346
347#[derive(Debug)]
348pub struct VoteResult {
349    pub matches: Vec<Test>,
350    pub non_matches: Vec<Test>,
351    pub no_consensus: Vec<Test>,
352}
353
354#[cfg(test)]
355mod tests {
356    use super::*;
357
358    #[test]
359    fn parse_single_test() {
360        let input = "test tests::test_1 ... ok";
361        let expected = vec!["test tests::test_1 ... ok"];
362        assert_eq!(parse_test_output(input), expected);
363    }
364
365    #[test]
366    fn parse_double_test() {
367        let input = "Hello\nWorld";
368        let expected = vec!["Hello", "World"];
369        assert_eq!(parse_test_output(input), expected);
370    }
371
372    #[test]
373    fn extract_test_name() {
374        let input = vec![String::from("test tests::test_1 ... ok")];
375        let mut expected = BTreeSet::new();
376        expected.insert(String::from("tests::test_1"));
377        assert_eq!(get_test_names(&input), expected);
378    }
379
380    #[test]
381    fn get_consensus() {
382        let test_1 = Test {
383            name: "test_1".to_string(),
384            toolkit: "nightly".to_string(),
385            result: true,
386            output: "test output".to_string(),
387            hash: 42,
388        };
389
390        let test_2 = Test {
391            name: "test_2".to_string(),
392            toolkit: "nightly".to_string(),
393            result: false,
394            output: "test output".to_string(),
395            hash: 42,
396        };
397
398        let test_3 = Test {
399            name: "test_3".to_string(),
400            toolkit: "nightly".to_string(),
401            result: false,
402            output: "test output".to_string(),
403            hash: 12,
404        };
405
406        let tests: Vec<Test> = vec![test_1, test_2, test_3];
407        let consensus = get_consensus_hash(&tests);
408        assert_eq!(consensus, Some(42));
409    }
410
411    #[test]
412    fn get_test_result_normal() {
413        let test_name = "testing::test_name";
414        let lines = ["test testing::test_name ... ok".to_string()];
415
416        assert!(get_test_result(&test_name, &lines).unwrap());
417    }
418
419    #[test]
420    fn get_test_result_test_panics() {
421        let test_name = "testing::test_name";
422        let lines = ["test testing::test_name - should panic ... ok".to_string()];
423
424        assert!(get_test_result(&test_name, &lines).unwrap());
425    }
426
427    #[test]
428    fn vote_all_consensus() {
429        let test_1 = Test {
430            name: "test_name".to_string(),
431            toolkit: "nightly_1".to_string(),
432            result: true,
433            output: "this is the output".to_string(),
434            hash: 42,
435        };
436        let test_2 = Test {
437            name: "test_name".to_string(),
438            toolkit: "nightly_2".to_string(),
439            result: true,
440            output: "this is the output".to_string(),
441            hash: 42,
442        };
443        let test_3 = Test {
444            name: "test_name".to_string(),
445            toolkit: "nightly_3".to_string(),
446            result: true,
447            output: "this is the output".to_string(),
448            hash: 42,
449        };
450        let tests = vec![test_1, test_2, test_3];
451        let votes = vote(tests).unwrap();
452        assert_eq!(votes.matches.len(), 3);
453        assert_eq!(votes.non_matches.len(), 0);
454        assert_eq!(votes.no_consensus.len(), 0);
455    }
456
457    #[test]
458    fn vote_all_unmatched() {
459        let test_1 = Test {
460            name: "test_name".to_string(),
461            toolkit: "nightly_1".to_string(),
462            result: true,
463            output: "this is the output".to_string(),
464            hash: 42,
465        };
466        let test_2 = Test {
467            name: "test_name".to_string(),
468            toolkit: "nightly_2".to_string(),
469            result: true,
470            output: "this is the output".to_string(),
471            hash: 42,
472        };
473        let test_3 = Test {
474            name: "test_name".to_string(),
475            toolkit: "nightly_3".to_string(),
476            result: true,
477            output: "this is the output".to_string(),
478            hash: 12,
479        };
480        let tests = vec![test_1, test_2, test_3];
481        let votes = vote(tests).unwrap();
482        assert_eq!(votes.matches.len(), 2);
483        assert_eq!(votes.non_matches.len(), 1);
484        assert_eq!(votes.no_consensus.len(), 0);
485    }
486
487    #[test]
488    fn vote_no_consensus() {
489        let test_1 = Test {
490            name: "test_name".to_string(),
491            toolkit: "nightly_1".to_string(),
492            result: true,
493            output: "this is the output".to_string(),
494            hash: 42,
495        };
496        let test_2 = Test {
497            name: "test_name".to_string(),
498            toolkit: "nightly_2".to_string(),
499            result: true,
500            output: "this is the output".to_string(),
501            hash: 44,
502        };
503        let test_3 = Test {
504            name: "test_name".to_string(),
505            toolkit: "nightly_3".to_string(),
506            result: true,
507            output: "this is the output".to_string(),
508            hash: 12,
509        };
510        let tests = vec![test_1, test_2, test_3];
511        let votes = vote(tests).unwrap();
512        assert_eq!(votes.matches.len(), 0);
513        assert_eq!(votes.non_matches.len(), 0);
514        assert_eq!(votes.no_consensus.len(), 3);
515    }
516
517    #[test]
518    fn test_output_generation() {
519        let output =
520            generate_test_output_output(&"test_name", &"this is the output", Some("tester"));
521        let expected = "\n\t---- test test_name @ tester stdout ----\n\tthis is the output\n";
522        assert_eq!(output, expected);
523    }
524
525    #[test]
526    fn test_output_generation_no_output() {
527        let output = generate_test_output_output(&"test_name", &"", Some("tester"));
528        let expected = "\n\t---- test test_name @ tester stdout ----\n\t\n";
529        assert_eq!(output, expected);
530    }
531
532    #[test]
533    fn test_pass_result_generation() {
534        let output = generate_test_result_output(&"test_name", true, Some("tester"));
535        let expected = "test test_name @ tester ... ok";
536        assert_eq!(output, expected);
537    }
538
539    #[test]
540    fn test_failure_result_generation() {
541        let output = generate_test_result_output(&"test_name", false, Some("tester"));
542        let expected = "test test_name @ tester ... FAILED";
543        assert_eq!(output, expected);
544    }
545
546    #[test]
547    fn consensus_map_normal_generation() {
548        let test_1 = Test {
549            name: "test_name".to_string(),
550            toolkit: "nightly_1".to_string(),
551            result: true,
552            output: "this is the output".to_string(),
553            hash: 42,
554        };
555        let test_2 = Test {
556            name: "test_name".to_string(),
557            toolkit: "nightly_2".to_string(),
558            result: true,
559            output: "this is the output".to_string(),
560            hash: 44,
561        };
562        let test_3 = Test {
563            name: "test_name".to_string(),
564            toolkit: "nightly_3".to_string(),
565            result: true,
566            output: "this is the output".to_string(),
567            hash: 42,
568        };
569        let tests = vec![test_1, test_2, test_3];
570
571        assert_eq!(format!("{:?}", generate_consensus_map(&tests)),
572            "{\"test_name\": Consensus { name: \"test_name\", result: true, output: \"this is the output\" }}");
573    }
574
575    #[test]
576    fn consensus_result_normal_generation() {
577        let test_1 = Test {
578            name: "test_name".to_string(),
579            toolkit: "nightly_1".to_string(),
580            result: true,
581            output: "this is the output".to_string(),
582            hash: 42,
583        };
584        let test_2 = Test {
585            name: "test_name".to_string(),
586            toolkit: "nightly_2".to_string(),
587            result: true,
588            output: "this is the output".to_string(),
589            hash: 44,
590        };
591        let test_3 = Test {
592            name: "test_name".to_string(),
593            toolkit: "nightly_3".to_string(),
594            result: true,
595            output: "this is the output".to_string(),
596            hash: 42,
597        };
598        let tests = vec![test_1, test_2, test_3];
599        let map = generate_consensus_map(&tests);
600        assert_eq!(get_consensus_results(&map),
601            "Consensus Test Results...\ntest test_name @ consensus ... ok\n\t---- test test_name @ consensus stdout ----\n\tthis is the output\n\n");
602    }
603
604    #[test]
605    fn dissenting_result_normal_generation() {
606        let test_1 = Test {
607            name: "test_name".to_string(),
608            toolkit: "nightly_1".to_string(),
609            result: true,
610            output: "this is the output".to_string(),
611            hash: 42,
612        };
613        let test_2 = Test {
614            name: "test_name".to_string(),
615            toolkit: "nightly_2".to_string(),
616            result: true,
617            output: "this is the output".to_string(),
618            hash: 44,
619        };
620        let test_3 = Test {
621            name: "test_name".to_string(),
622            toolkit: "nightly_3".to_string(),
623            result: true,
624            output: "this is the output".to_string(),
625            hash: 42,
626        };
627        let test_4 = test_2.clone();
628        let tests = vec![test_1, test_2, test_3];
629        let map = generate_consensus_map(&tests);
630        assert_eq!(get_dissenting_results(vec![test_4], &map),
631            "Dissenting Test Results...\ntest test_name @ consensus ... ok\ntest test_name @ nightly_2 ... ok\n\t---- test test_name @ consensus stdout ----\n\tthis is the output\n\n\t---- test test_name @ nightly_2 stdout ----\n\tthis is the output\n\n");
632    }
633
634    #[test]
635    fn no_consensus_result_normal_generation() {
636        let test_1 = Test {
637            name: "test_name".to_string(),
638            toolkit: "nightly_1".to_string(),
639            result: true,
640            output: "this is the output".to_string(),
641            hash: 12,
642        };
643        let test_2 = Test {
644            name: "test_name".to_string(),
645            toolkit: "nightly_2".to_string(),
646            result: true,
647            output: "this is the output".to_string(),
648            hash: 44,
649        };
650        let tests = vec![test_1, test_2];
651        assert_eq!(get_no_consensus_results(tests),
652            "No Consensus Results...\ntest test_name @ nightly_1 ... ok\n\t---- test test_name @ nightly_1 stdout ----\n\tthis is the output\n\ntest test_name @ nightly_2 ... ok\n\t---- test test_name @ nightly_2 stdout ----\n\tthis is the output\n\n");
653    }
654
655    #[test]
656    fn multiple_test_outputs() {
657        let output = "---- tests::test_1 stdout ----
658this is the first chunk of output
659
660---- tests::test_2 stdout ----
661this is the second chunk of output
662
663---- tests::test_3 stdout ----
664this is the third chunk of output
665
666successes:"
667            .to_string();
668
669        let output_map = generate_output_map(&output);
670        assert_eq!(
671            output_map.get("tests::test_1").unwrap(),
672            "this is the first chunk of output"
673        );
674        assert_eq!(
675            output_map.get("tests::test_2").unwrap(),
676            "this is the second chunk of output"
677        );
678        assert_eq!(
679            output_map.get("tests::test_3").unwrap(),
680            "this is the third chunk of output"
681        );
682    }
683}
cargo_referendum/lib.rs

cargo_referendum/
lib.rs
