1use std::borrow::Cow;
4use std::collections::BTreeMap;
5use std::collections::HashMap;
6use std::fs;
7use std::io::BufRead;
8use std::path::Component;
9use std::path::Path;
10use std::path::PathBuf;
11
12use anyhow::Context;
13use anyhow::Result;
14use anyhow::bail;
15use indexmap::IndexMap;
16use itertools::Itertools;
17use rev_buf_reader::RevBufReader;
18use wdl_analysis::document::Document;
19use wdl_analysis::document::Task;
20use wdl_analysis::types::Type;
21use wdl_ast::Diagnostic;
22use wdl_ast::Span;
23use wdl_ast::SupportedVersion;
24use wdl_ast::v1::TASK_REQUIREMENT_RETURN_CODES;
25use wdl_ast::v1::TASK_REQUIREMENT_RETURN_CODES_ALIAS;
26
27use crate::CompoundValue;
28use crate::Outputs;
29use crate::PrimitiveValue;
30use crate::TaskExecutionResult;
31use crate::Value;
32use crate::http::Downloader;
33use crate::http::Location;
34use crate::path::EvaluationPath;
35use crate::stdlib::download_file;
36
37pub mod v1;
38
39const MAX_STDERR_LINES: usize = 10;
41
42#[derive(Debug, Clone)]
44pub struct CallLocation {
45 pub document: Document,
47 pub span: Span,
49}
50
51#[derive(Debug)]
53pub struct SourceError {
54 pub document: Document,
56 pub diagnostic: Diagnostic,
58 pub backtrace: Vec<CallLocation>,
65}
66
67#[derive(Debug)]
69pub enum EvaluationError {
70 Source(Box<SourceError>),
72 Other(anyhow::Error),
74}
75
76impl EvaluationError {
77 pub fn new(document: Document, diagnostic: Diagnostic) -> Self {
79 Self::Source(Box::new(SourceError {
80 document,
81 diagnostic,
82 backtrace: Default::default(),
83 }))
84 }
85}
86
87impl From<anyhow::Error> for EvaluationError {
88 fn from(e: anyhow::Error) -> Self {
89 Self::Other(e)
90 }
91}
92
93pub type EvaluationResult<T> = Result<T, EvaluationError>;
95
96pub trait EvaluationContext: Send + Sync {
98 fn version(&self) -> SupportedVersion;
100
101 fn resolve_name(&self, name: &str, span: Span) -> Result<Value, Diagnostic>;
103
104 fn resolve_type_name(&self, name: &str, span: Span) -> Result<Type, Diagnostic>;
106
107 fn work_dir(&self) -> Option<&EvaluationPath>;
111
112 fn temp_dir(&self) -> &Path;
114
115 fn stdout(&self) -> Option<&Value>;
119
120 fn stderr(&self) -> Option<&Value>;
124
125 fn task(&self) -> Option<&Task>;
129
130 fn translate_path(&self, path: &str) -> Option<Cow<'_, Path>>;
134
135 fn downloader(&self) -> &dyn Downloader;
137}
138
139#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
141pub struct ScopeIndex(usize);
142
143impl ScopeIndex {
144 pub const fn new(index: usize) -> Self {
146 Self(index)
147 }
148}
149
150impl From<usize> for ScopeIndex {
151 fn from(index: usize) -> Self {
152 Self(index)
153 }
154}
155
156impl From<ScopeIndex> for usize {
157 fn from(index: ScopeIndex) -> Self {
158 index.0
159 }
160}
161
162#[derive(Default, Debug)]
164pub struct Scope {
165 parent: Option<ScopeIndex>,
169 names: IndexMap<String, Value>,
171}
172
173impl Scope {
174 pub fn new(parent: ScopeIndex) -> Self {
176 Self {
177 parent: Some(parent),
178 names: Default::default(),
179 }
180 }
181
182 pub fn insert(&mut self, name: impl Into<String>, value: impl Into<Value>) {
184 let prev = self.names.insert(name.into(), value.into());
185 assert!(prev.is_none(), "conflicting name in scope");
186 }
187
188 pub fn local(&self) -> impl Iterator<Item = (&str, &Value)> + use<'_> {
190 self.names.iter().map(|(k, v)| (k.as_str(), v))
191 }
192
193 pub(crate) fn get_mut(&mut self, name: &str) -> Option<&mut Value> {
195 self.names.get_mut(name)
196 }
197
198 pub(crate) fn clear(&mut self) {
200 self.parent = None;
201 self.names.clear();
202 }
203
204 pub(crate) fn set_parent(&mut self, parent: ScopeIndex) {
206 self.parent = Some(parent);
207 }
208}
209
210impl From<Scope> for IndexMap<String, Value> {
211 fn from(scope: Scope) -> Self {
212 scope.names
213 }
214}
215
216#[derive(Debug, Clone, Copy)]
218pub struct ScopeRef<'a> {
219 scopes: &'a [Scope],
221 index: ScopeIndex,
223}
224
225impl<'a> ScopeRef<'a> {
226 pub fn new(scopes: &'a [Scope], index: impl Into<ScopeIndex>) -> Self {
228 Self {
229 scopes,
230 index: index.into(),
231 }
232 }
233
234 pub fn parent(&self) -> Option<Self> {
238 self.scopes[self.index.0].parent.map(|p| Self {
239 scopes: self.scopes,
240 index: p,
241 })
242 }
243
244 pub fn names(&self) -> impl Iterator<Item = (&str, &Value)> + use<'_> {
246 self.scopes[self.index.0]
247 .names
248 .iter()
249 .map(|(n, name)| (n.as_str(), name))
250 }
251
252 pub fn for_each(&self, mut cb: impl FnMut(&str, &Value) -> Result<()>) -> Result<()> {
257 let mut current = Some(self.index);
258
259 while let Some(index) = current {
260 for (n, v) in self.scopes[index.0].local() {
261 cb(n, v)?;
262 }
263
264 current = self.scopes[index.0].parent;
265 }
266
267 Ok(())
268 }
269
270 pub fn local(&self, name: &str) -> Option<&Value> {
274 self.scopes[self.index.0].names.get(name)
275 }
276
277 pub fn lookup(&self, name: &str) -> Option<&Value> {
281 let mut current = Some(self.index);
282
283 while let Some(index) = current {
284 if let Some(name) = self.scopes[index.0].names.get(name) {
285 return Some(name);
286 }
287
288 current = self.scopes[index.0].parent;
289 }
290
291 None
292 }
293}
294
295#[derive(Debug)]
297pub struct EvaluatedTask {
298 attempt_dir: PathBuf,
300 result: TaskExecutionResult,
302 outputs: EvaluationResult<Outputs>,
310}
311
312impl EvaluatedTask {
313 fn new(attempt_dir: PathBuf, result: TaskExecutionResult) -> anyhow::Result<Self> {
317 Ok(Self {
318 result,
319 attempt_dir,
320 outputs: Ok(Default::default()),
321 })
322 }
323
324 pub fn exit_code(&self) -> i32 {
326 self.result.exit_code
327 }
328
329 pub fn attempt_dir(&self) -> &Path {
331 &self.attempt_dir
332 }
333
334 pub fn inputs(&self) -> &[Input] {
336 &self.result.inputs
337 }
338
339 pub fn work_dir(&self) -> &EvaluationPath {
341 &self.result.work_dir
342 }
343
344 pub fn stdout(&self) -> &Value {
346 &self.result.stdout
347 }
348
349 pub fn stderr(&self) -> &Value {
351 &self.result.stderr
352 }
353
354 pub fn outputs(&self) -> &EvaluationResult<Outputs> {
362 &self.outputs
363 }
364
365 pub fn into_result(self) -> EvaluationResult<Outputs> {
371 self.outputs
372 }
373
374 async fn handle_exit(
378 &self,
379 requirements: &HashMap<String, Value>,
380 downloader: &dyn Downloader,
381 ) -> anyhow::Result<()> {
382 let mut error = true;
383 if let Some(return_codes) = requirements
384 .get(TASK_REQUIREMENT_RETURN_CODES)
385 .or_else(|| requirements.get(TASK_REQUIREMENT_RETURN_CODES_ALIAS))
386 {
387 match return_codes {
388 Value::Primitive(PrimitiveValue::String(s)) if s.as_ref() == "*" => {
389 error = false;
390 }
391 Value::Primitive(PrimitiveValue::String(s)) => {
392 bail!(
393 "invalid return code value `{s}`: only `*` is accepted when the return \
394 code is specified as a string"
395 );
396 }
397 Value::Primitive(PrimitiveValue::Integer(ok)) => {
398 if self.result.exit_code == i32::try_from(*ok).unwrap_or_default() {
399 error = false;
400 }
401 }
402 Value::Compound(CompoundValue::Array(codes)) => {
403 error = !codes.as_slice().iter().any(|v| {
404 v.as_integer()
405 .map(|i| i32::try_from(i).unwrap_or_default() == self.result.exit_code)
406 .unwrap_or(false)
407 });
408 }
409 _ => unreachable!("unexpected return codes value"),
410 }
411 } else {
412 error = self.result.exit_code != 0;
413 }
414
415 if error {
416 let stderr = download_file(downloader, None, self.stderr().as_file().unwrap())
419 .await
420 .ok()
421 .and_then(|l| {
422 fs::File::open(l).ok().map(|f| {
423 let reader = RevBufReader::new(f);
425 let lines: Vec<_> = reader
426 .lines()
427 .take(MAX_STDERR_LINES)
428 .map_while(|l| l.ok())
429 .collect();
430
431 lines
433 .iter()
434 .rev()
435 .format_with("\n", |l, f| f(&format_args!(" {l}")))
436 .to_string()
437 })
438 })
439 .unwrap_or_default();
440
441 bail!(
443 "process terminated with exit code {code}: see `{stdout_path}` and \
444 `{stderr_path}` for task output and the related files in \
445 `{dir}`{header}{stderr}{trailer}",
446 code = self.result.exit_code,
447 dir = self.attempt_dir().display(),
448 stdout_path = self.stdout().as_file().expect("must be file"),
449 stderr_path = self.stderr().as_file().expect("must be file"),
450 header = if stderr.is_empty() {
451 Cow::Borrowed("")
452 } else {
453 format!("\n\ntask stderr output (last {MAX_STDERR_LINES} lines):\n\n").into()
454 },
455 trailer = if stderr.is_empty() { "" } else { "\n" }
456 );
457 }
458
459 Ok(())
460 }
461}
462
463#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
465pub enum InputKind {
466 File,
468 Directory,
470}
471
472impl From<InputKind> for crankshaft::engine::task::input::Type {
473 fn from(value: InputKind) -> Self {
474 match value {
475 InputKind::File => Self::File,
476 InputKind::Directory => Self::Directory,
477 }
478 }
479}
480
481#[derive(Debug, Clone)]
483pub struct Input {
484 kind: InputKind,
486 path: EvaluationPath,
488 location: Option<Location<'static>>,
492 guest_path: Option<String>,
494}
495
496impl Input {
497 pub fn new(kind: InputKind, path: EvaluationPath) -> Self {
499 Self {
500 kind,
501 path,
502 location: None,
503 guest_path: None,
504 }
505 }
506
507 pub fn from_primitive(value: &PrimitiveValue) -> Result<Self> {
509 let (kind, path) = match value {
510 PrimitiveValue::File(path) => (InputKind::File, path),
511 PrimitiveValue::Directory(path) => (InputKind::Directory, path),
512 _ => bail!("value is not a `File` or `Directory`"),
513 };
514
515 Ok(Self {
516 kind,
517 path: path.parse()?,
518 location: None,
519 guest_path: None,
520 })
521 }
522
523 pub fn kind(&self) -> InputKind {
525 self.kind
526 }
527
528 pub fn path(&self) -> &EvaluationPath {
530 &self.path
531 }
532
533 pub fn location(&self) -> Option<&Path> {
535 self.location.as_deref()
536 }
537
538 pub fn set_location(&mut self, location: Location<'static>) {
540 self.location = Some(location);
541 }
542
543 pub fn guest_path(&self) -> Option<&str> {
545 self.guest_path.as_deref()
546 }
547
548 pub fn set_guest_path(&mut self, path: impl Into<String>) {
550 self.guest_path = Some(path.into());
551 }
552}
553
554#[derive(Debug)]
556struct InputTrieNode<'a> {
557 children: BTreeMap<&'a str, Self>,
561 id: usize,
565 input: Option<(usize, &'a Input)>,
571}
572
573impl InputTrieNode<'_> {
574 fn new(id: usize) -> Self {
576 Self {
577 children: Default::default(),
578 id,
579 input: None,
580 }
581 }
582
583 fn calculate_guest_paths(
585 &self,
586 root: &str,
587 parent_id: usize,
588 paths: &mut Vec<(usize, String)>,
589 ) -> Result<()> {
590 if let Some((index, input)) = self.input {
592 let file_name = input.path.file_name()?.unwrap_or("");
593
594 let guest_path = if file_name.is_empty() {
596 format!(
597 "{root}{sep}{parent_id}/.root",
598 root = root,
599 sep = if root.as_bytes().last() == Some(&b'/') {
600 ""
601 } else {
602 "/"
603 }
604 )
605 } else {
606 format!(
607 "{root}{sep}{parent_id}/{file_name}",
608 root = root,
609 sep = if root.as_bytes().last() == Some(&b'/') {
610 ""
611 } else {
612 "/"
613 },
614 )
615 };
616
617 paths.push((index, guest_path));
618 }
619
620 for child in self.children.values() {
622 child.calculate_guest_paths(root, self.id, paths)?;
623 }
624
625 Ok(())
626 }
627}
628
629#[derive(Debug)]
635pub struct InputTrie<'a> {
636 urls: BTreeMap<&'a str, InputTrieNode<'a>>,
642 paths: BTreeMap<&'a str, InputTrieNode<'a>>,
648 next_id: usize,
650 count: usize,
652}
653
654impl<'a> InputTrie<'a> {
655 pub fn insert(&mut self, input: &'a Input) -> Result<()> {
657 let node = match &input.path {
658 EvaluationPath::Local(path) => {
659 if path.is_relative() {
663 self.count += 1;
664 return Ok(());
665 }
666
667 let mut components = path.components();
668
669 let component = components
670 .next()
671 .context("input path cannot be empty")?
672 .as_os_str()
673 .to_str()
674 .with_context(|| {
675 format!("input path `{path}` is not UTF-8", path = path.display())
676 })?;
677 let mut node = self.paths.entry(component).or_insert_with(|| {
678 let node = InputTrieNode::new(self.next_id);
679 self.next_id += 1;
680 node
681 });
682
683 for component in components {
684 match component {
685 Component::CurDir | Component::ParentDir => {
686 bail!(
687 "input path `{path}` may not contain `.` or `..`",
688 path = path.display()
689 );
690 }
691 _ => {}
692 }
693
694 let component = component.as_os_str().to_str().with_context(|| {
695 format!("input path `{path}` is not UTF-8", path = path.display())
696 })?;
697 node = node.children.entry(component).or_insert_with(|| {
698 let node = InputTrieNode::new(self.next_id);
699 self.next_id += 1;
700 node
701 });
702 }
703
704 node
705 }
706 EvaluationPath::Remote(url) => {
707 let mut node = self.urls.entry(url.scheme()).or_insert_with(|| {
709 let node = InputTrieNode::new(self.next_id);
710 self.next_id += 1;
711 node
712 });
713
714 node = node.children.entry(url.authority()).or_insert_with(|| {
716 let node = InputTrieNode::new(self.next_id);
717 self.next_id += 1;
718 node
719 });
720
721 if let Some(segments) = url.path_segments() {
723 for segment in segments {
724 node = node.children.entry(segment).or_insert_with(|| {
725 let node = InputTrieNode::new(self.next_id);
726 self.next_id += 1;
727 node
728 });
729 }
730 }
731
732 node
734 }
735 };
736
737 node.input = Some((self.count, input));
738 self.count += 1;
739 Ok(())
740 }
741
742 pub fn calculate_guest_paths(&self, root: &str) -> Result<Vec<(usize, String)>> {
747 let mut paths = Vec::with_capacity(self.count);
748 for child in self.urls.values() {
749 child.calculate_guest_paths(root, 0, &mut paths)?;
750 }
751
752 for child in self.paths.values() {
753 child.calculate_guest_paths(root, 0, &mut paths)?;
754 }
755
756 Ok(paths)
757 }
758}
759
760impl Default for InputTrie<'_> {
761 fn default() -> Self {
762 Self {
763 urls: Default::default(),
764 paths: Default::default(),
765 next_id: 1,
767 count: 0,
768 }
769 }
770}
771
772#[cfg(test)]
773mod test {
774 use pretty_assertions::assert_eq;
775
776 use super::*;
777
778 #[test]
779 fn empty_trie() {
780 let empty = InputTrie::default();
781 let paths = empty.calculate_guest_paths("/mnt/").unwrap();
782 assert!(paths.is_empty());
783 }
784
785 #[cfg(unix)]
786 #[test]
787 fn non_empty_trie_unix() {
788 let mut trie = InputTrie::default();
789 let inputs = [
790 Input::new(InputKind::Directory, "/".parse().unwrap()),
791 Input::new(InputKind::File, "/foo/bar/foo.txt".parse().unwrap()),
792 Input::new(InputKind::File, "/foo/bar/bar.txt".parse().unwrap()),
793 Input::new(InputKind::File, "/foo/baz/foo.txt".parse().unwrap()),
794 Input::new(InputKind::File, "/foo/baz/bar.txt".parse().unwrap()),
795 Input::new(InputKind::File, "/bar/foo/foo.txt".parse().unwrap()),
796 Input::new(InputKind::File, "/bar/foo/bar.txt".parse().unwrap()),
797 Input::new(InputKind::Directory, "/baz".parse().unwrap()),
798 Input::new(InputKind::File, "https://example.com/".parse().unwrap()),
799 Input::new(
800 InputKind::File,
801 "https://example.com/foo/bar/foo.txt".parse().unwrap(),
802 ),
803 Input::new(
804 InputKind::File,
805 "https://example.com/foo/bar/bar.txt".parse().unwrap(),
806 ),
807 Input::new(
808 InputKind::File,
809 "https://example.com/foo/baz/foo.txt".parse().unwrap(),
810 ),
811 Input::new(
812 InputKind::File,
813 "https://example.com/foo/baz/bar.txt".parse().unwrap(),
814 ),
815 Input::new(
816 InputKind::File,
817 "https://example.com/bar/foo/foo.txt".parse().unwrap(),
818 ),
819 Input::new(
820 InputKind::File,
821 "https://example.com/bar/foo/bar.txt".parse().unwrap(),
822 ),
823 Input::new(InputKind::File, "https://foo.com/bar".parse().unwrap()),
824 ];
825
826 for input in &inputs {
827 trie.insert(input).unwrap();
828 }
829
830 let paths = trie.calculate_guest_paths("/mnt/").unwrap();
835 let paths: Vec<_> = paths
836 .iter()
837 .map(|(index, guest)| (inputs[*index].path().to_str().unwrap(), guest.as_str()))
838 .collect();
839
840 assert_eq!(
841 paths,
842 [
843 ("https://example.com/", "/mnt/15/.root"),
844 ("https://example.com/bar/foo/bar.txt", "/mnt/25/bar.txt"),
845 ("https://example.com/bar/foo/foo.txt", "/mnt/25/foo.txt"),
846 ("https://example.com/foo/bar/bar.txt", "/mnt/18/bar.txt"),
847 ("https://example.com/foo/bar/foo.txt", "/mnt/18/foo.txt"),
848 ("https://example.com/foo/baz/bar.txt", "/mnt/21/bar.txt"),
849 ("https://example.com/foo/baz/foo.txt", "/mnt/21/foo.txt"),
850 ("https://foo.com/bar", "/mnt/28/bar"),
851 ("/", "/mnt/0/.root"),
852 ("/bar/foo/bar.txt", "/mnt/10/bar.txt"),
853 ("/bar/foo/foo.txt", "/mnt/10/foo.txt"),
854 ("/baz", "/mnt/1/baz"),
855 ("/foo/bar/bar.txt", "/mnt/3/bar.txt"),
856 ("/foo/bar/foo.txt", "/mnt/3/foo.txt"),
857 ("/foo/baz/bar.txt", "/mnt/6/bar.txt"),
858 ("/foo/baz/foo.txt", "/mnt/6/foo.txt"),
859 ]
860 );
861 }
862
863 #[cfg(windows)]
864 #[test]
865 fn non_empty_trie_windows() {
866 let mut trie = InputTrie::default();
867 let inputs = [
868 Input::new(InputKind::Directory, "C:\\".parse().unwrap()),
869 Input::new(InputKind::File, "C:\\foo\\bar\\foo.txt".parse().unwrap()),
870 Input::new(InputKind::File, "C:\\foo\\bar\\bar.txt".parse().unwrap()),
871 Input::new(InputKind::File, "C:\\foo\\baz\\foo.txt".parse().unwrap()),
872 Input::new(InputKind::File, "C:\\foo\\baz\\bar.txt".parse().unwrap()),
873 Input::new(InputKind::File, "C:\\bar\\foo\\foo.txt".parse().unwrap()),
874 Input::new(InputKind::File, "C:\\bar\\foo\\bar.txt".parse().unwrap()),
875 Input::new(InputKind::Directory, "C:\\baz".parse().unwrap()),
876 Input::new(InputKind::File, "https://example.com/".parse().unwrap()),
877 Input::new(
878 InputKind::File,
879 "https://example.com/foo/bar/foo.txt".parse().unwrap(),
880 ),
881 Input::new(
882 InputKind::File,
883 "https://example.com/foo/bar/bar.txt".parse().unwrap(),
884 ),
885 Input::new(
886 InputKind::File,
887 "https://example.com/foo/baz/foo.txt".parse().unwrap(),
888 ),
889 Input::new(
890 InputKind::File,
891 "https://example.com/foo/baz/bar.txt".parse().unwrap(),
892 ),
893 Input::new(
894 InputKind::File,
895 "https://example.com/bar/foo/foo.txt".parse().unwrap(),
896 ),
897 Input::new(
898 InputKind::File,
899 "https://example.com/bar/foo/bar.txt".parse().unwrap(),
900 ),
901 Input::new(InputKind::File, "https://foo.com/bar".parse().unwrap()),
902 ];
903
904 for input in &inputs {
905 trie.insert(input).unwrap();
906 }
907
908 let paths = trie.calculate_guest_paths("/mnt/").unwrap();
913 let paths: Vec<_> = paths
914 .iter()
915 .map(|(index, guest)| (inputs[*index].path().to_str().unwrap(), guest.as_str()))
916 .collect();
917
918 assert_eq!(
919 paths,
920 [
921 ("https://example.com/", "/mnt/16/.root"),
922 ("https://example.com/bar/foo/bar.txt", "/mnt/26/bar.txt"),
923 ("https://example.com/bar/foo/foo.txt", "/mnt/26/foo.txt"),
924 ("https://example.com/foo/bar/bar.txt", "/mnt/19/bar.txt"),
925 ("https://example.com/foo/bar/foo.txt", "/mnt/19/foo.txt"),
926 ("https://example.com/foo/baz/bar.txt", "/mnt/22/bar.txt"),
927 ("https://example.com/foo/baz/foo.txt", "/mnt/22/foo.txt"),
928 ("https://foo.com/bar", "/mnt/29/bar"),
929 ("C:\\", "/mnt/1/.root"),
930 ("C:\\bar\\foo\\bar.txt", "/mnt/11/bar.txt"),
931 ("C:\\bar\\foo\\foo.txt", "/mnt/11/foo.txt"),
932 ("C:\\baz", "/mnt/2/baz"),
933 ("C:\\foo\\bar\\bar.txt", "/mnt/4/bar.txt"),
934 ("C:\\foo\\bar\\foo.txt", "/mnt/4/foo.txt"),
935 ("C:\\foo\\baz\\bar.txt", "/mnt/7/bar.txt"),
936 ("C:\\foo\\baz\\foo.txt", "/mnt/7/foo.txt"),
937 ]
938 );
939 }
940}