Skip to main content

opal/gitlab/
parser.rs

1// TODO: looks like there's a buttload of crap going on here - one there's a pipeline graph - great
2// - bad there's also a parses with sparse parsing function, wtf
3
4use crate::{GitLabRemoteConfig, env, git, runtime};
5use std::collections::HashMap;
6use std::convert::TryFrom;
7use std::fs;
8use std::path::{Path, PathBuf};
9use std::time::Duration;
10
11use anyhow::{Context, Result, anyhow, bail};
12use globset::Glob;
13use humantime;
14use petgraph::graph::{DiGraph, NodeIndex};
15use serde::Deserialize;
16use serde_yaml::value::TaggedValue;
17use serde_yaml::{Mapping, Value};
18use tracing::warn;
19
20use super::graph::{
21    ArtifactConfig, ArtifactWhen, CacheConfig, CacheKey, CachePolicy, DependencySource,
22    EnvironmentAction, EnvironmentConfig, ExternalDependency, ImageConfig, Job, JobDependency,
23    ParallelConfig, ParallelMatrixEntry, ParallelVariable, PipelineDefaults, PipelineGraph,
24    RetryPolicy, ServiceConfig, StageGroup, WorkflowConfig,
25};
26use super::rules::JobRule;
27
28impl PipelineGraph {
29    pub fn from_path(path: impl AsRef<Path>) -> Result<Self> {
30        Self::from_path_with_gitlab(path, None)
31    }
32
33    pub fn from_path_with_gitlab(
34        path: impl AsRef<Path>,
35        gitlab: Option<&GitLabRemoteConfig>,
36    ) -> Result<Self> {
37        Self::from_path_with_env(path, std::env::vars().collect(), gitlab)
38    }
39
40    fn from_path_with_env(
41        path: impl AsRef<Path>,
42        host_env: HashMap<String, String>,
43        gitlab: Option<&GitLabRemoteConfig>,
44    ) -> Result<Self> {
45        let path = path.as_ref();
46        // TODO: why's there a random file system operation in the middle of some function?
47        // move this to it's own thing such that this function accepts path as a param.
48        let canonical =
49            fs::canonicalize(path).with_context(|| format!("failed to resolve {:?}", path))?;
50        let include_root = git::repository_root(&canonical)
51            .unwrap_or_else(|_| canonical.parent().unwrap_or(Path::new(".")).to_path_buf());
52        let include_env = env::build_include_lookup(&canonical, &host_env);
53        let mut stack = Vec::new();
54        let root = load_pipeline_file(
55            &canonical,
56            &include_root,
57            &include_env,
58            gitlab,
59            None,
60            &mut stack,
61        )?;
62        let root = resolve_yaml_merge_keys(resolve_reference_tags(root)?)?;
63        Self::from_mapping(root)
64    }
65
66    pub fn from_yaml_str(contents: &str) -> Result<Self> {
67        let root: Mapping = serde_yaml::from_str(contents)?;
68        let root = resolve_yaml_merge_keys(resolve_reference_tags(root)?)?;
69        Self::from_mapping(root)
70    }
71
72    fn from_mapping(root: Mapping) -> Result<Self> {
73        let mut stage_names: Vec<String> = Vec::new();
74        let mut defaults = PipelineDefaults::default();
75        let mut workflow: Option<WorkflowConfig> = None;
76        let mut filters = super::graph::PipelineFilters::default();
77
78        let mut job_defs: HashMap<String, Value> = HashMap::new();
79        let mut job_names: Vec<String> = Vec::new();
80
81        for (key, value) in root {
82            match key {
83                Value::String(name) if name == "stages" => {
84                    stage_names = parse_stages(value)?;
85                }
86                Value::String(name) if name == "cache" => {
87                    defaults.cache = parse_cache_value(value)?;
88                }
89                Value::String(name) if name == "image" => {
90                    defaults.image = Some(parse_image(value)?);
91                }
92                Value::String(name) if name == "variables" => {
93                    let vars = parse_variables_map(value)?;
94                    defaults.variables.extend(vars);
95                }
96                Value::String(name) if name == "default" => {
97                    parse_default_block(&mut defaults, value)?;
98                }
99                Value::String(name) if name == "workflow" => {
100                    workflow = parse_workflow(value)?;
101                }
102                Value::String(name) if name == "only" => {
103                    filters.only = parse_filter_list(value, "only")?;
104                }
105                Value::String(name) if name == "except" => {
106                    filters.except = parse_filter_list(value, "except")?;
107                }
108                Value::String(name) => {
109                    if is_reserved_keyword(&name) {
110                        continue;
111                    }
112
113                    match value {
114                        Value::Mapping(map) => {
115                            job_defs.insert(name.clone(), Value::Mapping(map.clone()));
116                            if !name.starts_with('.') {
117                                job_names.push(name);
118                            }
119                        }
120                        other => bail!("job '{name}' must be defined as a mapping, got {other:?}"),
121                    }
122                }
123                other => bail!("expected string keys in pipeline, got {other:?}"),
124            }
125        }
126
127        build_graph(
128            defaults,
129            workflow,
130            filters,
131            stage_names,
132            job_names,
133            job_defs,
134        )
135    }
136}
137
138impl std::str::FromStr for PipelineGraph {
139    type Err = anyhow::Error;
140
141    fn from_str(s: &str) -> Result<Self, Self::Err> {
142        Self::from_yaml_str(s)
143    }
144}
145
146fn load_pipeline_file(
147    path: &Path,
148    include_root: &Path,
149    include_env: &HashMap<String, String>,
150    gitlab: Option<&GitLabRemoteConfig>,
151    project_context: Option<&ProjectIncludeContext>,
152    stack: &mut Vec<PathBuf>,
153) -> Result<Mapping> {
154    if stack.iter().any(|p| p == path) {
155        bail!("include cycle detected involving {:?}", path);
156    }
157    stack.push(path.to_path_buf());
158
159    let content = fs::read_to_string(path).with_context(|| format!("failed to read {:?}", path))?;
160    let mut root: Mapping = serde_yaml::from_str(&content)?;
161    let include_key = Value::String("include".to_string());
162    let mut combined = Mapping::new();
163
164    // TODO: this does too much - refactor and split accordingly
165    if let Some(include_value) = root.remove(&include_key) {
166        let includes = parse_include_entries(include_value)?;
167        for include in includes {
168            match include {
169                IncludeEntry::Local(path) => {
170                    let include = expand_include_path(&path, include_env);
171                    for resolved in
172                        resolve_include_paths(include_root, &include, project_context, include_env)?
173                    {
174                        validate_include_extension(&resolved)?;
175                        let canonical = fs::canonicalize(&resolved)
176                            .with_context(|| format!("failed to resolve include {:?}", resolved))?;
177                        let included = load_pipeline_file(
178                            &canonical,
179                            include_root,
180                            include_env,
181                            gitlab,
182                            project_context,
183                            stack,
184                        )?;
185                        combined = merge_mappings(combined, included);
186                    }
187                }
188                IncludeEntry::Project {
189                    project,
190                    reference,
191                    files,
192                } => {
193                    let gitlab = gitlab.ok_or_else(|| {
194                        anyhow!(
195                            "include:project requires GitLab credentials/configuration (use --gitlab-token and optionally --gitlab-base-url)"
196                        )
197                    })?;
198                    let resolved_ref = reference.unwrap_or_else(|| "HEAD".to_string());
199                    let project_root =
200                        project_include_root(&runtime::cache_root(), &project, &resolved_ref);
201                    let project_context = ProjectIncludeContext {
202                        gitlab: gitlab.clone(),
203                        project: project.clone(),
204                        reference: resolved_ref.clone(),
205                    };
206                    for file in files {
207                        let fetched = fetch_project_include_file(
208                            gitlab,
209                            &project_root,
210                            &project,
211                            &resolved_ref,
212                            &file,
213                            include_env,
214                        )?;
215                        let included = load_pipeline_file(
216                            &fetched,
217                            &project_root,
218                            include_env,
219                            Some(gitlab),
220                            Some(&project_context),
221                            stack,
222                        )?;
223                        combined = merge_mappings(combined, included);
224                    }
225                }
226            }
227        }
228    }
229
230    combined = merge_mappings(combined, root);
231    stack.pop();
232    Ok(combined)
233}
234
235fn resolve_include_path(include_root: &Path, include: &Path) -> PathBuf {
236    if let Ok(stripped) = include.strip_prefix(Path::new("/")) {
237        include_root.join(stripped)
238    } else if include.is_absolute() {
239        include.to_path_buf()
240    } else {
241        include_root.join(include)
242    }
243}
244
245fn expand_include_path(include: &Path, include_env: &HashMap<String, String>) -> PathBuf {
246    let expanded = env::expand_value(&include.to_string_lossy(), include_env);
247    PathBuf::from(expanded)
248}
249
250#[derive(Debug, Clone)]
251struct ProjectIncludeContext {
252    gitlab: GitLabRemoteConfig,
253    project: String,
254    reference: String,
255}
256
257fn resolve_include_paths(
258    include_root: &Path,
259    include: &Path,
260    project_context: Option<&ProjectIncludeContext>,
261    include_env: &HashMap<String, String>,
262) -> Result<Vec<PathBuf>> {
263    if !include_has_glob(include) {
264        let resolved = resolve_include_path(include_root, include);
265        if resolved.exists() || project_context.is_none() {
266            return Ok(vec![resolved]);
267        }
268        let Some(project_context) = project_context else {
269            return Ok(vec![resolved]);
270        };
271        let fetched = fetch_project_include_file(
272            &project_context.gitlab,
273            include_root,
274            &project_context.project,
275            &project_context.reference,
276            include,
277            include_env,
278        )?;
279        return Ok(vec![fetched]);
280    }
281
282    let pattern = include_glob_pattern(include);
283    let matcher = Glob::new(&pattern)
284        .with_context(|| format!("invalid include glob '{pattern}'"))?
285        .compile_matcher();
286    let mut matches = Vec::new();
287    for entry in walkdir::WalkDir::new(include_root).follow_links(false) {
288        let entry = entry?;
289        if !entry.file_type().is_file() {
290            continue;
291        }
292        let rel = entry
293            .path()
294            .strip_prefix(include_root)
295            .unwrap_or(entry.path());
296        if matcher.is_match(rel) {
297            matches.push(entry.path().to_path_buf());
298        }
299    }
300    matches.sort();
301    matches.dedup();
302    if matches.is_empty() && project_context.is_some() {
303        bail!("wildcard local includes inside include:project are not supported yet");
304    }
305    if matches.is_empty() {
306        bail!("include glob '{pattern}' matched no files");
307    }
308    Ok(matches)
309}
310
311fn include_has_glob(include: &Path) -> bool {
312    include
313        .to_string_lossy()
314        .chars()
315        .any(|ch| matches!(ch, '*' | '?' | '['))
316}
317
318fn include_glob_pattern(include: &Path) -> String {
319    include
320        .strip_prefix(Path::new("/"))
321        .unwrap_or(include)
322        .to_string_lossy()
323        .to_string()
324}
325
326fn validate_include_extension(path: &Path) -> Result<()> {
327    match path.extension().and_then(|ext| ext.to_str()) {
328        Some("yml") | Some("yaml") => Ok(()),
329        _ => bail!(
330            "include path '{}' must reference a .yml or .yaml file",
331            path.display()
332        ),
333    }
334}
335
336fn resolve_yaml_merge_keys(root: Mapping) -> Result<Mapping> {
337    let resolved = resolve_yaml_merge_value(Value::Mapping(root))?;
338    match resolved {
339        Value::Mapping(map) => Ok(map),
340        other => bail!(
341            "pipeline root must be a mapping after resolving YAML merge keys, got {}",
342            value_kind(&other)
343        ),
344    }
345}
346
347fn resolve_yaml_merge_value(value: Value) -> Result<Value> {
348    match value {
349        Value::Mapping(map) => Ok(Value::Mapping(resolve_yaml_merge_mapping(map)?)),
350        Value::Sequence(entries) => Ok(Value::Sequence(
351            entries
352                .into_iter()
353                .map(resolve_yaml_merge_value)
354                .collect::<Result<Vec<_>>>()?,
355        )),
356        other => Ok(other),
357    }
358}
359
360fn resolve_yaml_merge_mapping(map: Mapping) -> Result<Mapping> {
361    let merge_key = Value::String("<<".to_string());
362    let mut merged = Mapping::new();
363
364    if let Some(merge_value) = map.get(&merge_key).cloned() {
365        match resolve_yaml_merge_value(merge_value)? {
366            Value::Mapping(parent) => {
367                for (key, value) in parent {
368                    merged.insert(key, value);
369                }
370            }
371            Value::Sequence(entries) => {
372                for entry in entries {
373                    let Value::Mapping(parent) = resolve_yaml_merge_value(entry)? else {
374                        bail!("YAML merge key expects a mapping or list of mappings");
375                    };
376                    for (key, value) in parent {
377                        merged.insert(key, value);
378                    }
379                }
380            }
381            other => {
382                bail!(
383                    "YAML merge key expects a mapping or list of mappings, got {}",
384                    value_kind(&other)
385                );
386            }
387        }
388    }
389
390    for (key, value) in map {
391        if key == merge_key {
392            continue;
393        }
394        merged.insert(key, resolve_yaml_merge_value(value)?);
395    }
396
397    Ok(merged)
398}
399
400fn resolve_reference_tags(root: Mapping) -> Result<Mapping> {
401    let root_value = Value::Mapping(root);
402    let mut visiting = Vec::new();
403    let resolved = resolve_references(&root_value, &root_value, &mut visiting)?;
404    match resolved {
405        Value::Mapping(map) => Ok(map),
406        other => bail!(
407            "pipeline root must be a mapping after resolving !reference tags, got {}",
408            value_kind(&other)
409        ),
410    }
411}
412
413type ReferencePath = Vec<ReferenceSegment>;
414
415#[derive(Clone, PartialEq, Eq)]
416enum ReferenceSegment {
417    Key(String),
418    Index(usize),
419}
420
421fn resolve_references(
422    value: &Value,
423    root: &Value,
424    visiting: &mut Vec<ReferencePath>,
425) -> Result<Value> {
426    match value {
427        Value::Tagged(tagged) => {
428            if tagged.tag == "reference" {
429                let path = parse_reference_path(&tagged.value)?;
430                if visiting.iter().any(|current| current == &path) {
431                    bail!(
432                        "detected recursive !reference {}",
433                        describe_reference_path(&path)
434                    );
435                }
436                visiting.push(path.clone());
437                let target = follow_reference_path(root, &path).with_context(|| {
438                    format!(
439                        "failed to resolve !reference {}",
440                        describe_reference_path(&path)
441                    )
442                })?;
443                let resolved = resolve_references(target, root, visiting)?;
444                visiting.pop();
445                Ok(resolved)
446            } else {
447                let resolved_value = resolve_references(&tagged.value, root, visiting)?;
448                Ok(Value::Tagged(Box::new(TaggedValue {
449                    tag: tagged.tag.clone(),
450                    value: resolved_value,
451                })))
452            }
453        }
454        Value::Mapping(map) => {
455            let mut resolved = Mapping::with_capacity(map.len());
456            for (key, val) in map.iter() {
457                let resolved_key = resolve_references(key, root, visiting)?;
458                let resolved_val = resolve_references(val, root, visiting)?;
459                resolved.insert(resolved_key, resolved_val);
460            }
461            Ok(Value::Mapping(resolved))
462        }
463        Value::Sequence(seq) => {
464            let mut resolved = Vec::with_capacity(seq.len());
465            for entry in seq.iter() {
466                resolved.push(resolve_references(entry, root, visiting)?);
467            }
468            Ok(Value::Sequence(resolved))
469        }
470        other => Ok(other.clone()),
471    }
472}
473
474fn parse_reference_path(value: &Value) -> Result<ReferencePath> {
475    let entries = match value {
476        Value::Sequence(entries) => entries,
477        other => bail!(
478            "!reference expects a sequence path, got {}",
479            value_kind(other)
480        ),
481    };
482    if entries.is_empty() {
483        bail!("!reference path must contain at least one entry");
484    }
485    let mut path = Vec::with_capacity(entries.len());
486    for entry in entries.iter() {
487        match entry {
488            Value::String(name) => path.push(ReferenceSegment::Key(name.clone())),
489            Value::Number(number) => {
490                let index_u64 = number
491                    .as_u64()
492                    .ok_or_else(|| anyhow!("!reference indices must be non-negative integers"))?;
493                let index = usize::try_from(index_u64).map_err(|_| {
494                    anyhow!("!reference index {index_u64} is too large for this platform")
495                })?;
496                path.push(ReferenceSegment::Index(index));
497            }
498            other => bail!(
499                "!reference path entries must be strings or integers, got {}",
500                value_kind(other)
501            ),
502        }
503    }
504    Ok(path)
505}
506
507fn follow_reference_path<'a>(root: &'a Value, path: &[ReferenceSegment]) -> Result<&'a Value> {
508    let mut current = root;
509    for segment in path {
510        current = match segment {
511            ReferenceSegment::Key(name) => {
512                let mapping = value_as_mapping(current).ok_or_else(|| {
513                    anyhow!(
514                        "!reference {} expected a mapping before '{}', found {}",
515                        describe_reference_path(path),
516                        name,
517                        value_kind(current)
518                    )
519                })?;
520                mapping.get(name.as_str()).ok_or_else(|| {
521                    anyhow!(
522                        "!reference {} key '{}' not found",
523                        describe_reference_path(path),
524                        name
525                    )
526                })?
527            }
528            ReferenceSegment::Index(idx) => {
529                let sequence = value_as_sequence(current).ok_or_else(|| {
530                    anyhow!(
531                        "!reference {} expected a sequence before index {}, found {}",
532                        describe_reference_path(path),
533                        idx,
534                        value_kind(current)
535                    )
536                })?;
537                sequence.get(*idx).ok_or_else(|| {
538                    anyhow!(
539                        "!reference {} index {} out of bounds (len {})",
540                        describe_reference_path(path),
541                        idx,
542                        sequence.len()
543                    )
544                })?
545            }
546        };
547    }
548    Ok(current)
549}
550
551fn describe_reference_path(path: &[ReferenceSegment]) -> String {
552    let mut parts = Vec::with_capacity(path.len());
553    for segment in path {
554        match segment {
555            ReferenceSegment::Key(name) => parts.push(name.clone()),
556            ReferenceSegment::Index(idx) => parts.push(idx.to_string()),
557        }
558    }
559    format!("[{}]", parts.join(", "))
560}
561
562fn value_as_mapping(value: &Value) -> Option<&Mapping> {
563    match value {
564        Value::Mapping(map) => Some(map),
565        Value::Tagged(tagged) => value_as_mapping(&tagged.value),
566        _ => None,
567    }
568}
569
570fn value_as_sequence(value: &Value) -> Option<&Vec<Value>> {
571    match value {
572        Value::Sequence(seq) => Some(seq),
573        Value::Tagged(tagged) => value_as_sequence(&tagged.value),
574        _ => None,
575    }
576}
577
578fn value_kind(value: &Value) -> &'static str {
579    match value {
580        Value::Null => "null",
581        Value::Bool(_) => "bool",
582        Value::Number(_) => "number",
583        Value::String(_) => "string",
584        Value::Sequence(_) => "sequence",
585        Value::Mapping(_) => "mapping",
586        Value::Tagged(tagged) => value_kind(&tagged.value),
587    }
588}
589
590fn parse_stages(value: Value) -> Result<Vec<String>> {
591    match value {
592        Value::Sequence(entries) => entries
593            .into_iter()
594            .map(|val| match val {
595                Value::String(name) => Ok(name),
596                other => bail!("stage value must be string, got {other:?}"),
597            })
598            .collect(),
599        other => bail!("stages must be a sequence, got {other:?}"),
600    }
601}
602
603fn parse_default_block(defaults: &mut PipelineDefaults, value: Value) -> Result<()> {
604    let mapping = match value {
605        Value::Mapping(map) => map,
606        other => bail!("default section must be a mapping, got {other:?}"),
607    };
608
609    for (key, value) in mapping {
610        match key {
611            Value::String(name) if name == "before_script" => {
612                defaults.before_script = parse_string_list(value, "before_script")?;
613            }
614            Value::String(name) if name == "after_script" => {
615                defaults.after_script = parse_string_list(value, "after_script")?;
616            }
617            Value::String(name) if name == "image" => {
618                defaults.image = Some(parse_image(value)?);
619            }
620            Value::String(name) if name == "variables" => {
621                let vars = parse_variables_map(value)?;
622                defaults.variables.extend(vars);
623            }
624            Value::String(name) if name == "cache" => {
625                defaults.cache = parse_cache_value(value)?;
626            }
627            Value::String(name) if name == "services" => {
628                defaults.services = parse_services_value(value, "services")?;
629            }
630            Value::String(name) if name == "timeout" => {
631                defaults.timeout = parse_timeout_value(value, "default.timeout")?;
632            }
633            Value::String(name) if name == "retry" => {
634                let raw: RawRetry =
635                    serde_yaml::from_value(value).context("failed to parse default.retry")?;
636                defaults.retry = raw.into_policy(&RetryPolicy::default(), "default.retry")?;
637            }
638            Value::String(name) if name == "interruptible" => {
639                defaults.interruptible = extract_bool(value, "default.interruptible")?;
640            }
641            Value::String(_) => {
642                // ignore other default keywords for now
643            }
644            other => bail!("default keys must be strings, got {other:?}"),
645        }
646    }
647
648    Ok(())
649}
650
651fn parse_workflow(value: Value) -> Result<Option<WorkflowConfig>> {
652    let mapping = match value {
653        Value::Mapping(map) => map,
654        other => bail!("workflow section must be a mapping, got {other:?}"),
655    };
656    let key = Value::String("rules".to_string());
657    let Some(rules_value) = mapping.get(&key) else {
658        return Ok(None);
659    };
660    let rules: Vec<JobRule> =
661        serde_yaml::from_value(rules_value.clone()).context("failed to parse workflow.rules")?;
662    Ok(Some(WorkflowConfig { rules }))
663}
664
665fn is_reserved_keyword(name: &str) -> bool {
666    matches!(
667        name,
668        "stages"
669            | "default"
670            | "include"
671            | "cache"
672            | "variables"
673            | "workflow"
674            | "spec"
675            | "image"
676            | "services"
677            | "before_script"
678            | "after_script"
679            | "only"
680            | "except"
681    )
682}
683
684fn parse_image(value: Value) -> Result<ImageConfig> {
685    match value {
686        Value::String(name) => Ok(ImageConfig {
687            name,
688            docker_platform: None,
689            docker_user: None,
690            entrypoint: Vec::new(),
691        }),
692        Value::Mapping(mut map) => {
693            if let Some(val) = map.remove(Value::String("name".to_string())) {
694                let name = extract_string(val, "image name")?;
695                let entrypoint = map
696                    .remove(Value::String("entrypoint".to_string()))
697                    .map(|value| {
698                        serde_yaml::from_value::<ServiceCommand>(value)
699                            .map(ServiceCommand::into_vec)
700                    })
701                    .transpose()
702                    .context("failed to parse image.entrypoint")?
703                    .unwrap_or_default();
704                let docker_cfg = map
705                    .remove(Value::String("docker".to_string()))
706                    .map(|value| parse_docker_executor_config(value, "image.docker"))
707                    .transpose()?;
708                Ok(ImageConfig {
709                    name,
710                    docker_platform: docker_cfg.as_ref().and_then(|cfg| cfg.platform.clone()),
711                    docker_user: docker_cfg.and_then(|cfg| cfg.user),
712                    entrypoint,
713                })
714            } else {
715                bail!("image mapping must include 'name'")
716            }
717        }
718        other => bail!("image must be a string or mapping, got {other:?}"),
719    }
720}
721
722struct DockerExecutorConfig {
723    platform: Option<String>,
724    user: Option<String>,
725}
726
727fn parse_docker_executor_config(value: Value, field: &str) -> Result<DockerExecutorConfig> {
728    let map = match value {
729        Value::Mapping(map) => map,
730        other => bail!("{field} must be a mapping, got {other:?}"),
731    };
732    let platform_key = Value::String("platform".to_string());
733    let user_key = Value::String("user".to_string());
734    let platform = map
735        .get(&platform_key)
736        .cloned()
737        .map(|value| extract_string(value, &format!("{field}.platform")))
738        .transpose()?;
739    let user = map
740        .get(&user_key)
741        .cloned()
742        .map(|value| extract_string(value, &format!("{field}.user")))
743        .transpose()?;
744    if platform.is_none() && user.is_none() {
745        bail!("{field} must include 'platform' or 'user'");
746    }
747    Ok(DockerExecutorConfig { platform, user })
748}
749
750fn extract_string(value: Value, what: &str) -> Result<String> {
751    match value {
752        Value::String(text) => Ok(text),
753        other => bail!("{what} must be a string, got {other:?}"),
754    }
755}
756
757type ParsedJobSpec = (
758    RawJob,
759    Option<ImageConfig>,
760    HashMap<String, String>,
761    Vec<CacheConfig>,
762    Vec<ServiceConfig>,
763    Option<ParallelConfig>,
764    Vec<String>,
765    Vec<String>,
766);
767
768fn parse_job(value: Value) -> Result<ParsedJobSpec> {
769    match value {
770        Value::Mapping(mut map) => {
771            let image_value = map.remove(Value::String("image".to_string()));
772            let variables_value = map.remove(Value::String("variables".to_string()));
773            let cache_value = map.remove(Value::String("cache".to_string()));
774            let services_value = map.remove(Value::String("services".to_string()));
775            let parallel_value = map.remove(Value::String("parallel".to_string()));
776            let only_value = map.remove(Value::String("only".to_string()));
777            let except_value = map.remove(Value::String("except".to_string()));
778            let job_spec: RawJob = serde_yaml::from_value(Value::Mapping(map))?;
779            let image = image_value.map(parse_image).transpose()?;
780            let variables = variables_value
781                .map(parse_variables_map)
782                .transpose()?
783                .unwrap_or_default();
784            let cache = cache_value
785                .map(parse_cache_value)
786                .transpose()?
787                .unwrap_or_default();
788            let services = services_value
789                .map(|value| parse_services_value(value, "services"))
790                .transpose()?
791                .unwrap_or_default();
792            let parallel = parallel_value.map(parse_parallel_value).transpose()?;
793            let only = only_value
794                .map(|value| parse_filter_list(value, "only"))
795                .transpose()?
796                .unwrap_or_default();
797            let except = except_value
798                .map(|value| parse_filter_list(value, "except"))
799                .transpose()?
800                .unwrap_or_default();
801            Ok((
802                job_spec, image, variables, cache, services, parallel, only, except,
803            ))
804        }
805        other => bail!("job definition must be a mapping, got {other:?}"),
806    }
807}
808
809fn parse_string_list(value: Value, field: &str) -> Result<Vec<String>> {
810    match value {
811        Value::Sequence(entries) => {
812            let mut out = Vec::new();
813            for entry in entries {
814                let text = yaml_command_string(entry)
815                    .map_err(|err| anyhow!("{field} entries must be strings ({err})"))?;
816                out.push(text);
817            }
818            Ok(out)
819        }
820        Value::Null => Ok(Vec::new()),
821        other => {
822            let text = yaml_command_string(other)
823                .map_err(|err| anyhow!("{field} must be a string or sequence ({err})"))?;
824            Ok(vec![text])
825        }
826    }
827}
828
829fn parse_cache_value(value: Value) -> Result<Vec<CacheConfig>> {
830    match value {
831        Value::Sequence(entries) => entries
832            .into_iter()
833            .map(parse_cache_entry)
834            .collect::<Result<Vec<_>>>(),
835        Value::Null => Ok(Vec::new()),
836        other => Ok(vec![parse_cache_entry(other)?]),
837    }
838}
839
840fn yaml_command_string(value: Value) -> Result<String, String> {
841    match value {
842        Value::String(text) => Ok(text),
843        Value::Number(number) => Ok(number.to_string()),
844        Value::Bool(boolean) => Ok(boolean.to_string()),
845        Value::Null => Ok(String::new()),
846        Value::Mapping(map) => mapping_command_string(map),
847        other => Err(format!("got {other:?}")),
848    }
849}
850
851fn mapping_command_string(map: Mapping) -> Result<String, String> {
852    if map.len() != 1 {
853        return Err(format!(
854            "mapping entries must contain exactly one command, got {map:?}"
855        ));
856    }
857    let (key, value) = map
858        .into_iter()
859        .next()
860        .ok_or_else(|| "mapping entries must contain exactly one command".to_string())?;
861    let key_text = match key {
862        Value::String(text) => text,
863        other => return Err(format!("mapping keys must be strings, got {other:?}")),
864    };
865    let value_text = yaml_command_string(value)?;
866    if value_text.is_empty() {
867        Ok(format!("{key_text}:"))
868    } else {
869        Ok(format!("{key_text}: {value_text}"))
870    }
871}
872
873fn parse_cache_entry(value: Value) -> Result<CacheConfig> {
874    let raw: CacheEntryRaw = match value {
875        Value::Mapping(_) => serde_yaml::from_value(value)?,
876        other => bail!("cache entry must be a mapping, got {other:?}"),
877    };
878    let key = parse_cache_key(raw.key)?;
879    let fallback_keys = raw.fallback_keys;
880    let paths = if raw.paths.is_empty() {
881        bail!("cache entry must specify at least one path");
882    } else {
883        raw.paths
884    };
885    let policy = raw
886        .policy
887        .as_deref()
888        .map(CachePolicy::from_str)
889        .unwrap_or(CachePolicy::PullPush);
890    Ok(CacheConfig {
891        key,
892        fallback_keys,
893        paths,
894        policy,
895    })
896}
897
898fn parse_cache_key(raw: Option<CacheKeyRaw>) -> Result<CacheKey> {
899    let Some(raw) = raw else {
900        return Ok(CacheKey::default());
901    };
902
903    match raw {
904        CacheKeyRaw::Literal(value) => Ok(CacheKey::Literal(value)),
905        CacheKeyRaw::Detailed(details) => {
906            if details.files.is_empty() {
907                bail!("cache key map must include at least one file in 'files'");
908            }
909            if details.files.len() > 2 {
910                bail!("cache key map supports at most two files");
911            }
912            Ok(CacheKey::Files {
913                files: details.files,
914                prefix: details.prefix.filter(|value| !value.is_empty()),
915            })
916        }
917    }
918}
919
920fn parse_variables_map(value: Value) -> Result<HashMap<String, String>> {
921    let mapping = match value {
922        Value::Mapping(map) => map,
923        other => bail!("variables must be a mapping, got {other:?}"),
924    };
925
926    let mut vars = HashMap::new();
927    for (key, val) in mapping {
928        let name = match key {
929            Value::String(s) => s,
930            other => bail!("variable names must be strings, got {other:?}"),
931        };
932        let value = extract_variable_value(val, &format!("variable '{name}'"))?;
933        vars.insert(name, value);
934    }
935
936    Ok(vars)
937}
938
939fn extract_variable_value(value: Value, what: &str) -> Result<String> {
940    match value {
941        Value::String(text) => Ok(text),
942        Value::Bool(flag) => Ok(flag.to_string()),
943        Value::Number(num) => Ok(num.to_string()),
944        Value::Null => Ok(String::new()),
945        Value::Mapping(mut map) => {
946            let key = Value::String("value".to_string());
947            if let Some(entry) = map.remove(&key) {
948                extract_variable_value(entry, what)
949            } else {
950                bail!("{what} mapping must include 'value'")
951            }
952        }
953        other => bail!("{what} must be a string/bool/number, got {other:?}"),
954    }
955}
956
957fn parse_services_value(value: Value, field: &str) -> Result<Vec<ServiceConfig>> {
958    let entries = match value {
959        Value::Sequence(seq) => seq,
960        Value::Null => return Ok(Vec::new()),
961        other => vec![other],
962    };
963    let mut services = Vec::new();
964    for entry in entries {
965        let raw: RawService = serde_yaml::from_value(entry)
966            .with_context(|| format!("failed to parse {field} entry"))?;
967        let config = match raw {
968            RawService::Simple(image) => ServiceConfig {
969                image,
970                aliases: Vec::new(),
971                docker_platform: None,
972                docker_user: None,
973                entrypoint: Vec::new(),
974                command: Vec::new(),
975                variables: HashMap::new(),
976            },
977            RawService::Detailed(details) => details.into_config()?,
978        };
979        services.push(config);
980    }
981    Ok(services)
982}
983
984fn parse_parallel_value(value: Value) -> Result<ParallelConfig> {
985    match value {
986        Value::Number(num) => {
987            let count = num
988                .as_u64()
989                .ok_or_else(|| anyhow!("parallel count must be positive integer"))?;
990            if count == 0 {
991                bail!("parallel count must be greater than zero");
992            }
993            Ok(ParallelConfig::Count(count.try_into().unwrap_or(u32::MAX)))
994        }
995        Value::Mapping(mut map) => {
996            let matrix_key = Value::String("matrix".to_string());
997            let Some(entries) = map.remove(&matrix_key) else {
998                bail!("parallel mapping must include 'matrix'");
999            };
1000            let matrices = parse_parallel_matrix(entries)?;
1001            Ok(ParallelConfig::Matrix(matrices))
1002        }
1003        other => bail!("parallel must be an integer or mapping, got {other:?}"),
1004    }
1005}
1006
1007fn parse_parallel_matrix(value: Value) -> Result<Vec<ParallelMatrixEntry>> {
1008    match value {
1009        Value::Sequence(entries) => entries
1010            .into_iter()
1011            .map(parse_parallel_matrix_entry)
1012            .collect(),
1013        other => Ok(vec![parse_parallel_matrix_entry(other)?]),
1014    }
1015}
1016
1017fn parse_parallel_matrix_entry(value: Value) -> Result<ParallelMatrixEntry> {
1018    let mapping = match value {
1019        Value::Mapping(map) => map,
1020        other => bail!("parallel matrix entries must be mappings, got {other:?}"),
1021    };
1022    let mut variables = Vec::new();
1023    for (key, value) in mapping {
1024        let name = match key {
1025            Value::String(name) => name,
1026            other => bail!("parallel matrix variable names must be strings, got {other:?}"),
1027        };
1028        let values = match value {
1029            Value::String(text) => vec![text],
1030            Value::Sequence(entries) => entries
1031                .into_iter()
1032                .map(|entry| match entry {
1033                    Value::String(text) => Ok(text),
1034                    other => bail!("parallel matrix values must be strings, got {other:?}"),
1035                })
1036                .collect::<Result<Vec<_>>>()?,
1037            other => bail!("parallel matrix values must be string or list, got {other:?}"),
1038        };
1039        if values.is_empty() {
1040            bail!(
1041                "parallel matrix variable '{}' must have at least one value",
1042                name
1043            );
1044        }
1045        variables.push(ParallelVariable { name, values });
1046    }
1047    if variables.is_empty() {
1048        bail!("parallel matrix entries must define at least one variable");
1049    }
1050    Ok(ParallelMatrixEntry { variables })
1051}
1052
1053fn parse_filter_list(value: Value, field: &str) -> Result<Vec<String>> {
1054    match value {
1055        Value::String(text) => Ok(vec![text]),
1056        Value::Sequence(entries) => entries
1057            .into_iter()
1058            .map(|entry| match entry {
1059                Value::String(text) => Ok(text),
1060                other => bail!("{field} entries must be strings, got {other:?}"),
1061            })
1062            .collect(),
1063        Value::Mapping(mut map) => {
1064            let variables_key = Value::String("variables".to_string());
1065            if let Some(variables) = map.remove(&variables_key) {
1066                if !map.is_empty() {
1067                    bail!("{field} mapping supports only 'variables'");
1068                }
1069                parse_variable_filter_list(variables, field)
1070            } else {
1071                bail!("{field} mapping supports only 'variables'");
1072            }
1073        }
1074        Value::Null => Ok(Vec::new()),
1075        other => bail!("{field} must be a string or list, got {other:?}"),
1076    }
1077}
1078
1079fn parse_variable_filter_list(value: Value, field: &str) -> Result<Vec<String>> {
1080    let expressions = match value {
1081        Value::String(text) => vec![text],
1082        Value::Sequence(entries) => entries
1083            .into_iter()
1084            .map(|entry| match entry {
1085                Value::String(text) => Ok(text),
1086                other => bail!("{field}.variables entries must be strings, got {other:?}"),
1087            })
1088            .collect::<Result<Vec<_>>>()?,
1089        Value::Null => Vec::new(),
1090        other => bail!("{field}.variables must be a string or list, got {other:?}"),
1091    };
1092
1093    Ok(expressions
1094        .into_iter()
1095        .map(|expr| format!("__opal_variables__:{expr}"))
1096        .collect())
1097}
1098
1099fn parse_timeout_value(value: Value, field: &str) -> Result<Option<Duration>> {
1100    match value {
1101        Value::Null => Ok(None),
1102        Value::String(text) => parse_timeout_str(&text, field).map(Some),
1103        other => bail!("{field} must be a string or null, got {other:?}"),
1104    }
1105}
1106
1107fn parse_optional_timeout(raw: &Option<String>, field: &str) -> Result<Option<Duration>> {
1108    if let Some(text) = raw {
1109        Ok(Some(parse_timeout_str(text, field)?))
1110    } else {
1111        Ok(None)
1112    }
1113}
1114
1115fn parse_timeout_str(text: &str, field: &str) -> Result<Duration> {
1116    humantime::parse_duration(text).with_context(|| format!("invalid duration for {field}: {text}"))
1117}
1118
1119fn extract_bool(value: Value, field: &str) -> Result<bool> {
1120    match value {
1121        Value::Bool(b) => Ok(b),
1122        other => bail!("{field} must be a boolean, got {other:?}"),
1123    }
1124}
1125
1126// TODO: just no, this does too many things, also it should probably be part of the PipelinGraph,
1127// since it's... building the pipeline graph.
1128fn build_graph(
1129    defaults: PipelineDefaults,
1130    workflow: Option<WorkflowConfig>,
1131    filters: super::graph::PipelineFilters,
1132    stage_names: Vec<String>,
1133    job_names: Vec<String>,
1134    job_defs: HashMap<String, Value>,
1135) -> Result<PipelineGraph> {
1136    let mut graph = DiGraph::<Job, ()>::new();
1137    let mut stages: Vec<StageGroup> = stage_names
1138        .into_iter()
1139        .map(|name| StageGroup {
1140            name,
1141            jobs: Vec::new(),
1142        })
1143        .collect();
1144    let mut name_to_index: HashMap<String, NodeIndex> = HashMap::new();
1145    let mut pending_needs: Vec<(String, NodeIndex, Vec<JobDependency>)> = Vec::new();
1146
1147    if stages.is_empty() {
1148        stages.push(StageGroup {
1149            name: "default".to_string(),
1150            jobs: Vec::new(),
1151        });
1152    }
1153
1154    let mut resolved_defs: HashMap<String, Mapping> = HashMap::new();
1155
1156    for job_name in job_names {
1157        let merged_map =
1158            resolve_job_definition(&job_name, &job_defs, &mut resolved_defs, &mut Vec::new())?;
1159        let (
1160            job_spec,
1161            job_image,
1162            job_variables,
1163            job_cache,
1164            job_services,
1165            job_parallel,
1166            only,
1167            except,
1168        ) = parse_job(Value::Mapping(merged_map))?;
1169        let inherit_defaults = job_inherit_defaults(&job_spec);
1170        let stage_name = job_spec.stage.unwrap_or_else(|| {
1171            stages
1172                .first()
1173                .map(|stage| stage.name.as_str())
1174                .unwrap_or("default")
1175                .to_string()
1176        });
1177        let stage_index = ensure_stage(&mut stages, &stage_name);
1178        let commands = job_spec.script.into_commands();
1179        if commands.is_empty() {
1180            bail!(
1181                "job '{}' must define a script (directly or via extends)",
1182                job_name
1183            );
1184        }
1185        let (raw_needs, explicit_needs) = match job_spec.needs {
1186            Some(entries) => (entries, true),
1187            None => (Vec::new(), false),
1188        };
1189        let needs: Vec<JobDependency> = raw_needs
1190            .into_iter()
1191            .filter_map(|need| need.into_dependency(&job_name))
1192            .collect();
1193        let dependencies = job_spec.dependencies;
1194        let before_script = job_spec.before_script.map(Script::into_commands);
1195        let after_script = job_spec.after_script.map(Script::into_commands);
1196        let artifacts = job_spec.artifacts.into_config(&job_name)?;
1197        let cache_entries = if job_cache.is_empty() && inherit_defaults.cache {
1198            defaults.cache.clone()
1199        } else {
1200            job_cache
1201        };
1202        let services = if job_services.is_empty() && inherit_defaults.services {
1203            defaults.services.clone()
1204        } else {
1205            job_services
1206        };
1207        let timeout =
1208            parse_optional_timeout(&job_spec.timeout, &format!("job '{}'.timeout", job_name))?.or(
1209                if inherit_defaults.timeout {
1210                    defaults.timeout
1211                } else {
1212                    None
1213                },
1214            );
1215        let retry_base = if inherit_defaults.retry {
1216            defaults.retry.clone()
1217        } else {
1218            RetryPolicy::default()
1219        };
1220        let retry = job_spec
1221            .retry
1222            .map(|raw| raw.into_policy(&retry_base, &format!("job '{}'.retry", job_name)))
1223            .transpose()?
1224            .unwrap_or(retry_base);
1225        let interruptible = job_spec
1226            .interruptible
1227            .unwrap_or(if inherit_defaults.interruptible {
1228                defaults.interruptible
1229            } else {
1230                false
1231            });
1232        let resource_group = job_spec.resource_group.clone();
1233        let parallel = job_parallel;
1234
1235        let environment = job_spec.environment.as_ref().map(|env| {
1236            let action = match env.action.as_deref() {
1237                Some("prepare") => EnvironmentAction::Prepare,
1238                Some("stop") => EnvironmentAction::Stop,
1239                Some("verify") => EnvironmentAction::Verify,
1240                Some("access") => EnvironmentAction::Access,
1241                _ => EnvironmentAction::Start,
1242            };
1243            let name = if env.name.is_empty() {
1244                job_name.clone()
1245            } else {
1246                env.name.clone()
1247            };
1248            EnvironmentConfig {
1249                name,
1250                url: env.url.clone(),
1251                on_stop: env.on_stop.clone(),
1252                auto_stop_in: parse_optional_timeout(
1253                    &env.auto_stop_in,
1254                    &format!("job '{}'.environment.auto_stop_in", job_name),
1255                )
1256                .ok()
1257                .flatten(),
1258                action,
1259            }
1260        });
1261
1262        let inherited_image = if job_image.is_none() && inherit_defaults.image {
1263            defaults.image.clone()
1264        } else {
1265            job_image
1266        };
1267
1268        let node = graph.add_node(Job {
1269            name: job_name.clone(),
1270            stage: stage_name,
1271            commands,
1272            needs: needs.clone(),
1273            explicit_needs,
1274            dependencies: dependencies.clone(),
1275            before_script,
1276            after_script,
1277            inherit_default_image: inherit_defaults.image,
1278            inherit_default_before_script: inherit_defaults.before_script,
1279            inherit_default_after_script: inherit_defaults.after_script,
1280            inherit_default_cache: inherit_defaults.cache,
1281            inherit_default_services: inherit_defaults.services,
1282            inherit_default_timeout: inherit_defaults.timeout,
1283            inherit_default_retry: inherit_defaults.retry,
1284            inherit_default_interruptible: inherit_defaults.interruptible,
1285            when: job_spec.when.clone(),
1286            rules: job_spec.rules.clone(),
1287            artifacts,
1288            cache: cache_entries,
1289            image: inherited_image,
1290            variables: job_variables,
1291            services,
1292            timeout,
1293            retry,
1294            interruptible,
1295            resource_group,
1296            parallel,
1297            only,
1298            except,
1299            tags: job_spec.tags.clone(),
1300            environment,
1301        });
1302
1303        name_to_index.insert(job_name.clone(), node);
1304        pending_needs.push((job_name, node, needs));
1305
1306        let stage = stages
1307            .get_mut(stage_index)
1308            .ok_or_else(|| anyhow!("internal error: stage index {} missing", stage_index))?;
1309        stage.jobs.push(node);
1310    }
1311
1312    // TODO; for for for for fuck off - refactor
1313
1314    for (job_name, job_idx, needs) in pending_needs {
1315        for dependency in needs {
1316            if !matches!(dependency.source, DependencySource::Local) {
1317                continue;
1318            }
1319            let Some(dependency_idx) = name_to_index.get(&dependency.job).copied() else {
1320                if dependency.optional {
1321                    continue;
1322                }
1323                return Err(anyhow::anyhow!(
1324                    "job '{}' declared unknown dependency '{}'",
1325                    job_name,
1326                    dependency.job
1327                ));
1328            };
1329
1330            graph.add_edge(dependency_idx, job_idx, ());
1331        }
1332    }
1333
1334    Ok(PipelineGraph {
1335        graph,
1336        stages,
1337        defaults,
1338        workflow,
1339        filters,
1340    })
1341}
1342
1343struct JobInheritDefaults {
1344    image: bool,
1345    before_script: bool,
1346    after_script: bool,
1347    cache: bool,
1348    services: bool,
1349    timeout: bool,
1350    retry: bool,
1351    interruptible: bool,
1352}
1353
1354impl Default for JobInheritDefaults {
1355    fn default() -> Self {
1356        Self {
1357            image: true,
1358            before_script: true,
1359            after_script: true,
1360            cache: true,
1361            services: true,
1362            timeout: true,
1363            retry: true,
1364            interruptible: true,
1365        }
1366    }
1367}
1368
1369fn job_inherit_defaults(job: &RawJob) -> JobInheritDefaults {
1370    let mut inherit = JobInheritDefaults::default();
1371    if let Some(raw_inherit) = &job.inherit
1372        && let Some(default) = &raw_inherit.default
1373    {
1374        match default {
1375            RawInheritDefault::Bool(value) => {
1376                inherit.image = *value;
1377                inherit.before_script = *value;
1378                inherit.after_script = *value;
1379                inherit.cache = *value;
1380                inherit.services = *value;
1381                inherit.timeout = *value;
1382                inherit.retry = *value;
1383                inherit.interruptible = *value;
1384            }
1385            RawInheritDefault::List(entries) => {
1386                inherit.image = entries.iter().any(|entry| entry == "image");
1387                inherit.before_script = entries.iter().any(|entry| entry == "before_script");
1388                inherit.after_script = entries.iter().any(|entry| entry == "after_script");
1389                inherit.cache = entries.iter().any(|entry| entry == "cache");
1390                inherit.services = entries.iter().any(|entry| entry == "services");
1391                inherit.timeout = entries.iter().any(|entry| entry == "timeout");
1392                inherit.retry = entries.iter().any(|entry| entry == "retry");
1393                inherit.interruptible = entries.iter().any(|entry| entry == "interruptible");
1394            }
1395        }
1396    }
1397    inherit
1398}
1399
1400fn resolve_job_definition(
1401    name: &str,
1402    job_defs: &HashMap<String, Value>,
1403    cache: &mut HashMap<String, Mapping>,
1404    stack: &mut Vec<String>,
1405) -> Result<Mapping> {
1406    if let Some(resolved) = cache.get(name) {
1407        return Ok(resolved.clone());
1408    }
1409
1410    if stack.iter().any(|entry| entry == name) {
1411        bail!("job '{}' has cyclical extends", name);
1412    }
1413
1414    let value = match job_defs.get(name) {
1415        Some(v) => v,
1416        None => {
1417            let requester = stack.last().cloned().unwrap_or_else(|| name.to_string());
1418            bail!("job '{requester}' extends unknown job/template '{name}'");
1419        }
1420    };
1421
1422    let map = match value {
1423        Value::Mapping(map) => map.clone(),
1424        other => bail!("job '{name}' must be defined as mapping, got {other:?}"),
1425    };
1426
1427    stack.push(name.to_string());
1428
1429    let extends_key = Value::String("extends".to_string());
1430    let extends = map.get(&extends_key).map(parse_extends_list).transpose()?;
1431
1432    let mut merged = Mapping::new();
1433    if let Some(parents) = extends {
1434        for parent_name in parents {
1435            let parent_map = resolve_job_definition(&parent_name, job_defs, cache, stack)?;
1436            merged = merge_mappings(merged, parent_map);
1437        }
1438    }
1439
1440    let mut child_map = map;
1441    child_map.remove(&extends_key);
1442    merged = merge_mappings(merged, child_map);
1443
1444    stack.pop();
1445    cache.insert(name.to_string(), merged.clone());
1446    Ok(merged)
1447}
1448
1449fn parse_extends_list(value: &Value) -> Result<Vec<String>> {
1450    match value {
1451        Value::String(name) => Ok(vec![name.clone()]),
1452        Value::Sequence(seq) => seq
1453            .iter()
1454            .map(|val| match val {
1455                Value::String(name) => Ok(name.clone()),
1456                other => bail!("extends entries must be strings, got {other:?}"),
1457            })
1458            .collect(),
1459        other => bail!("extends must be string or sequence, got {other:?}"),
1460    }
1461}
1462
1463fn merge_mappings(mut base: Mapping, addition: Mapping) -> Mapping {
1464    for (key, value) in addition {
1465        base.insert(key, value);
1466    }
1467    base
1468}
1469
1470fn parse_include_entries(value: Value) -> Result<Vec<IncludeEntry>> {
1471    match value {
1472        Value::String(path) => Ok(vec![IncludeEntry::Local(PathBuf::from(path))]),
1473        Value::Sequence(entries) => {
1474            let mut paths = Vec::new();
1475            for entry in entries {
1476                paths.extend(parse_include_entry(entry)?);
1477            }
1478            Ok(paths)
1479        }
1480        Value::Mapping(_) => parse_include_entry(value),
1481        other => bail!("include must be a string or list, got {other:?}"),
1482    }
1483}
1484
1485#[derive(Debug, Clone)]
1486enum IncludeEntry {
1487    Local(PathBuf),
1488    Project {
1489        project: String,
1490        reference: Option<String>,
1491        files: Vec<PathBuf>,
1492    },
1493}
1494
1495// TODO - refactor, does way too much - garbage
1496fn parse_include_entry(value: Value) -> Result<Vec<IncludeEntry>> {
1497    match value {
1498        Value::String(path) => Ok(vec![IncludeEntry::Local(PathBuf::from(path))]),
1499        Value::Mapping(map) => {
1500            let local_key = Value::String("local".to_string());
1501            let file_key = Value::String("file".to_string());
1502            let files_key = Value::String("files".to_string());
1503            let project_key = Value::String("project".to_string());
1504            let remote_key = Value::String("remote".to_string());
1505            let template_key = Value::String("template".to_string());
1506            let component_key = Value::String("component".to_string());
1507            if map.contains_key(&remote_key) {
1508                bail!("include:remote is not supported yet");
1509            }
1510            if map.contains_key(&template_key) {
1511                bail!("include:template is not supported yet");
1512            }
1513            if map.contains_key(&component_key) {
1514                bail!("include:component is not supported yet");
1515            }
1516            if let Some(Value::String(local)) = map.get(&local_key) {
1517                Ok(vec![IncludeEntry::Local(PathBuf::from(local))])
1518            } else if let Some(Value::String(project)) = map.get(&project_key) {
1519                let reference = map
1520                    .get(Value::String("ref".to_string()))
1521                    .map(|value| match value {
1522                        Value::String(text) => Ok(text.clone()),
1523                        other => bail!("include:project ref must be a string, got {other:?}"),
1524                    })
1525                    .transpose()?;
1526                let files = parse_project_include_files(map.get(&file_key), map.get(&files_key))?;
1527                Ok(vec![IncludeEntry::Project {
1528                    project: project.clone(),
1529                    reference,
1530                    files,
1531                }])
1532            } else if let Some(Value::String(file)) = map.get(&file_key) {
1533                Ok(vec![IncludeEntry::Local(PathBuf::from(file))])
1534            } else if let Some(Value::Sequence(files)) = map.get(&files_key) {
1535                let mut paths = Vec::new();
1536                for entry in files {
1537                    match entry {
1538                        Value::String(path) => paths.push(IncludeEntry::Local(PathBuf::from(path))),
1539                        other => bail!("include 'files' entries must be strings, got {other:?}"),
1540                    }
1541                }
1542                Ok(paths)
1543            } else {
1544                bail!("only 'local' or 'file(s)' includes are supported");
1545            }
1546        }
1547        other => bail!("unsupported include entry {other:?}"),
1548    }
1549}
1550
1551fn parse_project_include_files(
1552    file_value: Option<&Value>,
1553    files_value: Option<&Value>,
1554) -> Result<Vec<PathBuf>> {
1555    if files_value.is_some() {
1556        bail!("include:project must use 'file', not 'files'");
1557    }
1558    let Some(value) = file_value else {
1559        bail!("include:project requires a 'file' entry");
1560    };
1561    match value {
1562        Value::String(path) => Ok(vec![PathBuf::from(path)]),
1563        Value::Sequence(entries) => entries
1564            .iter()
1565            .map(|entry| match entry {
1566                Value::String(path) => Ok(PathBuf::from(path)),
1567                other => bail!("include:project file entries must be strings, got {other:?}"),
1568            })
1569            .collect(),
1570        other => bail!("include:project file must be a string or list, got {other:?}"),
1571    }
1572}
1573
1574fn project_include_root(cache_root: &Path, project: &str, reference: &str) -> PathBuf {
1575    cache_root
1576        .join("includes")
1577        .join(percent_encode(project))
1578        .join(sanitize_reference(reference))
1579}
1580
1581// TODO: this does so much, it creates new paths, expands, formats hardcoded shit from the API. awful
1582fn fetch_project_include_file(
1583    gitlab: &GitLabRemoteConfig,
1584    project_root: &Path,
1585    project: &str,
1586    reference: &str,
1587    file: &Path,
1588    include_env: &HashMap<String, String>,
1589) -> Result<PathBuf> {
1590    let expanded = expand_include_path(file, include_env);
1591    validate_include_extension(&expanded)?;
1592    let relative = expanded.strip_prefix(Path::new("/")).unwrap_or(&expanded);
1593    let target = project_root.join(relative);
1594    if target.exists() {
1595        return Ok(target);
1596    }
1597    if let Some(parent) = target.parent() {
1598        fs::create_dir_all(parent)
1599            .with_context(|| format!("failed to create {}", parent.display()))?;
1600    }
1601    let base = gitlab.base_url.trim_end_matches('/');
1602    let project_id = percent_encode(project);
1603    let file_id = percent_encode(&relative.to_string_lossy());
1604    let ref_id = percent_encode(reference);
1605    let url =
1606        format!("{base}/api/v4/projects/{project_id}/repository/files/{file_id}/raw?ref={ref_id}");
1607    let status = std::process::Command::new("curl")
1608        .arg("--fail")
1609        .arg("-sS")
1610        .arg("-L")
1611        .arg("-H")
1612        .arg(format!("PRIVATE-TOKEN: {}", gitlab.token))
1613        .arg("-o")
1614        .arg(&target)
1615        .arg(&url)
1616        .status()
1617        .with_context(|| "failed to invoke curl to resolve include:project")?;
1618    if !status.success() {
1619        return Err(anyhow!(
1620            "curl failed to resolve include:project from {} (status {})",
1621            url,
1622            status.code().unwrap_or(-1)
1623        ));
1624    }
1625    Ok(target)
1626}
1627
1628fn percent_encode(value: &str) -> String {
1629    let mut encoded = String::new();
1630    for byte in value.bytes() {
1631        match byte {
1632            b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' => {
1633                encoded.push(byte as char)
1634            }
1635            _ => encoded.push_str(&format!("%{:02X}", byte)),
1636        }
1637    }
1638    encoded
1639}
1640
1641fn sanitize_reference(reference: &str) -> String {
1642    let mut slug = String::new();
1643    for ch in reference.chars() {
1644        if ch.is_ascii_alphanumeric() {
1645            slug.push(ch.to_ascii_lowercase());
1646        } else if matches!(ch, '-' | '_' | '.') {
1647            slug.push(ch);
1648        } else {
1649            slug.push('-');
1650        }
1651    }
1652    if slug.is_empty() {
1653        slug.push_str("ref");
1654    }
1655    slug
1656}
1657
1658fn ensure_stage(stages: &mut Vec<StageGroup>, stage_name: &str) -> usize {
1659    if let Some(pos) = stages.iter().position(|stage| stage.name == stage_name) {
1660        pos
1661    } else {
1662        stages.push(StageGroup {
1663            name: stage_name.to_string(),
1664            jobs: Vec::new(),
1665        });
1666        stages.len() - 1
1667    }
1668}
1669
1670#[derive(Debug, Deserialize)]
1671struct RawJob {
1672    #[serde(default)]
1673    before_script: Option<Script>,
1674    #[serde(default)]
1675    after_script: Option<Script>,
1676    #[serde(default)]
1677    stage: Option<String>,
1678    #[serde(default)]
1679    script: Script,
1680    #[serde(default)]
1681    when: Option<String>,
1682    #[serde(default)]
1683    needs: Option<Vec<Need>>,
1684    #[serde(default)]
1685    dependencies: Vec<String>,
1686    #[serde(default)]
1687    rules: Vec<JobRule>,
1688    #[serde(default)]
1689    artifacts: RawArtifacts,
1690    #[serde(default)]
1691    timeout: Option<String>,
1692    #[serde(default)]
1693    retry: Option<RawRetry>,
1694    #[serde(default)]
1695    interruptible: Option<bool>,
1696    #[serde(default)]
1697    resource_group: Option<String>,
1698    #[serde(default)]
1699    inherit: Option<RawInherit>,
1700    #[serde(default)]
1701    tags: Vec<String>,
1702    #[serde(default)]
1703    environment: Option<RawEnvironment>,
1704}
1705
1706#[derive(Debug, Deserialize, Default)]
1707struct RawEnvironment {
1708    #[serde(default)]
1709    name: String,
1710    #[serde(default)]
1711    url: Option<String>,
1712    #[serde(default)]
1713    on_stop: Option<String>,
1714    #[serde(default)]
1715    auto_stop_in: Option<String>,
1716    #[serde(default)]
1717    action: Option<String>,
1718}
1719
1720#[derive(Debug, Default, Deserialize)]
1721#[serde(transparent)]
1722struct Script(StringList);
1723
1724impl Script {
1725    fn into_commands(self) -> Vec<String> {
1726        self.0.into_vec()
1727    }
1728}
1729
1730#[derive(Debug, Deserialize, Default)]
1731struct RawArtifacts {
1732    #[serde(default)]
1733    name: Option<String>,
1734    #[serde(default)]
1735    paths: Vec<PathBuf>,
1736    #[serde(default)]
1737    exclude: StringList,
1738    #[serde(default)]
1739    untracked: bool,
1740    #[serde(default)]
1741    when: Option<String>,
1742    #[serde(default)]
1743    expire_in: Option<String>,
1744    #[serde(default)]
1745    reports: RawArtifactReports,
1746}
1747
1748impl RawArtifacts {
1749    fn into_config(self, job_name: &str) -> Result<ArtifactConfig> {
1750        validate_artifact_excludes(&self.exclude.0, job_name)?;
1751        Ok(ArtifactConfig {
1752            name: self.name,
1753            paths: self.paths,
1754            exclude: self.exclude.into_vec(),
1755            untracked: self.untracked,
1756            when: parse_artifact_when(self.when.as_deref(), job_name)?,
1757            expire_in: parse_optional_timeout(
1758                &self.expire_in,
1759                &format!("job '{}'.artifacts.expire_in", job_name),
1760            )?,
1761            report_dotenv: self.reports.dotenv,
1762        })
1763    }
1764}
1765
1766#[derive(Debug, Deserialize, Default)]
1767struct RawArtifactReports {
1768    #[serde(default)]
1769    dotenv: Option<PathBuf>,
1770}
1771
1772fn validate_artifact_excludes(patterns: &[String], job_name: &str) -> Result<()> {
1773    for pattern in patterns {
1774        Glob::new(pattern).with_context(|| {
1775            format!(
1776                "job '{}' has invalid artifacts.exclude pattern '{}'",
1777                job_name, pattern
1778            )
1779        })?;
1780    }
1781    Ok(())
1782}
1783
1784fn parse_artifact_when(value: Option<&str>, job_name: &str) -> Result<ArtifactWhen> {
1785    match value.unwrap_or("on_success") {
1786        "on_success" => Ok(ArtifactWhen::OnSuccess),
1787        "on_failure" => Ok(ArtifactWhen::OnFailure),
1788        "always" => Ok(ArtifactWhen::Always),
1789        other => bail!(
1790            "job '{}' has unsupported artifacts.when value '{}'",
1791            job_name,
1792            other
1793        ),
1794    }
1795}
1796
1797#[derive(Debug, Deserialize)]
1798#[serde(untagged)]
1799enum RawService {
1800    Simple(String),
1801    Detailed(Box<RawServiceConfig>),
1802}
1803
1804#[derive(Debug, Deserialize)]
1805struct RawServiceConfig {
1806    #[serde(default)]
1807    name: Option<String>,
1808    #[serde(default)]
1809    image: Option<String>,
1810    #[serde(default)]
1811    alias: Option<String>,
1812    #[serde(default)]
1813    docker: Option<Value>,
1814    #[serde(default)]
1815    entrypoint: ServiceCommand,
1816    #[serde(default)]
1817    command: ServiceCommand,
1818    #[serde(default)]
1819    variables: HashMap<String, String>,
1820}
1821
1822impl RawServiceConfig {
1823    fn into_config(self) -> Result<ServiceConfig> {
1824        let image = self
1825            .image
1826            .or(self.name)
1827            .ok_or_else(|| anyhow!("service entry must specify an image (name)"))?;
1828        let docker = self
1829            .docker
1830            .map(|value| parse_docker_executor_config(value, "services.docker"))
1831            .transpose()?;
1832        Ok(ServiceConfig {
1833            image,
1834            aliases: parse_service_aliases(self.alias),
1835            docker_platform: docker.as_ref().and_then(|cfg| cfg.platform.clone()),
1836            docker_user: docker.and_then(|cfg| cfg.user),
1837            entrypoint: self.entrypoint.into_vec(),
1838            command: self.command.into_vec(),
1839            variables: self.variables,
1840        })
1841    }
1842}
1843
1844fn parse_service_aliases(alias: Option<String>) -> Vec<String> {
1845    alias
1846        .into_iter()
1847        .flat_map(|raw| {
1848            raw.split(',')
1849                .map(str::trim)
1850                .map(str::to_string)
1851                .collect::<Vec<_>>()
1852        })
1853        .filter(|value| !value.is_empty())
1854        .collect()
1855}
1856
1857#[cfg(test)]
1858mod service_alias_tests {
1859    use super::parse_service_aliases;
1860
1861    #[test]
1862    fn parse_service_aliases_splits_comma_separated_values() {
1863        assert_eq!(
1864            parse_service_aliases(Some("db,postgres,pg".into())),
1865            vec!["db", "postgres", "pg"]
1866        );
1867    }
1868}
1869
1870#[derive(Debug, Deserialize)]
1871#[serde(untagged)]
1872enum RawRetry {
1873    Simple(u32),
1874    Detailed(RawRetryConfig),
1875}
1876
1877impl RawRetry {
1878    fn into_policy(self, base: &RetryPolicy, field: &str) -> Result<RetryPolicy> {
1879        match self {
1880            RawRetry::Simple(max) => {
1881                validate_retry_max(max, field)?;
1882                Ok(RetryPolicy {
1883                    max,
1884                    when: base.when.clone(),
1885                    exit_codes: base.exit_codes.clone(),
1886                })
1887            }
1888            RawRetry::Detailed(cfg) => cfg.into_policy(base, field),
1889        }
1890    }
1891}
1892
1893#[derive(Debug, Deserialize)]
1894struct RawRetryConfig {
1895    #[serde(default)]
1896    max: Option<u32>,
1897    #[serde(default)]
1898    when: StringList,
1899    #[serde(default)]
1900    exit_codes: IntList,
1901}
1902
1903impl RawRetryConfig {
1904    fn into_policy(self, base: &RetryPolicy, field: &str) -> Result<RetryPolicy> {
1905        let mut policy = base.clone();
1906        if let Some(max) = self.max {
1907            validate_retry_max(max, &format!("{field}.max"))?;
1908            policy.max = max;
1909        }
1910        if !self.when.0.is_empty() {
1911            validate_retry_when(&self.when.0, &format!("{field}.when"))?;
1912            policy.when = self.when.into_vec();
1913        }
1914        if !self.exit_codes.0.is_empty() {
1915            validate_retry_exit_codes(&self.exit_codes.0, &format!("{field}.exit_codes"))?;
1916            policy.exit_codes = self.exit_codes.into_vec();
1917        }
1918        Ok(policy)
1919    }
1920}
1921
1922fn validate_retry_max(max: u32, field: &str) -> Result<()> {
1923    if max > 2 {
1924        bail!("{field} must be 0, 1, or 2");
1925    }
1926    Ok(())
1927}
1928
1929fn validate_retry_when(conditions: &[String], field: &str) -> Result<()> {
1930    for condition in conditions {
1931        if !SUPPORTED_RETRY_WHEN_VALUES.contains(&condition.as_str()) {
1932            bail!("{field} has unsupported retry condition '{condition}'");
1933        }
1934    }
1935    Ok(())
1936}
1937
1938fn validate_retry_exit_codes(codes: &[i32], field: &str) -> Result<()> {
1939    for code in codes {
1940        if *code < 0 {
1941            bail!("{field} must contain non-negative integers");
1942        }
1943    }
1944    Ok(())
1945}
1946
1947const SUPPORTED_RETRY_WHEN_VALUES: &[&str] = &[
1948    "always",
1949    "unknown_failure",
1950    "script_failure",
1951    "api_failure",
1952    "stuck_or_timeout_failure",
1953    "runner_system_failure",
1954    "runner_unsupported",
1955    "stale_schedule",
1956    "job_execution_timeout",
1957    "archived_failure",
1958    "unmet_prerequisites",
1959    "scheduler_failure",
1960    "data_integrity_failure",
1961];
1962
1963#[derive(Debug, Default)]
1964struct StringList(Vec<String>);
1965
1966impl StringList {
1967    fn into_vec(self) -> Vec<String> {
1968        self.0
1969    }
1970}
1971
1972#[derive(Debug, Default)]
1973struct IntList(Vec<i32>);
1974
1975impl IntList {
1976    fn into_vec(self) -> Vec<i32> {
1977        self.0
1978    }
1979}
1980
1981#[derive(Debug, Default)]
1982struct ServiceCommand(Vec<String>);
1983
1984impl ServiceCommand {
1985    fn into_vec(self) -> Vec<String> {
1986        self.0
1987    }
1988}
1989
1990impl<'de> serde::Deserialize<'de> for ServiceCommand {
1991    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
1992    where
1993        D: serde::Deserializer<'de>,
1994    {
1995        let list = StringList::deserialize(deserializer)?;
1996        Ok(ServiceCommand(list.into_vec()))
1997    }
1998}
1999impl<'de> serde::Deserialize<'de> for StringList {
2000    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
2001    where
2002        D: serde::de::Deserializer<'de>,
2003    {
2004        let value = Value::deserialize(deserializer)?;
2005        let items = match value {
2006            Value::Sequence(entries) => {
2007                let mut commands = Vec::new();
2008                for entry in entries {
2009                    let cmd = yaml_command_string(entry).map_err(serde::de::Error::custom)?;
2010                    commands.push(cmd);
2011                }
2012                commands
2013            }
2014            Value::Null => Vec::new(),
2015            other => vec![yaml_command_string(other).map_err(serde::de::Error::custom)?],
2016        };
2017        Ok(StringList(items))
2018    }
2019}
2020
2021impl<'de> serde::Deserialize<'de> for IntList {
2022    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
2023    where
2024        D: serde::de::Deserializer<'de>,
2025    {
2026        let value = Value::deserialize(deserializer)?;
2027        let items = match value {
2028            Value::Sequence(entries) => {
2029                let mut codes = Vec::new();
2030                for entry in entries {
2031                    match entry {
2032                        Value::Number(number) => {
2033                            let code = number.as_i64().ok_or_else(|| {
2034                                serde::de::Error::custom("retry exit codes must be integers")
2035                            })?;
2036                            let code = i32::try_from(code).map_err(|_| {
2037                                serde::de::Error::custom(
2038                                    "retry exit code is too large for this platform",
2039                                )
2040                            })?;
2041                            codes.push(code);
2042                        }
2043                        other => {
2044                            return Err(serde::de::Error::custom(format!(
2045                                "retry exit codes must be integers, got {other:?}"
2046                            )));
2047                        }
2048                    }
2049                }
2050                codes
2051            }
2052            Value::Null => Vec::new(),
2053            Value::Number(number) => {
2054                let code = number
2055                    .as_i64()
2056                    .ok_or_else(|| serde::de::Error::custom("retry exit codes must be integers"))?;
2057                vec![i32::try_from(code).map_err(|_| {
2058                    serde::de::Error::custom("retry exit code is too large for this platform")
2059                })?]
2060            }
2061            other => {
2062                return Err(serde::de::Error::custom(format!(
2063                    "retry exit codes must be an integer or list, got {other:?}"
2064                )));
2065            }
2066        };
2067        Ok(IntList(items))
2068    }
2069}
2070
2071#[derive(Debug, Deserialize, Default)]
2072struct CacheEntryRaw {
2073    key: Option<CacheKeyRaw>,
2074    #[serde(default)]
2075    fallback_keys: Vec<String>,
2076    #[serde(default)]
2077    paths: Vec<PathBuf>,
2078    #[serde(default)]
2079    policy: Option<String>,
2080}
2081
2082#[derive(Debug, Deserialize)]
2083#[serde(untagged)]
2084enum CacheKeyRaw {
2085    Literal(String),
2086    Detailed(CacheKeyDetailedRaw),
2087}
2088
2089#[derive(Debug, Deserialize, Default)]
2090#[serde(deny_unknown_fields)]
2091struct CacheKeyDetailedRaw {
2092    #[serde(default)]
2093    files: Vec<PathBuf>,
2094    #[serde(default)]
2095    prefix: Option<String>,
2096}
2097
2098#[derive(Debug, Deserialize)]
2099#[serde(untagged)]
2100enum Need {
2101    Name(String),
2102    Config(NeedConfig),
2103}
2104
2105#[derive(Debug, Deserialize)]
2106struct NeedConfig {
2107    job: String,
2108    #[serde(default = "default_artifacts_true")]
2109    artifacts: bool,
2110    #[serde(default)]
2111    optional: bool,
2112    #[serde(default)]
2113    project: Option<String>,
2114    #[serde(rename = "ref")]
2115    reference: Option<String>,
2116    #[serde(default)]
2117    parallel: Option<NeedParallelRaw>,
2118}
2119
2120impl Need {
2121    fn into_dependency(self, owner: &str) -> Option<JobDependency> {
2122        match self {
2123            Need::Name(job) => {
2124                if let Some((base, values)) = parse_inline_variant_reference(&job) {
2125                    Some(JobDependency {
2126                        job: base,
2127                        needs_artifacts: true,
2128                        optional: false,
2129                        source: DependencySource::Local,
2130                        parallel: None,
2131                        inline_variant: Some(values),
2132                    })
2133                } else {
2134                    Some(JobDependency {
2135                        job,
2136                        needs_artifacts: true,
2137                        optional: false,
2138                        source: DependencySource::Local,
2139                        parallel: None,
2140                        inline_variant: None,
2141                    })
2142                }
2143            }
2144            Need::Config(cfg) => {
2145                let NeedConfig {
2146                    job,
2147                    artifacts,
2148                    optional,
2149                    project,
2150                    reference,
2151                    parallel,
2152                } = cfg;
2153                if let Some(project) = project {
2154                    let reference = reference.unwrap_or_default();
2155                    if reference.is_empty() {
2156                        warn!(
2157                            job = owner,
2158                            dependency = %job,
2159                            "needs:project for '{}' is missing required 'ref'",
2160                            project
2161                        );
2162                        return None;
2163                    }
2164                    Some(JobDependency {
2165                        job,
2166                        needs_artifacts: artifacts,
2167                        optional,
2168                        source: DependencySource::External(ExternalDependency {
2169                            project,
2170                            reference,
2171                        }),
2172                        parallel: None,
2173                        inline_variant: None,
2174                    })
2175                } else {
2176                    let parallel_filters =
2177                        parallel.and_then(|raw| match raw.into_filters(owner, &job) {
2178                            Ok(filters) => Some(filters),
2179                            Err(err) => {
2180                                warn!(
2181                                    job = owner,
2182                                    dependency = %job,
2183                                    error = %err,
2184                                    "invalid needs.parallel configuration"
2185                                );
2186                                None
2187                            }
2188                        });
2189                    let (job_name, inline_variant) = parse_inline_variant_reference(&job)
2190                        .map_or((job, None), |(base, values)| (base, Some(values)));
2191                    Some(JobDependency {
2192                        job: job_name,
2193                        needs_artifacts: artifacts,
2194                        optional,
2195                        source: DependencySource::Local,
2196                        parallel: parallel_filters,
2197                        inline_variant,
2198                    })
2199                }
2200            }
2201        }
2202    }
2203}
2204
2205fn parse_inline_variant_reference(value: &str) -> Option<(String, Vec<String>)> {
2206    let trimmed = value.trim();
2207    let (base, list) = trimmed.split_once(':')?;
2208    let payload = list.trim();
2209    if !payload.starts_with('[') {
2210        return None;
2211    }
2212    let values: Vec<String> = serde_yaml::from_str(payload).ok()?;
2213    Some((base.trim().to_string(), values))
2214}
2215
2216#[derive(Debug, Deserialize)]
2217struct NeedParallelRaw {
2218    #[serde(default)]
2219    matrix: Vec<HashMap<String, Value>>,
2220}
2221
2222impl NeedParallelRaw {
2223    // TODO: this function has nested for loops - you're going to get the quadratic award
2224    fn into_filters(self, owner: &str, dependency: &str) -> Result<Vec<HashMap<String, String>>> {
2225        let mut filters = Vec::new();
2226        for entry in self.matrix {
2227            let mut filter = HashMap::new();
2228            for (name, value) in entry {
2229                let value = match value {
2230                    Value::String(text) => text,
2231                    other => bail!(
2232                        "job '{}' dependency '{}' parallel values must be strings, got {other:?}",
2233                        owner,
2234                        dependency
2235                    ),
2236                };
2237                filter.insert(name, value);
2238            }
2239            if filter.is_empty() {
2240                bail!(
2241                    "job '{}' dependency '{}' parallel matrix entries must include variables",
2242                    owner,
2243                    dependency
2244                );
2245            }
2246            filters.push(filter);
2247        }
2248        if filters.is_empty() {
2249            bail!(
2250                "job '{}' dependency '{}' parallel matrix must include at least one entry",
2251                owner,
2252                dependency
2253            );
2254        }
2255        Ok(filters)
2256    }
2257}
2258
2259fn default_artifacts_true() -> bool {
2260    true
2261}
2262
2263#[cfg(test)]
2264mod tests {
2265    use super::*;
2266    use anyhow::{Result, anyhow};
2267    use std::fs;
2268    use tempfile::tempdir;
2269
2270    #[test]
2271    fn parses_stage_and_job_order() -> Result<()> {
2272        let yaml = r#"
2273    stages:
2274      - build
2275      - test
2276
2277    build-job:
2278      stage: build
2279      script:
2280        - echo build
2281
2282    second-build:
2283      stage: build
2284      script: echo build 2
2285
2286    test-job:
2287      stage: test
2288      script:
2289        - echo test
2290    "#;
2291
2292        let pipeline = PipelineGraph::from_yaml_str(yaml)?;
2293        assert_eq!(pipeline.stages.len(), 2);
2294        assert_eq!(pipeline.stages[0].name, "build");
2295        assert_eq!(pipeline.stages[1].name, "test");
2296
2297        let build_jobs: Vec<&Job> = pipeline.stages[0]
2298            .jobs
2299            .iter()
2300            .map(|idx| &pipeline.graph[*idx])
2301            .collect();
2302        assert_eq!(build_jobs.len(), 2);
2303        assert_eq!(build_jobs[0].name, "build-job");
2304        assert_eq!(build_jobs[1].name, "second-build");
2305
2306        let test_jobs: Vec<&Job> = pipeline.stages[1]
2307            .jobs
2308            .iter()
2309            .map(|idx| &pipeline.graph[*idx])
2310            .collect();
2311        assert_eq!(test_jobs.len(), 1);
2312        assert_eq!(test_jobs[0].name, "test-job");
2313        Ok(())
2314    }
2315
2316    #[test]
2317    fn resolves_reference_tags() -> Result<()> {
2318        let yaml = r#"
2319    stages: [build]
2320
2321    .shared:
2322      script:
2323        - echo shared
2324      variables:
2325        SHARED_VAR: shared-value
2326
2327    build-job:
2328      stage: build
2329      script: !reference [.shared, script]
2330      variables:
2331        COPIED: !reference [.shared, variables, SHARED_VAR]
2332    "#;
2333
2334        let pipeline = PipelineGraph::from_yaml_str(yaml)?;
2335        assert_eq!(pipeline.stages.len(), 1);
2336        let build_stage = &pipeline.stages[0];
2337        assert_eq!(build_stage.jobs.len(), 1);
2338        let job = &pipeline.graph[build_stage.jobs[0]];
2339        assert_eq!(job.commands, vec!["echo shared"]);
2340        assert_eq!(
2341            job.variables.get("COPIED").map(|value| value.as_str()),
2342            Some("shared-value")
2343        );
2344        Ok(())
2345    }
2346
2347    #[test]
2348    fn includes_local_fragment() -> Result<()> {
2349        let dir = tempdir()?;
2350        let fragment_path = dir.path().join("fragment.yml");
2351        fs::write(
2352            &fragment_path,
2353            r#"
2354fragment-job:
2355  stage: build
2356  script:
2357    - echo fragment
2358"#,
2359        )?;
2360
2361        let main_path = dir.path().join("main.yml");
2362        fs::write(
2363            &main_path,
2364            r#"
2365stages:
2366  - build
2367include:
2368  - local: fragment.yml
2369
2370main-job:
2371  stage: build
2372  script:
2373    - echo main
2374"#,
2375        )?;
2376
2377        let pipeline = PipelineGraph::from_path(&main_path)?;
2378        assert_eq!(pipeline.stages.len(), 1);
2379        assert_eq!(pipeline.stages[0].jobs.len(), 2);
2380        assert!(
2381            pipeline
2382                .graph
2383                .node_weights()
2384                .any(|job| job.name == "fragment-job")
2385        );
2386        assert!(
2387            pipeline
2388                .graph
2389                .node_weights()
2390                .any(|job| job.name == "main-job")
2391        );
2392        Ok(())
2393    }
2394
2395    #[test]
2396    fn includes_local_paths_from_repo_root() -> Result<()> {
2397        let dir = tempdir()?;
2398        git2::Repository::init(dir.path())?;
2399
2400        let fragment_path = dir.path().join("fragment.yml");
2401        fs::write(
2402            &fragment_path,
2403            r#"
2404fragment-job:
2405  stage: build
2406  script:
2407    - echo fragment
2408"#,
2409        )?;
2410
2411        let ci_dir = dir.path().join("ci");
2412        fs::create_dir_all(&ci_dir)?;
2413        let main_path = ci_dir.join("main.yml");
2414        fs::write(
2415            &main_path,
2416            r#"
2417stages:
2418  - build
2419include:
2420  - local: fragment.yml
2421
2422main-job:
2423  stage: build
2424  script:
2425    - echo main
2426"#,
2427        )?;
2428
2429        let pipeline = PipelineGraph::from_path(&main_path)?;
2430        assert!(
2431            pipeline
2432                .graph
2433                .node_weights()
2434                .any(|job| job.name == "fragment-job")
2435        );
2436        assert!(
2437            pipeline
2438                .graph
2439                .node_weights()
2440                .any(|job| job.name == "main-job")
2441        );
2442        Ok(())
2443    }
2444
2445    #[test]
2446    fn includes_local_glob_from_repo_root() -> Result<()> {
2447        let dir = tempdir()?;
2448        git2::Repository::init(dir.path())?;
2449
2450        let configs_dir = dir.path().join("configs");
2451        fs::create_dir_all(&configs_dir)?;
2452        fs::write(
2453            configs_dir.join("a.yml"),
2454            r#"
2455job-a:
2456  stage: build
2457  script:
2458    - echo a
2459"#,
2460        )?;
2461        fs::write(
2462            configs_dir.join("b.yml"),
2463            r#"
2464job-b:
2465  stage: build
2466  script:
2467    - echo b
2468"#,
2469        )?;
2470
2471        let ci_dir = dir.path().join("ci");
2472        fs::create_dir_all(&ci_dir)?;
2473        let main_path = ci_dir.join("main.yml");
2474        fs::write(
2475            &main_path,
2476            r#"
2477stages:
2478  - build
2479include:
2480  - local: configs/*.yml
2481
2482main-job:
2483  stage: build
2484  script:
2485    - echo main
2486"#,
2487        )?;
2488
2489        let pipeline = PipelineGraph::from_path(&main_path)?;
2490        assert!(pipeline.graph.node_weights().any(|job| job.name == "job-a"));
2491        assert!(pipeline.graph.node_weights().any(|job| job.name == "job-b"));
2492        assert!(
2493            pipeline
2494                .graph
2495                .node_weights()
2496                .any(|job| job.name == "main-job")
2497        );
2498        Ok(())
2499    }
2500
2501    #[test]
2502    fn includes_local_paths_expand_environment_variables() -> Result<()> {
2503        let dir = tempdir()?;
2504        git2::Repository::init(dir.path())?;
2505
2506        fs::write(
2507            dir.path().join("fragment.yml"),
2508            r#"
2509fragment-job:
2510  stage: build
2511  script:
2512    - echo fragment
2513"#,
2514        )?;
2515
2516        let ci_dir = dir.path().join("ci");
2517        fs::create_dir_all(&ci_dir)?;
2518        let main_path = ci_dir.join("main.yml");
2519        fs::write(
2520            &main_path,
2521            r#"
2522stages:
2523  - build
2524include:
2525  - local: $INCLUDE_FILE
2526
2527main-job:
2528  stage: build
2529  script:
2530    - echo main
2531"#,
2532        )?;
2533
2534        let pipeline = PipelineGraph::from_path_with_env(
2535            &main_path,
2536            HashMap::from([("INCLUDE_FILE".to_string(), "fragment.yml".to_string())]),
2537            None,
2538        )?;
2539
2540        assert!(
2541            pipeline
2542                .graph
2543                .node_weights()
2544                .any(|job| job.name == "fragment-job")
2545        );
2546        assert!(
2547            pipeline
2548                .graph
2549                .node_weights()
2550                .any(|job| job.name == "main-job")
2551        );
2552        Ok(())
2553    }
2554
2555    #[test]
2556    fn includes_local_files_list_from_repo_root() -> Result<()> {
2557        let dir = tempdir()?;
2558        git2::Repository::init(dir.path())?;
2559
2560        let parts_dir = dir.path().join("parts");
2561        fs::create_dir_all(&parts_dir)?;
2562        fs::write(
2563            parts_dir.join("a.yml"),
2564            r#"
2565job-a:
2566  stage: build
2567  script:
2568    - echo a
2569"#,
2570        )?;
2571        fs::write(
2572            parts_dir.join("b.yml"),
2573            r#"
2574job-b:
2575  stage: build
2576  script:
2577    - echo b
2578"#,
2579        )?;
2580
2581        let main_path = dir.path().join("main.yml");
2582        fs::write(
2583            &main_path,
2584            r#"
2585stages:
2586  - build
2587include:
2588  files:
2589    - /parts/a.yml
2590    - /parts/b.yml
2591
2592main-job:
2593  stage: build
2594  script:
2595    - echo main
2596"#,
2597        )?;
2598
2599        let pipeline = PipelineGraph::from_path(&main_path)?;
2600        assert!(pipeline.graph.node_weights().any(|job| job.name == "job-a"));
2601        assert!(pipeline.graph.node_weights().any(|job| job.name == "job-b"));
2602        assert!(
2603            pipeline
2604                .graph
2605                .node_weights()
2606                .any(|job| job.name == "main-job")
2607        );
2608        Ok(())
2609    }
2610
2611    #[test]
2612    fn includes_local_non_yaml_file_errors() {
2613        let dir = tempdir().expect("tempdir");
2614        git2::Repository::init(dir.path()).expect("init repo");
2615
2616        fs::write(dir.path().join("fragment.txt"), "not yaml").expect("write fragment");
2617
2618        let main_path = dir.path().join("main.yml");
2619        fs::write(
2620            &main_path,
2621            r#"
2622stages:
2623  - build
2624include:
2625  - local: /fragment.txt
2626
2627main-job:
2628  stage: build
2629  script:
2630    - echo main
2631"#,
2632        )
2633        .expect("write main");
2634
2635        let err = PipelineGraph::from_path(&main_path).expect_err("non-yaml include must error");
2636        assert!(
2637            err.to_string()
2638                .contains("must reference a .yml or .yaml file")
2639        );
2640    }
2641
2642    #[test]
2643    fn includes_file_alias_as_local_path() -> Result<()> {
2644        let dir = tempdir()?;
2645        git2::Repository::init(dir.path())?;
2646
2647        fs::write(
2648            dir.path().join("fragment.yml"),
2649            r#"
2650fragment-job:
2651  stage: build
2652  script:
2653    - echo fragment
2654"#,
2655        )?;
2656
2657        let main_path = dir.path().join("main.yml");
2658        fs::write(
2659            &main_path,
2660            r#"
2661stages:
2662  - build
2663include:
2664  - file: /fragment.yml
2665
2666main-job:
2667  stage: build
2668  script:
2669    - echo main
2670"#,
2671        )?;
2672
2673        let pipeline = PipelineGraph::from_path(&main_path)?;
2674        assert!(
2675            pipeline
2676                .graph
2677                .node_weights()
2678                .any(|job| job.name == "fragment-job")
2679        );
2680        assert!(
2681            pipeline
2682                .graph
2683                .node_weights()
2684                .any(|job| job.name == "main-job")
2685        );
2686        Ok(())
2687    }
2688
2689    #[test]
2690    fn retry_max_above_two_errors() {
2691        let dir = tempdir().expect("tempdir");
2692        let main_path = dir.path().join("main.yml");
2693        fs::write(
2694            &main_path,
2695            r#"
2696stages:
2697  - build
2698
2699build:
2700  stage: build
2701  retry: 3
2702  script:
2703    - echo hi
2704"#,
2705        )
2706        .expect("write main");
2707
2708        let err = PipelineGraph::from_path(&main_path).expect_err("retry max must error");
2709        assert!(
2710            err.to_string()
2711                .contains("job 'build'.retry must be 0, 1, or 2")
2712        );
2713    }
2714
2715    #[test]
2716    fn retry_exit_codes_parses_single_value() {
2717        let dir = tempdir().expect("tempdir");
2718        let main_path = dir.path().join("main.yml");
2719        fs::write(
2720            &main_path,
2721            r#"
2722stages:
2723  - build
2724
2725build:
2726  stage: build
2727  retry:
2728    max: 1
2729    exit_codes: 137
2730  script:
2731    - echo hi
2732"#,
2733        )
2734        .expect("write main");
2735
2736        let pipeline = PipelineGraph::from_path(&main_path).expect("retry exit_codes must parse");
2737        let job = pipeline
2738            .graph
2739            .node_weights()
2740            .find(|job| job.name == "build")
2741            .expect("build job present");
2742        assert_eq!(job.retry.max, 1);
2743        assert_eq!(job.retry.exit_codes, vec![137]);
2744    }
2745
2746    #[test]
2747    fn retry_exit_codes_rejects_negative_values() {
2748        let dir = tempdir().expect("tempdir");
2749        let main_path = dir.path().join("main.yml");
2750        fs::write(
2751            &main_path,
2752            r#"
2753stages:
2754  - build
2755
2756build:
2757  stage: build
2758  retry:
2759    max: 1
2760    exit_codes:
2761      - -1
2762  script:
2763    - echo hi
2764"#,
2765        )
2766        .expect("write main");
2767
2768        let err = PipelineGraph::from_path(&main_path).expect_err("negative exit code must error");
2769        assert!(
2770            err.to_string()
2771                .contains("job 'build'.retry.exit_codes must contain non-negative integers")
2772        );
2773    }
2774
2775    #[test]
2776    fn parses_artifacts_reports_dotenv() {
2777        let dir = tempdir().expect("tempdir");
2778        let main_path = dir.path().join("main.yml");
2779        fs::write(
2780            &main_path,
2781            r#"
2782stages:
2783  - build
2784
2785build:
2786  stage: build
2787  script:
2788    - echo hi
2789  artifacts:
2790    reports:
2791      dotenv: tests-temp/dotenv/build.env
2792"#,
2793        )
2794        .expect("write main");
2795
2796        let pipeline = PipelineGraph::from_path(&main_path).expect("pipeline parses");
2797        let job = pipeline
2798            .graph
2799            .node_weights()
2800            .find(|job| job.name == "build")
2801            .expect("build job present");
2802
2803        assert_eq!(
2804            job.artifacts.report_dotenv.as_deref(),
2805            Some(std::path::Path::new("tests-temp/dotenv/build.env"))
2806        );
2807    }
2808
2809    #[test]
2810    fn yaml_merge_keys_work_in_variables_mapping() {
2811        let pipeline = PipelineGraph::from_yaml_str(
2812            r#"
2813stages:
2814  - test
2815
2816.base-vars: &base_vars
2817  SHARED_FLAG: from-anchor
2818  OVERRIDE_ME: old
2819
2820merged-vars:
2821  stage: test
2822  variables:
2823    <<: *base_vars
2824    OVERRIDE_ME: new
2825  script:
2826    - echo ok
2827"#,
2828        )
2829        .expect("pipeline parses");
2830
2831        let job = pipeline
2832            .graph
2833            .node_weights()
2834            .find(|job| job.name == "merged-vars")
2835            .expect("job present");
2836
2837        assert_eq!(
2838            job.variables.get("SHARED_FLAG").map(String::as_str),
2839            Some("from-anchor")
2840        );
2841        assert_eq!(
2842            job.variables.get("OVERRIDE_ME").map(String::as_str),
2843            Some("new")
2844        );
2845    }
2846
2847    #[test]
2848    fn yaml_merge_keys_work_in_job_mapping() {
2849        let pipeline = PipelineGraph::from_yaml_str(
2850            r#"
2851stages:
2852  - test
2853
2854.job-template: &job_template
2855  stage: test
2856  script:
2857    - echo ok
2858  variables:
2859    INNER_FLAG: inner
2860
2861merged-job:
2862  <<: *job_template
2863  variables:
2864    <<: { INNER_FLAG: inner, EXTRA_FLAG: extra }
2865"#,
2866        )
2867        .expect("pipeline parses");
2868
2869        let job = pipeline
2870            .graph
2871            .node_weights()
2872            .find(|job| job.name == "merged-job")
2873            .expect("job present");
2874
2875        assert_eq!(job.stage, "test");
2876        assert_eq!(job.commands, vec!["echo ok".to_string()]);
2877        assert_eq!(
2878            job.variables.get("INNER_FLAG").map(String::as_str),
2879            Some("inner")
2880        );
2881        assert_eq!(
2882            job.variables.get("EXTRA_FLAG").map(String::as_str),
2883            Some("extra")
2884        );
2885    }
2886
2887    #[test]
2888    fn parses_image_docker_platform() {
2889        let pipeline = PipelineGraph::from_yaml_str(
2890            r#"
2891stages:
2892  - test
2893
2894platform-job:
2895  stage: test
2896  image:
2897    name: docker.io/library/alpine:3.19
2898    docker:
2899      platform: linux/arm64/v8
2900  script:
2901    - echo ok
2902"#,
2903        )
2904        .expect("pipeline parses");
2905
2906        let job = pipeline
2907            .graph
2908            .node_weights()
2909            .find(|job| job.name == "platform-job")
2910            .expect("job present");
2911
2912        let image = job.image.as_ref().expect("image present");
2913        assert_eq!(image.name, "docker.io/library/alpine:3.19");
2914        assert_eq!(image.docker_platform.as_deref(), Some("linux/arm64/v8"));
2915    }
2916
2917    #[test]
2918    fn parses_image_entrypoint_and_docker_user() {
2919        let pipeline = PipelineGraph::from_yaml_str(
2920            r#"
2921stages:
2922  - test
2923
2924platform-job:
2925  stage: test
2926  image:
2927    name: docker.io/library/alpine:3.19
2928    entrypoint: [""]
2929    docker:
2930      user: 1000:1000
2931  script:
2932    - echo ok
2933"#,
2934        )
2935        .expect("pipeline parses");
2936
2937        let job = pipeline
2938            .graph
2939            .node_weights()
2940            .find(|job| job.name == "platform-job")
2941            .expect("job present");
2942
2943        let image = job.image.as_ref().expect("image present");
2944        assert_eq!(image.entrypoint, vec![""]);
2945        assert_eq!(image.docker_user.as_deref(), Some("1000:1000"));
2946    }
2947
2948    #[test]
2949    fn parses_service_docker_platform_and_user() {
2950        let pipeline = PipelineGraph::from_yaml_str(
2951            r#"
2952stages:
2953  - test
2954
2955service-job:
2956  stage: test
2957  image: docker.io/library/alpine:3.19
2958  services:
2959    - name: docker.io/library/redis:7.2
2960      alias: cache
2961      docker:
2962        platform: linux/arm64/v8
2963        user: 1000:1000
2964  script:
2965    - echo ok
2966"#,
2967        )
2968        .expect("pipeline parses");
2969
2970        let job = pipeline
2971            .graph
2972            .node_weights()
2973            .find(|job| job.name == "service-job")
2974            .expect("job present");
2975
2976        let service = job.services.first().expect("service present");
2977        assert_eq!(service.image, "docker.io/library/redis:7.2");
2978        assert_eq!(service.aliases, vec!["cache"]);
2979        assert_eq!(service.docker_platform.as_deref(), Some("linux/arm64/v8"));
2980        assert_eq!(service.docker_user.as_deref(), Some("1000:1000"));
2981    }
2982
2983    #[test]
2984    fn inherit_default_controls_modeled_default_keywords() {
2985        let pipeline = PipelineGraph::from_yaml_str(
2986            r#"
2987stages:
2988  - test
2989
2990default:
2991  image: docker.io/library/alpine:3.19
2992  before_script:
2993    - echo before
2994  after_script:
2995    - echo after
2996  cache:
2997    key: default-cache
2998    paths:
2999      - tests-temp/default-cache/
3000  services:
3001    - docker.io/library/redis:7.2
3002  timeout: 10m
3003  retry: 2
3004  interruptible: true
3005
3006inherit-none:
3007  stage: test
3008  inherit:
3009    default: false
3010  image: docker.io/library/alpine:3.19
3011  script:
3012    - echo none
3013
3014inherit-some:
3015  stage: test
3016  inherit:
3017    default: [image, retry, interruptible]
3018  script:
3019    - echo some
3020"#,
3021        )
3022        .expect("pipeline parses");
3023
3024        let none = pipeline
3025            .graph
3026            .node_weights()
3027            .find(|job| job.name == "inherit-none")
3028            .expect("job present");
3029        assert!(!none.inherit_default_image);
3030        assert!(!none.inherit_default_before_script);
3031        assert!(!none.inherit_default_after_script);
3032        assert!(!none.inherit_default_cache);
3033        assert!(!none.inherit_default_services);
3034        assert!(!none.inherit_default_timeout);
3035        assert!(!none.inherit_default_retry);
3036        assert!(!none.inherit_default_interruptible);
3037        assert!(none.cache.is_empty());
3038        assert!(none.services.is_empty());
3039        assert_eq!(none.timeout, None);
3040        assert_eq!(none.retry.max, 0);
3041        assert!(!none.interruptible);
3042
3043        let some = pipeline
3044            .graph
3045            .node_weights()
3046            .find(|job| job.name == "inherit-some")
3047            .expect("job present");
3048        assert!(some.inherit_default_image);
3049        assert!(!some.inherit_default_before_script);
3050        assert!(!some.inherit_default_after_script);
3051        assert!(!some.inherit_default_cache);
3052        assert!(!some.inherit_default_services);
3053        assert!(!some.inherit_default_timeout);
3054        assert!(some.inherit_default_retry);
3055        assert!(some.inherit_default_interruptible);
3056        assert_eq!(
3057            some.image.as_ref().map(|image| image.name.as_str()),
3058            Some("docker.io/library/alpine:3.19")
3059        );
3060        assert!(some.cache.is_empty());
3061        assert!(some.services.is_empty());
3062        assert_eq!(some.timeout, None);
3063        assert_eq!(some.retry.max, 2);
3064        assert!(some.interruptible);
3065    }
3066
3067    #[test]
3068    fn include_cycle_errors() -> Result<()> {
3069        let dir = tempdir()?;
3070        let a_path = dir.path().join("a.yml");
3071        let b_path = dir.path().join("b.yml");
3072        fs::write(
3073            &a_path,
3074            format!(
3075                "
3076include:
3077  - local: {}
3078job-a:
3079  stage: build
3080  script: echo a
3081",
3082                b_path
3083                    .file_name()
3084                    .ok_or_else(|| anyhow!("missing b.yml filename"))?
3085                    .to_string_lossy()
3086            ),
3087        )?;
3088        fs::write(
3089            &b_path,
3090            format!(
3091                "
3092include:
3093  - local: {}
3094job-b:
3095  stage: build
3096  script: echo b
3097",
3098                a_path
3099                    .file_name()
3100                    .ok_or_else(|| anyhow!("missing a.yml filename"))?
3101                    .to_string_lossy()
3102            ),
3103        )?;
3104
3105        let err = PipelineGraph::from_path(&a_path).expect_err("cycle must error");
3106        assert!(err.to_string().contains("include cycle"));
3107        Ok(())
3108    }
3109
3110    #[test]
3111    fn records_needs_dependencies() {
3112        let yaml = r#"
3113stages:
3114  - build
3115  - deploy
3116
3117build-job:
3118  stage: build
3119  script:
3120    - echo build
3121
3122deploy-job:
3123  stage: deploy
3124  needs:
3125    - build-job
3126  script:
3127    - echo deploy
3128"#;
3129
3130        let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3131        let build_idx = find_job(&pipeline, "build-job");
3132        let deploy_idx = find_job(&pipeline, "deploy-job");
3133
3134        assert_eq!(
3135            pipeline.graph[deploy_idx].needs[0].job,
3136            "build-job".to_string()
3137        );
3138        assert!(pipeline.graph[deploy_idx].needs[0].needs_artifacts);
3139        assert!(pipeline.graph.contains_edge(build_idx, deploy_idx));
3140    }
3141
3142    #[test]
3143    fn parses_needs_without_artifacts() {
3144        let yaml = r#"
3145stages:
3146  - build
3147  - test
3148
3149build-job:
3150  stage: build
3151  script:
3152    - echo build
3153
3154test-job:
3155  stage: test
3156  needs:
3157    - job: build-job
3158      artifacts: false
3159  script:
3160    - echo test
3161"#;
3162
3163        let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3164        let test_idx = find_job(&pipeline, "test-job");
3165        assert_eq!(pipeline.graph[test_idx].needs.len(), 1);
3166        let need = &pipeline.graph[test_idx].needs[0];
3167        assert_eq!(need.job, "build-job");
3168        assert!(!need.needs_artifacts);
3169        assert!(!need.optional);
3170    }
3171
3172    #[test]
3173    fn parses_optional_needs() {
3174        let yaml = r#"
3175stages:
3176  - build
3177  - test
3178
3179build-job:
3180  stage: build
3181  script:
3182    - echo build
3183
3184maybe-job:
3185  stage: build
3186  script:
3187    - echo maybe
3188
3189test-job:
3190  stage: test
3191  needs:
3192    - build-job
3193    - job: maybe-job
3194      optional: true
3195  script:
3196    - echo test
3197"#;
3198
3199        let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3200        let test_idx = find_job(&pipeline, "test-job");
3201        assert_eq!(pipeline.graph[test_idx].needs.len(), 2);
3202        let need0 = &pipeline.graph[test_idx].needs[0];
3203        assert_eq!(need0.job, "build-job");
3204        assert!(!need0.optional);
3205        let need1 = &pipeline.graph[test_idx].needs[1];
3206        assert_eq!(need1.job, "maybe-job");
3207        assert!(need1.optional);
3208    }
3209
3210    #[test]
3211    fn parses_artifacts_paths() {
3212        let yaml = r#"
3213stages:
3214  - build
3215
3216build-job:
3217  stage: build
3218  script:
3219    - echo build
3220  artifacts:
3221    paths:
3222      - vendor/
3223      - output/report.txt
3224"#;
3225
3226        let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3227        let build_idx = find_job(&pipeline, "build-job");
3228        let job = &pipeline.graph[build_idx];
3229        assert_eq!(job.artifacts.paths.len(), 2);
3230        assert_eq!(job.artifacts.paths[0], PathBuf::from("vendor"));
3231        assert_eq!(job.artifacts.paths[1], PathBuf::from("output/report.txt"));
3232        assert_eq!(job.artifacts.when, ArtifactWhen::OnSuccess);
3233    }
3234
3235    #[test]
3236    fn parses_artifacts_when() {
3237        let yaml = r#"
3238stages:
3239  - build
3240
3241build-job:
3242  stage: build
3243  script:
3244    - echo build
3245  artifacts:
3246    when: always
3247    paths:
3248      - output/report.txt
3249"#;
3250
3251        let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3252        let build_idx = find_job(&pipeline, "build-job");
3253        let job = &pipeline.graph[build_idx];
3254        assert_eq!(job.artifacts.when, ArtifactWhen::Always);
3255    }
3256
3257    #[test]
3258    fn parses_artifacts_exclude() {
3259        let yaml = r#"
3260stages:
3261  - build
3262
3263build-job:
3264  stage: build
3265  script:
3266    - echo build
3267  artifacts:
3268    paths:
3269      - tests-temp/output/
3270    exclude:
3271      - tests-temp/output/**/*.log
3272      - tests-temp/output/ignore.txt
3273"#;
3274
3275        let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3276        let build_idx = find_job(&pipeline, "build-job");
3277        let job = &pipeline.graph[build_idx];
3278        assert_eq!(
3279            job.artifacts.exclude,
3280            vec![
3281                "tests-temp/output/**/*.log".to_string(),
3282                "tests-temp/output/ignore.txt".to_string()
3283            ]
3284        );
3285    }
3286
3287    #[test]
3288    fn parses_cache_fallback_keys() {
3289        let yaml = r#"
3290stages:
3291  - build
3292
3293build-job:
3294  stage: build
3295  script:
3296    - echo build
3297  cache:
3298    key: cache-$CI_COMMIT_REF_SLUG
3299    fallback_keys:
3300      - cache-$CI_DEFAULT_BRANCH
3301      - cache-default
3302    paths:
3303      - vendor/
3304"#;
3305
3306        let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3307        let build_idx = find_job(&pipeline, "build-job");
3308        let job = &pipeline.graph[build_idx];
3309        assert_eq!(
3310            job.cache[0].fallback_keys,
3311            vec![
3312                "cache-$CI_DEFAULT_BRANCH".to_string(),
3313                "cache-default".to_string()
3314            ]
3315        );
3316    }
3317
3318    #[test]
3319    fn parses_cache_key_files_mapping() {
3320        let yaml = r#"
3321stages:
3322  - build
3323
3324build-job:
3325  stage: build
3326  script:
3327    - echo build
3328  cache:
3329    key:
3330      files:
3331        - Cargo.lock
3332      prefix: $CI_JOB_NAME
3333    paths:
3334      - vendor/
3335"#;
3336
3337        let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3338        let build_idx = find_job(&pipeline, "build-job");
3339        let job = &pipeline.graph[build_idx];
3340        assert_eq!(
3341            job.cache[0].key,
3342            CacheKey::Files {
3343                files: vec![PathBuf::from("Cargo.lock")],
3344                prefix: Some("$CI_JOB_NAME".to_string()),
3345            }
3346        );
3347    }
3348
3349    #[test]
3350    fn errors_when_cache_key_files_has_more_than_two_paths() {
3351        let yaml = r#"
3352stages:
3353  - build
3354
3355build-job:
3356  stage: build
3357  script:
3358    - echo build
3359  cache:
3360    key:
3361      files:
3362        - Cargo.lock
3363        - package-lock.json
3364        - yarn.lock
3365    paths:
3366      - vendor/
3367"#;
3368
3369        let err = PipelineGraph::from_yaml_str(yaml).expect_err("pipeline should fail");
3370        assert!(
3371            err.to_string()
3372                .contains("cache key map supports at most two files"),
3373            "unexpected error: {err:#}"
3374        );
3375    }
3376
3377    #[test]
3378    fn parses_artifacts_untracked() {
3379        let yaml = r#"
3380stages:
3381  - build
3382
3383build-job:
3384  stage: build
3385  script:
3386    - echo build
3387  artifacts:
3388    untracked: true
3389    exclude:
3390      - tests-temp/output/**/*.log
3391"#;
3392
3393        let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3394        let build_idx = find_job(&pipeline, "build-job");
3395        let job = &pipeline.graph[build_idx];
3396        assert!(job.artifacts.untracked);
3397        assert_eq!(
3398            job.artifacts.exclude,
3399            vec!["tests-temp/output/**/*.log".to_string()]
3400        );
3401    }
3402
3403    #[test]
3404    fn parses_pipeline_and_job_images() {
3405        let yaml = r#"
3406image: rust:latest
3407stages:
3408  - build
3409  - test
3410
3411build-job:
3412  stage: build
3413  image: rust:slim
3414  script:
3415    - echo build
3416
3417test-job:
3418  stage: test
3419  script:
3420    - echo test
3421"#;
3422
3423        let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3424        assert_eq!(
3425            pipeline
3426                .defaults
3427                .image
3428                .as_ref()
3429                .map(|image| image.name.as_str()),
3430            Some("rust:latest")
3431        );
3432        let build_idx = find_job(&pipeline, "build-job");
3433        assert_eq!(
3434            pipeline.graph[build_idx]
3435                .image
3436                .as_ref()
3437                .map(|image| image.name.as_str()),
3438            Some("rust:slim")
3439        );
3440        let test_idx = find_job(&pipeline, "test-job");
3441        assert_eq!(
3442            pipeline.graph[test_idx]
3443                .image
3444                .as_ref()
3445                .map(|image| image.name.as_str()),
3446            Some("rust:latest")
3447        );
3448    }
3449
3450    #[test]
3451    fn ignores_default_section_as_job() {
3452        let yaml = r#"
3453stages:
3454  - build
3455
3456default:
3457  image: alpine:latest
3458  before_script:
3459    - echo before
3460  after_script:
3461    - echo after
3462  variables:
3463    GLOBAL_DEFAULT: foo
3464
3465build-job:
3466  stage: build
3467  script:
3468    - echo build
3469"#;
3470
3471        let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3472        assert_eq!(pipeline.stages.len(), 1);
3473        assert_eq!(pipeline.stages[0].jobs.len(), 1);
3474        assert_eq!(
3475            pipeline.defaults.before_script,
3476            vec!["echo before".to_string()]
3477        );
3478        assert_eq!(
3479            pipeline.defaults.after_script,
3480            vec!["echo after".to_string()]
3481        );
3482        assert_eq!(
3483            pipeline
3484                .defaults
3485                .variables
3486                .get("GLOBAL_DEFAULT")
3487                .map(String::as_str),
3488            Some("foo")
3489        );
3490        let job_idx = pipeline.stages[0].jobs[0];
3491        assert_eq!(pipeline.graph[job_idx].name, "build-job");
3492    }
3493
3494    #[test]
3495    fn parses_global_hooks() {
3496        let yaml = r#"
3497stages:
3498  - build
3499
3500default:
3501  before_script:
3502    - echo before-one
3503    - echo before-two
3504  after_script: echo after
3505
3506build-job:
3507  stage: build
3508  script: echo body
3509"#;
3510
3511        let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3512        assert_eq!(
3513            pipeline.defaults.before_script,
3514            vec!["echo before-one".to_string(), "echo before-two".to_string()]
3515        );
3516        assert_eq!(
3517            pipeline.defaults.after_script,
3518            vec!["echo after".to_string()]
3519        );
3520    }
3521
3522    #[test]
3523    fn parses_variable_scopes() {
3524        let yaml = r#"
3525variables:
3526  GLOBAL_VAR: foo
3527
3528default:
3529  variables:
3530    DEFAULT_VAR: bar
3531
3532stages:
3533  - build
3534
3535build-job:
3536  stage: build
3537  variables:
3538    JOB_VAR: baz
3539  script:
3540    - echo job
3541"#;
3542
3543        let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3544        assert_eq!(
3545            pipeline
3546                .defaults
3547                .variables
3548                .get("GLOBAL_VAR")
3549                .map(String::as_str),
3550            Some("foo")
3551        );
3552        assert_eq!(
3553            pipeline
3554                .defaults
3555                .variables
3556                .get("DEFAULT_VAR")
3557                .map(String::as_str),
3558            Some("bar")
3559        );
3560        let job_idx = find_job(&pipeline, "build-job");
3561        assert_eq!(
3562            pipeline.graph[job_idx]
3563                .variables
3564                .get("JOB_VAR")
3565                .map(String::as_str),
3566            Some("baz")
3567        );
3568    }
3569
3570    #[test]
3571    fn ignores_non_job_sections_without_scripts() {
3572        let yaml = r#"
3573stages:
3574  - build
3575
3576workflow:
3577  rules:
3578    - if: $CI_PIPELINE_SOURCE == "push"
3579
3580build-job:
3581  stage: build
3582  script:
3583    - echo build
3584"#;
3585
3586        let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3587        assert_eq!(pipeline.stages.len(), 1);
3588        assert_eq!(pipeline.stages[0].jobs.len(), 1);
3589        let job_idx = pipeline.stages[0].jobs[0];
3590        assert_eq!(pipeline.graph[job_idx].name, "build-job");
3591    }
3592
3593    #[test]
3594    fn errors_when_job_missing_script() {
3595        let yaml = r#"
3596stages:
3597  - build
3598
3599broken-job:
3600  stage: build
3601"#;
3602
3603        let err = PipelineGraph::from_yaml_str(yaml).expect_err("missing script should error");
3604        assert!(err.to_string().contains("must define a script"));
3605    }
3606
3607    #[test]
3608    fn ignores_hidden_jobs_starting_with_dot() {
3609        let yaml = r#"
3610stages:
3611  - build
3612
3613.template:
3614  script:
3615    - echo template
3616
3617build-job:
3618  stage: build
3619  script:
3620    - echo build
3621"#;
3622
3623        let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3624        assert_eq!(pipeline.stages[0].jobs.len(), 1);
3625        let job_idx = pipeline.stages[0].jobs[0];
3626        assert_eq!(pipeline.graph[job_idx].name, "build-job");
3627    }
3628
3629    #[test]
3630    fn job_can_extend_hidden_template() {
3631        let yaml = r#"
3632stages:
3633  - build
3634
3635.base-template:
3636  stage: build
3637  script:
3638    - echo from template
3639  artifacts:
3640    paths:
3641      - template.txt
3642
3643child-job:
3644  extends: .base-template
3645"#;
3646
3647        let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3648        let job_idx = find_job(&pipeline, "child-job");
3649        let job = &pipeline.graph[job_idx];
3650        assert_eq!(job.stage, "build");
3651        assert_eq!(job.commands, vec!["echo from template"]);
3652        assert_eq!(job.artifacts.paths, vec![PathBuf::from("template.txt")]);
3653    }
3654
3655    #[test]
3656    fn job_merges_multiple_extends_in_order() {
3657        let yaml = r#"
3658stages:
3659  - test
3660
3661.lint-template:
3662  script:
3663    - echo lint
3664  artifacts:
3665    paths:
3666      - lint.txt
3667
3668.test-template:
3669  stage: test
3670  script:
3671    - echo tests
3672  artifacts:
3673    paths:
3674      - tests.txt
3675
3676combined:
3677  extends:
3678    - .lint-template
3679    - .test-template
3680"#;
3681
3682        let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3683        let job_idx = find_job(&pipeline, "combined");
3684        let job = &pipeline.graph[job_idx];
3685        assert_eq!(job.commands, vec!["echo tests"]);
3686        assert_eq!(job.artifacts.paths, vec![PathBuf::from("tests.txt")]);
3687        assert_eq!(job.stage, "test");
3688    }
3689
3690    #[test]
3691    fn errors_on_extends_cycle() {
3692        let yaml = r#"
3693stages:
3694  - build
3695
3696.a:
3697  extends: .b
3698  script:
3699    - echo a
3700
3701.b:
3702  extends: .a
3703  script:
3704    - echo b
3705
3706job:
3707  extends: .a
3708"#;
3709
3710        let err = PipelineGraph::from_yaml_str(yaml).expect_err("cycle must error");
3711        assert!(err.to_string().contains("cyclical extends"));
3712    }
3713
3714    #[test]
3715    fn errors_on_unknown_extended_template() {
3716        let yaml = r#"
3717stages:
3718  - build
3719
3720job:
3721  stage: build
3722  extends: .missing
3723"#;
3724
3725        let err = PipelineGraph::from_yaml_str(yaml).expect_err("unknown template must error");
3726        assert!(err.to_string().contains("unknown job/template '.missing'"));
3727    }
3728
3729    fn find_job(graph: &PipelineGraph, name: &str) -> NodeIndex {
3730        graph
3731            .graph
3732            .node_indices()
3733            .find(|&idx| graph.graph[idx].name == name)
3734            .expect("job must exist")
3735    }
3736}
3737#[derive(Debug, Deserialize, Default)]
3738struct RawInherit {
3739    #[serde(default)]
3740    default: Option<RawInheritDefault>,
3741}
3742
3743#[derive(Debug, Deserialize)]
3744#[serde(untagged)]
3745enum RawInheritDefault {
3746    Bool(bool),
3747    List(Vec<String>),
3748}