1use crate::{GitLabRemoteConfig, env, git, runtime};
5use std::collections::HashMap;
6use std::convert::TryFrom;
7use std::fs;
8use std::path::{Path, PathBuf};
9use std::time::Duration;
10
11use anyhow::{Context, Result, anyhow, bail};
12use globset::Glob;
13use humantime;
14use petgraph::graph::{DiGraph, NodeIndex};
15use serde::Deserialize;
16use serde_yaml::value::TaggedValue;
17use serde_yaml::{Mapping, Value};
18use tracing::warn;
19
20use super::graph::{
21 ArtifactConfig, ArtifactWhen, CacheConfig, CacheKey, CachePolicy, DependencySource,
22 EnvironmentAction, EnvironmentConfig, ExternalDependency, ImageConfig, Job, JobDependency,
23 ParallelConfig, ParallelMatrixEntry, ParallelVariable, PipelineDefaults, PipelineGraph,
24 RetryPolicy, ServiceConfig, StageGroup, WorkflowConfig,
25};
26use super::rules::JobRule;
27
28impl PipelineGraph {
29 pub fn from_path(path: impl AsRef<Path>) -> Result<Self> {
30 Self::from_path_with_gitlab(path, None)
31 }
32
33 pub fn from_path_with_gitlab(
34 path: impl AsRef<Path>,
35 gitlab: Option<&GitLabRemoteConfig>,
36 ) -> Result<Self> {
37 Self::from_path_with_env(path, std::env::vars().collect(), gitlab)
38 }
39
40 fn from_path_with_env(
41 path: impl AsRef<Path>,
42 host_env: HashMap<String, String>,
43 gitlab: Option<&GitLabRemoteConfig>,
44 ) -> Result<Self> {
45 let path = path.as_ref();
46 let canonical =
49 fs::canonicalize(path).with_context(|| format!("failed to resolve {:?}", path))?;
50 let include_root = git::repository_root(&canonical)
51 .unwrap_or_else(|_| canonical.parent().unwrap_or(Path::new(".")).to_path_buf());
52 let include_env = env::build_include_lookup(&canonical, &host_env);
53 let mut stack = Vec::new();
54 let root = load_pipeline_file(
55 &canonical,
56 &include_root,
57 &include_env,
58 gitlab,
59 None,
60 &mut stack,
61 )?;
62 let root = resolve_yaml_merge_keys(resolve_reference_tags(root)?)?;
63 Self::from_mapping(root)
64 }
65
66 pub fn from_yaml_str(contents: &str) -> Result<Self> {
67 let root: Mapping = serde_yaml::from_str(contents)?;
68 let root = resolve_yaml_merge_keys(resolve_reference_tags(root)?)?;
69 Self::from_mapping(root)
70 }
71
72 fn from_mapping(root: Mapping) -> Result<Self> {
73 let mut stage_names: Vec<String> = Vec::new();
74 let mut defaults = PipelineDefaults::default();
75 let mut workflow: Option<WorkflowConfig> = None;
76 let mut filters = super::graph::PipelineFilters::default();
77
78 let mut job_defs: HashMap<String, Value> = HashMap::new();
79 let mut job_names: Vec<String> = Vec::new();
80
81 for (key, value) in root {
82 match key {
83 Value::String(name) if name == "stages" => {
84 stage_names = parse_stages(value)?;
85 }
86 Value::String(name) if name == "cache" => {
87 defaults.cache = parse_cache_value(value)?;
88 }
89 Value::String(name) if name == "image" => {
90 defaults.image = Some(parse_image(value)?);
91 }
92 Value::String(name) if name == "variables" => {
93 let vars = parse_variables_map(value)?;
94 defaults.variables.extend(vars);
95 }
96 Value::String(name) if name == "default" => {
97 parse_default_block(&mut defaults, value)?;
98 }
99 Value::String(name) if name == "workflow" => {
100 workflow = parse_workflow(value)?;
101 }
102 Value::String(name) if name == "only" => {
103 filters.only = parse_filter_list(value, "only")?;
104 }
105 Value::String(name) if name == "except" => {
106 filters.except = parse_filter_list(value, "except")?;
107 }
108 Value::String(name) => {
109 if is_reserved_keyword(&name) {
110 continue;
111 }
112
113 match value {
114 Value::Mapping(map) => {
115 job_defs.insert(name.clone(), Value::Mapping(map.clone()));
116 if !name.starts_with('.') {
117 job_names.push(name);
118 }
119 }
120 other => bail!("job '{name}' must be defined as a mapping, got {other:?}"),
121 }
122 }
123 other => bail!("expected string keys in pipeline, got {other:?}"),
124 }
125 }
126
127 build_graph(
128 defaults,
129 workflow,
130 filters,
131 stage_names,
132 job_names,
133 job_defs,
134 )
135 }
136}
137
138impl std::str::FromStr for PipelineGraph {
139 type Err = anyhow::Error;
140
141 fn from_str(s: &str) -> Result<Self, Self::Err> {
142 Self::from_yaml_str(s)
143 }
144}
145
146fn load_pipeline_file(
147 path: &Path,
148 include_root: &Path,
149 include_env: &HashMap<String, String>,
150 gitlab: Option<&GitLabRemoteConfig>,
151 project_context: Option<&ProjectIncludeContext>,
152 stack: &mut Vec<PathBuf>,
153) -> Result<Mapping> {
154 if stack.iter().any(|p| p == path) {
155 bail!("include cycle detected involving {:?}", path);
156 }
157 stack.push(path.to_path_buf());
158
159 let content = fs::read_to_string(path).with_context(|| format!("failed to read {:?}", path))?;
160 let mut root: Mapping = serde_yaml::from_str(&content)?;
161 let include_key = Value::String("include".to_string());
162 let mut combined = Mapping::new();
163
164 if let Some(include_value) = root.remove(&include_key) {
166 let includes = parse_include_entries(include_value)?;
167 for include in includes {
168 match include {
169 IncludeEntry::Local(path) => {
170 let include = expand_include_path(&path, include_env);
171 for resolved in
172 resolve_include_paths(include_root, &include, project_context, include_env)?
173 {
174 validate_include_extension(&resolved)?;
175 let canonical = fs::canonicalize(&resolved)
176 .with_context(|| format!("failed to resolve include {:?}", resolved))?;
177 let included = load_pipeline_file(
178 &canonical,
179 include_root,
180 include_env,
181 gitlab,
182 project_context,
183 stack,
184 )?;
185 combined = merge_mappings(combined, included);
186 }
187 }
188 IncludeEntry::Project {
189 project,
190 reference,
191 files,
192 } => {
193 let gitlab = gitlab.ok_or_else(|| {
194 anyhow!(
195 "include:project requires GitLab credentials/configuration (use --gitlab-token and optionally --gitlab-base-url)"
196 )
197 })?;
198 let resolved_ref = reference.unwrap_or_else(|| "HEAD".to_string());
199 let project_root =
200 project_include_root(&runtime::cache_root(), &project, &resolved_ref);
201 let project_context = ProjectIncludeContext {
202 gitlab: gitlab.clone(),
203 project: project.clone(),
204 reference: resolved_ref.clone(),
205 };
206 for file in files {
207 let fetched = fetch_project_include_file(
208 gitlab,
209 &project_root,
210 &project,
211 &resolved_ref,
212 &file,
213 include_env,
214 )?;
215 let included = load_pipeline_file(
216 &fetched,
217 &project_root,
218 include_env,
219 Some(gitlab),
220 Some(&project_context),
221 stack,
222 )?;
223 combined = merge_mappings(combined, included);
224 }
225 }
226 }
227 }
228 }
229
230 combined = merge_mappings(combined, root);
231 stack.pop();
232 Ok(combined)
233}
234
235fn resolve_include_path(include_root: &Path, include: &Path) -> PathBuf {
236 if let Ok(stripped) = include.strip_prefix(Path::new("/")) {
237 include_root.join(stripped)
238 } else if include.is_absolute() {
239 include.to_path_buf()
240 } else {
241 include_root.join(include)
242 }
243}
244
245fn expand_include_path(include: &Path, include_env: &HashMap<String, String>) -> PathBuf {
246 let expanded = env::expand_value(&include.to_string_lossy(), include_env);
247 PathBuf::from(expanded)
248}
249
250#[derive(Debug, Clone)]
251struct ProjectIncludeContext {
252 gitlab: GitLabRemoteConfig,
253 project: String,
254 reference: String,
255}
256
257fn resolve_include_paths(
258 include_root: &Path,
259 include: &Path,
260 project_context: Option<&ProjectIncludeContext>,
261 include_env: &HashMap<String, String>,
262) -> Result<Vec<PathBuf>> {
263 if !include_has_glob(include) {
264 let resolved = resolve_include_path(include_root, include);
265 if resolved.exists() || project_context.is_none() {
266 return Ok(vec![resolved]);
267 }
268 let Some(project_context) = project_context else {
269 return Ok(vec![resolved]);
270 };
271 let fetched = fetch_project_include_file(
272 &project_context.gitlab,
273 include_root,
274 &project_context.project,
275 &project_context.reference,
276 include,
277 include_env,
278 )?;
279 return Ok(vec![fetched]);
280 }
281
282 let pattern = include_glob_pattern(include);
283 let matcher = Glob::new(&pattern)
284 .with_context(|| format!("invalid include glob '{pattern}'"))?
285 .compile_matcher();
286 let mut matches = Vec::new();
287 for entry in walkdir::WalkDir::new(include_root).follow_links(false) {
288 let entry = entry?;
289 if !entry.file_type().is_file() {
290 continue;
291 }
292 let rel = entry
293 .path()
294 .strip_prefix(include_root)
295 .unwrap_or(entry.path());
296 if matcher.is_match(rel) {
297 matches.push(entry.path().to_path_buf());
298 }
299 }
300 matches.sort();
301 matches.dedup();
302 if matches.is_empty() && project_context.is_some() {
303 bail!("wildcard local includes inside include:project are not supported yet");
304 }
305 if matches.is_empty() {
306 bail!("include glob '{pattern}' matched no files");
307 }
308 Ok(matches)
309}
310
311fn include_has_glob(include: &Path) -> bool {
312 include
313 .to_string_lossy()
314 .chars()
315 .any(|ch| matches!(ch, '*' | '?' | '['))
316}
317
318fn include_glob_pattern(include: &Path) -> String {
319 include
320 .strip_prefix(Path::new("/"))
321 .unwrap_or(include)
322 .to_string_lossy()
323 .to_string()
324}
325
326fn validate_include_extension(path: &Path) -> Result<()> {
327 match path.extension().and_then(|ext| ext.to_str()) {
328 Some("yml") | Some("yaml") => Ok(()),
329 _ => bail!(
330 "include path '{}' must reference a .yml or .yaml file",
331 path.display()
332 ),
333 }
334}
335
336fn resolve_yaml_merge_keys(root: Mapping) -> Result<Mapping> {
337 let resolved = resolve_yaml_merge_value(Value::Mapping(root))?;
338 match resolved {
339 Value::Mapping(map) => Ok(map),
340 other => bail!(
341 "pipeline root must be a mapping after resolving YAML merge keys, got {}",
342 value_kind(&other)
343 ),
344 }
345}
346
347fn resolve_yaml_merge_value(value: Value) -> Result<Value> {
348 match value {
349 Value::Mapping(map) => Ok(Value::Mapping(resolve_yaml_merge_mapping(map)?)),
350 Value::Sequence(entries) => Ok(Value::Sequence(
351 entries
352 .into_iter()
353 .map(resolve_yaml_merge_value)
354 .collect::<Result<Vec<_>>>()?,
355 )),
356 other => Ok(other),
357 }
358}
359
360fn resolve_yaml_merge_mapping(map: Mapping) -> Result<Mapping> {
361 let merge_key = Value::String("<<".to_string());
362 let mut merged = Mapping::new();
363
364 if let Some(merge_value) = map.get(&merge_key).cloned() {
365 match resolve_yaml_merge_value(merge_value)? {
366 Value::Mapping(parent) => {
367 for (key, value) in parent {
368 merged.insert(key, value);
369 }
370 }
371 Value::Sequence(entries) => {
372 for entry in entries {
373 let Value::Mapping(parent) = resolve_yaml_merge_value(entry)? else {
374 bail!("YAML merge key expects a mapping or list of mappings");
375 };
376 for (key, value) in parent {
377 merged.insert(key, value);
378 }
379 }
380 }
381 other => {
382 bail!(
383 "YAML merge key expects a mapping or list of mappings, got {}",
384 value_kind(&other)
385 );
386 }
387 }
388 }
389
390 for (key, value) in map {
391 if key == merge_key {
392 continue;
393 }
394 merged.insert(key, resolve_yaml_merge_value(value)?);
395 }
396
397 Ok(merged)
398}
399
400fn resolve_reference_tags(root: Mapping) -> Result<Mapping> {
401 let root_value = Value::Mapping(root);
402 let mut visiting = Vec::new();
403 let resolved = resolve_references(&root_value, &root_value, &mut visiting)?;
404 match resolved {
405 Value::Mapping(map) => Ok(map),
406 other => bail!(
407 "pipeline root must be a mapping after resolving !reference tags, got {}",
408 value_kind(&other)
409 ),
410 }
411}
412
413type ReferencePath = Vec<ReferenceSegment>;
414
415#[derive(Clone, PartialEq, Eq)]
416enum ReferenceSegment {
417 Key(String),
418 Index(usize),
419}
420
421fn resolve_references(
422 value: &Value,
423 root: &Value,
424 visiting: &mut Vec<ReferencePath>,
425) -> Result<Value> {
426 match value {
427 Value::Tagged(tagged) => {
428 if tagged.tag == "reference" {
429 let path = parse_reference_path(&tagged.value)?;
430 if visiting.iter().any(|current| current == &path) {
431 bail!(
432 "detected recursive !reference {}",
433 describe_reference_path(&path)
434 );
435 }
436 visiting.push(path.clone());
437 let target = follow_reference_path(root, &path).with_context(|| {
438 format!(
439 "failed to resolve !reference {}",
440 describe_reference_path(&path)
441 )
442 })?;
443 let resolved = resolve_references(target, root, visiting)?;
444 visiting.pop();
445 Ok(resolved)
446 } else {
447 let resolved_value = resolve_references(&tagged.value, root, visiting)?;
448 Ok(Value::Tagged(Box::new(TaggedValue {
449 tag: tagged.tag.clone(),
450 value: resolved_value,
451 })))
452 }
453 }
454 Value::Mapping(map) => {
455 let mut resolved = Mapping::with_capacity(map.len());
456 for (key, val) in map.iter() {
457 let resolved_key = resolve_references(key, root, visiting)?;
458 let resolved_val = resolve_references(val, root, visiting)?;
459 resolved.insert(resolved_key, resolved_val);
460 }
461 Ok(Value::Mapping(resolved))
462 }
463 Value::Sequence(seq) => {
464 let mut resolved = Vec::with_capacity(seq.len());
465 for entry in seq.iter() {
466 resolved.push(resolve_references(entry, root, visiting)?);
467 }
468 Ok(Value::Sequence(resolved))
469 }
470 other => Ok(other.clone()),
471 }
472}
473
474fn parse_reference_path(value: &Value) -> Result<ReferencePath> {
475 let entries = match value {
476 Value::Sequence(entries) => entries,
477 other => bail!(
478 "!reference expects a sequence path, got {}",
479 value_kind(other)
480 ),
481 };
482 if entries.is_empty() {
483 bail!("!reference path must contain at least one entry");
484 }
485 let mut path = Vec::with_capacity(entries.len());
486 for entry in entries.iter() {
487 match entry {
488 Value::String(name) => path.push(ReferenceSegment::Key(name.clone())),
489 Value::Number(number) => {
490 let index_u64 = number
491 .as_u64()
492 .ok_or_else(|| anyhow!("!reference indices must be non-negative integers"))?;
493 let index = usize::try_from(index_u64).map_err(|_| {
494 anyhow!("!reference index {index_u64} is too large for this platform")
495 })?;
496 path.push(ReferenceSegment::Index(index));
497 }
498 other => bail!(
499 "!reference path entries must be strings or integers, got {}",
500 value_kind(other)
501 ),
502 }
503 }
504 Ok(path)
505}
506
507fn follow_reference_path<'a>(root: &'a Value, path: &[ReferenceSegment]) -> Result<&'a Value> {
508 let mut current = root;
509 for segment in path {
510 current = match segment {
511 ReferenceSegment::Key(name) => {
512 let mapping = value_as_mapping(current).ok_or_else(|| {
513 anyhow!(
514 "!reference {} expected a mapping before '{}', found {}",
515 describe_reference_path(path),
516 name,
517 value_kind(current)
518 )
519 })?;
520 mapping.get(name.as_str()).ok_or_else(|| {
521 anyhow!(
522 "!reference {} key '{}' not found",
523 describe_reference_path(path),
524 name
525 )
526 })?
527 }
528 ReferenceSegment::Index(idx) => {
529 let sequence = value_as_sequence(current).ok_or_else(|| {
530 anyhow!(
531 "!reference {} expected a sequence before index {}, found {}",
532 describe_reference_path(path),
533 idx,
534 value_kind(current)
535 )
536 })?;
537 sequence.get(*idx).ok_or_else(|| {
538 anyhow!(
539 "!reference {} index {} out of bounds (len {})",
540 describe_reference_path(path),
541 idx,
542 sequence.len()
543 )
544 })?
545 }
546 };
547 }
548 Ok(current)
549}
550
551fn describe_reference_path(path: &[ReferenceSegment]) -> String {
552 let mut parts = Vec::with_capacity(path.len());
553 for segment in path {
554 match segment {
555 ReferenceSegment::Key(name) => parts.push(name.clone()),
556 ReferenceSegment::Index(idx) => parts.push(idx.to_string()),
557 }
558 }
559 format!("[{}]", parts.join(", "))
560}
561
562fn value_as_mapping(value: &Value) -> Option<&Mapping> {
563 match value {
564 Value::Mapping(map) => Some(map),
565 Value::Tagged(tagged) => value_as_mapping(&tagged.value),
566 _ => None,
567 }
568}
569
570fn value_as_sequence(value: &Value) -> Option<&Vec<Value>> {
571 match value {
572 Value::Sequence(seq) => Some(seq),
573 Value::Tagged(tagged) => value_as_sequence(&tagged.value),
574 _ => None,
575 }
576}
577
578fn value_kind(value: &Value) -> &'static str {
579 match value {
580 Value::Null => "null",
581 Value::Bool(_) => "bool",
582 Value::Number(_) => "number",
583 Value::String(_) => "string",
584 Value::Sequence(_) => "sequence",
585 Value::Mapping(_) => "mapping",
586 Value::Tagged(tagged) => value_kind(&tagged.value),
587 }
588}
589
590fn parse_stages(value: Value) -> Result<Vec<String>> {
591 match value {
592 Value::Sequence(entries) => entries
593 .into_iter()
594 .map(|val| match val {
595 Value::String(name) => Ok(name),
596 other => bail!("stage value must be string, got {other:?}"),
597 })
598 .collect(),
599 other => bail!("stages must be a sequence, got {other:?}"),
600 }
601}
602
603fn parse_default_block(defaults: &mut PipelineDefaults, value: Value) -> Result<()> {
604 let mapping = match value {
605 Value::Mapping(map) => map,
606 other => bail!("default section must be a mapping, got {other:?}"),
607 };
608
609 for (key, value) in mapping {
610 match key {
611 Value::String(name) if name == "before_script" => {
612 defaults.before_script = parse_string_list(value, "before_script")?;
613 }
614 Value::String(name) if name == "after_script" => {
615 defaults.after_script = parse_string_list(value, "after_script")?;
616 }
617 Value::String(name) if name == "image" => {
618 defaults.image = Some(parse_image(value)?);
619 }
620 Value::String(name) if name == "variables" => {
621 let vars = parse_variables_map(value)?;
622 defaults.variables.extend(vars);
623 }
624 Value::String(name) if name == "cache" => {
625 defaults.cache = parse_cache_value(value)?;
626 }
627 Value::String(name) if name == "services" => {
628 defaults.services = parse_services_value(value, "services")?;
629 }
630 Value::String(name) if name == "timeout" => {
631 defaults.timeout = parse_timeout_value(value, "default.timeout")?;
632 }
633 Value::String(name) if name == "retry" => {
634 let raw: RawRetry =
635 serde_yaml::from_value(value).context("failed to parse default.retry")?;
636 defaults.retry = raw.into_policy(&RetryPolicy::default(), "default.retry")?;
637 }
638 Value::String(name) if name == "interruptible" => {
639 defaults.interruptible = extract_bool(value, "default.interruptible")?;
640 }
641 Value::String(_) => {
642 }
644 other => bail!("default keys must be strings, got {other:?}"),
645 }
646 }
647
648 Ok(())
649}
650
651fn parse_workflow(value: Value) -> Result<Option<WorkflowConfig>> {
652 let mapping = match value {
653 Value::Mapping(map) => map,
654 other => bail!("workflow section must be a mapping, got {other:?}"),
655 };
656 let key = Value::String("rules".to_string());
657 let Some(rules_value) = mapping.get(&key) else {
658 return Ok(None);
659 };
660 let rules: Vec<JobRule> =
661 serde_yaml::from_value(rules_value.clone()).context("failed to parse workflow.rules")?;
662 Ok(Some(WorkflowConfig { rules }))
663}
664
665fn is_reserved_keyword(name: &str) -> bool {
666 matches!(
667 name,
668 "stages"
669 | "default"
670 | "include"
671 | "cache"
672 | "variables"
673 | "workflow"
674 | "spec"
675 | "image"
676 | "services"
677 | "before_script"
678 | "after_script"
679 | "only"
680 | "except"
681 )
682}
683
684fn parse_image(value: Value) -> Result<ImageConfig> {
685 match value {
686 Value::String(name) => Ok(ImageConfig {
687 name,
688 docker_platform: None,
689 docker_user: None,
690 entrypoint: Vec::new(),
691 }),
692 Value::Mapping(mut map) => {
693 if let Some(val) = map.remove(Value::String("name".to_string())) {
694 let name = extract_string(val, "image name")?;
695 let entrypoint = map
696 .remove(Value::String("entrypoint".to_string()))
697 .map(|value| {
698 serde_yaml::from_value::<ServiceCommand>(value)
699 .map(ServiceCommand::into_vec)
700 })
701 .transpose()
702 .context("failed to parse image.entrypoint")?
703 .unwrap_or_default();
704 let docker_cfg = map
705 .remove(Value::String("docker".to_string()))
706 .map(|value| parse_docker_executor_config(value, "image.docker"))
707 .transpose()?;
708 Ok(ImageConfig {
709 name,
710 docker_platform: docker_cfg.as_ref().and_then(|cfg| cfg.platform.clone()),
711 docker_user: docker_cfg.and_then(|cfg| cfg.user),
712 entrypoint,
713 })
714 } else {
715 bail!("image mapping must include 'name'")
716 }
717 }
718 other => bail!("image must be a string or mapping, got {other:?}"),
719 }
720}
721
722struct DockerExecutorConfig {
723 platform: Option<String>,
724 user: Option<String>,
725}
726
727fn parse_docker_executor_config(value: Value, field: &str) -> Result<DockerExecutorConfig> {
728 let map = match value {
729 Value::Mapping(map) => map,
730 other => bail!("{field} must be a mapping, got {other:?}"),
731 };
732 let platform_key = Value::String("platform".to_string());
733 let user_key = Value::String("user".to_string());
734 let platform = map
735 .get(&platform_key)
736 .cloned()
737 .map(|value| extract_string(value, &format!("{field}.platform")))
738 .transpose()?;
739 let user = map
740 .get(&user_key)
741 .cloned()
742 .map(|value| extract_string(value, &format!("{field}.user")))
743 .transpose()?;
744 if platform.is_none() && user.is_none() {
745 bail!("{field} must include 'platform' or 'user'");
746 }
747 Ok(DockerExecutorConfig { platform, user })
748}
749
750fn extract_string(value: Value, what: &str) -> Result<String> {
751 match value {
752 Value::String(text) => Ok(text),
753 other => bail!("{what} must be a string, got {other:?}"),
754 }
755}
756
757type ParsedJobSpec = (
758 RawJob,
759 Option<ImageConfig>,
760 HashMap<String, String>,
761 Vec<CacheConfig>,
762 Vec<ServiceConfig>,
763 Option<ParallelConfig>,
764 Vec<String>,
765 Vec<String>,
766);
767
768fn parse_job(value: Value) -> Result<ParsedJobSpec> {
769 match value {
770 Value::Mapping(mut map) => {
771 let image_value = map.remove(Value::String("image".to_string()));
772 let variables_value = map.remove(Value::String("variables".to_string()));
773 let cache_value = map.remove(Value::String("cache".to_string()));
774 let services_value = map.remove(Value::String("services".to_string()));
775 let parallel_value = map.remove(Value::String("parallel".to_string()));
776 let only_value = map.remove(Value::String("only".to_string()));
777 let except_value = map.remove(Value::String("except".to_string()));
778 let job_spec: RawJob = serde_yaml::from_value(Value::Mapping(map))?;
779 let image = image_value.map(parse_image).transpose()?;
780 let variables = variables_value
781 .map(parse_variables_map)
782 .transpose()?
783 .unwrap_or_default();
784 let cache = cache_value
785 .map(parse_cache_value)
786 .transpose()?
787 .unwrap_or_default();
788 let services = services_value
789 .map(|value| parse_services_value(value, "services"))
790 .transpose()?
791 .unwrap_or_default();
792 let parallel = parallel_value.map(parse_parallel_value).transpose()?;
793 let only = only_value
794 .map(|value| parse_filter_list(value, "only"))
795 .transpose()?
796 .unwrap_or_default();
797 let except = except_value
798 .map(|value| parse_filter_list(value, "except"))
799 .transpose()?
800 .unwrap_or_default();
801 Ok((
802 job_spec, image, variables, cache, services, parallel, only, except,
803 ))
804 }
805 other => bail!("job definition must be a mapping, got {other:?}"),
806 }
807}
808
809fn parse_string_list(value: Value, field: &str) -> Result<Vec<String>> {
810 match value {
811 Value::Sequence(entries) => {
812 let mut out = Vec::new();
813 for entry in entries {
814 let text = yaml_command_string(entry)
815 .map_err(|err| anyhow!("{field} entries must be strings ({err})"))?;
816 out.push(text);
817 }
818 Ok(out)
819 }
820 Value::Null => Ok(Vec::new()),
821 other => {
822 let text = yaml_command_string(other)
823 .map_err(|err| anyhow!("{field} must be a string or sequence ({err})"))?;
824 Ok(vec![text])
825 }
826 }
827}
828
829fn parse_cache_value(value: Value) -> Result<Vec<CacheConfig>> {
830 match value {
831 Value::Sequence(entries) => entries
832 .into_iter()
833 .map(parse_cache_entry)
834 .collect::<Result<Vec<_>>>(),
835 Value::Null => Ok(Vec::new()),
836 other => Ok(vec![parse_cache_entry(other)?]),
837 }
838}
839
840fn yaml_command_string(value: Value) -> Result<String, String> {
841 match value {
842 Value::String(text) => Ok(text),
843 Value::Number(number) => Ok(number.to_string()),
844 Value::Bool(boolean) => Ok(boolean.to_string()),
845 Value::Null => Ok(String::new()),
846 Value::Mapping(map) => mapping_command_string(map),
847 other => Err(format!("got {other:?}")),
848 }
849}
850
851fn mapping_command_string(map: Mapping) -> Result<String, String> {
852 if map.len() != 1 {
853 return Err(format!(
854 "mapping entries must contain exactly one command, got {map:?}"
855 ));
856 }
857 let (key, value) = map
858 .into_iter()
859 .next()
860 .ok_or_else(|| "mapping entries must contain exactly one command".to_string())?;
861 let key_text = match key {
862 Value::String(text) => text,
863 other => return Err(format!("mapping keys must be strings, got {other:?}")),
864 };
865 let value_text = yaml_command_string(value)?;
866 if value_text.is_empty() {
867 Ok(format!("{key_text}:"))
868 } else {
869 Ok(format!("{key_text}: {value_text}"))
870 }
871}
872
873fn parse_cache_entry(value: Value) -> Result<CacheConfig> {
874 let raw: CacheEntryRaw = match value {
875 Value::Mapping(_) => serde_yaml::from_value(value)?,
876 other => bail!("cache entry must be a mapping, got {other:?}"),
877 };
878 let key = parse_cache_key(raw.key)?;
879 let fallback_keys = raw.fallback_keys;
880 let paths = if raw.paths.is_empty() {
881 bail!("cache entry must specify at least one path");
882 } else {
883 raw.paths
884 };
885 let policy = raw
886 .policy
887 .as_deref()
888 .map(CachePolicy::from_str)
889 .unwrap_or(CachePolicy::PullPush);
890 Ok(CacheConfig {
891 key,
892 fallback_keys,
893 paths,
894 policy,
895 })
896}
897
898fn parse_cache_key(raw: Option<CacheKeyRaw>) -> Result<CacheKey> {
899 let Some(raw) = raw else {
900 return Ok(CacheKey::default());
901 };
902
903 match raw {
904 CacheKeyRaw::Literal(value) => Ok(CacheKey::Literal(value)),
905 CacheKeyRaw::Detailed(details) => {
906 if details.files.is_empty() {
907 bail!("cache key map must include at least one file in 'files'");
908 }
909 if details.files.len() > 2 {
910 bail!("cache key map supports at most two files");
911 }
912 Ok(CacheKey::Files {
913 files: details.files,
914 prefix: details.prefix.filter(|value| !value.is_empty()),
915 })
916 }
917 }
918}
919
920fn parse_variables_map(value: Value) -> Result<HashMap<String, String>> {
921 let mapping = match value {
922 Value::Mapping(map) => map,
923 other => bail!("variables must be a mapping, got {other:?}"),
924 };
925
926 let mut vars = HashMap::new();
927 for (key, val) in mapping {
928 let name = match key {
929 Value::String(s) => s,
930 other => bail!("variable names must be strings, got {other:?}"),
931 };
932 let value = extract_variable_value(val, &format!("variable '{name}'"))?;
933 vars.insert(name, value);
934 }
935
936 Ok(vars)
937}
938
939fn extract_variable_value(value: Value, what: &str) -> Result<String> {
940 match value {
941 Value::String(text) => Ok(text),
942 Value::Bool(flag) => Ok(flag.to_string()),
943 Value::Number(num) => Ok(num.to_string()),
944 Value::Null => Ok(String::new()),
945 Value::Mapping(mut map) => {
946 let key = Value::String("value".to_string());
947 if let Some(entry) = map.remove(&key) {
948 extract_variable_value(entry, what)
949 } else {
950 bail!("{what} mapping must include 'value'")
951 }
952 }
953 other => bail!("{what} must be a string/bool/number, got {other:?}"),
954 }
955}
956
957fn parse_services_value(value: Value, field: &str) -> Result<Vec<ServiceConfig>> {
958 let entries = match value {
959 Value::Sequence(seq) => seq,
960 Value::Null => return Ok(Vec::new()),
961 other => vec![other],
962 };
963 let mut services = Vec::new();
964 for entry in entries {
965 let raw: RawService = serde_yaml::from_value(entry)
966 .with_context(|| format!("failed to parse {field} entry"))?;
967 let config = match raw {
968 RawService::Simple(image) => ServiceConfig {
969 image,
970 aliases: Vec::new(),
971 docker_platform: None,
972 docker_user: None,
973 entrypoint: Vec::new(),
974 command: Vec::new(),
975 variables: HashMap::new(),
976 },
977 RawService::Detailed(details) => details.into_config()?,
978 };
979 services.push(config);
980 }
981 Ok(services)
982}
983
984fn parse_parallel_value(value: Value) -> Result<ParallelConfig> {
985 match value {
986 Value::Number(num) => {
987 let count = num
988 .as_u64()
989 .ok_or_else(|| anyhow!("parallel count must be positive integer"))?;
990 if count == 0 {
991 bail!("parallel count must be greater than zero");
992 }
993 Ok(ParallelConfig::Count(count.try_into().unwrap_or(u32::MAX)))
994 }
995 Value::Mapping(mut map) => {
996 let matrix_key = Value::String("matrix".to_string());
997 let Some(entries) = map.remove(&matrix_key) else {
998 bail!("parallel mapping must include 'matrix'");
999 };
1000 let matrices = parse_parallel_matrix(entries)?;
1001 Ok(ParallelConfig::Matrix(matrices))
1002 }
1003 other => bail!("parallel must be an integer or mapping, got {other:?}"),
1004 }
1005}
1006
1007fn parse_parallel_matrix(value: Value) -> Result<Vec<ParallelMatrixEntry>> {
1008 match value {
1009 Value::Sequence(entries) => entries
1010 .into_iter()
1011 .map(parse_parallel_matrix_entry)
1012 .collect(),
1013 other => Ok(vec![parse_parallel_matrix_entry(other)?]),
1014 }
1015}
1016
1017fn parse_parallel_matrix_entry(value: Value) -> Result<ParallelMatrixEntry> {
1018 let mapping = match value {
1019 Value::Mapping(map) => map,
1020 other => bail!("parallel matrix entries must be mappings, got {other:?}"),
1021 };
1022 let mut variables = Vec::new();
1023 for (key, value) in mapping {
1024 let name = match key {
1025 Value::String(name) => name,
1026 other => bail!("parallel matrix variable names must be strings, got {other:?}"),
1027 };
1028 let values = match value {
1029 Value::String(text) => vec![text],
1030 Value::Sequence(entries) => entries
1031 .into_iter()
1032 .map(|entry| match entry {
1033 Value::String(text) => Ok(text),
1034 other => bail!("parallel matrix values must be strings, got {other:?}"),
1035 })
1036 .collect::<Result<Vec<_>>>()?,
1037 other => bail!("parallel matrix values must be string or list, got {other:?}"),
1038 };
1039 if values.is_empty() {
1040 bail!(
1041 "parallel matrix variable '{}' must have at least one value",
1042 name
1043 );
1044 }
1045 variables.push(ParallelVariable { name, values });
1046 }
1047 if variables.is_empty() {
1048 bail!("parallel matrix entries must define at least one variable");
1049 }
1050 Ok(ParallelMatrixEntry { variables })
1051}
1052
1053fn parse_filter_list(value: Value, field: &str) -> Result<Vec<String>> {
1054 match value {
1055 Value::String(text) => Ok(vec![text]),
1056 Value::Sequence(entries) => entries
1057 .into_iter()
1058 .map(|entry| match entry {
1059 Value::String(text) => Ok(text),
1060 other => bail!("{field} entries must be strings, got {other:?}"),
1061 })
1062 .collect(),
1063 Value::Mapping(mut map) => {
1064 let variables_key = Value::String("variables".to_string());
1065 if let Some(variables) = map.remove(&variables_key) {
1066 if !map.is_empty() {
1067 bail!("{field} mapping supports only 'variables'");
1068 }
1069 parse_variable_filter_list(variables, field)
1070 } else {
1071 bail!("{field} mapping supports only 'variables'");
1072 }
1073 }
1074 Value::Null => Ok(Vec::new()),
1075 other => bail!("{field} must be a string or list, got {other:?}"),
1076 }
1077}
1078
1079fn parse_variable_filter_list(value: Value, field: &str) -> Result<Vec<String>> {
1080 let expressions = match value {
1081 Value::String(text) => vec![text],
1082 Value::Sequence(entries) => entries
1083 .into_iter()
1084 .map(|entry| match entry {
1085 Value::String(text) => Ok(text),
1086 other => bail!("{field}.variables entries must be strings, got {other:?}"),
1087 })
1088 .collect::<Result<Vec<_>>>()?,
1089 Value::Null => Vec::new(),
1090 other => bail!("{field}.variables must be a string or list, got {other:?}"),
1091 };
1092
1093 Ok(expressions
1094 .into_iter()
1095 .map(|expr| format!("__opal_variables__:{expr}"))
1096 .collect())
1097}
1098
1099fn parse_timeout_value(value: Value, field: &str) -> Result<Option<Duration>> {
1100 match value {
1101 Value::Null => Ok(None),
1102 Value::String(text) => parse_timeout_str(&text, field).map(Some),
1103 other => bail!("{field} must be a string or null, got {other:?}"),
1104 }
1105}
1106
1107fn parse_optional_timeout(raw: &Option<String>, field: &str) -> Result<Option<Duration>> {
1108 if let Some(text) = raw {
1109 Ok(Some(parse_timeout_str(text, field)?))
1110 } else {
1111 Ok(None)
1112 }
1113}
1114
1115fn parse_timeout_str(text: &str, field: &str) -> Result<Duration> {
1116 humantime::parse_duration(text).with_context(|| format!("invalid duration for {field}: {text}"))
1117}
1118
1119fn extract_bool(value: Value, field: &str) -> Result<bool> {
1120 match value {
1121 Value::Bool(b) => Ok(b),
1122 other => bail!("{field} must be a boolean, got {other:?}"),
1123 }
1124}
1125
1126fn build_graph(
1129 defaults: PipelineDefaults,
1130 workflow: Option<WorkflowConfig>,
1131 filters: super::graph::PipelineFilters,
1132 stage_names: Vec<String>,
1133 job_names: Vec<String>,
1134 job_defs: HashMap<String, Value>,
1135) -> Result<PipelineGraph> {
1136 let mut graph = DiGraph::<Job, ()>::new();
1137 let mut stages: Vec<StageGroup> = stage_names
1138 .into_iter()
1139 .map(|name| StageGroup {
1140 name,
1141 jobs: Vec::new(),
1142 })
1143 .collect();
1144 let mut name_to_index: HashMap<String, NodeIndex> = HashMap::new();
1145 let mut pending_needs: Vec<(String, NodeIndex, Vec<JobDependency>)> = Vec::new();
1146
1147 if stages.is_empty() {
1148 stages.push(StageGroup {
1149 name: "default".to_string(),
1150 jobs: Vec::new(),
1151 });
1152 }
1153
1154 let mut resolved_defs: HashMap<String, Mapping> = HashMap::new();
1155
1156 for job_name in job_names {
1157 let merged_map =
1158 resolve_job_definition(&job_name, &job_defs, &mut resolved_defs, &mut Vec::new())?;
1159 let (
1160 job_spec,
1161 job_image,
1162 job_variables,
1163 job_cache,
1164 job_services,
1165 job_parallel,
1166 only,
1167 except,
1168 ) = parse_job(Value::Mapping(merged_map))?;
1169 let inherit_defaults = job_inherit_defaults(&job_spec);
1170 let stage_name = job_spec.stage.unwrap_or_else(|| {
1171 stages
1172 .first()
1173 .map(|stage| stage.name.as_str())
1174 .unwrap_or("default")
1175 .to_string()
1176 });
1177 let stage_index = ensure_stage(&mut stages, &stage_name);
1178 let commands = job_spec.script.into_commands();
1179 if commands.is_empty() {
1180 bail!(
1181 "job '{}' must define a script (directly or via extends)",
1182 job_name
1183 );
1184 }
1185 let (raw_needs, explicit_needs) = match job_spec.needs {
1186 Some(entries) => (entries, true),
1187 None => (Vec::new(), false),
1188 };
1189 let needs: Vec<JobDependency> = raw_needs
1190 .into_iter()
1191 .filter_map(|need| need.into_dependency(&job_name))
1192 .collect();
1193 let dependencies = job_spec.dependencies;
1194 let before_script = job_spec.before_script.map(Script::into_commands);
1195 let after_script = job_spec.after_script.map(Script::into_commands);
1196 let artifacts = job_spec.artifacts.into_config(&job_name)?;
1197 let cache_entries = if job_cache.is_empty() && inherit_defaults.cache {
1198 defaults.cache.clone()
1199 } else {
1200 job_cache
1201 };
1202 let services = if job_services.is_empty() && inherit_defaults.services {
1203 defaults.services.clone()
1204 } else {
1205 job_services
1206 };
1207 let timeout =
1208 parse_optional_timeout(&job_spec.timeout, &format!("job '{}'.timeout", job_name))?.or(
1209 if inherit_defaults.timeout {
1210 defaults.timeout
1211 } else {
1212 None
1213 },
1214 );
1215 let retry_base = if inherit_defaults.retry {
1216 defaults.retry.clone()
1217 } else {
1218 RetryPolicy::default()
1219 };
1220 let retry = job_spec
1221 .retry
1222 .map(|raw| raw.into_policy(&retry_base, &format!("job '{}'.retry", job_name)))
1223 .transpose()?
1224 .unwrap_or(retry_base);
1225 let interruptible = job_spec
1226 .interruptible
1227 .unwrap_or(if inherit_defaults.interruptible {
1228 defaults.interruptible
1229 } else {
1230 false
1231 });
1232 let resource_group = job_spec.resource_group.clone();
1233 let parallel = job_parallel;
1234
1235 let environment = job_spec.environment.as_ref().map(|env| {
1236 let action = match env.action.as_deref() {
1237 Some("prepare") => EnvironmentAction::Prepare,
1238 Some("stop") => EnvironmentAction::Stop,
1239 Some("verify") => EnvironmentAction::Verify,
1240 Some("access") => EnvironmentAction::Access,
1241 _ => EnvironmentAction::Start,
1242 };
1243 let name = if env.name.is_empty() {
1244 job_name.clone()
1245 } else {
1246 env.name.clone()
1247 };
1248 EnvironmentConfig {
1249 name,
1250 url: env.url.clone(),
1251 on_stop: env.on_stop.clone(),
1252 auto_stop_in: parse_optional_timeout(
1253 &env.auto_stop_in,
1254 &format!("job '{}'.environment.auto_stop_in", job_name),
1255 )
1256 .ok()
1257 .flatten(),
1258 action,
1259 }
1260 });
1261
1262 let inherited_image = if job_image.is_none() && inherit_defaults.image {
1263 defaults.image.clone()
1264 } else {
1265 job_image
1266 };
1267
1268 let node = graph.add_node(Job {
1269 name: job_name.clone(),
1270 stage: stage_name,
1271 commands,
1272 needs: needs.clone(),
1273 explicit_needs,
1274 dependencies: dependencies.clone(),
1275 before_script,
1276 after_script,
1277 inherit_default_image: inherit_defaults.image,
1278 inherit_default_before_script: inherit_defaults.before_script,
1279 inherit_default_after_script: inherit_defaults.after_script,
1280 inherit_default_cache: inherit_defaults.cache,
1281 inherit_default_services: inherit_defaults.services,
1282 inherit_default_timeout: inherit_defaults.timeout,
1283 inherit_default_retry: inherit_defaults.retry,
1284 inherit_default_interruptible: inherit_defaults.interruptible,
1285 when: job_spec.when.clone(),
1286 rules: job_spec.rules.clone(),
1287 artifacts,
1288 cache: cache_entries,
1289 image: inherited_image,
1290 variables: job_variables,
1291 services,
1292 timeout,
1293 retry,
1294 interruptible,
1295 resource_group,
1296 parallel,
1297 only,
1298 except,
1299 tags: job_spec.tags.clone(),
1300 environment,
1301 });
1302
1303 name_to_index.insert(job_name.clone(), node);
1304 pending_needs.push((job_name, node, needs));
1305
1306 let stage = stages
1307 .get_mut(stage_index)
1308 .ok_or_else(|| anyhow!("internal error: stage index {} missing", stage_index))?;
1309 stage.jobs.push(node);
1310 }
1311
1312 for (job_name, job_idx, needs) in pending_needs {
1315 for dependency in needs {
1316 if !matches!(dependency.source, DependencySource::Local) {
1317 continue;
1318 }
1319 let Some(dependency_idx) = name_to_index.get(&dependency.job).copied() else {
1320 if dependency.optional {
1321 continue;
1322 }
1323 return Err(anyhow::anyhow!(
1324 "job '{}' declared unknown dependency '{}'",
1325 job_name,
1326 dependency.job
1327 ));
1328 };
1329
1330 graph.add_edge(dependency_idx, job_idx, ());
1331 }
1332 }
1333
1334 Ok(PipelineGraph {
1335 graph,
1336 stages,
1337 defaults,
1338 workflow,
1339 filters,
1340 })
1341}
1342
1343struct JobInheritDefaults {
1344 image: bool,
1345 before_script: bool,
1346 after_script: bool,
1347 cache: bool,
1348 services: bool,
1349 timeout: bool,
1350 retry: bool,
1351 interruptible: bool,
1352}
1353
1354impl Default for JobInheritDefaults {
1355 fn default() -> Self {
1356 Self {
1357 image: true,
1358 before_script: true,
1359 after_script: true,
1360 cache: true,
1361 services: true,
1362 timeout: true,
1363 retry: true,
1364 interruptible: true,
1365 }
1366 }
1367}
1368
1369fn job_inherit_defaults(job: &RawJob) -> JobInheritDefaults {
1370 let mut inherit = JobInheritDefaults::default();
1371 if let Some(raw_inherit) = &job.inherit
1372 && let Some(default) = &raw_inherit.default
1373 {
1374 match default {
1375 RawInheritDefault::Bool(value) => {
1376 inherit.image = *value;
1377 inherit.before_script = *value;
1378 inherit.after_script = *value;
1379 inherit.cache = *value;
1380 inherit.services = *value;
1381 inherit.timeout = *value;
1382 inherit.retry = *value;
1383 inherit.interruptible = *value;
1384 }
1385 RawInheritDefault::List(entries) => {
1386 inherit.image = entries.iter().any(|entry| entry == "image");
1387 inherit.before_script = entries.iter().any(|entry| entry == "before_script");
1388 inherit.after_script = entries.iter().any(|entry| entry == "after_script");
1389 inherit.cache = entries.iter().any(|entry| entry == "cache");
1390 inherit.services = entries.iter().any(|entry| entry == "services");
1391 inherit.timeout = entries.iter().any(|entry| entry == "timeout");
1392 inherit.retry = entries.iter().any(|entry| entry == "retry");
1393 inherit.interruptible = entries.iter().any(|entry| entry == "interruptible");
1394 }
1395 }
1396 }
1397 inherit
1398}
1399
1400fn resolve_job_definition(
1401 name: &str,
1402 job_defs: &HashMap<String, Value>,
1403 cache: &mut HashMap<String, Mapping>,
1404 stack: &mut Vec<String>,
1405) -> Result<Mapping> {
1406 if let Some(resolved) = cache.get(name) {
1407 return Ok(resolved.clone());
1408 }
1409
1410 if stack.iter().any(|entry| entry == name) {
1411 bail!("job '{}' has cyclical extends", name);
1412 }
1413
1414 let value = match job_defs.get(name) {
1415 Some(v) => v,
1416 None => {
1417 let requester = stack.last().cloned().unwrap_or_else(|| name.to_string());
1418 bail!("job '{requester}' extends unknown job/template '{name}'");
1419 }
1420 };
1421
1422 let map = match value {
1423 Value::Mapping(map) => map.clone(),
1424 other => bail!("job '{name}' must be defined as mapping, got {other:?}"),
1425 };
1426
1427 stack.push(name.to_string());
1428
1429 let extends_key = Value::String("extends".to_string());
1430 let extends = map.get(&extends_key).map(parse_extends_list).transpose()?;
1431
1432 let mut merged = Mapping::new();
1433 if let Some(parents) = extends {
1434 for parent_name in parents {
1435 let parent_map = resolve_job_definition(&parent_name, job_defs, cache, stack)?;
1436 merged = merge_mappings(merged, parent_map);
1437 }
1438 }
1439
1440 let mut child_map = map;
1441 child_map.remove(&extends_key);
1442 merged = merge_mappings(merged, child_map);
1443
1444 stack.pop();
1445 cache.insert(name.to_string(), merged.clone());
1446 Ok(merged)
1447}
1448
1449fn parse_extends_list(value: &Value) -> Result<Vec<String>> {
1450 match value {
1451 Value::String(name) => Ok(vec![name.clone()]),
1452 Value::Sequence(seq) => seq
1453 .iter()
1454 .map(|val| match val {
1455 Value::String(name) => Ok(name.clone()),
1456 other => bail!("extends entries must be strings, got {other:?}"),
1457 })
1458 .collect(),
1459 other => bail!("extends must be string or sequence, got {other:?}"),
1460 }
1461}
1462
1463fn merge_mappings(mut base: Mapping, addition: Mapping) -> Mapping {
1464 for (key, value) in addition {
1465 base.insert(key, value);
1466 }
1467 base
1468}
1469
1470fn parse_include_entries(value: Value) -> Result<Vec<IncludeEntry>> {
1471 match value {
1472 Value::String(path) => Ok(vec![IncludeEntry::Local(PathBuf::from(path))]),
1473 Value::Sequence(entries) => {
1474 let mut paths = Vec::new();
1475 for entry in entries {
1476 paths.extend(parse_include_entry(entry)?);
1477 }
1478 Ok(paths)
1479 }
1480 Value::Mapping(_) => parse_include_entry(value),
1481 other => bail!("include must be a string or list, got {other:?}"),
1482 }
1483}
1484
1485#[derive(Debug, Clone)]
1486enum IncludeEntry {
1487 Local(PathBuf),
1488 Project {
1489 project: String,
1490 reference: Option<String>,
1491 files: Vec<PathBuf>,
1492 },
1493}
1494
1495fn parse_include_entry(value: Value) -> Result<Vec<IncludeEntry>> {
1497 match value {
1498 Value::String(path) => Ok(vec![IncludeEntry::Local(PathBuf::from(path))]),
1499 Value::Mapping(map) => {
1500 let local_key = Value::String("local".to_string());
1501 let file_key = Value::String("file".to_string());
1502 let files_key = Value::String("files".to_string());
1503 let project_key = Value::String("project".to_string());
1504 let remote_key = Value::String("remote".to_string());
1505 let template_key = Value::String("template".to_string());
1506 let component_key = Value::String("component".to_string());
1507 if map.contains_key(&remote_key) {
1508 bail!("include:remote is not supported yet");
1509 }
1510 if map.contains_key(&template_key) {
1511 bail!("include:template is not supported yet");
1512 }
1513 if map.contains_key(&component_key) {
1514 bail!("include:component is not supported yet");
1515 }
1516 if let Some(Value::String(local)) = map.get(&local_key) {
1517 Ok(vec![IncludeEntry::Local(PathBuf::from(local))])
1518 } else if let Some(Value::String(project)) = map.get(&project_key) {
1519 let reference = map
1520 .get(Value::String("ref".to_string()))
1521 .map(|value| match value {
1522 Value::String(text) => Ok(text.clone()),
1523 other => bail!("include:project ref must be a string, got {other:?}"),
1524 })
1525 .transpose()?;
1526 let files = parse_project_include_files(map.get(&file_key), map.get(&files_key))?;
1527 Ok(vec![IncludeEntry::Project {
1528 project: project.clone(),
1529 reference,
1530 files,
1531 }])
1532 } else if let Some(Value::String(file)) = map.get(&file_key) {
1533 Ok(vec![IncludeEntry::Local(PathBuf::from(file))])
1534 } else if let Some(Value::Sequence(files)) = map.get(&files_key) {
1535 let mut paths = Vec::new();
1536 for entry in files {
1537 match entry {
1538 Value::String(path) => paths.push(IncludeEntry::Local(PathBuf::from(path))),
1539 other => bail!("include 'files' entries must be strings, got {other:?}"),
1540 }
1541 }
1542 Ok(paths)
1543 } else {
1544 bail!("only 'local' or 'file(s)' includes are supported");
1545 }
1546 }
1547 other => bail!("unsupported include entry {other:?}"),
1548 }
1549}
1550
1551fn parse_project_include_files(
1552 file_value: Option<&Value>,
1553 files_value: Option<&Value>,
1554) -> Result<Vec<PathBuf>> {
1555 if files_value.is_some() {
1556 bail!("include:project must use 'file', not 'files'");
1557 }
1558 let Some(value) = file_value else {
1559 bail!("include:project requires a 'file' entry");
1560 };
1561 match value {
1562 Value::String(path) => Ok(vec![PathBuf::from(path)]),
1563 Value::Sequence(entries) => entries
1564 .iter()
1565 .map(|entry| match entry {
1566 Value::String(path) => Ok(PathBuf::from(path)),
1567 other => bail!("include:project file entries must be strings, got {other:?}"),
1568 })
1569 .collect(),
1570 other => bail!("include:project file must be a string or list, got {other:?}"),
1571 }
1572}
1573
1574fn project_include_root(cache_root: &Path, project: &str, reference: &str) -> PathBuf {
1575 cache_root
1576 .join("includes")
1577 .join(percent_encode(project))
1578 .join(sanitize_reference(reference))
1579}
1580
1581fn fetch_project_include_file(
1583 gitlab: &GitLabRemoteConfig,
1584 project_root: &Path,
1585 project: &str,
1586 reference: &str,
1587 file: &Path,
1588 include_env: &HashMap<String, String>,
1589) -> Result<PathBuf> {
1590 let expanded = expand_include_path(file, include_env);
1591 validate_include_extension(&expanded)?;
1592 let relative = expanded.strip_prefix(Path::new("/")).unwrap_or(&expanded);
1593 let target = project_root.join(relative);
1594 if target.exists() {
1595 return Ok(target);
1596 }
1597 if let Some(parent) = target.parent() {
1598 fs::create_dir_all(parent)
1599 .with_context(|| format!("failed to create {}", parent.display()))?;
1600 }
1601 let base = gitlab.base_url.trim_end_matches('/');
1602 let project_id = percent_encode(project);
1603 let file_id = percent_encode(&relative.to_string_lossy());
1604 let ref_id = percent_encode(reference);
1605 let url =
1606 format!("{base}/api/v4/projects/{project_id}/repository/files/{file_id}/raw?ref={ref_id}");
1607 let status = std::process::Command::new("curl")
1608 .arg("--fail")
1609 .arg("-sS")
1610 .arg("-L")
1611 .arg("-H")
1612 .arg(format!("PRIVATE-TOKEN: {}", gitlab.token))
1613 .arg("-o")
1614 .arg(&target)
1615 .arg(&url)
1616 .status()
1617 .with_context(|| "failed to invoke curl to resolve include:project")?;
1618 if !status.success() {
1619 return Err(anyhow!(
1620 "curl failed to resolve include:project from {} (status {})",
1621 url,
1622 status.code().unwrap_or(-1)
1623 ));
1624 }
1625 Ok(target)
1626}
1627
1628fn percent_encode(value: &str) -> String {
1629 let mut encoded = String::new();
1630 for byte in value.bytes() {
1631 match byte {
1632 b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' => {
1633 encoded.push(byte as char)
1634 }
1635 _ => encoded.push_str(&format!("%{:02X}", byte)),
1636 }
1637 }
1638 encoded
1639}
1640
1641fn sanitize_reference(reference: &str) -> String {
1642 let mut slug = String::new();
1643 for ch in reference.chars() {
1644 if ch.is_ascii_alphanumeric() {
1645 slug.push(ch.to_ascii_lowercase());
1646 } else if matches!(ch, '-' | '_' | '.') {
1647 slug.push(ch);
1648 } else {
1649 slug.push('-');
1650 }
1651 }
1652 if slug.is_empty() {
1653 slug.push_str("ref");
1654 }
1655 slug
1656}
1657
1658fn ensure_stage(stages: &mut Vec<StageGroup>, stage_name: &str) -> usize {
1659 if let Some(pos) = stages.iter().position(|stage| stage.name == stage_name) {
1660 pos
1661 } else {
1662 stages.push(StageGroup {
1663 name: stage_name.to_string(),
1664 jobs: Vec::new(),
1665 });
1666 stages.len() - 1
1667 }
1668}
1669
1670#[derive(Debug, Deserialize)]
1671struct RawJob {
1672 #[serde(default)]
1673 before_script: Option<Script>,
1674 #[serde(default)]
1675 after_script: Option<Script>,
1676 #[serde(default)]
1677 stage: Option<String>,
1678 #[serde(default)]
1679 script: Script,
1680 #[serde(default)]
1681 when: Option<String>,
1682 #[serde(default)]
1683 needs: Option<Vec<Need>>,
1684 #[serde(default)]
1685 dependencies: Vec<String>,
1686 #[serde(default)]
1687 rules: Vec<JobRule>,
1688 #[serde(default)]
1689 artifacts: RawArtifacts,
1690 #[serde(default)]
1691 timeout: Option<String>,
1692 #[serde(default)]
1693 retry: Option<RawRetry>,
1694 #[serde(default)]
1695 interruptible: Option<bool>,
1696 #[serde(default)]
1697 resource_group: Option<String>,
1698 #[serde(default)]
1699 inherit: Option<RawInherit>,
1700 #[serde(default)]
1701 tags: Vec<String>,
1702 #[serde(default)]
1703 environment: Option<RawEnvironment>,
1704}
1705
1706#[derive(Debug, Deserialize, Default)]
1707struct RawEnvironment {
1708 #[serde(default)]
1709 name: String,
1710 #[serde(default)]
1711 url: Option<String>,
1712 #[serde(default)]
1713 on_stop: Option<String>,
1714 #[serde(default)]
1715 auto_stop_in: Option<String>,
1716 #[serde(default)]
1717 action: Option<String>,
1718}
1719
1720#[derive(Debug, Default, Deserialize)]
1721#[serde(transparent)]
1722struct Script(StringList);
1723
1724impl Script {
1725 fn into_commands(self) -> Vec<String> {
1726 self.0.into_vec()
1727 }
1728}
1729
1730#[derive(Debug, Deserialize, Default)]
1731struct RawArtifacts {
1732 #[serde(default)]
1733 name: Option<String>,
1734 #[serde(default)]
1735 paths: Vec<PathBuf>,
1736 #[serde(default)]
1737 exclude: StringList,
1738 #[serde(default)]
1739 untracked: bool,
1740 #[serde(default)]
1741 when: Option<String>,
1742 #[serde(default)]
1743 expire_in: Option<String>,
1744 #[serde(default)]
1745 reports: RawArtifactReports,
1746}
1747
1748impl RawArtifacts {
1749 fn into_config(self, job_name: &str) -> Result<ArtifactConfig> {
1750 validate_artifact_excludes(&self.exclude.0, job_name)?;
1751 Ok(ArtifactConfig {
1752 name: self.name,
1753 paths: self.paths,
1754 exclude: self.exclude.into_vec(),
1755 untracked: self.untracked,
1756 when: parse_artifact_when(self.when.as_deref(), job_name)?,
1757 expire_in: parse_optional_timeout(
1758 &self.expire_in,
1759 &format!("job '{}'.artifacts.expire_in", job_name),
1760 )?,
1761 report_dotenv: self.reports.dotenv,
1762 })
1763 }
1764}
1765
1766#[derive(Debug, Deserialize, Default)]
1767struct RawArtifactReports {
1768 #[serde(default)]
1769 dotenv: Option<PathBuf>,
1770}
1771
1772fn validate_artifact_excludes(patterns: &[String], job_name: &str) -> Result<()> {
1773 for pattern in patterns {
1774 Glob::new(pattern).with_context(|| {
1775 format!(
1776 "job '{}' has invalid artifacts.exclude pattern '{}'",
1777 job_name, pattern
1778 )
1779 })?;
1780 }
1781 Ok(())
1782}
1783
1784fn parse_artifact_when(value: Option<&str>, job_name: &str) -> Result<ArtifactWhen> {
1785 match value.unwrap_or("on_success") {
1786 "on_success" => Ok(ArtifactWhen::OnSuccess),
1787 "on_failure" => Ok(ArtifactWhen::OnFailure),
1788 "always" => Ok(ArtifactWhen::Always),
1789 other => bail!(
1790 "job '{}' has unsupported artifacts.when value '{}'",
1791 job_name,
1792 other
1793 ),
1794 }
1795}
1796
1797#[derive(Debug, Deserialize)]
1798#[serde(untagged)]
1799enum RawService {
1800 Simple(String),
1801 Detailed(Box<RawServiceConfig>),
1802}
1803
1804#[derive(Debug, Deserialize)]
1805struct RawServiceConfig {
1806 #[serde(default)]
1807 name: Option<String>,
1808 #[serde(default)]
1809 image: Option<String>,
1810 #[serde(default)]
1811 alias: Option<String>,
1812 #[serde(default)]
1813 docker: Option<Value>,
1814 #[serde(default)]
1815 entrypoint: ServiceCommand,
1816 #[serde(default)]
1817 command: ServiceCommand,
1818 #[serde(default)]
1819 variables: HashMap<String, String>,
1820}
1821
1822impl RawServiceConfig {
1823 fn into_config(self) -> Result<ServiceConfig> {
1824 let image = self
1825 .image
1826 .or(self.name)
1827 .ok_or_else(|| anyhow!("service entry must specify an image (name)"))?;
1828 let docker = self
1829 .docker
1830 .map(|value| parse_docker_executor_config(value, "services.docker"))
1831 .transpose()?;
1832 Ok(ServiceConfig {
1833 image,
1834 aliases: parse_service_aliases(self.alias),
1835 docker_platform: docker.as_ref().and_then(|cfg| cfg.platform.clone()),
1836 docker_user: docker.and_then(|cfg| cfg.user),
1837 entrypoint: self.entrypoint.into_vec(),
1838 command: self.command.into_vec(),
1839 variables: self.variables,
1840 })
1841 }
1842}
1843
1844fn parse_service_aliases(alias: Option<String>) -> Vec<String> {
1845 alias
1846 .into_iter()
1847 .flat_map(|raw| {
1848 raw.split(',')
1849 .map(str::trim)
1850 .map(str::to_string)
1851 .collect::<Vec<_>>()
1852 })
1853 .filter(|value| !value.is_empty())
1854 .collect()
1855}
1856
1857#[cfg(test)]
1858mod service_alias_tests {
1859 use super::parse_service_aliases;
1860
1861 #[test]
1862 fn parse_service_aliases_splits_comma_separated_values() {
1863 assert_eq!(
1864 parse_service_aliases(Some("db,postgres,pg".into())),
1865 vec!["db", "postgres", "pg"]
1866 );
1867 }
1868}
1869
1870#[derive(Debug, Deserialize)]
1871#[serde(untagged)]
1872enum RawRetry {
1873 Simple(u32),
1874 Detailed(RawRetryConfig),
1875}
1876
1877impl RawRetry {
1878 fn into_policy(self, base: &RetryPolicy, field: &str) -> Result<RetryPolicy> {
1879 match self {
1880 RawRetry::Simple(max) => {
1881 validate_retry_max(max, field)?;
1882 Ok(RetryPolicy {
1883 max,
1884 when: base.when.clone(),
1885 exit_codes: base.exit_codes.clone(),
1886 })
1887 }
1888 RawRetry::Detailed(cfg) => cfg.into_policy(base, field),
1889 }
1890 }
1891}
1892
1893#[derive(Debug, Deserialize)]
1894struct RawRetryConfig {
1895 #[serde(default)]
1896 max: Option<u32>,
1897 #[serde(default)]
1898 when: StringList,
1899 #[serde(default)]
1900 exit_codes: IntList,
1901}
1902
1903impl RawRetryConfig {
1904 fn into_policy(self, base: &RetryPolicy, field: &str) -> Result<RetryPolicy> {
1905 let mut policy = base.clone();
1906 if let Some(max) = self.max {
1907 validate_retry_max(max, &format!("{field}.max"))?;
1908 policy.max = max;
1909 }
1910 if !self.when.0.is_empty() {
1911 validate_retry_when(&self.when.0, &format!("{field}.when"))?;
1912 policy.when = self.when.into_vec();
1913 }
1914 if !self.exit_codes.0.is_empty() {
1915 validate_retry_exit_codes(&self.exit_codes.0, &format!("{field}.exit_codes"))?;
1916 policy.exit_codes = self.exit_codes.into_vec();
1917 }
1918 Ok(policy)
1919 }
1920}
1921
1922fn validate_retry_max(max: u32, field: &str) -> Result<()> {
1923 if max > 2 {
1924 bail!("{field} must be 0, 1, or 2");
1925 }
1926 Ok(())
1927}
1928
1929fn validate_retry_when(conditions: &[String], field: &str) -> Result<()> {
1930 for condition in conditions {
1931 if !SUPPORTED_RETRY_WHEN_VALUES.contains(&condition.as_str()) {
1932 bail!("{field} has unsupported retry condition '{condition}'");
1933 }
1934 }
1935 Ok(())
1936}
1937
1938fn validate_retry_exit_codes(codes: &[i32], field: &str) -> Result<()> {
1939 for code in codes {
1940 if *code < 0 {
1941 bail!("{field} must contain non-negative integers");
1942 }
1943 }
1944 Ok(())
1945}
1946
1947const SUPPORTED_RETRY_WHEN_VALUES: &[&str] = &[
1948 "always",
1949 "unknown_failure",
1950 "script_failure",
1951 "api_failure",
1952 "stuck_or_timeout_failure",
1953 "runner_system_failure",
1954 "runner_unsupported",
1955 "stale_schedule",
1956 "job_execution_timeout",
1957 "archived_failure",
1958 "unmet_prerequisites",
1959 "scheduler_failure",
1960 "data_integrity_failure",
1961];
1962
1963#[derive(Debug, Default)]
1964struct StringList(Vec<String>);
1965
1966impl StringList {
1967 fn into_vec(self) -> Vec<String> {
1968 self.0
1969 }
1970}
1971
1972#[derive(Debug, Default)]
1973struct IntList(Vec<i32>);
1974
1975impl IntList {
1976 fn into_vec(self) -> Vec<i32> {
1977 self.0
1978 }
1979}
1980
1981#[derive(Debug, Default)]
1982struct ServiceCommand(Vec<String>);
1983
1984impl ServiceCommand {
1985 fn into_vec(self) -> Vec<String> {
1986 self.0
1987 }
1988}
1989
1990impl<'de> serde::Deserialize<'de> for ServiceCommand {
1991 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
1992 where
1993 D: serde::Deserializer<'de>,
1994 {
1995 let list = StringList::deserialize(deserializer)?;
1996 Ok(ServiceCommand(list.into_vec()))
1997 }
1998}
1999impl<'de> serde::Deserialize<'de> for StringList {
2000 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
2001 where
2002 D: serde::de::Deserializer<'de>,
2003 {
2004 let value = Value::deserialize(deserializer)?;
2005 let items = match value {
2006 Value::Sequence(entries) => {
2007 let mut commands = Vec::new();
2008 for entry in entries {
2009 let cmd = yaml_command_string(entry).map_err(serde::de::Error::custom)?;
2010 commands.push(cmd);
2011 }
2012 commands
2013 }
2014 Value::Null => Vec::new(),
2015 other => vec![yaml_command_string(other).map_err(serde::de::Error::custom)?],
2016 };
2017 Ok(StringList(items))
2018 }
2019}
2020
2021impl<'de> serde::Deserialize<'de> for IntList {
2022 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
2023 where
2024 D: serde::de::Deserializer<'de>,
2025 {
2026 let value = Value::deserialize(deserializer)?;
2027 let items = match value {
2028 Value::Sequence(entries) => {
2029 let mut codes = Vec::new();
2030 for entry in entries {
2031 match entry {
2032 Value::Number(number) => {
2033 let code = number.as_i64().ok_or_else(|| {
2034 serde::de::Error::custom("retry exit codes must be integers")
2035 })?;
2036 let code = i32::try_from(code).map_err(|_| {
2037 serde::de::Error::custom(
2038 "retry exit code is too large for this platform",
2039 )
2040 })?;
2041 codes.push(code);
2042 }
2043 other => {
2044 return Err(serde::de::Error::custom(format!(
2045 "retry exit codes must be integers, got {other:?}"
2046 )));
2047 }
2048 }
2049 }
2050 codes
2051 }
2052 Value::Null => Vec::new(),
2053 Value::Number(number) => {
2054 let code = number
2055 .as_i64()
2056 .ok_or_else(|| serde::de::Error::custom("retry exit codes must be integers"))?;
2057 vec![i32::try_from(code).map_err(|_| {
2058 serde::de::Error::custom("retry exit code is too large for this platform")
2059 })?]
2060 }
2061 other => {
2062 return Err(serde::de::Error::custom(format!(
2063 "retry exit codes must be an integer or list, got {other:?}"
2064 )));
2065 }
2066 };
2067 Ok(IntList(items))
2068 }
2069}
2070
2071#[derive(Debug, Deserialize, Default)]
2072struct CacheEntryRaw {
2073 key: Option<CacheKeyRaw>,
2074 #[serde(default)]
2075 fallback_keys: Vec<String>,
2076 #[serde(default)]
2077 paths: Vec<PathBuf>,
2078 #[serde(default)]
2079 policy: Option<String>,
2080}
2081
2082#[derive(Debug, Deserialize)]
2083#[serde(untagged)]
2084enum CacheKeyRaw {
2085 Literal(String),
2086 Detailed(CacheKeyDetailedRaw),
2087}
2088
2089#[derive(Debug, Deserialize, Default)]
2090#[serde(deny_unknown_fields)]
2091struct CacheKeyDetailedRaw {
2092 #[serde(default)]
2093 files: Vec<PathBuf>,
2094 #[serde(default)]
2095 prefix: Option<String>,
2096}
2097
2098#[derive(Debug, Deserialize)]
2099#[serde(untagged)]
2100enum Need {
2101 Name(String),
2102 Config(NeedConfig),
2103}
2104
2105#[derive(Debug, Deserialize)]
2106struct NeedConfig {
2107 job: String,
2108 #[serde(default = "default_artifacts_true")]
2109 artifacts: bool,
2110 #[serde(default)]
2111 optional: bool,
2112 #[serde(default)]
2113 project: Option<String>,
2114 #[serde(rename = "ref")]
2115 reference: Option<String>,
2116 #[serde(default)]
2117 parallel: Option<NeedParallelRaw>,
2118}
2119
2120impl Need {
2121 fn into_dependency(self, owner: &str) -> Option<JobDependency> {
2122 match self {
2123 Need::Name(job) => {
2124 if let Some((base, values)) = parse_inline_variant_reference(&job) {
2125 Some(JobDependency {
2126 job: base,
2127 needs_artifacts: true,
2128 optional: false,
2129 source: DependencySource::Local,
2130 parallel: None,
2131 inline_variant: Some(values),
2132 })
2133 } else {
2134 Some(JobDependency {
2135 job,
2136 needs_artifacts: true,
2137 optional: false,
2138 source: DependencySource::Local,
2139 parallel: None,
2140 inline_variant: None,
2141 })
2142 }
2143 }
2144 Need::Config(cfg) => {
2145 let NeedConfig {
2146 job,
2147 artifacts,
2148 optional,
2149 project,
2150 reference,
2151 parallel,
2152 } = cfg;
2153 if let Some(project) = project {
2154 let reference = reference.unwrap_or_default();
2155 if reference.is_empty() {
2156 warn!(
2157 job = owner,
2158 dependency = %job,
2159 "needs:project for '{}' is missing required 'ref'",
2160 project
2161 );
2162 return None;
2163 }
2164 Some(JobDependency {
2165 job,
2166 needs_artifacts: artifacts,
2167 optional,
2168 source: DependencySource::External(ExternalDependency {
2169 project,
2170 reference,
2171 }),
2172 parallel: None,
2173 inline_variant: None,
2174 })
2175 } else {
2176 let parallel_filters =
2177 parallel.and_then(|raw| match raw.into_filters(owner, &job) {
2178 Ok(filters) => Some(filters),
2179 Err(err) => {
2180 warn!(
2181 job = owner,
2182 dependency = %job,
2183 error = %err,
2184 "invalid needs.parallel configuration"
2185 );
2186 None
2187 }
2188 });
2189 let (job_name, inline_variant) = parse_inline_variant_reference(&job)
2190 .map_or((job, None), |(base, values)| (base, Some(values)));
2191 Some(JobDependency {
2192 job: job_name,
2193 needs_artifacts: artifacts,
2194 optional,
2195 source: DependencySource::Local,
2196 parallel: parallel_filters,
2197 inline_variant,
2198 })
2199 }
2200 }
2201 }
2202 }
2203}
2204
2205fn parse_inline_variant_reference(value: &str) -> Option<(String, Vec<String>)> {
2206 let trimmed = value.trim();
2207 let (base, list) = trimmed.split_once(':')?;
2208 let payload = list.trim();
2209 if !payload.starts_with('[') {
2210 return None;
2211 }
2212 let values: Vec<String> = serde_yaml::from_str(payload).ok()?;
2213 Some((base.trim().to_string(), values))
2214}
2215
2216#[derive(Debug, Deserialize)]
2217struct NeedParallelRaw {
2218 #[serde(default)]
2219 matrix: Vec<HashMap<String, Value>>,
2220}
2221
2222impl NeedParallelRaw {
2223 fn into_filters(self, owner: &str, dependency: &str) -> Result<Vec<HashMap<String, String>>> {
2225 let mut filters = Vec::new();
2226 for entry in self.matrix {
2227 let mut filter = HashMap::new();
2228 for (name, value) in entry {
2229 let value = match value {
2230 Value::String(text) => text,
2231 other => bail!(
2232 "job '{}' dependency '{}' parallel values must be strings, got {other:?}",
2233 owner,
2234 dependency
2235 ),
2236 };
2237 filter.insert(name, value);
2238 }
2239 if filter.is_empty() {
2240 bail!(
2241 "job '{}' dependency '{}' parallel matrix entries must include variables",
2242 owner,
2243 dependency
2244 );
2245 }
2246 filters.push(filter);
2247 }
2248 if filters.is_empty() {
2249 bail!(
2250 "job '{}' dependency '{}' parallel matrix must include at least one entry",
2251 owner,
2252 dependency
2253 );
2254 }
2255 Ok(filters)
2256 }
2257}
2258
2259fn default_artifacts_true() -> bool {
2260 true
2261}
2262
2263#[cfg(test)]
2264mod tests {
2265 use super::*;
2266 use anyhow::{Result, anyhow};
2267 use std::fs;
2268 use tempfile::tempdir;
2269
2270 #[test]
2271 fn parses_stage_and_job_order() -> Result<()> {
2272 let yaml = r#"
2273 stages:
2274 - build
2275 - test
2276
2277 build-job:
2278 stage: build
2279 script:
2280 - echo build
2281
2282 second-build:
2283 stage: build
2284 script: echo build 2
2285
2286 test-job:
2287 stage: test
2288 script:
2289 - echo test
2290 "#;
2291
2292 let pipeline = PipelineGraph::from_yaml_str(yaml)?;
2293 assert_eq!(pipeline.stages.len(), 2);
2294 assert_eq!(pipeline.stages[0].name, "build");
2295 assert_eq!(pipeline.stages[1].name, "test");
2296
2297 let build_jobs: Vec<&Job> = pipeline.stages[0]
2298 .jobs
2299 .iter()
2300 .map(|idx| &pipeline.graph[*idx])
2301 .collect();
2302 assert_eq!(build_jobs.len(), 2);
2303 assert_eq!(build_jobs[0].name, "build-job");
2304 assert_eq!(build_jobs[1].name, "second-build");
2305
2306 let test_jobs: Vec<&Job> = pipeline.stages[1]
2307 .jobs
2308 .iter()
2309 .map(|idx| &pipeline.graph[*idx])
2310 .collect();
2311 assert_eq!(test_jobs.len(), 1);
2312 assert_eq!(test_jobs[0].name, "test-job");
2313 Ok(())
2314 }
2315
2316 #[test]
2317 fn resolves_reference_tags() -> Result<()> {
2318 let yaml = r#"
2319 stages: [build]
2320
2321 .shared:
2322 script:
2323 - echo shared
2324 variables:
2325 SHARED_VAR: shared-value
2326
2327 build-job:
2328 stage: build
2329 script: !reference [.shared, script]
2330 variables:
2331 COPIED: !reference [.shared, variables, SHARED_VAR]
2332 "#;
2333
2334 let pipeline = PipelineGraph::from_yaml_str(yaml)?;
2335 assert_eq!(pipeline.stages.len(), 1);
2336 let build_stage = &pipeline.stages[0];
2337 assert_eq!(build_stage.jobs.len(), 1);
2338 let job = &pipeline.graph[build_stage.jobs[0]];
2339 assert_eq!(job.commands, vec!["echo shared"]);
2340 assert_eq!(
2341 job.variables.get("COPIED").map(|value| value.as_str()),
2342 Some("shared-value")
2343 );
2344 Ok(())
2345 }
2346
2347 #[test]
2348 fn includes_local_fragment() -> Result<()> {
2349 let dir = tempdir()?;
2350 let fragment_path = dir.path().join("fragment.yml");
2351 fs::write(
2352 &fragment_path,
2353 r#"
2354fragment-job:
2355 stage: build
2356 script:
2357 - echo fragment
2358"#,
2359 )?;
2360
2361 let main_path = dir.path().join("main.yml");
2362 fs::write(
2363 &main_path,
2364 r#"
2365stages:
2366 - build
2367include:
2368 - local: fragment.yml
2369
2370main-job:
2371 stage: build
2372 script:
2373 - echo main
2374"#,
2375 )?;
2376
2377 let pipeline = PipelineGraph::from_path(&main_path)?;
2378 assert_eq!(pipeline.stages.len(), 1);
2379 assert_eq!(pipeline.stages[0].jobs.len(), 2);
2380 assert!(
2381 pipeline
2382 .graph
2383 .node_weights()
2384 .any(|job| job.name == "fragment-job")
2385 );
2386 assert!(
2387 pipeline
2388 .graph
2389 .node_weights()
2390 .any(|job| job.name == "main-job")
2391 );
2392 Ok(())
2393 }
2394
2395 #[test]
2396 fn includes_local_paths_from_repo_root() -> Result<()> {
2397 let dir = tempdir()?;
2398 git2::Repository::init(dir.path())?;
2399
2400 let fragment_path = dir.path().join("fragment.yml");
2401 fs::write(
2402 &fragment_path,
2403 r#"
2404fragment-job:
2405 stage: build
2406 script:
2407 - echo fragment
2408"#,
2409 )?;
2410
2411 let ci_dir = dir.path().join("ci");
2412 fs::create_dir_all(&ci_dir)?;
2413 let main_path = ci_dir.join("main.yml");
2414 fs::write(
2415 &main_path,
2416 r#"
2417stages:
2418 - build
2419include:
2420 - local: fragment.yml
2421
2422main-job:
2423 stage: build
2424 script:
2425 - echo main
2426"#,
2427 )?;
2428
2429 let pipeline = PipelineGraph::from_path(&main_path)?;
2430 assert!(
2431 pipeline
2432 .graph
2433 .node_weights()
2434 .any(|job| job.name == "fragment-job")
2435 );
2436 assert!(
2437 pipeline
2438 .graph
2439 .node_weights()
2440 .any(|job| job.name == "main-job")
2441 );
2442 Ok(())
2443 }
2444
2445 #[test]
2446 fn includes_local_glob_from_repo_root() -> Result<()> {
2447 let dir = tempdir()?;
2448 git2::Repository::init(dir.path())?;
2449
2450 let configs_dir = dir.path().join("configs");
2451 fs::create_dir_all(&configs_dir)?;
2452 fs::write(
2453 configs_dir.join("a.yml"),
2454 r#"
2455job-a:
2456 stage: build
2457 script:
2458 - echo a
2459"#,
2460 )?;
2461 fs::write(
2462 configs_dir.join("b.yml"),
2463 r#"
2464job-b:
2465 stage: build
2466 script:
2467 - echo b
2468"#,
2469 )?;
2470
2471 let ci_dir = dir.path().join("ci");
2472 fs::create_dir_all(&ci_dir)?;
2473 let main_path = ci_dir.join("main.yml");
2474 fs::write(
2475 &main_path,
2476 r#"
2477stages:
2478 - build
2479include:
2480 - local: configs/*.yml
2481
2482main-job:
2483 stage: build
2484 script:
2485 - echo main
2486"#,
2487 )?;
2488
2489 let pipeline = PipelineGraph::from_path(&main_path)?;
2490 assert!(pipeline.graph.node_weights().any(|job| job.name == "job-a"));
2491 assert!(pipeline.graph.node_weights().any(|job| job.name == "job-b"));
2492 assert!(
2493 pipeline
2494 .graph
2495 .node_weights()
2496 .any(|job| job.name == "main-job")
2497 );
2498 Ok(())
2499 }
2500
2501 #[test]
2502 fn includes_local_paths_expand_environment_variables() -> Result<()> {
2503 let dir = tempdir()?;
2504 git2::Repository::init(dir.path())?;
2505
2506 fs::write(
2507 dir.path().join("fragment.yml"),
2508 r#"
2509fragment-job:
2510 stage: build
2511 script:
2512 - echo fragment
2513"#,
2514 )?;
2515
2516 let ci_dir = dir.path().join("ci");
2517 fs::create_dir_all(&ci_dir)?;
2518 let main_path = ci_dir.join("main.yml");
2519 fs::write(
2520 &main_path,
2521 r#"
2522stages:
2523 - build
2524include:
2525 - local: $INCLUDE_FILE
2526
2527main-job:
2528 stage: build
2529 script:
2530 - echo main
2531"#,
2532 )?;
2533
2534 let pipeline = PipelineGraph::from_path_with_env(
2535 &main_path,
2536 HashMap::from([("INCLUDE_FILE".to_string(), "fragment.yml".to_string())]),
2537 None,
2538 )?;
2539
2540 assert!(
2541 pipeline
2542 .graph
2543 .node_weights()
2544 .any(|job| job.name == "fragment-job")
2545 );
2546 assert!(
2547 pipeline
2548 .graph
2549 .node_weights()
2550 .any(|job| job.name == "main-job")
2551 );
2552 Ok(())
2553 }
2554
2555 #[test]
2556 fn includes_local_files_list_from_repo_root() -> Result<()> {
2557 let dir = tempdir()?;
2558 git2::Repository::init(dir.path())?;
2559
2560 let parts_dir = dir.path().join("parts");
2561 fs::create_dir_all(&parts_dir)?;
2562 fs::write(
2563 parts_dir.join("a.yml"),
2564 r#"
2565job-a:
2566 stage: build
2567 script:
2568 - echo a
2569"#,
2570 )?;
2571 fs::write(
2572 parts_dir.join("b.yml"),
2573 r#"
2574job-b:
2575 stage: build
2576 script:
2577 - echo b
2578"#,
2579 )?;
2580
2581 let main_path = dir.path().join("main.yml");
2582 fs::write(
2583 &main_path,
2584 r#"
2585stages:
2586 - build
2587include:
2588 files:
2589 - /parts/a.yml
2590 - /parts/b.yml
2591
2592main-job:
2593 stage: build
2594 script:
2595 - echo main
2596"#,
2597 )?;
2598
2599 let pipeline = PipelineGraph::from_path(&main_path)?;
2600 assert!(pipeline.graph.node_weights().any(|job| job.name == "job-a"));
2601 assert!(pipeline.graph.node_weights().any(|job| job.name == "job-b"));
2602 assert!(
2603 pipeline
2604 .graph
2605 .node_weights()
2606 .any(|job| job.name == "main-job")
2607 );
2608 Ok(())
2609 }
2610
2611 #[test]
2612 fn includes_local_non_yaml_file_errors() {
2613 let dir = tempdir().expect("tempdir");
2614 git2::Repository::init(dir.path()).expect("init repo");
2615
2616 fs::write(dir.path().join("fragment.txt"), "not yaml").expect("write fragment");
2617
2618 let main_path = dir.path().join("main.yml");
2619 fs::write(
2620 &main_path,
2621 r#"
2622stages:
2623 - build
2624include:
2625 - local: /fragment.txt
2626
2627main-job:
2628 stage: build
2629 script:
2630 - echo main
2631"#,
2632 )
2633 .expect("write main");
2634
2635 let err = PipelineGraph::from_path(&main_path).expect_err("non-yaml include must error");
2636 assert!(
2637 err.to_string()
2638 .contains("must reference a .yml or .yaml file")
2639 );
2640 }
2641
2642 #[test]
2643 fn includes_file_alias_as_local_path() -> Result<()> {
2644 let dir = tempdir()?;
2645 git2::Repository::init(dir.path())?;
2646
2647 fs::write(
2648 dir.path().join("fragment.yml"),
2649 r#"
2650fragment-job:
2651 stage: build
2652 script:
2653 - echo fragment
2654"#,
2655 )?;
2656
2657 let main_path = dir.path().join("main.yml");
2658 fs::write(
2659 &main_path,
2660 r#"
2661stages:
2662 - build
2663include:
2664 - file: /fragment.yml
2665
2666main-job:
2667 stage: build
2668 script:
2669 - echo main
2670"#,
2671 )?;
2672
2673 let pipeline = PipelineGraph::from_path(&main_path)?;
2674 assert!(
2675 pipeline
2676 .graph
2677 .node_weights()
2678 .any(|job| job.name == "fragment-job")
2679 );
2680 assert!(
2681 pipeline
2682 .graph
2683 .node_weights()
2684 .any(|job| job.name == "main-job")
2685 );
2686 Ok(())
2687 }
2688
2689 #[test]
2690 fn retry_max_above_two_errors() {
2691 let dir = tempdir().expect("tempdir");
2692 let main_path = dir.path().join("main.yml");
2693 fs::write(
2694 &main_path,
2695 r#"
2696stages:
2697 - build
2698
2699build:
2700 stage: build
2701 retry: 3
2702 script:
2703 - echo hi
2704"#,
2705 )
2706 .expect("write main");
2707
2708 let err = PipelineGraph::from_path(&main_path).expect_err("retry max must error");
2709 assert!(
2710 err.to_string()
2711 .contains("job 'build'.retry must be 0, 1, or 2")
2712 );
2713 }
2714
2715 #[test]
2716 fn retry_exit_codes_parses_single_value() {
2717 let dir = tempdir().expect("tempdir");
2718 let main_path = dir.path().join("main.yml");
2719 fs::write(
2720 &main_path,
2721 r#"
2722stages:
2723 - build
2724
2725build:
2726 stage: build
2727 retry:
2728 max: 1
2729 exit_codes: 137
2730 script:
2731 - echo hi
2732"#,
2733 )
2734 .expect("write main");
2735
2736 let pipeline = PipelineGraph::from_path(&main_path).expect("retry exit_codes must parse");
2737 let job = pipeline
2738 .graph
2739 .node_weights()
2740 .find(|job| job.name == "build")
2741 .expect("build job present");
2742 assert_eq!(job.retry.max, 1);
2743 assert_eq!(job.retry.exit_codes, vec![137]);
2744 }
2745
2746 #[test]
2747 fn retry_exit_codes_rejects_negative_values() {
2748 let dir = tempdir().expect("tempdir");
2749 let main_path = dir.path().join("main.yml");
2750 fs::write(
2751 &main_path,
2752 r#"
2753stages:
2754 - build
2755
2756build:
2757 stage: build
2758 retry:
2759 max: 1
2760 exit_codes:
2761 - -1
2762 script:
2763 - echo hi
2764"#,
2765 )
2766 .expect("write main");
2767
2768 let err = PipelineGraph::from_path(&main_path).expect_err("negative exit code must error");
2769 assert!(
2770 err.to_string()
2771 .contains("job 'build'.retry.exit_codes must contain non-negative integers")
2772 );
2773 }
2774
2775 #[test]
2776 fn parses_artifacts_reports_dotenv() {
2777 let dir = tempdir().expect("tempdir");
2778 let main_path = dir.path().join("main.yml");
2779 fs::write(
2780 &main_path,
2781 r#"
2782stages:
2783 - build
2784
2785build:
2786 stage: build
2787 script:
2788 - echo hi
2789 artifacts:
2790 reports:
2791 dotenv: tests-temp/dotenv/build.env
2792"#,
2793 )
2794 .expect("write main");
2795
2796 let pipeline = PipelineGraph::from_path(&main_path).expect("pipeline parses");
2797 let job = pipeline
2798 .graph
2799 .node_weights()
2800 .find(|job| job.name == "build")
2801 .expect("build job present");
2802
2803 assert_eq!(
2804 job.artifacts.report_dotenv.as_deref(),
2805 Some(std::path::Path::new("tests-temp/dotenv/build.env"))
2806 );
2807 }
2808
2809 #[test]
2810 fn yaml_merge_keys_work_in_variables_mapping() {
2811 let pipeline = PipelineGraph::from_yaml_str(
2812 r#"
2813stages:
2814 - test
2815
2816.base-vars: &base_vars
2817 SHARED_FLAG: from-anchor
2818 OVERRIDE_ME: old
2819
2820merged-vars:
2821 stage: test
2822 variables:
2823 <<: *base_vars
2824 OVERRIDE_ME: new
2825 script:
2826 - echo ok
2827"#,
2828 )
2829 .expect("pipeline parses");
2830
2831 let job = pipeline
2832 .graph
2833 .node_weights()
2834 .find(|job| job.name == "merged-vars")
2835 .expect("job present");
2836
2837 assert_eq!(
2838 job.variables.get("SHARED_FLAG").map(String::as_str),
2839 Some("from-anchor")
2840 );
2841 assert_eq!(
2842 job.variables.get("OVERRIDE_ME").map(String::as_str),
2843 Some("new")
2844 );
2845 }
2846
2847 #[test]
2848 fn yaml_merge_keys_work_in_job_mapping() {
2849 let pipeline = PipelineGraph::from_yaml_str(
2850 r#"
2851stages:
2852 - test
2853
2854.job-template: &job_template
2855 stage: test
2856 script:
2857 - echo ok
2858 variables:
2859 INNER_FLAG: inner
2860
2861merged-job:
2862 <<: *job_template
2863 variables:
2864 <<: { INNER_FLAG: inner, EXTRA_FLAG: extra }
2865"#,
2866 )
2867 .expect("pipeline parses");
2868
2869 let job = pipeline
2870 .graph
2871 .node_weights()
2872 .find(|job| job.name == "merged-job")
2873 .expect("job present");
2874
2875 assert_eq!(job.stage, "test");
2876 assert_eq!(job.commands, vec!["echo ok".to_string()]);
2877 assert_eq!(
2878 job.variables.get("INNER_FLAG").map(String::as_str),
2879 Some("inner")
2880 );
2881 assert_eq!(
2882 job.variables.get("EXTRA_FLAG").map(String::as_str),
2883 Some("extra")
2884 );
2885 }
2886
2887 #[test]
2888 fn parses_image_docker_platform() {
2889 let pipeline = PipelineGraph::from_yaml_str(
2890 r#"
2891stages:
2892 - test
2893
2894platform-job:
2895 stage: test
2896 image:
2897 name: docker.io/library/alpine:3.19
2898 docker:
2899 platform: linux/arm64/v8
2900 script:
2901 - echo ok
2902"#,
2903 )
2904 .expect("pipeline parses");
2905
2906 let job = pipeline
2907 .graph
2908 .node_weights()
2909 .find(|job| job.name == "platform-job")
2910 .expect("job present");
2911
2912 let image = job.image.as_ref().expect("image present");
2913 assert_eq!(image.name, "docker.io/library/alpine:3.19");
2914 assert_eq!(image.docker_platform.as_deref(), Some("linux/arm64/v8"));
2915 }
2916
2917 #[test]
2918 fn parses_image_entrypoint_and_docker_user() {
2919 let pipeline = PipelineGraph::from_yaml_str(
2920 r#"
2921stages:
2922 - test
2923
2924platform-job:
2925 stage: test
2926 image:
2927 name: docker.io/library/alpine:3.19
2928 entrypoint: [""]
2929 docker:
2930 user: 1000:1000
2931 script:
2932 - echo ok
2933"#,
2934 )
2935 .expect("pipeline parses");
2936
2937 let job = pipeline
2938 .graph
2939 .node_weights()
2940 .find(|job| job.name == "platform-job")
2941 .expect("job present");
2942
2943 let image = job.image.as_ref().expect("image present");
2944 assert_eq!(image.entrypoint, vec![""]);
2945 assert_eq!(image.docker_user.as_deref(), Some("1000:1000"));
2946 }
2947
2948 #[test]
2949 fn parses_service_docker_platform_and_user() {
2950 let pipeline = PipelineGraph::from_yaml_str(
2951 r#"
2952stages:
2953 - test
2954
2955service-job:
2956 stage: test
2957 image: docker.io/library/alpine:3.19
2958 services:
2959 - name: docker.io/library/redis:7.2
2960 alias: cache
2961 docker:
2962 platform: linux/arm64/v8
2963 user: 1000:1000
2964 script:
2965 - echo ok
2966"#,
2967 )
2968 .expect("pipeline parses");
2969
2970 let job = pipeline
2971 .graph
2972 .node_weights()
2973 .find(|job| job.name == "service-job")
2974 .expect("job present");
2975
2976 let service = job.services.first().expect("service present");
2977 assert_eq!(service.image, "docker.io/library/redis:7.2");
2978 assert_eq!(service.aliases, vec!["cache"]);
2979 assert_eq!(service.docker_platform.as_deref(), Some("linux/arm64/v8"));
2980 assert_eq!(service.docker_user.as_deref(), Some("1000:1000"));
2981 }
2982
2983 #[test]
2984 fn inherit_default_controls_modeled_default_keywords() {
2985 let pipeline = PipelineGraph::from_yaml_str(
2986 r#"
2987stages:
2988 - test
2989
2990default:
2991 image: docker.io/library/alpine:3.19
2992 before_script:
2993 - echo before
2994 after_script:
2995 - echo after
2996 cache:
2997 key: default-cache
2998 paths:
2999 - tests-temp/default-cache/
3000 services:
3001 - docker.io/library/redis:7.2
3002 timeout: 10m
3003 retry: 2
3004 interruptible: true
3005
3006inherit-none:
3007 stage: test
3008 inherit:
3009 default: false
3010 image: docker.io/library/alpine:3.19
3011 script:
3012 - echo none
3013
3014inherit-some:
3015 stage: test
3016 inherit:
3017 default: [image, retry, interruptible]
3018 script:
3019 - echo some
3020"#,
3021 )
3022 .expect("pipeline parses");
3023
3024 let none = pipeline
3025 .graph
3026 .node_weights()
3027 .find(|job| job.name == "inherit-none")
3028 .expect("job present");
3029 assert!(!none.inherit_default_image);
3030 assert!(!none.inherit_default_before_script);
3031 assert!(!none.inherit_default_after_script);
3032 assert!(!none.inherit_default_cache);
3033 assert!(!none.inherit_default_services);
3034 assert!(!none.inherit_default_timeout);
3035 assert!(!none.inherit_default_retry);
3036 assert!(!none.inherit_default_interruptible);
3037 assert!(none.cache.is_empty());
3038 assert!(none.services.is_empty());
3039 assert_eq!(none.timeout, None);
3040 assert_eq!(none.retry.max, 0);
3041 assert!(!none.interruptible);
3042
3043 let some = pipeline
3044 .graph
3045 .node_weights()
3046 .find(|job| job.name == "inherit-some")
3047 .expect("job present");
3048 assert!(some.inherit_default_image);
3049 assert!(!some.inherit_default_before_script);
3050 assert!(!some.inherit_default_after_script);
3051 assert!(!some.inherit_default_cache);
3052 assert!(!some.inherit_default_services);
3053 assert!(!some.inherit_default_timeout);
3054 assert!(some.inherit_default_retry);
3055 assert!(some.inherit_default_interruptible);
3056 assert_eq!(
3057 some.image.as_ref().map(|image| image.name.as_str()),
3058 Some("docker.io/library/alpine:3.19")
3059 );
3060 assert!(some.cache.is_empty());
3061 assert!(some.services.is_empty());
3062 assert_eq!(some.timeout, None);
3063 assert_eq!(some.retry.max, 2);
3064 assert!(some.interruptible);
3065 }
3066
3067 #[test]
3068 fn include_cycle_errors() -> Result<()> {
3069 let dir = tempdir()?;
3070 let a_path = dir.path().join("a.yml");
3071 let b_path = dir.path().join("b.yml");
3072 fs::write(
3073 &a_path,
3074 format!(
3075 "
3076include:
3077 - local: {}
3078job-a:
3079 stage: build
3080 script: echo a
3081",
3082 b_path
3083 .file_name()
3084 .ok_or_else(|| anyhow!("missing b.yml filename"))?
3085 .to_string_lossy()
3086 ),
3087 )?;
3088 fs::write(
3089 &b_path,
3090 format!(
3091 "
3092include:
3093 - local: {}
3094job-b:
3095 stage: build
3096 script: echo b
3097",
3098 a_path
3099 .file_name()
3100 .ok_or_else(|| anyhow!("missing a.yml filename"))?
3101 .to_string_lossy()
3102 ),
3103 )?;
3104
3105 let err = PipelineGraph::from_path(&a_path).expect_err("cycle must error");
3106 assert!(err.to_string().contains("include cycle"));
3107 Ok(())
3108 }
3109
3110 #[test]
3111 fn records_needs_dependencies() {
3112 let yaml = r#"
3113stages:
3114 - build
3115 - deploy
3116
3117build-job:
3118 stage: build
3119 script:
3120 - echo build
3121
3122deploy-job:
3123 stage: deploy
3124 needs:
3125 - build-job
3126 script:
3127 - echo deploy
3128"#;
3129
3130 let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3131 let build_idx = find_job(&pipeline, "build-job");
3132 let deploy_idx = find_job(&pipeline, "deploy-job");
3133
3134 assert_eq!(
3135 pipeline.graph[deploy_idx].needs[0].job,
3136 "build-job".to_string()
3137 );
3138 assert!(pipeline.graph[deploy_idx].needs[0].needs_artifacts);
3139 assert!(pipeline.graph.contains_edge(build_idx, deploy_idx));
3140 }
3141
3142 #[test]
3143 fn parses_needs_without_artifacts() {
3144 let yaml = r#"
3145stages:
3146 - build
3147 - test
3148
3149build-job:
3150 stage: build
3151 script:
3152 - echo build
3153
3154test-job:
3155 stage: test
3156 needs:
3157 - job: build-job
3158 artifacts: false
3159 script:
3160 - echo test
3161"#;
3162
3163 let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3164 let test_idx = find_job(&pipeline, "test-job");
3165 assert_eq!(pipeline.graph[test_idx].needs.len(), 1);
3166 let need = &pipeline.graph[test_idx].needs[0];
3167 assert_eq!(need.job, "build-job");
3168 assert!(!need.needs_artifacts);
3169 assert!(!need.optional);
3170 }
3171
3172 #[test]
3173 fn parses_optional_needs() {
3174 let yaml = r#"
3175stages:
3176 - build
3177 - test
3178
3179build-job:
3180 stage: build
3181 script:
3182 - echo build
3183
3184maybe-job:
3185 stage: build
3186 script:
3187 - echo maybe
3188
3189test-job:
3190 stage: test
3191 needs:
3192 - build-job
3193 - job: maybe-job
3194 optional: true
3195 script:
3196 - echo test
3197"#;
3198
3199 let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3200 let test_idx = find_job(&pipeline, "test-job");
3201 assert_eq!(pipeline.graph[test_idx].needs.len(), 2);
3202 let need0 = &pipeline.graph[test_idx].needs[0];
3203 assert_eq!(need0.job, "build-job");
3204 assert!(!need0.optional);
3205 let need1 = &pipeline.graph[test_idx].needs[1];
3206 assert_eq!(need1.job, "maybe-job");
3207 assert!(need1.optional);
3208 }
3209
3210 #[test]
3211 fn parses_artifacts_paths() {
3212 let yaml = r#"
3213stages:
3214 - build
3215
3216build-job:
3217 stage: build
3218 script:
3219 - echo build
3220 artifacts:
3221 paths:
3222 - vendor/
3223 - output/report.txt
3224"#;
3225
3226 let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3227 let build_idx = find_job(&pipeline, "build-job");
3228 let job = &pipeline.graph[build_idx];
3229 assert_eq!(job.artifacts.paths.len(), 2);
3230 assert_eq!(job.artifacts.paths[0], PathBuf::from("vendor"));
3231 assert_eq!(job.artifacts.paths[1], PathBuf::from("output/report.txt"));
3232 assert_eq!(job.artifacts.when, ArtifactWhen::OnSuccess);
3233 }
3234
3235 #[test]
3236 fn parses_artifacts_when() {
3237 let yaml = r#"
3238stages:
3239 - build
3240
3241build-job:
3242 stage: build
3243 script:
3244 - echo build
3245 artifacts:
3246 when: always
3247 paths:
3248 - output/report.txt
3249"#;
3250
3251 let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3252 let build_idx = find_job(&pipeline, "build-job");
3253 let job = &pipeline.graph[build_idx];
3254 assert_eq!(job.artifacts.when, ArtifactWhen::Always);
3255 }
3256
3257 #[test]
3258 fn parses_artifacts_exclude() {
3259 let yaml = r#"
3260stages:
3261 - build
3262
3263build-job:
3264 stage: build
3265 script:
3266 - echo build
3267 artifacts:
3268 paths:
3269 - tests-temp/output/
3270 exclude:
3271 - tests-temp/output/**/*.log
3272 - tests-temp/output/ignore.txt
3273"#;
3274
3275 let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3276 let build_idx = find_job(&pipeline, "build-job");
3277 let job = &pipeline.graph[build_idx];
3278 assert_eq!(
3279 job.artifacts.exclude,
3280 vec![
3281 "tests-temp/output/**/*.log".to_string(),
3282 "tests-temp/output/ignore.txt".to_string()
3283 ]
3284 );
3285 }
3286
3287 #[test]
3288 fn parses_cache_fallback_keys() {
3289 let yaml = r#"
3290stages:
3291 - build
3292
3293build-job:
3294 stage: build
3295 script:
3296 - echo build
3297 cache:
3298 key: cache-$CI_COMMIT_REF_SLUG
3299 fallback_keys:
3300 - cache-$CI_DEFAULT_BRANCH
3301 - cache-default
3302 paths:
3303 - vendor/
3304"#;
3305
3306 let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3307 let build_idx = find_job(&pipeline, "build-job");
3308 let job = &pipeline.graph[build_idx];
3309 assert_eq!(
3310 job.cache[0].fallback_keys,
3311 vec![
3312 "cache-$CI_DEFAULT_BRANCH".to_string(),
3313 "cache-default".to_string()
3314 ]
3315 );
3316 }
3317
3318 #[test]
3319 fn parses_cache_key_files_mapping() {
3320 let yaml = r#"
3321stages:
3322 - build
3323
3324build-job:
3325 stage: build
3326 script:
3327 - echo build
3328 cache:
3329 key:
3330 files:
3331 - Cargo.lock
3332 prefix: $CI_JOB_NAME
3333 paths:
3334 - vendor/
3335"#;
3336
3337 let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3338 let build_idx = find_job(&pipeline, "build-job");
3339 let job = &pipeline.graph[build_idx];
3340 assert_eq!(
3341 job.cache[0].key,
3342 CacheKey::Files {
3343 files: vec![PathBuf::from("Cargo.lock")],
3344 prefix: Some("$CI_JOB_NAME".to_string()),
3345 }
3346 );
3347 }
3348
3349 #[test]
3350 fn errors_when_cache_key_files_has_more_than_two_paths() {
3351 let yaml = r#"
3352stages:
3353 - build
3354
3355build-job:
3356 stage: build
3357 script:
3358 - echo build
3359 cache:
3360 key:
3361 files:
3362 - Cargo.lock
3363 - package-lock.json
3364 - yarn.lock
3365 paths:
3366 - vendor/
3367"#;
3368
3369 let err = PipelineGraph::from_yaml_str(yaml).expect_err("pipeline should fail");
3370 assert!(
3371 err.to_string()
3372 .contains("cache key map supports at most two files"),
3373 "unexpected error: {err:#}"
3374 );
3375 }
3376
3377 #[test]
3378 fn parses_artifacts_untracked() {
3379 let yaml = r#"
3380stages:
3381 - build
3382
3383build-job:
3384 stage: build
3385 script:
3386 - echo build
3387 artifacts:
3388 untracked: true
3389 exclude:
3390 - tests-temp/output/**/*.log
3391"#;
3392
3393 let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3394 let build_idx = find_job(&pipeline, "build-job");
3395 let job = &pipeline.graph[build_idx];
3396 assert!(job.artifacts.untracked);
3397 assert_eq!(
3398 job.artifacts.exclude,
3399 vec!["tests-temp/output/**/*.log".to_string()]
3400 );
3401 }
3402
3403 #[test]
3404 fn parses_pipeline_and_job_images() {
3405 let yaml = r#"
3406image: rust:latest
3407stages:
3408 - build
3409 - test
3410
3411build-job:
3412 stage: build
3413 image: rust:slim
3414 script:
3415 - echo build
3416
3417test-job:
3418 stage: test
3419 script:
3420 - echo test
3421"#;
3422
3423 let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3424 assert_eq!(
3425 pipeline
3426 .defaults
3427 .image
3428 .as_ref()
3429 .map(|image| image.name.as_str()),
3430 Some("rust:latest")
3431 );
3432 let build_idx = find_job(&pipeline, "build-job");
3433 assert_eq!(
3434 pipeline.graph[build_idx]
3435 .image
3436 .as_ref()
3437 .map(|image| image.name.as_str()),
3438 Some("rust:slim")
3439 );
3440 let test_idx = find_job(&pipeline, "test-job");
3441 assert_eq!(
3442 pipeline.graph[test_idx]
3443 .image
3444 .as_ref()
3445 .map(|image| image.name.as_str()),
3446 Some("rust:latest")
3447 );
3448 }
3449
3450 #[test]
3451 fn ignores_default_section_as_job() {
3452 let yaml = r#"
3453stages:
3454 - build
3455
3456default:
3457 image: alpine:latest
3458 before_script:
3459 - echo before
3460 after_script:
3461 - echo after
3462 variables:
3463 GLOBAL_DEFAULT: foo
3464
3465build-job:
3466 stage: build
3467 script:
3468 - echo build
3469"#;
3470
3471 let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3472 assert_eq!(pipeline.stages.len(), 1);
3473 assert_eq!(pipeline.stages[0].jobs.len(), 1);
3474 assert_eq!(
3475 pipeline.defaults.before_script,
3476 vec!["echo before".to_string()]
3477 );
3478 assert_eq!(
3479 pipeline.defaults.after_script,
3480 vec!["echo after".to_string()]
3481 );
3482 assert_eq!(
3483 pipeline
3484 .defaults
3485 .variables
3486 .get("GLOBAL_DEFAULT")
3487 .map(String::as_str),
3488 Some("foo")
3489 );
3490 let job_idx = pipeline.stages[0].jobs[0];
3491 assert_eq!(pipeline.graph[job_idx].name, "build-job");
3492 }
3493
3494 #[test]
3495 fn parses_global_hooks() {
3496 let yaml = r#"
3497stages:
3498 - build
3499
3500default:
3501 before_script:
3502 - echo before-one
3503 - echo before-two
3504 after_script: echo after
3505
3506build-job:
3507 stage: build
3508 script: echo body
3509"#;
3510
3511 let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3512 assert_eq!(
3513 pipeline.defaults.before_script,
3514 vec!["echo before-one".to_string(), "echo before-two".to_string()]
3515 );
3516 assert_eq!(
3517 pipeline.defaults.after_script,
3518 vec!["echo after".to_string()]
3519 );
3520 }
3521
3522 #[test]
3523 fn parses_variable_scopes() {
3524 let yaml = r#"
3525variables:
3526 GLOBAL_VAR: foo
3527
3528default:
3529 variables:
3530 DEFAULT_VAR: bar
3531
3532stages:
3533 - build
3534
3535build-job:
3536 stage: build
3537 variables:
3538 JOB_VAR: baz
3539 script:
3540 - echo job
3541"#;
3542
3543 let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3544 assert_eq!(
3545 pipeline
3546 .defaults
3547 .variables
3548 .get("GLOBAL_VAR")
3549 .map(String::as_str),
3550 Some("foo")
3551 );
3552 assert_eq!(
3553 pipeline
3554 .defaults
3555 .variables
3556 .get("DEFAULT_VAR")
3557 .map(String::as_str),
3558 Some("bar")
3559 );
3560 let job_idx = find_job(&pipeline, "build-job");
3561 assert_eq!(
3562 pipeline.graph[job_idx]
3563 .variables
3564 .get("JOB_VAR")
3565 .map(String::as_str),
3566 Some("baz")
3567 );
3568 }
3569
3570 #[test]
3571 fn ignores_non_job_sections_without_scripts() {
3572 let yaml = r#"
3573stages:
3574 - build
3575
3576workflow:
3577 rules:
3578 - if: $CI_PIPELINE_SOURCE == "push"
3579
3580build-job:
3581 stage: build
3582 script:
3583 - echo build
3584"#;
3585
3586 let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3587 assert_eq!(pipeline.stages.len(), 1);
3588 assert_eq!(pipeline.stages[0].jobs.len(), 1);
3589 let job_idx = pipeline.stages[0].jobs[0];
3590 assert_eq!(pipeline.graph[job_idx].name, "build-job");
3591 }
3592
3593 #[test]
3594 fn errors_when_job_missing_script() {
3595 let yaml = r#"
3596stages:
3597 - build
3598
3599broken-job:
3600 stage: build
3601"#;
3602
3603 let err = PipelineGraph::from_yaml_str(yaml).expect_err("missing script should error");
3604 assert!(err.to_string().contains("must define a script"));
3605 }
3606
3607 #[test]
3608 fn ignores_hidden_jobs_starting_with_dot() {
3609 let yaml = r#"
3610stages:
3611 - build
3612
3613.template:
3614 script:
3615 - echo template
3616
3617build-job:
3618 stage: build
3619 script:
3620 - echo build
3621"#;
3622
3623 let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3624 assert_eq!(pipeline.stages[0].jobs.len(), 1);
3625 let job_idx = pipeline.stages[0].jobs[0];
3626 assert_eq!(pipeline.graph[job_idx].name, "build-job");
3627 }
3628
3629 #[test]
3630 fn job_can_extend_hidden_template() {
3631 let yaml = r#"
3632stages:
3633 - build
3634
3635.base-template:
3636 stage: build
3637 script:
3638 - echo from template
3639 artifacts:
3640 paths:
3641 - template.txt
3642
3643child-job:
3644 extends: .base-template
3645"#;
3646
3647 let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3648 let job_idx = find_job(&pipeline, "child-job");
3649 let job = &pipeline.graph[job_idx];
3650 assert_eq!(job.stage, "build");
3651 assert_eq!(job.commands, vec!["echo from template"]);
3652 assert_eq!(job.artifacts.paths, vec![PathBuf::from("template.txt")]);
3653 }
3654
3655 #[test]
3656 fn job_merges_multiple_extends_in_order() {
3657 let yaml = r#"
3658stages:
3659 - test
3660
3661.lint-template:
3662 script:
3663 - echo lint
3664 artifacts:
3665 paths:
3666 - lint.txt
3667
3668.test-template:
3669 stage: test
3670 script:
3671 - echo tests
3672 artifacts:
3673 paths:
3674 - tests.txt
3675
3676combined:
3677 extends:
3678 - .lint-template
3679 - .test-template
3680"#;
3681
3682 let pipeline = PipelineGraph::from_yaml_str(yaml).expect("pipeline parses");
3683 let job_idx = find_job(&pipeline, "combined");
3684 let job = &pipeline.graph[job_idx];
3685 assert_eq!(job.commands, vec!["echo tests"]);
3686 assert_eq!(job.artifacts.paths, vec![PathBuf::from("tests.txt")]);
3687 assert_eq!(job.stage, "test");
3688 }
3689
3690 #[test]
3691 fn errors_on_extends_cycle() {
3692 let yaml = r#"
3693stages:
3694 - build
3695
3696.a:
3697 extends: .b
3698 script:
3699 - echo a
3700
3701.b:
3702 extends: .a
3703 script:
3704 - echo b
3705
3706job:
3707 extends: .a
3708"#;
3709
3710 let err = PipelineGraph::from_yaml_str(yaml).expect_err("cycle must error");
3711 assert!(err.to_string().contains("cyclical extends"));
3712 }
3713
3714 #[test]
3715 fn errors_on_unknown_extended_template() {
3716 let yaml = r#"
3717stages:
3718 - build
3719
3720job:
3721 stage: build
3722 extends: .missing
3723"#;
3724
3725 let err = PipelineGraph::from_yaml_str(yaml).expect_err("unknown template must error");
3726 assert!(err.to_string().contains("unknown job/template '.missing'"));
3727 }
3728
3729 fn find_job(graph: &PipelineGraph, name: &str) -> NodeIndex {
3730 graph
3731 .graph
3732 .node_indices()
3733 .find(|&idx| graph.graph[idx].name == name)
3734 .expect("job must exist")
3735 }
3736}
3737#[derive(Debug, Deserialize, Default)]
3738struct RawInherit {
3739 #[serde(default)]
3740 default: Option<RawInheritDefault>,
3741}
3742
3743#[derive(Debug, Deserialize)]
3744#[serde(untagged)]
3745enum RawInheritDefault {
3746 Bool(bool),
3747 List(Vec<String>),
3748}