1use indexmap::IndexMap;
36use serde::{de::Error as _, Deserialize, Deserializer, Serialize, Serializer};
37use std::collections::HashMap;
38
39use crate::types::{Pipeline, QedStep};
40
41pub type MatrixCoord = IndexMap<String, toml::Value>;
45
46#[derive(Debug, Clone, Default, PartialEq)]
48pub struct MatrixSpec {
49 pub dimensions: IndexMap<String, Vec<toml::Value>>,
51 pub include: Vec<MatrixCoord>,
53 pub exclude: Vec<MatrixCoord>,
55}
56
57impl MatrixSpec {
58 pub fn is_empty(&self) -> bool {
62 self.dimensions.is_empty() && self.include.is_empty()
63 }
64}
65
66impl<'de> Deserialize<'de> for MatrixSpec {
67 fn deserialize<D>(d: D) -> Result<Self, D::Error>
68 where
69 D: Deserializer<'de>,
70 {
71 let value = toml::Value::deserialize(d)?;
72 let table = match value {
73 toml::Value::Table(t) => t,
74 _ => return Err(D::Error::custom("[matrix] must be a TOML table")),
75 };
76 let mut dimensions: IndexMap<String, Vec<toml::Value>> = IndexMap::new();
77 let mut include: Vec<MatrixCoord> = Vec::new();
78 let mut exclude: Vec<MatrixCoord> = Vec::new();
79 for (key, val) in table {
80 match key.as_str() {
81 "include" | "exclude" => {
82 let rows = match val {
83 toml::Value::Array(arr) => arr,
84 _ => {
85 return Err(D::Error::custom(format!(
86 "matrix.{key} must be an array of tables"
87 )));
88 }
89 };
90 let mut out = Vec::with_capacity(rows.len());
91 for (i, row) in rows.into_iter().enumerate() {
92 let row_table = match row {
93 toml::Value::Table(t) => t,
94 _ => {
95 return Err(D::Error::custom(format!(
96 "matrix.{key}[{i}] must be a table"
97 )));
98 }
99 };
100 let mut coord = IndexMap::new();
101 for (k, v) in row_table {
102 coord.insert(k, v);
103 }
104 out.push(coord);
105 }
106 if key == "include" {
107 include = out;
108 } else {
109 exclude = out;
110 }
111 }
112 _ => {
113 let arr = match val {
114 toml::Value::Array(a) => a,
115 _ => {
116 return Err(D::Error::custom(format!(
117 "matrix dimension `{key}` must be an array"
118 )));
119 }
120 };
121 dimensions.insert(key, arr);
122 }
123 }
124 }
125 Ok(MatrixSpec {
126 dimensions,
127 include,
128 exclude,
129 })
130 }
131}
132
133impl Serialize for MatrixSpec {
134 fn serialize<S>(&self, s: S) -> Result<S::Ok, S::Error>
135 where
136 S: Serializer,
137 {
138 let mut table = toml::value::Table::new();
142 for (k, v) in &self.dimensions {
143 table.insert(k.clone(), toml::Value::Array(v.clone()));
144 }
145 if !self.include.is_empty() {
146 table.insert(
147 "include".to_string(),
148 toml::Value::Array(self.include.iter().map(coord_to_value).collect()),
149 );
150 }
151 if !self.exclude.is_empty() {
152 table.insert(
153 "exclude".to_string(),
154 toml::Value::Array(self.exclude.iter().map(coord_to_value).collect()),
155 );
156 }
157 toml::Value::Table(table).serialize(s)
158 }
159}
160
161fn coord_to_value(coord: &MatrixCoord) -> toml::Value {
162 let mut t = toml::value::Table::new();
163 for (k, v) in coord {
164 t.insert(k.clone(), v.clone());
165 }
166 toml::Value::Table(t)
167}
168
169pub fn expand_matrix(spec: &MatrixSpec) -> Vec<MatrixCoord> {
172 let dim_keys: Vec<String> = spec.dimensions.keys().cloned().collect();
173 let dim_vals: Vec<Vec<toml::Value>> = spec.dimensions.values().cloned().collect();
174
175 let mut rows: Vec<MatrixCoord> = if dim_keys.is_empty() {
177 Vec::new()
178 } else {
179 cartesian(&dim_keys, &dim_vals)
180 };
181
182 let original_keys: std::collections::HashSet<&str> =
184 dim_keys.iter().map(|s| s.as_str()).collect();
185 for inc in &spec.include {
186 if dim_keys.is_empty() {
187 rows.push(inc.clone());
188 continue;
189 }
190 let (orig_part, new_part): (MatrixCoord, MatrixCoord) = inc
191 .iter()
192 .map(|(k, v)| (k.clone(), v.clone()))
193 .partition(|(k, _)| original_keys.contains(k.as_str()));
194
195 if orig_part.is_empty() {
196 rows.push(new_part);
197 continue;
198 }
199
200 let mut matched_any = false;
201 for row in rows.iter_mut() {
202 if orig_part.iter().all(|(k, v)| row.get(k) == Some(v)) {
203 matched_any = true;
204 for (k, v) in &new_part {
205 if !row.contains_key(k) {
206 row.insert(k.clone(), v.clone());
207 }
208 }
209 }
210 }
211 if !matched_any {
212 let mut row = IndexMap::new();
213 row.extend(orig_part);
214 row.extend(new_part);
215 rows.push(row);
216 }
217 }
218
219 if !spec.exclude.is_empty() {
221 rows.retain(|row| {
222 !spec
223 .exclude
224 .iter()
225 .any(|ex| ex.iter().all(|(k, ev)| row.get(k) == Some(ev)))
226 });
227 }
228
229 rows
230}
231
232fn cartesian(keys: &[String], vals: &[Vec<toml::Value>]) -> Vec<MatrixCoord> {
233 let mut out: Vec<MatrixCoord> = vec![IndexMap::new()];
234 for (k, vs) in keys.iter().zip(vals.iter()) {
235 let mut next = Vec::with_capacity(out.len() * vs.len().max(1));
236 for row in &out {
237 if vs.is_empty() {
238 next.push(row.clone());
239 continue;
240 }
241 for v in vs {
242 let mut nr = row.clone();
243 nr.insert(k.clone(), v.clone());
244 next.push(nr);
245 }
246 }
247 out = next;
248 }
249 out
250}
251
252#[derive(Debug, Clone)]
255pub struct PlannedJob {
256 pub coord: Option<MatrixCoord>,
257 pub pipeline: Pipeline,
258}
259
260impl PlannedJob {
261 pub fn label(&self) -> String {
265 match &self.coord {
266 None => self.pipeline.name.clone(),
267 Some(coord) => coord
268 .iter()
269 .map(|(k, v)| format!("{k}={}", toml_value_to_str(v)))
270 .collect::<Vec<_>>()
271 .join(" "),
272 }
273 }
274}
275
276pub fn plan(pipeline: &Pipeline) -> Vec<PlannedJob> {
284 let pipeline_rows: Vec<Option<MatrixCoord>> = match &pipeline.matrix {
285 Some(spec) if !spec.is_empty() => {
286 let rows = expand_matrix(spec);
287 if rows.is_empty() {
288 vec![None]
289 } else {
290 rows.into_iter().map(Some).collect()
291 }
292 }
293 _ => vec![None],
294 };
295
296 pipeline_rows
297 .into_iter()
298 .map(|coord| {
299 let mut clone = pipeline.clone();
300 clone.matrix = None;
304
305 let mut expanded_steps: Vec<QedStep> = Vec::with_capacity(clone.steps.len());
310 for step in clone.steps.drain(..) {
311 expanded_steps.extend(expand_step(step));
312 }
313 clone.steps = expanded_steps;
314
315 if let Some(row) = &coord {
316 apply_matrix_to_pipeline(&mut clone, row);
317 }
318 PlannedJob {
319 coord,
320 pipeline: clone,
321 }
322 })
323 .collect()
324}
325
326fn expand_step(step: QedStep) -> Vec<QedStep> {
327 let spec = match step.matrix.clone() {
328 Some(s) if !s.is_empty() => s,
329 _ => return vec![strip_step_matrix(step)],
330 };
331 let rows = expand_matrix(&spec);
332 if rows.is_empty() {
333 return vec![strip_step_matrix(step)];
334 }
335 rows.into_iter()
336 .map(|row| {
337 let mut copy = strip_step_matrix(step.clone());
338 let suffix = row
342 .iter()
343 .map(|(k, v)| format!("{k}={}", toml_value_to_str(v)))
344 .collect::<Vec<_>>()
345 .join(" ");
346 if !suffix.is_empty() {
347 copy.name = format!("{} [{}]", copy.name, suffix);
348 }
349 apply_matrix_to_step(&mut copy, &row);
350 copy
351 })
352 .collect()
353}
354
355fn strip_step_matrix(mut step: QedStep) -> QedStep {
356 step.matrix = None;
357 step
358}
359
360fn apply_matrix_to_pipeline(pipeline: &mut Pipeline, coord: &MatrixCoord) {
361 for step in &mut pipeline.steps {
362 apply_matrix_to_step(step, coord);
363 }
364}
365
366fn apply_matrix_to_step(step: &mut QedStep, coord: &MatrixCoord) {
367 let lookup: HashMap<&str, String> = coord
368 .iter()
369 .map(|(k, v)| (k.as_str(), toml_value_to_str(v)))
370 .collect();
371 for arg in &mut step.argv {
372 *arg = substitute_matrix(arg, &lookup);
373 }
374 for value in step.env.values_mut() {
375 *value = substitute_matrix(value, &lookup);
376 }
377 if let Some(cwd) = &mut step.cwd {
378 *cwd = substitute_matrix(cwd, &lookup);
379 }
380 if let Some(platform) = &mut step.platform {
384 if let Some(target) = &mut platform.target {
385 *target = substitute_matrix(target, &lookup);
386 }
387 if let Some(cp) = &mut platform.container_platform {
388 *cp = substitute_matrix(cp, &lookup);
389 }
390 }
391}
392
393pub fn toml_value_to_str(v: &toml::Value) -> String {
399 match v {
400 toml::Value::String(s) => s.clone(),
401 toml::Value::Integer(i) => i.to_string(),
402 toml::Value::Float(f) => f.to_string(),
403 toml::Value::Boolean(b) => b.to_string(),
404 toml::Value::Datetime(d) => d.to_string(),
405 toml::Value::Array(_) | toml::Value::Table(_) => v.to_string(),
406 }
407}
408
409pub fn substitute_matrix(input: &str, coord: &HashMap<&str, String>) -> String {
414 let mut out = String::with_capacity(input.len());
415 let bytes = input.as_bytes();
416 let mut i = 0;
417 while i < bytes.len() {
418 if i + 3 < bytes.len() && &bytes[i..i + 3] == b"${{" {
419 if let Some(end) = find_close(bytes, i + 3) {
421 let body = &input[i + 3..end];
422 let trimmed = body.trim();
423 if let Some(key) = trimmed.strip_prefix("matrix.") {
424 let key = key.trim();
425 if let Some(val) = coord.get(key) {
426 out.push_str(val);
427 i = end + 2;
428 continue;
429 }
430 }
431 out.push_str(&input[i..end + 2]);
436 i = end + 2;
437 continue;
438 }
439 }
440 out.push(input[i..].chars().next().unwrap());
441 i += input[i..].chars().next().unwrap().len_utf8();
442 }
443 out
444}
445
446fn find_close(bytes: &[u8], start: usize) -> Option<usize> {
447 let mut i = start;
448 while i + 1 < bytes.len() {
449 if &bytes[i..i + 2] == b"}}" {
450 return Some(i);
451 }
452 i += 1;
453 }
454 None
455}
456
457#[cfg(test)]
458mod tests {
459 use super::*;
460
461 fn s(v: &str) -> toml::Value {
462 toml::Value::String(v.into())
463 }
464
465 fn coord(pairs: &[(&str, &str)]) -> MatrixCoord {
466 pairs
467 .iter()
468 .map(|(k, v)| ((*k).to_string(), s(v)))
469 .collect()
470 }
471
472 #[test]
473 fn cartesian_two_dims() {
474 let spec: MatrixSpec = toml::from_str(
475 r#"
476os = ["linux", "macos"]
477rust = ["stable", "beta"]
478"#,
479 )
480 .unwrap();
481 let rows = expand_matrix(&spec);
482 assert_eq!(rows.len(), 4);
483 assert_eq!(rows[0].get("os"), Some(&s("linux")));
484 assert_eq!(rows[0].get("rust"), Some(&s("stable")));
485 assert_eq!(rows[1].get("rust"), Some(&s("beta")));
486 assert_eq!(rows[2].get("os"), Some(&s("macos")));
487 }
488
489 #[test]
490 fn include_extends_matching_combination() {
491 let spec: MatrixSpec = toml::from_str(
492 r#"
493os = ["linux", "macos"]
494rust = ["stable"]
495include = [{ os = "linux", rust = "stable", extra = "special" }]
496"#,
497 )
498 .unwrap();
499 let rows = expand_matrix(&spec);
500 assert_eq!(rows.len(), 2);
501 let linux = rows
502 .iter()
503 .find(|r| r.get("os") == Some(&s("linux")))
504 .unwrap();
505 assert_eq!(linux.get("extra"), Some(&s("special")));
506 let macos = rows
507 .iter()
508 .find(|r| r.get("os") == Some(&s("macos")))
509 .unwrap();
510 assert!(macos.get("extra").is_none());
511 }
512
513 #[test]
514 fn include_with_no_anchor_appends_standalone_row() {
515 let spec: MatrixSpec = toml::from_str(
516 r#"
517os = ["linux"]
518include = [{ os = "windows", arch = "x86_64" }]
519"#,
520 )
521 .unwrap();
522 let rows = expand_matrix(&spec);
523 assert_eq!(rows.len(), 2);
524 assert_eq!(rows[1].get("os"), Some(&s("windows")));
525 assert_eq!(rows[1].get("arch"), Some(&s("x86_64")));
526 }
527
528 #[test]
529 fn exclude_drops_matching_rows() {
530 let spec: MatrixSpec = toml::from_str(
531 r#"
532os = ["linux", "macos"]
533arch = ["x86_64", "aarch64"]
534exclude = [{ os = "macos", arch = "x86_64" }]
535"#,
536 )
537 .unwrap();
538 let rows = expand_matrix(&spec);
539 assert_eq!(rows.len(), 3);
540 let any_excluded = rows
541 .iter()
542 .any(|r| r.get("os") == Some(&s("macos")) && r.get("arch") == Some(&s("x86_64")));
543 assert!(!any_excluded);
544 }
545
546 #[test]
547 fn noisetable_release_apple_expands_to_seven_rows() {
548 let spec: MatrixSpec = toml::from_str(
552 r#"
553target = ["winit", "macos-native", "ios-sim", "ios-device", "vst"]
554arch = ["x86_64", "aarch64"]
555exclude = [
556 { target = "ios-sim", arch = "x86_64" },
557 { target = "ios-device", arch = "x86_64" },
558 { target = "macos-native", arch = "x86_64" },
559 { target = "macos-native", arch = "aarch64" },
560]
561include = [
562 { target = "macos-native", arch = "universal" },
563]
564"#,
565 )
566 .unwrap();
567 let rows = expand_matrix(&spec);
568 assert_eq!(rows.len(), 7, "rows: {rows:?}");
569 let last = rows.last().unwrap();
572 assert_eq!(last.get("target"), Some(&s("macos-native")));
573 assert_eq!(last.get("arch"), Some(&s("universal")));
574 for row in &rows {
576 let is_ios = matches!(
577 row.get("target"),
578 Some(toml::Value::String(t)) if t.starts_with("ios-"),
579 );
580 if is_ios {
581 assert_eq!(row.get("arch"), Some(&s("aarch64")));
582 }
583 }
584 }
585
586 #[test]
587 fn substitute_matrix_replaces_known_keys() {
588 let c = coord(&[("arch", "aarch64"), ("target", "macos-native")]);
589 let lookup: HashMap<&str, String> = c
590 .iter()
591 .map(|(k, v)| (k.as_str(), toml_value_to_str(v)))
592 .collect();
593 let out = substitute_matrix(
594 "cargo build --target ${{ matrix.arch }}-apple-${{ matrix.target }}",
595 &lookup,
596 );
597 assert_eq!(out, "cargo build --target aarch64-apple-macos-native");
598 }
599
600 #[test]
601 fn substitute_matrix_leaves_unknown_expressions_alone() {
602 let c = coord(&[("arch", "aarch64")]);
603 let lookup: HashMap<&str, String> = c
604 .iter()
605 .map(|(k, v)| (k.as_str(), toml_value_to_str(v)))
606 .collect();
607 let out = substitute_matrix(
610 "tag=${{ matrix.arch }}-${{ steps.x.outputs.sha }}-${{ matrix.unknown }}",
611 &lookup,
612 );
613 assert_eq!(
614 out,
615 "tag=aarch64-${{ steps.x.outputs.sha }}-${{ matrix.unknown }}"
616 );
617 }
618
619 #[test]
620 fn round_trips_through_toml() {
621 let original: MatrixSpec = toml::from_str(
622 r#"
623os = ["linux", "macos"]
624arch = ["x86_64", "aarch64"]
625include = [{ os = "linux", arch = "aarch64", extra = "qemu" }]
626exclude = [{ os = "macos", arch = "x86_64" }]
627"#,
628 )
629 .unwrap();
630 let serialized = toml::to_string(&original).unwrap();
631 let reparsed: MatrixSpec = toml::from_str(&serialized).unwrap();
632 assert_eq!(reparsed, original);
633 }
634
635 #[test]
636 fn matrix_spec_with_only_include_works() {
637 let spec: MatrixSpec = toml::from_str(
638 r#"
639include = [
640 { os = "linux", target = "x86_64-unknown-linux-gnu" },
641 { os = "linux", target = "aarch64-unknown-linux-musl" },
642]
643"#,
644 )
645 .unwrap();
646 let rows = expand_matrix(&spec);
647 assert_eq!(rows.len(), 2);
648 assert_eq!(rows[0].get("target"), Some(&s("x86_64-unknown-linux-gnu")));
649 }
650
651 #[test]
652 fn plan_with_no_matrix_returns_single_job() {
653 let pipeline = test_pipeline(None, vec![test_step("build", &["echo", "hi"], None)]);
654 let jobs = plan(&pipeline);
655 assert_eq!(jobs.len(), 1);
656 assert!(jobs[0].coord.is_none());
657 assert_eq!(jobs[0].pipeline.steps.len(), 1);
658 assert_eq!(jobs[0].pipeline.steps[0].argv, vec!["echo", "hi"]);
659 }
660
661 #[test]
662 fn plan_with_pipeline_matrix_substitutes_argv() {
663 let matrix: MatrixSpec = toml::from_str(
664 r#"
665arch = ["x86_64", "aarch64"]
666"#,
667 )
668 .unwrap();
669 let step = test_step(
670 "build",
671 &[
672 "cargo",
673 "build",
674 "--target",
675 "${{ matrix.arch }}-apple-darwin",
676 ],
677 None,
678 );
679 let pipeline = test_pipeline(Some(matrix), vec![step]);
680 let jobs = plan(&pipeline);
681 assert_eq!(jobs.len(), 2);
682 assert_eq!(
683 jobs[0].pipeline.steps[0].argv,
684 vec!["cargo", "build", "--target", "x86_64-apple-darwin"],
685 );
686 assert_eq!(
687 jobs[1].pipeline.steps[0].argv,
688 vec!["cargo", "build", "--target", "aarch64-apple-darwin"],
689 );
690 assert!(jobs[0].pipeline.matrix.is_none());
693 }
694
695 #[test]
696 fn plan_expands_step_level_matrix_inside_single_job() {
697 let step_matrix: MatrixSpec = toml::from_str(
698 r#"
699target = ["x86_64", "aarch64"]
700"#,
701 )
702 .unwrap();
703 let mut step = test_step(
704 "check",
705 &["cargo", "check", "--target", "${{ matrix.target }}"],
706 None,
707 );
708 step.matrix = Some(step_matrix);
709 let pipeline = test_pipeline(None, vec![step]);
710 let jobs = plan(&pipeline);
711 assert_eq!(jobs.len(), 1);
712 assert_eq!(jobs[0].pipeline.steps.len(), 2);
713 assert!(jobs[0].pipeline.steps[0].name.contains("target=x86_64"));
714 assert_eq!(jobs[0].pipeline.steps[0].argv.last().unwrap(), "x86_64");
715 assert_eq!(jobs[0].pipeline.steps[1].argv.last().unwrap(), "aarch64");
716 }
717
718 #[test]
719 fn plan_combines_pipeline_and_step_matrix() {
720 let pm: MatrixSpec = toml::from_str(r#"os = ["linux"]"#).unwrap();
722 let sm: MatrixSpec = toml::from_str(r#"arch = ["x86", "arm"]"#).unwrap();
723 let mut step = test_step(
724 "build",
725 &["build", "${{ matrix.os }}/${{ matrix.arch }}"],
726 None,
727 );
728 step.matrix = Some(sm);
729 let pipeline = test_pipeline(Some(pm), vec![step]);
730 let jobs = plan(&pipeline);
731 assert_eq!(jobs.len(), 1);
732 assert_eq!(jobs[0].pipeline.steps.len(), 2);
733 assert_eq!(jobs[0].pipeline.steps[0].argv[1], "linux/x86");
736 assert_eq!(jobs[0].pipeline.steps[1].argv[1], "linux/arm");
737 }
738
739 #[test]
740 fn planned_job_label_uses_coord_pairs() {
741 let pipeline = test_pipeline(
742 Some(toml::from_str(r#"arch = ["x86_64"]"#).unwrap()),
743 vec![test_step("build", &[], None)],
744 );
745 let jobs = plan(&pipeline);
746 assert_eq!(jobs[0].label(), "arch=x86_64");
747 }
748
749 fn test_pipeline(matrix: Option<MatrixSpec>, steps: Vec<QedStep>) -> Pipeline {
752 Pipeline {
753 name: "test".into(),
754 label: "Test".into(),
755 steps,
756 params: Default::default(),
757 on_success: Default::default(),
758 on_fail: Default::default(),
759 triggers: Default::default(),
760 concurrency_key: None,
761 placement: Default::default(),
762 workspace: crate::types::WorkspaceMode::default(),
763 wraps: None,
764 matrix,
765 toolchain: None,
766 binds: Vec::new(),
767 on_change: Vec::new(),
768 finally: Vec::new(),
769 }
770 }
771
772 fn test_step(name: &str, argv: &[&str], env_pairs: Option<&[(&str, &str)]>) -> QedStep {
773 QedStep {
774 background: false,
775 background_until: None,
776 wait_for: None,
777 manifest_stitch: None,
778 name: name.into(),
779 argv: argv.iter().map(|s| (*s).into()).collect(),
780 cwd: None,
781 env: env_pairs
782 .map(|p| p.iter().map(|(k, v)| ((*k).into(), (*v).into())).collect())
783 .unwrap_or_default(),
784 timeout: None,
785 on_fail: Default::default(),
786 produces: Default::default(),
787 runtime: None,
788 kind: Default::default(),
789 image: None,
790 tag: None,
791 push: false,
792 platforms: Vec::new(),
793 binary_path: None,
794 triple: None,
795 package: None,
796 context: None,
797 load: false,
798 sub_pipeline: None,
799 outputs: Default::default(),
800 gha_workflow: None,
801 import: None,
802 matrix: None,
803 enabled: true,
804 activation: crate::types::StepActivation::Active,
805 if_cond: None,
806 platform: None,
807 toolchain: None,
808 }
809 }
810
811 #[test]
812 fn step_matrix_substitutes_lifted_platform_target() {
813 let mut step = test_step("build", &["cargo", "zigbuild", "--target", "${{ matrix.target }}"], None);
816 step.platform = Some(crate::platform::PlatformSpec {
817 target: Some("${{ matrix.target }}".into()),
818 container_platform: None,
819 native: false,
820 });
821 let mut spec = MatrixSpec::default();
822 spec.dimensions.insert(
823 "target".into(),
824 vec![
825 toml::Value::String("x86_64-unknown-linux-musl".into()),
826 toml::Value::String("aarch64-unknown-linux-musl".into()),
827 ],
828 );
829 step.matrix = Some(spec);
830
831 let fanned = expand_step(step);
832 assert_eq!(fanned.len(), 2);
833 let targets: Vec<&str> = fanned
834 .iter()
835 .map(|s| s.platform.as_ref().unwrap().target.as_deref().unwrap())
836 .collect();
837 assert_eq!(targets, vec!["x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl"]);
838 assert!(fanned[0].argv.contains(&"x86_64-unknown-linux-musl".to_string()));
840 }
841}