opal/
env.rs

1use crate::git;
2use crate::model::{EnvironmentSpec, JobSpec};
3use crate::naming::job_name_slug;
4use crate::secrets::SecretsStore;
5use anyhow::{Context, Result};
6use globset::{Glob, GlobSetBuilder};
7use std::collections::HashMap;
8use std::env;
9use std::path::Path;
10
11pub fn build_include_lookup(
12    workdir: &Path,
13    host_env: &HashMap<String, String>,
14) -> HashMap<String, String> {
15    let mut lookup = host_env.clone();
16    for (key, value) in inferred_ci_env(workdir, host_env) {
17        lookup.entry(key).or_insert(value);
18    }
19    lookup
20}
21
22pub fn collect_env_vars(patterns: &[String]) -> Result<Vec<(String, String)>> {
23    if patterns.is_empty() {
24        return Ok(Vec::new());
25    }
26
27    let mut builder = GlobSetBuilder::new();
28    for pattern in patterns {
29        let glob =
30            Glob::new(pattern).with_context(|| format!("invalid --env pattern '{pattern}'"))?;
31        builder.add(glob);
32    }
33    let matcher = builder.build()?;
34
35    let vars = env::vars()
36        .filter(|(key, _)| matcher.is_match(key))
37        .collect();
38    Ok(vars)
39}
40
41#[allow(clippy::too_many_arguments)]
42pub fn build_job_env(
43    base_env: &[(String, String)],
44    default_vars: &HashMap<String, String>,
45    job: &JobSpec,
46    secrets: &SecretsStore,
47    host_workdir: &Path,
48    container_workdir: &Path,
49    container_root: &Path,
50    run_id: &str,
51    host_env: &HashMap<String, String>,
52) -> Vec<(String, String)> {
53    let mut env = Vec::new();
54    let mut push = |key: &str, value: &str| {
55        if let Some(existing) = env.iter_mut().find(|(k, _)| k == key) {
56            existing.1 = value.to_string();
57        } else {
58            env.push((key.to_string(), value.to_string()));
59        }
60    };
61
62    for (key, value) in base_env {
63        push(key, value);
64    }
65    for (key, value) in default_vars {
66        push(key, value);
67    }
68    for (key, value) in &job.variables {
69        push(key, value);
70    }
71
72    push("CI", "true");
73    push("GITLAB_CI", "true");
74    push("CI_JOB_NAME", &job.name);
75    push("CI_JOB_NAME_SLUG", &job_name_slug(&job.name));
76    push("CI_JOB_STAGE", &job.stage);
77    push("CI_PROJECT_DIR", &container_workdir.display().to_string());
78    push("CI_BUILDS_DIR", &container_root.display().to_string());
79    push("CI_PIPELINE_ID", run_id);
80    push("OPAL_IN_OPAL", "1");
81
82    for (key, value) in inferred_ci_env(host_workdir, host_env) {
83        push(&key, &value);
84    }
85
86    if let Some(timeout) = job.timeout {
87        push("CI_JOB_TIMEOUT", &timeout.as_secs().to_string());
88    }
89
90    if secrets.has_secrets() {
91        secrets.extend_env(&mut env);
92    }
93
94    expand_env_list(&mut env[..], host_env);
95
96    env
97}
98
99pub fn expand_env_list(env: &mut [(String, String)], host_env: &HashMap<String, String>) {
100    let mut lookup: HashMap<String, String> = HashMap::with_capacity(host_env.len() + env.len());
101    for (key, value) in host_env {
102        lookup.insert(key.clone(), value.clone());
103    }
104    for (key, value) in env.iter() {
105        lookup.entry(key.clone()).or_insert_with(|| value.clone());
106    }
107    for (key, value) in env.iter_mut() {
108        let expanded = expand_value(value, &lookup);
109        *value = expanded.clone();
110        lookup.insert(key.clone(), expanded);
111    }
112}
113
114pub fn expand_environment(
115    environment: &EnvironmentSpec,
116    lookup: &HashMap<String, String>,
117) -> EnvironmentSpec {
118    EnvironmentSpec {
119        name: expand_value(&environment.name, lookup),
120        url: environment
121            .url
122            .as_ref()
123            .map(|value| expand_value(value, lookup)),
124        on_stop: environment
125            .on_stop
126            .as_ref()
127            .map(|value| expand_value(value, lookup)),
128        auto_stop_in: environment.auto_stop_in,
129        action: environment.action,
130    }
131}
132
133pub fn expand_value(value: &str, lookup: &HashMap<String, String>) -> String {
134    let chars: Vec<char> = value.chars().collect();
135    let mut idx = 0;
136    let mut output = String::new();
137    while idx < chars.len() {
138        let ch = chars[idx];
139        if ch == '$' && idx + 1 < chars.len() {
140            match chars[idx + 1] {
141                '$' => {
142                    output.push('$');
143                    idx += 2;
144                    continue;
145                }
146                '{' => {
147                    let mut end = idx + 2;
148                    while end < chars.len() && chars[end] != '}' {
149                        end += 1;
150                    }
151                    if end < chars.len() {
152                        let expr: String = chars[idx + 2..end].iter().collect();
153                        if let Some((name, default)) = expr.split_once(":-") {
154                            if let Some(val) = lookup.get(name).filter(|val| !val.is_empty()) {
155                                output.push_str(val);
156                            } else {
157                                output.push_str(&expand_value(default, lookup));
158                            }
159                        } else if let Some(val) = lookup.get(&expr) {
160                            output.push_str(val);
161                        }
162                        idx = end + 1;
163                        continue;
164                    }
165                }
166                c if is_var_char(c) => {
167                    let mut end = idx + 1;
168                    while end < chars.len() && is_var_char(chars[end]) {
169                        end += 1;
170                    }
171                    let name: String = chars[idx + 1..end].iter().collect();
172                    if let Some(val) = lookup.get(&name) {
173                        output.push_str(val);
174                    }
175                    idx = end;
176                    continue;
177                }
178                _ => {}
179            }
180        }
181        output.push(ch);
182        idx += 1;
183    }
184    output
185}
186
187fn is_var_char(ch: char) -> bool {
188    ch == '_' || ch.is_ascii_alphanumeric()
189}
190
191fn inferred_ci_env(workdir: &Path, host_env: &HashMap<String, String>) -> Vec<(String, String)> {
192    let mut inferred = Vec::new();
193
194    insert_inferred_env(
195        &mut inferred,
196        "CI_PIPELINE_SOURCE",
197        host_env,
198        Some(|| Ok("push".to_string())),
199    );
200    if let Some(branch) = host_env
201        .get("CI_COMMIT_BRANCH")
202        .filter(|value| !value.is_empty())
203    {
204        inferred.push(("CI_COMMIT_BRANCH".into(), branch.clone()));
205    } else if host_env
206        .get("CI_COMMIT_TAG")
207        .filter(|value| !value.is_empty())
208        .or_else(|| {
209            host_env
210                .get("GIT_COMMIT_TAG")
211                .filter(|value| !value.is_empty())
212        })
213        .is_none()
214        && let Ok(branch) = git::current_branch(workdir)
215        && !branch.is_empty()
216    {
217        inferred.push(("CI_COMMIT_BRANCH".into(), branch));
218    }
219    if let Some(tag) = host_env
220        .get("CI_COMMIT_TAG")
221        .filter(|value| !value.is_empty())
222        .or_else(|| {
223            host_env
224                .get("GIT_COMMIT_TAG")
225                .filter(|value| !value.is_empty())
226        })
227    {
228        inferred.push(("CI_COMMIT_TAG".into(), tag.clone()));
229    } else if let Ok(tag) = git::current_tag(workdir)
230        && !tag.is_empty()
231    {
232        inferred.push(("CI_COMMIT_TAG".into(), tag));
233    }
234    insert_inferred_env(
235        &mut inferred,
236        "CI_DEFAULT_BRANCH",
237        host_env,
238        Some(|| git::default_branch(workdir)),
239    );
240
241    if host_env
242        .get("CI_COMMIT_REF_NAME")
243        .is_none_or(|value| value.is_empty())
244    {
245        if let Some(tag) = host_env
246            .get("CI_COMMIT_TAG")
247            .filter(|value| !value.is_empty())
248            .cloned()
249            .or_else(|| {
250                inferred
251                    .iter()
252                    .find(|(key, _)| key == "CI_COMMIT_TAG")
253                    .map(|(_, value)| value.clone())
254            })
255        {
256            inferred.push(("CI_COMMIT_REF_NAME".into(), tag));
257        } else if let Some(branch) = host_env
258            .get("CI_COMMIT_BRANCH")
259            .filter(|value| !value.is_empty())
260            .cloned()
261            .or_else(|| {
262                inferred
263                    .iter()
264                    .find(|(key, _)| key == "CI_COMMIT_BRANCH")
265                    .map(|(_, value)| value.clone())
266            })
267        {
268            inferred.push(("CI_COMMIT_REF_NAME".into(), branch));
269        }
270    }
271    if host_env
272        .get("CI_COMMIT_REF_SLUG")
273        .is_none_or(|value| value.is_empty())
274        && let Some(ref_name) = host_env
275            .get("CI_COMMIT_REF_NAME")
276            .filter(|value| !value.is_empty())
277            .cloned()
278            .or_else(|| {
279                inferred
280                    .iter()
281                    .find(|(key, _)| key == "CI_COMMIT_REF_NAME")
282                    .map(|(_, value)| value.clone())
283            })
284    {
285        let slug = job_name_slug(&ref_name);
286        if !slug.is_empty() {
287            inferred.push(("CI_COMMIT_REF_SLUG".into(), slug));
288        }
289    }
290
291    inferred
292}
293
294fn insert_inferred_env<F>(
295    env: &mut Vec<(String, String)>,
296    key: &str,
297    host_env: &HashMap<String, String>,
298    fallback: Option<F>,
299) where
300    F: FnOnce() -> Result<String>,
301{
302    if let Some(value) = host_env.get(key).filter(|value| !value.is_empty()) {
303        env.push((key.to_string(), value.clone()));
304        return;
305    }
306    if let Some(fallback) = fallback
307        && let Ok(value) = fallback()
308        && !value.is_empty()
309    {
310        env.push((key.to_string(), value));
311    }
312}
313
314#[cfg(test)]
315mod tests {
316    use super::*;
317    use crate::git::test_support::init_repo_with_commit_and_tag;
318    use crate::model::{
319        ArtifactSpec, EnvironmentActionSpec, EnvironmentSpec, JobSpec, RetryPolicySpec,
320    };
321    use crate::secrets::SecretsStore;
322    use anyhow::Result;
323    use std::collections::HashMap;
324    use std::fs;
325    use std::path::PathBuf;
326    use std::time::{SystemTime, UNIX_EPOCH};
327
328    #[test]
329    fn expands_env_references() {
330        let job = JobSpec {
331            name: "lint".into(),
332            stage: "test".into(),
333            commands: Vec::new(),
334            needs: Vec::new(),
335            explicit_needs: false,
336            dependencies: Vec::new(),
337            before_script: None,
338            after_script: None,
339            inherit_default_before_script: true,
340            inherit_default_after_script: true,
341            inherit_default_image: true,
342            inherit_default_cache: true,
343            inherit_default_services: true,
344            inherit_default_timeout: true,
345            inherit_default_retry: true,
346            inherit_default_interruptible: true,
347            when: None,
348            rules: Vec::new(),
349            only: Vec::new(),
350            except: Vec::new(),
351            artifacts: ArtifactSpec::default(),
352            cache: Vec::new(),
353            image: None,
354            variables: HashMap::from([("CARGO_HOME".into(), "$CI_PROJECT_DIR/.cargo".into())]),
355            services: Vec::new(),
356            timeout: None,
357            retry: RetryPolicySpec::default(),
358            interruptible: false,
359            resource_group: None,
360            parallel: None,
361            tags: Vec::new(),
362            environment: None,
363        };
364        let env = build_job_env(
365            &[],
366            &HashMap::new(),
367            &job,
368            &SecretsStore::default(),
369            Path::new("/workspace"),
370            Path::new("/workspace"),
371            Path::new("/builds"),
372            "1",
373            &HashMap::from([("CI_PROJECT_DIR".into(), "/workspace".into())]),
374        );
375        let map: HashMap<_, _> = env.into_iter().collect();
376        assert_eq!(
377            map.get("CI_JOB_NAME_SLUG").map(String::as_str),
378            Some("lint")
379        );
380        assert_eq!(
381            map.get("CI_PROJECT_DIR").map(String::as_str),
382            Some("/workspace")
383        );
384        assert_eq!(
385            map.get("CARGO_HOME").map(String::as_str),
386            Some("/workspace/.cargo")
387        );
388    }
389
390    #[test]
391    fn expands_shell_style_default_fallbacks() {
392        let lookup = HashMap::from([
393            ("CI_COMMIT_REF_SLUG".into(), "main".into()),
394            ("CI_ENVIRONMENT_SLUG".into(), "review-main".into()),
395        ]);
396
397        assert_eq!(
398            expand_value("review/${CI_COMMIT_REF_SLUG:-local}", &lookup),
399            "review/main"
400        );
401        assert_eq!(
402            expand_value(
403                "https://${CI_ENVIRONMENT_SLUG:-fallback}.example.com",
404                &lookup
405            ),
406            "https://review-main.example.com"
407        );
408        assert_eq!(
409            expand_value("review/${MISSING_VAR:-local}", &lookup),
410            "review/local"
411        );
412    }
413
414    #[test]
415    fn expands_environment_metadata() {
416        let environment = EnvironmentSpec {
417            name: "review/${CI_COMMIT_REF_SLUG:-local}".into(),
418            url: Some("https://${CI_ENVIRONMENT_SLUG:-fallback}.example.com".into()),
419            on_stop: Some("stop-${CI_COMMIT_REF_SLUG:-local}".into()),
420            auto_stop_in: None,
421            action: EnvironmentActionSpec::Start,
422        };
423        let expanded = expand_environment(
424            &environment,
425            &HashMap::from([
426                ("CI_COMMIT_REF_SLUG".into(), "main".into()),
427                ("CI_ENVIRONMENT_SLUG".into(), "review-main".into()),
428            ]),
429        );
430
431        assert_eq!(expanded.name, "review/main");
432        assert_eq!(
433            expanded.url.as_deref(),
434            Some("https://review-main.example.com")
435        );
436        assert_eq!(expanded.on_stop.as_deref(), Some("stop-main"));
437    }
438
439    #[test]
440    fn infers_tagged_ref_vars_for_job_environment() -> Result<()> {
441        let dir = init_repo_with_commit_and_tag("v1.2.3")?;
442
443        let job = JobSpec {
444            name: "release-artifacts".into(),
445            stage: "release".into(),
446            commands: Vec::new(),
447            needs: Vec::new(),
448            explicit_needs: false,
449            dependencies: Vec::new(),
450            before_script: None,
451            after_script: None,
452            inherit_default_before_script: true,
453            inherit_default_after_script: true,
454            inherit_default_image: true,
455            inherit_default_cache: true,
456            inherit_default_services: true,
457            inherit_default_timeout: true,
458            inherit_default_retry: true,
459            inherit_default_interruptible: true,
460            when: None,
461            rules: Vec::new(),
462            only: Vec::new(),
463            except: Vec::new(),
464            artifacts: ArtifactSpec::default(),
465            cache: Vec::new(),
466            image: None,
467            variables: HashMap::new(),
468            services: Vec::new(),
469            timeout: None,
470            retry: RetryPolicySpec::default(),
471            interruptible: false,
472            resource_group: None,
473            parallel: None,
474            tags: Vec::new(),
475            environment: None,
476        };
477
478        let env = build_job_env(
479            &[],
480            &HashMap::new(),
481            &job,
482            &SecretsStore::default(),
483            dir.path(),
484            Path::new("/workspace"),
485            Path::new("/builds"),
486            "1",
487            &HashMap::new(),
488        );
489        let map: HashMap<_, _> = env.into_iter().collect();
490        assert_eq!(map.get("CI_COMMIT_TAG").map(String::as_str), Some("v1.2.3"));
491        assert_eq!(
492            map.get("CI_COMMIT_REF_NAME").map(String::as_str),
493            Some("v1.2.3")
494        );
495        Ok(())
496    }
497
498    #[test]
499    fn tagged_job_environment_does_not_infer_branch() -> Result<()> {
500        let dir = init_repo_with_commit_and_tag("v1.2.3")?;
501
502        let job = JobSpec {
503            name: "release-artifacts".into(),
504            stage: "release".into(),
505            commands: Vec::new(),
506            needs: Vec::new(),
507            explicit_needs: false,
508            dependencies: Vec::new(),
509            before_script: None,
510            after_script: None,
511            inherit_default_before_script: true,
512            inherit_default_after_script: true,
513            inherit_default_image: true,
514            inherit_default_cache: true,
515            inherit_default_services: true,
516            inherit_default_timeout: true,
517            inherit_default_retry: true,
518            inherit_default_interruptible: true,
519            when: None,
520            rules: Vec::new(),
521            only: Vec::new(),
522            except: Vec::new(),
523            artifacts: ArtifactSpec::default(),
524            cache: Vec::new(),
525            image: None,
526            variables: HashMap::new(),
527            services: Vec::new(),
528            timeout: None,
529            retry: RetryPolicySpec::default(),
530            interruptible: false,
531            resource_group: None,
532            parallel: None,
533            tags: Vec::new(),
534            environment: None,
535        };
536
537        let env = build_job_env(
538            &[],
539            &HashMap::new(),
540            &job,
541            &SecretsStore::default(),
542            dir.path(),
543            Path::new("/workspace"),
544            Path::new("/builds"),
545            "1",
546            &HashMap::from([
547                ("CI_COMMIT_TAG".into(), "v1.2.3".into()),
548                ("CI_COMMIT_REF_NAME".into(), "v1.2.3".into()),
549            ]),
550        );
551        let map: HashMap<_, _> = env.into_iter().collect();
552        assert_eq!(map.get("CI_COMMIT_TAG").map(String::as_str), Some("v1.2.3"));
553        assert!(!map.contains_key("CI_COMMIT_BRANCH"));
554        Ok(())
555    }
556
557    #[test]
558    fn secret_file_env_uses_absolute_container_path() -> Result<()> {
559        let temp_root = temp_path("env-secret-file");
560        let secrets_root = temp_root.join(".opal").join("env");
561        fs::create_dir_all(&secrets_root)?;
562        fs::write(secrets_root.join("API_TOKEN"), "super-secret")?;
563        let secrets = SecretsStore::load(&temp_root)?;
564        let job = JobSpec {
565            name: "lint".into(),
566            stage: "test".into(),
567            commands: Vec::new(),
568            needs: Vec::new(),
569            explicit_needs: false,
570            dependencies: Vec::new(),
571            before_script: None,
572            after_script: None,
573            inherit_default_before_script: true,
574            inherit_default_after_script: true,
575            inherit_default_image: true,
576            inherit_default_cache: true,
577            inherit_default_services: true,
578            inherit_default_timeout: true,
579            inherit_default_retry: true,
580            inherit_default_interruptible: true,
581            when: None,
582            rules: Vec::new(),
583            only: Vec::new(),
584            except: Vec::new(),
585            artifacts: ArtifactSpec::default(),
586            cache: Vec::new(),
587            image: None,
588            variables: HashMap::new(),
589            services: Vec::new(),
590            timeout: None,
591            retry: RetryPolicySpec::default(),
592            interruptible: false,
593            resource_group: None,
594            parallel: None,
595            tags: Vec::new(),
596            environment: None,
597        };
598
599        let env = build_job_env(
600            &[],
601            &HashMap::new(),
602            &job,
603            &secrets,
604            &temp_root,
605            Path::new("/builds/workspace"),
606            Path::new("/builds"),
607            "1",
608            &HashMap::new(),
609        );
610        let map: HashMap<_, _> = env.into_iter().collect();
611        assert_eq!(
612            map.get("API_TOKEN_FILE").map(String::as_str),
613            Some("/opal/secrets/API_TOKEN")
614        );
615
616        let _ = fs::remove_dir_all(temp_root);
617        Ok(())
618    }
619
620    #[test]
621    fn maps_git_commit_tag_into_ci_tag_variables() {
622        let job = JobSpec {
623            name: "release-artifacts".into(),
624            stage: "release".into(),
625            commands: Vec::new(),
626            needs: Vec::new(),
627            explicit_needs: false,
628            dependencies: Vec::new(),
629            before_script: None,
630            after_script: None,
631            inherit_default_before_script: true,
632            inherit_default_after_script: true,
633            inherit_default_image: true,
634            inherit_default_cache: true,
635            inherit_default_services: true,
636            inherit_default_timeout: true,
637            inherit_default_retry: true,
638            inherit_default_interruptible: true,
639            when: None,
640            rules: Vec::new(),
641            only: Vec::new(),
642            except: Vec::new(),
643            artifacts: ArtifactSpec::default(),
644            cache: Vec::new(),
645            image: None,
646            variables: HashMap::new(),
647            services: Vec::new(),
648            timeout: None,
649            retry: RetryPolicySpec::default(),
650            interruptible: false,
651            resource_group: None,
652            parallel: None,
653            tags: Vec::new(),
654            environment: None,
655        };
656
657        let env = build_job_env(
658            &[],
659            &HashMap::new(),
660            &job,
661            &SecretsStore::default(),
662            Path::new("/workspace"),
663            Path::new("/workspace"),
664            Path::new("/builds"),
665            "1",
666            &HashMap::from([("GIT_COMMIT_TAG".into(), "v9.9.9".into())]),
667        );
668        let map: HashMap<_, _> = env.into_iter().collect();
669        assert_eq!(map.get("CI_COMMIT_TAG").map(String::as_str), Some("v9.9.9"));
670        assert_eq!(
671            map.get("CI_COMMIT_REF_NAME").map(String::as_str),
672            Some("v9.9.9")
673        );
674        assert_eq!(
675            map.get("CI_COMMIT_REF_SLUG").map(String::as_str),
676            Some("v999")
677        );
678        assert!(!map.contains_key("CI_COMMIT_BRANCH"));
679    }
680
681    #[test]
682    fn build_job_env_includes_legacy_dotopal_secrets() -> Result<()> {
683        let temp_root = temp_path("env-legacy-secret-file");
684        let dotopal = temp_root.join(".opal");
685        fs::create_dir_all(&dotopal)?;
686        fs::write(dotopal.join("QUAY_USERNAME"), "robot-user")?;
687        let secrets = SecretsStore::load(&temp_root)?;
688        let job = JobSpec {
689            name: "container-release".into(),
690            stage: "publish".into(),
691            commands: Vec::new(),
692            needs: Vec::new(),
693            explicit_needs: false,
694            dependencies: Vec::new(),
695            before_script: None,
696            after_script: None,
697            inherit_default_before_script: true,
698            inherit_default_after_script: true,
699            inherit_default_image: true,
700            inherit_default_cache: true,
701            inherit_default_services: true,
702            inherit_default_timeout: true,
703            inherit_default_retry: true,
704            inherit_default_interruptible: true,
705            when: None,
706            rules: Vec::new(),
707            only: Vec::new(),
708            except: Vec::new(),
709            artifacts: ArtifactSpec::default(),
710            cache: Vec::new(),
711            image: None,
712            variables: HashMap::new(),
713            services: Vec::new(),
714            timeout: None,
715            retry: RetryPolicySpec::default(),
716            interruptible: false,
717            resource_group: None,
718            parallel: None,
719            tags: Vec::new(),
720            environment: None,
721        };
722
723        let env = build_job_env(
724            &[],
725            &HashMap::new(),
726            &job,
727            &secrets,
728            &temp_root,
729            Path::new("/builds/workspace"),
730            Path::new("/builds"),
731            "1",
732            &HashMap::new(),
733        );
734        let map: HashMap<_, _> = env.into_iter().collect();
735        assert_eq!(
736            map.get("QUAY_USERNAME").map(String::as_str),
737            Some("robot-user")
738        );
739
740        let _ = fs::remove_dir_all(temp_root);
741        Ok(())
742    }
743
744    fn temp_path(prefix: &str) -> PathBuf {
745        let nanos = SystemTime::now()
746            .duration_since(UNIX_EPOCH)
747            .expect("system time before epoch")
748            .as_nanos();
749        std::env::temp_dir().join(format!("opal-{prefix}-{nanos}"))
750    }
751}
opal/env.rs

opal/
env.rs
