1pub mod registry;
7
8#[cfg(feature = "secrets-aws-sm")]
9mod aws_sm;
10#[cfg(feature = "secrets-azure-kv")]
11mod azure_kv;
12#[cfg(feature = "secrets-gcp-sm")]
13mod gcp_sm;
14#[cfg(feature = "secrets-vault")]
15mod vault;
16
17use crate::config::PipelineConfig;
18use crate::error::{CliError, CliResult};
19use crate::interpolate::{self, Directive};
20use async_trait::async_trait;
21use futures::stream::{self, StreamExt, TryStreamExt};
22use serde_json::Value;
23use std::collections::{BTreeSet, HashMap};
24use std::sync::Arc;
25
26pub const SECRET_SCHEMES: &[&str] = &["vault", "aws-sm", "gcp-sm", "azure-kv"];
28
29pub type SecretRef = (String, String);
31
32#[async_trait]
33pub trait SecretResolver: Send + Sync {
34 fn scheme(&self) -> &'static str;
36 async fn resolve(&self, reference: &str) -> CliResult<String>;
38}
39
40#[allow(dead_code)] pub(crate) fn split_field(reference: &str) -> (&str, Option<&str>) {
43 match reference.split_once('#') {
44 Some((path, field)) => (path, Some(field)),
45 None => (reference, None),
46 }
47}
48
49#[allow(dead_code)] pub(crate) fn extract_field(
53 scheme: &str,
54 reference: &str,
55 body: &str,
56 field: &str,
57) -> CliResult<String> {
58 let json: Value = serde_json::from_str(body).map_err(|_| CliError::SecretNotJson {
59 scheme: scheme.to_owned(),
60 reference: reference.to_owned(),
61 })?;
62 let obj = json.as_object().ok_or_else(|| CliError::SecretNotJson {
63 scheme: scheme.to_owned(),
64 reference: reference.to_owned(),
65 })?;
66 match obj.get(field) {
67 Some(Value::String(s)) => Ok(s.clone()),
68 Some(other) => Ok(other.to_string()),
69 None => Err(CliError::SecretFieldMissing {
70 scheme: scheme.to_owned(),
71 reference: reference.to_owned(),
72 field: field.to_owned(),
73 available: obj.keys().cloned().collect(),
74 }),
75 }
76}
77
78fn for_each_string<F: FnMut(&str)>(value: &Value, f: &mut F) {
80 match value {
81 Value::String(s) => f(s),
82 Value::Array(a) => a.iter().for_each(|v| for_each_string(v, f)),
83 Value::Object(m) => m.values().for_each(|v| for_each_string(v, f)),
84 _ => {}
85 }
86}
87
88fn for_each_string_mut<F: FnMut(&mut String) -> CliResult<()>>(
90 value: &mut Value,
91 f: &mut F,
92) -> CliResult<()> {
93 match value {
94 Value::String(s) => f(s),
95 Value::Array(a) => a.iter_mut().try_for_each(|v| for_each_string_mut(v, f)),
96 Value::Object(m) => m.values_mut().try_for_each(|v| for_each_string_mut(v, f)),
97 _ => Ok(()),
98 }
99}
100
101fn collect_refs_in_str(s: &str, out: &mut BTreeSet<SecretRef>) {
103 for (_token, dir) in interpolate::iter_directives(s) {
104 if let Directive::LoadTime { prefix, body } = dir
105 && SECRET_SCHEMES.contains(&prefix)
106 {
107 out.insert((prefix.to_owned(), body.to_owned()));
108 }
109 }
110}
111
112pub fn collect_refs(value: &Value, out: &mut BTreeSet<SecretRef>) {
114 for_each_string(value, &mut |s| collect_refs_in_str(s, out));
115}
116
117pub fn substitute(value: &mut Value, cache: &HashMap<SecretRef, String>) -> CliResult<()> {
120 for_each_string_mut(value, &mut |s| {
121 let new = interpolate::rewrite(s, |body| match interpolate::classify_directive(body) {
122 Directive::LoadTime { prefix, body: b } if SECRET_SCHEMES.contains(&prefix) => {
123 Ok(Some(
124 cache
125 .get(&(prefix.to_owned(), b.to_owned()))
126 .cloned()
127 .expect("scan collected every secret ref before fetch"),
128 ))
129 }
130 _ => Ok(None),
131 })?;
132 *s = new;
133 Ok(())
134 })
135}
136
137#[derive(Default, Clone)]
139pub struct ResolverSet {
140 resolvers: HashMap<&'static str, Arc<dyn SecretResolver>>,
141}
142
143impl ResolverSet {
144 pub fn insert(&mut self, resolver: Arc<dyn SecretResolver>) {
145 self.resolvers.insert(resolver.scheme(), resolver);
146 }
147 fn get(&self, scheme: &str) -> Option<&Arc<dyn SecretResolver>> {
148 self.resolvers.get(scheme)
149 }
150}
151
152fn make_resolver(scheme: &str) -> CliResult<Arc<dyn SecretResolver>> {
156 match scheme {
157 #[cfg(feature = "secrets-vault")]
158 "vault" => Ok(Arc::new(vault::VaultResolver::from_env()?)),
159 #[cfg(feature = "secrets-aws-sm")]
160 "aws-sm" => Ok(Arc::new(aws_sm::AwsSmResolver::new())),
161 #[cfg(feature = "secrets-gcp-sm")]
162 "gcp-sm" => Ok(Arc::new(gcp_sm::GcpSmResolver::new())),
163 #[cfg(feature = "secrets-azure-kv")]
164 "azure-kv" => Ok(Arc::new(azure_kv::AzureKvResolver::new())),
165 other => Err(CliError::SecretBackendDisabled {
166 scheme: other.to_owned(),
167 }),
168 }
169}
170
171fn visit_config_values<F: FnMut(&Value)>(cfg: &PipelineConfig, mut f: F) {
174 if let Some(auth) = cfg.auth.as_ref() {
178 for spec in auth.values() {
179 f(spec);
180 }
181 }
182 if let Some(vars) = cfg.vars.as_ref() {
183 for v in vars.values() {
184 f(v);
185 }
186 }
187 for spec in cfg.pipeline.sources.values() {
188 f(&spec.config);
189 }
190 for spec in cfg.pipeline.sinks.values() {
191 f(&spec.config);
192 }
193 if let Some(spec) = cfg.pipeline.source.as_ref() {
194 f(&spec.config);
195 }
196 if let Some(spec) = cfg.pipeline.sink.as_ref() {
197 f(&spec.config);
198 }
199 for t in cfg.pipeline.transforms.iter() {
200 f(&t.config);
201 }
202 if let Some(s) = cfg.pipeline.state.as_ref() {
203 f(&s.config);
204 }
205 if let Some(d) = cfg.pipeline.dlq.as_ref() {
206 f(&d.sink.config);
207 }
208 for row in cfg.matrix.iter() {
209 if let Some(p) = row.source.as_ref()
210 && let Some(c) = p.config.as_ref()
211 {
212 f(c);
213 }
214 if let Some(p) = row.sink.as_ref()
215 && let Some(c) = p.config.as_ref()
216 {
217 f(c);
218 }
219 if let Some(ts) = row.transforms.as_ref() {
220 for t in ts.iter() {
221 f(&t.config);
222 }
223 }
224 if let Some(s) = row.state.as_ref() {
225 f(&s.config);
226 }
227 if let Some(Some(d)) = row.dlq.as_ref() {
228 f(&d.sink.config);
229 }
230 }
231}
232
233fn visit_config_values_mut<F: FnMut(&mut Value) -> CliResult<()>>(
235 cfg: &mut PipelineConfig,
236 mut f: F,
237) -> CliResult<()> {
238 if let Some(auth) = cfg.auth.as_mut() {
241 for spec in auth.values_mut() {
242 f(spec)?;
243 }
244 }
245 if let Some(vars) = cfg.vars.as_mut() {
246 for v in vars.values_mut() {
247 f(v)?;
248 }
249 }
250 for spec in cfg.pipeline.sources.values_mut() {
251 f(&mut spec.config)?;
252 }
253 for spec in cfg.pipeline.sinks.values_mut() {
254 f(&mut spec.config)?;
255 }
256 if let Some(spec) = cfg.pipeline.source.as_mut() {
257 f(&mut spec.config)?;
258 }
259 if let Some(spec) = cfg.pipeline.sink.as_mut() {
260 f(&mut spec.config)?;
261 }
262 for t in cfg.pipeline.transforms.iter_mut() {
263 f(&mut t.config)?;
264 }
265 if let Some(s) = cfg.pipeline.state.as_mut() {
266 f(&mut s.config)?;
267 }
268 if let Some(d) = cfg.pipeline.dlq.as_mut() {
269 f(&mut d.sink.config)?;
270 }
271 for row in cfg.matrix.iter_mut() {
272 if let Some(p) = row.source.as_mut()
273 && let Some(c) = p.config.as_mut()
274 {
275 f(c)?;
276 }
277 if let Some(p) = row.sink.as_mut()
278 && let Some(c) = p.config.as_mut()
279 {
280 f(c)?;
281 }
282 if let Some(ts) = row.transforms.as_mut() {
283 for t in ts.iter_mut() {
284 f(&mut t.config)?;
285 }
286 }
287 if let Some(s) = row.state.as_mut() {
288 f(&mut s.config)?;
289 }
290 if let Some(Some(d)) = row.dlq.as_mut() {
291 f(&mut d.sink.config)?;
292 }
293 }
294 Ok(())
295}
296
297pub(crate) fn scan_config(cfg: &PipelineConfig) -> BTreeSet<SecretRef> {
299 let mut refs = BTreeSet::new();
300 visit_config_values(cfg, |v| collect_refs(v, &mut refs));
301 refs
302}
303
304pub fn scan_path_refs(
306 path: &std::path::Path,
307 profile: Option<&str>,
308) -> CliResult<BTreeSet<SecretRef>> {
309 let cfg = PipelineConfig::from_path_tolerating_secrets(path, profile)?;
310 Ok(scan_config(&cfg))
311}
312
313pub fn ensure_no_secret_directives(cfg: &PipelineConfig) -> CliResult<()> {
316 if scan_config(cfg).is_empty() {
317 Ok(())
318 } else {
319 Err(CliError::SecretsRequireAsyncLoad)
320 }
321}
322
323pub async fn resolve_secrets(cfg: &mut PipelineConfig) -> CliResult<()> {
326 let refs = scan_config(cfg);
327 if refs.is_empty() {
328 return Ok(());
329 }
330 let mut set = ResolverSet::default();
331 let schemes: BTreeSet<&str> = refs.iter().map(|(s, _)| s.as_str()).collect();
332 for scheme in schemes {
333 set.insert(make_resolver(scheme)?);
334 }
335 resolve_secrets_with(cfg, &set).await
336}
337
338pub async fn resolve_secrets_with(cfg: &mut PipelineConfig, set: &ResolverSet) -> CliResult<()> {
341 let refs = scan_config(cfg);
342 if refs.is_empty() {
343 return Ok(());
344 }
345 let cache = fetch_all(&refs, set).await?;
346 visit_config_values_mut(cfg, |v| substitute(v, &cache))
347}
348
349async fn fetch_all(
352 refs: &BTreeSet<SecretRef>,
353 set: &ResolverSet,
354) -> CliResult<HashMap<SecretRef, String>> {
355 const MAX_CONCURRENCY: usize = 8;
356 let pairs: Vec<(SecretRef, String)> =
357 stream::iter(refs.iter().cloned())
358 .map(|(scheme, reference)| async move {
359 let resolver = Arc::clone(set.get(&scheme).ok_or_else(|| {
362 CliError::SecretBackendDisabled {
363 scheme: scheme.clone(),
364 }
365 })?);
366 let value = resolver.resolve(&reference).await?;
367 registry::register(&value);
368 Ok::<(SecretRef, String), CliError>(((scheme, reference), value))
369 })
370 .buffer_unordered(MAX_CONCURRENCY)
371 .try_collect()
372 .await?;
373 Ok(pairs.into_iter().collect())
374}
375
376#[cfg(test)]
377mod tests {
378 use super::*;
379 use serde_json::json;
380
381 #[test]
382 fn collects_unique_refs_and_ignores_other_directives() {
383 let v = json!({
384 "a": "${vault:secret/data/app#token}",
385 "b": "${aws-sm:prod/db#password}",
386 "c": "${vault:secret/data/app#token}",
387 "d": "${users.id}",
388 "e": "${env:HOME}",
389 "nested": ["${gcp-sm:projects/p/secrets/s/versions/latest}"]
390 });
391 let mut refs = BTreeSet::new();
392 collect_refs(&v, &mut refs);
393 assert_eq!(refs.len(), 3);
394 assert!(refs.contains(&("vault".into(), "secret/data/app#token".into())));
395 assert!(refs.contains(&("aws-sm".into(), "prod/db#password".into())));
396 assert!(refs.contains(&(
397 "gcp-sm".into(),
398 "projects/p/secrets/s/versions/latest".into()
399 )));
400 }
401
402 #[test]
403 fn substitutes_from_cache_and_preserves_runtime_refs() {
404 let mut v = json!({
405 "token": "Bearer ${vault:secret/data/app#token}",
406 "path": "/v1/${users.id}"
407 });
408 let mut cache = HashMap::new();
409 cache.insert(
410 ("vault".into(), "secret/data/app#token".into()),
411 "abc123".into(),
412 );
413 substitute(&mut v, &cache).unwrap();
414 assert_eq!(v["token"], "Bearer abc123");
415 assert_eq!(v["path"], "/v1/${users.id}");
416 }
417
418 #[test]
419 fn extract_field_picks_key_or_errors_with_available() {
420 let body = r#"{"username":"u","password":"p"}"#;
421 assert_eq!(
422 extract_field("aws-sm", "ref", body, "password").unwrap(),
423 "p"
424 );
425 match extract_field("aws-sm", "ref", body, "missing").unwrap_err() {
426 CliError::SecretFieldMissing { available, .. } => {
427 assert!(available.contains(&"username".to_string()));
428 }
429 other => panic!("expected SecretFieldMissing, got {other:?}"),
430 }
431 match extract_field("aws-sm", "ref", "not json", "x").unwrap_err() {
432 CliError::SecretNotJson { .. } => {}
433 other => panic!("expected SecretNotJson, got {other:?}"),
434 }
435 }
436
437 struct FakeResolver {
438 scheme: &'static str,
439 value: String,
440 }
441 #[async_trait]
442 impl SecretResolver for FakeResolver {
443 fn scheme(&self) -> &'static str {
444 self.scheme
445 }
446 async fn resolve(&self, _reference: &str) -> CliResult<String> {
447 Ok(self.value.clone())
448 }
449 }
450
451 #[tokio::test]
452 async fn resolve_secrets_with_substitutes_via_injected_resolvers() {
453 let mut set = ResolverSet::default();
454 set.insert(Arc::new(FakeResolver {
455 scheme: "vault",
456 value: "RESOLVED".into(),
457 }));
458 let cfg_yaml = r#"
459version: 1
460pipeline:
461 source: { type: rest, config: { base_url: https://x, auth: { type: bearer, config: { token: "${vault:secret/data/app#token}" } } } }
462 sink: { type: jsonl, config: { path: ./o.jsonl } }
463"#;
464 let mut cfg = PipelineConfig::from_text(cfg_yaml, std::path::Path::new("p.yaml")).unwrap();
465 resolve_secrets_with(&mut cfg, &set).await.unwrap();
466 let token = &cfg.pipeline.source.as_ref().unwrap().config["auth"]["config"]["token"];
467 assert_eq!(token, "RESOLVED");
468 }
469
470 #[tokio::test]
471 async fn resolve_secrets_resolves_auth_catalog_and_vars_block() {
472 let mut set = ResolverSet::default();
476 set.insert(Arc::new(FakeResolver {
477 scheme: "vault",
478 value: "RESOLVED".into(),
479 }));
480 let cfg_yaml = r#"
481version: 1
482vars:
483 shared_token: "${vault:secret/data/app#token}"
484auth:
485 idp: { type: static, config: { token: "${vault:secret/data/idp#token}" } }
486pipeline:
487 source: { type: rest, config: { base_url: https://x, auth: { ref: idp } } }
488 sink: { type: jsonl, config: { path: ./o.jsonl } }
489"#;
490 let mut cfg = PipelineConfig::from_text(cfg_yaml, std::path::Path::new("p.yaml")).unwrap();
491 resolve_secrets_with(&mut cfg, &set).await.unwrap();
492
493 let auth_token = &cfg.auth.as_ref().unwrap()["idp"]["config"]["token"];
494 assert_eq!(auth_token, "RESOLVED", "auth-catalog secret should resolve");
495
496 let var_value = &cfg.vars.as_ref().unwrap()["shared_token"];
497 assert_eq!(var_value, "RESOLVED", "vars-block secret should resolve");
498 }
499
500 #[tokio::test]
501 async fn scan_config_collects_refs_from_auth_and_vars() {
502 let cfg_yaml = r#"
505version: 1
506vars:
507 v: "${aws-sm:prod/api#key}"
508auth:
509 idp: { type: static, config: { token: "${vault:secret/data/idp#token}" } }
510pipeline:
511 source: { type: rest, config: { base_url: https://x, auth: { ref: idp } } }
512 sink: { type: jsonl, config: { path: ./o.jsonl } }
513"#;
514 let cfg = PipelineConfig::from_text(cfg_yaml, std::path::Path::new("p.yaml")).unwrap();
515 let refs = scan_config(&cfg);
516 assert!(refs.contains(&("vault".into(), "secret/data/idp#token".into())));
517 assert!(refs.contains(&("aws-sm".into(), "prod/api#key".into())));
518 }
519
520 #[tokio::test]
521 async fn resolve_secrets_errors_when_backend_not_built() {
522 let set = ResolverSet::default();
523 let cfg_yaml = r#"
524version: 1
525pipeline:
526 source: { type: rest, config: { url: "${vault:secret/x}" } }
527 sink: { type: jsonl, config: { path: ./o.jsonl } }
528"#;
529 let mut cfg = PipelineConfig::from_text(cfg_yaml, std::path::Path::new("p.yaml")).unwrap();
530 match resolve_secrets_with(&mut cfg, &set).await.unwrap_err() {
531 CliError::SecretBackendDisabled { scheme } => assert_eq!(scheme, "vault"),
532 other => panic!("expected SecretBackendDisabled, got {other:?}"),
533 }
534 }
535
536 #[test]
537 fn make_resolver_rejects_unknown_scheme() {
538 match make_resolver("not-a-scheme") {
539 Err(CliError::SecretBackendDisabled { scheme }) => assert_eq!(scheme, "not-a-scheme"),
540 Err(other) => panic!("expected SecretBackendDisabled, got {other:?}"),
541 Ok(_) => panic!("expected SecretBackendDisabled for an unknown scheme"),
542 }
543 }
544
545 #[cfg(feature = "secrets-aws-sm")]
546 #[test]
547 fn make_resolver_builds_compiled_in_aws_backend() {
548 let r = make_resolver("aws-sm").unwrap();
551 assert_eq!(r.scheme(), "aws-sm");
552 }
553
554 #[tokio::test]
555 async fn resolve_secrets_walks_matrix_row_state_dlq_and_transforms() {
556 let mut set = ResolverSet::default();
559 set.insert(Arc::new(FakeResolver {
560 scheme: "vault",
561 value: "R".into(),
562 }));
563 let cfg_yaml = r#"
564version: 1
565pipeline:
566 source: { type: csv, config: { path: ./in.csv } }
567 sink: { type: jsonl, config: { path: ./o.jsonl } }
568matrix:
569 - id: row1
570 source: { config: { path: "${vault:secret/src}" } }
571 sink: { config: { path: "${vault:secret/sink}" } }
572 state: { type: file, config: { path: "${vault:secret/state}" } }
573 transforms:
574 - type: set
575 config: { field: tag, value: "${vault:secret/tf}" }
576 dlq:
577 sink: { type: jsonl, config: { path: "${vault:secret/dlq}" } }
578"#;
579 let mut cfg = PipelineConfig::from_text(cfg_yaml, std::path::Path::new("p.yaml")).unwrap();
580 resolve_secrets_with(&mut cfg, &set).await.unwrap();
581
582 let row = &cfg.matrix[0];
583 assert_eq!(
584 row.source.as_ref().unwrap().config.as_ref().unwrap()["path"],
585 "R"
586 );
587 assert_eq!(
588 row.sink.as_ref().unwrap().config.as_ref().unwrap()["path"],
589 "R"
590 );
591 assert_eq!(row.state.as_ref().unwrap().config["path"], "R");
592 assert_eq!(row.transforms.as_ref().unwrap()[0].config["value"], "R");
593 let dlq = row.dlq.as_ref().unwrap().as_ref().unwrap();
594 assert_eq!(dlq.sink.config["path"], "R");
595 }
596
597 #[test]
598 fn scan_config_collects_refs_from_matrix_row_state_and_dlq() {
599 let cfg_yaml = r#"
601version: 1
602pipeline:
603 source: { type: csv, config: { path: ./in.csv } }
604 sink: { type: jsonl, config: { path: ./o.jsonl } }
605matrix:
606 - id: r
607 state: { type: file, config: { path: "${vault:secret/state}" } }
608 transforms:
609 - type: set
610 config: { field: t, value: "${aws-sm:tf/key}" }
611 dlq:
612 sink: { type: jsonl, config: { path: "${gcp-sm:projects/p/secrets/s/versions/1}" } }
613"#;
614 let cfg = PipelineConfig::from_text(cfg_yaml, std::path::Path::new("p.yaml")).unwrap();
615 let refs = scan_config(&cfg);
616 assert!(refs.contains(&("vault".into(), "secret/state".into())));
617 assert!(refs.contains(&("aws-sm".into(), "tf/key".into())));
618 assert!(refs.contains(&("gcp-sm".into(), "projects/p/secrets/s/versions/1".into())));
619 }
620
621 #[tokio::test]
622 async fn resolve_secrets_noop_when_no_directives() {
623 let cfg_yaml = r#"
626version: 1
627pipeline:
628 source: { type: csv, config: { path: ./in.csv } }
629 sink: { type: jsonl, config: { path: ./o.jsonl } }
630"#;
631 let mut cfg = PipelineConfig::from_text(cfg_yaml, std::path::Path::new("p.yaml")).unwrap();
632 resolve_secrets(&mut cfg).await.unwrap();
633 }
634
635 #[test]
636 fn split_field_splits_on_hash() {
637 assert_eq!(split_field("a/b#c"), ("a/b", Some("c")));
638 assert_eq!(split_field("a/b"), ("a/b", None));
639 }
640
641 #[test]
642 fn ensure_no_secret_directives_passes_when_clean() {
643 let cfg_yaml = r#"
644version: 1
645pipeline:
646 source: { type: csv, config: { path: ./in.csv } }
647 sink: { type: jsonl, config: { path: ./o.jsonl } }
648"#;
649 let cfg = PipelineConfig::from_text(cfg_yaml, std::path::Path::new("p.yaml")).unwrap();
650 assert!(ensure_no_secret_directives(&cfg).is_ok());
651 }
652
653 #[test]
654 fn ensure_no_secret_directives_flags_vault() {
655 let cfg_yaml = r#"
656version: 1
657pipeline:
658 source: { type: rest, config: { url: "${vault:secret/x}" } }
659 sink: { type: jsonl, config: { path: ./o.jsonl } }
660"#;
661 let cfg = PipelineConfig::from_text(cfg_yaml, std::path::Path::new("p.yaml")).unwrap();
662 assert!(matches!(
663 ensure_no_secret_directives(&cfg),
664 Err(CliError::SecretsRequireAsyncLoad)
665 ));
666 }
667}