dre_core/
profiles.rs

1//! `profiles.yml`: connection definitions, kept outside the project.
2//!
3//! Two sections keep database connections apart from delivery targets:
4//!
5//! ```yaml
6//! connections:
7//!   warehouse:
8//!     targets:
9//!       dev: {type: duckdb, path: dev.duckdb}
10//! destinations:
11//!   client_sftp:
12//!     targets:
13//!       dev: {type: sftp, host: sftp.example.com}
14//! ```
15//!
16//! A connection profile is referenced by `default_profile`/`profile:` (and a source's
17//! `profile:`), a destination profile by `output.destination.profile`; each is looked up in its
18//! own section only. Each profile lists one entry per target (environment). Each profile a run
19//! uses picks its entry by `--target`, else `DRE_TARGET` (either sets every profile), else the
20//! profile's own `target:`, else `dev`. A used profile without that entry is an error; a
21//! destination entry `{deliver: false}` deliberately delivers nowhere.
22//!
23//! DRE 0.1 called the connections section `sources:`; it still loads in 0.2.x, with a warning.
24//!
25//! Location: `--profiles-dir` > `DRE_PROFILES_DIR` > `~/.dre`, one file, never merged. These are
26//! read directly at startup, never through the `env_var()` Jinja function. `env_var()` calls
27//! *inside* the file are rendered at run time, just before a connection config is handed to a
28//! plugin.
29
30use std::collections::BTreeMap;
31use std::path::{Path, PathBuf};
32
33use schemars::JsonSchema;
34use serde::{Deserialize, Serialize};
35
36use crate::codes::Code;
37use crate::config::de::{self, Found, Located, Loose, Map, UnknownKeys};
38use crate::config::node;
39use crate::diag::Diagnostics;
40
41pub const PROFILES_FILE: &str = "profiles.yml";
42
43/// The built-in destination type: copying bytes to a local path needs no plugin (ADR 0003).
44/// `profile: local` works without a profiles.yml entry; defining a `local` profile overrides it.
45pub const LOCAL_TYPE: &str = "local";
46
47/// What `profile: local` resolves to when profiles.yml doesn't define it, for every target.
48pub static BUILTIN_LOCAL: std::sync::LazyLock<ProfileTarget> = std::sync::LazyLock::new(|| ProfileTarget {
49    kind: LOCAL_TYPE.into(),
50    fields: serde_json::Map::new(),
51});
52
53/// The section DRE 0.1 kept connections under; read with a warning in 0.2.x.
54pub const OLD_CONNECTIONS_SECTION: &str = "sources";
55
56/// The run's target (environment), and a profile's entry, when nothing chooses one.
57pub const DEFAULT_TARGET: &str = "dev";
58/// The environment variable choosing the run's target, below `--target`.
59pub const TARGET_ENV: &str = crate::settings::TARGET;
60
61/// Which section of profiles.yml a profile lives in.
62#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
63pub enum Role {
64    Connection,
65    Destination,
66}
67
68impl Role {
69    pub fn as_str(self) -> &'static str {
70        match self {
71            Role::Connection => "connection",
72            Role::Destination => "destination",
73        }
74    }
75    /// The profiles.yml section key.
76    pub fn section(self) -> &'static str {
77        match self {
78            Role::Connection => "connections",
79            Role::Destination => "destinations",
80        }
81    }
82}
83
84/// Where the run's target came from.
85#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
86pub enum TargetSource {
87    Flag,
88    Env,
89    #[default]
90    Default,
91}
92
93impl std::fmt::Display for TargetSource {
94    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
95        f.write_str(match self {
96            TargetSource::Flag => "--target",
97            TargetSource::Env => TARGET_ENV,
98            TargetSource::Default => "default",
99        })
100    }
101}
102
103/// The run's target: `--target`, else `DRE_TARGET`, else `dev`. It's `target.name` in templates.
104#[derive(Debug, Clone, PartialEq, Eq)]
105pub struct RunTarget {
106    pub name: String,
107    pub from: TargetSource,
108}
109
110impl Default for RunTarget {
111    fn default() -> Self {
112        RunTarget {
113            name: DEFAULT_TARGET.into(),
114            from: TargetSource::Default,
115        }
116    }
117}
118
119impl RunTarget {
120    /// `--target` (`flag`), else `DRE_TARGET`, else `dev`.
121    pub fn resolve(flag: Option<&str>) -> RunTarget {
122        if let Some(t) = flag.filter(|t| !t.is_empty()) {
123            return RunTarget {
124                name: t.to_string(),
125                from: TargetSource::Flag,
126            };
127        }
128        if let Some(t) = crate::settings::env(TARGET_ENV) {
129            return RunTarget {
130                name: t,
131                from: TargetSource::Env,
132            };
133        }
134        RunTarget::default()
135    }
136
137    /// Whether `--target` or `DRE_TARGET` chose it, which sets every profile's entry.
138    pub fn chosen(&self) -> bool {
139        self.from != TargetSource::Default
140    }
141}
142
143/// What a profile uses for the run: its entry, or why it has none.
144#[derive(Debug, Clone, Copy)]
145pub enum Entry<'a> {
146    Use(&'a ProfileTarget),
147    /// `{deliver: false}`: a destination that deliberately delivers nowhere on this target.
148    Nowhere,
149    /// The profile has no entry for its target.
150    Missing,
151    /// No such profile.
152    Unknown,
153}
154
155/// One environment of a profile: a plugin type and its connection fields.
156#[derive(Debug, Clone, Serialize)]
157pub struct ProfileTarget {
158    #[serde(rename = "type")]
159    pub kind: String,
160    /// Every other field, passed through to the plugin unvalidated by core.
161    #[serde(skip)]
162    pub fields: serde_json::Map<String, serde_json::Value>,
163}
164
165#[derive(Debug, Clone, Serialize)]
166pub struct Profile {
167    /// Its own default entry (`target:`), below `--target` and `DRE_TARGET`.
168    #[serde(skip_serializing_if = "Option::is_none")]
169    pub target: Option<String>,
170    /// The entries that connect or deliver.
171    pub targets: BTreeMap<String, ProfileTarget>,
172    /// The destination entries written `{deliver: false}`.
173    #[serde(skip_serializing_if = "std::collections::BTreeSet::is_empty")]
174    pub nowhere: std::collections::BTreeSet<String>,
175}
176
177impl Profile {
178    /// Every entry's name, delivering or not.
179    pub fn entry_names(&self) -> Vec<String> {
180        let mut names: Vec<String> = self.targets.keys().chain(&self.nowhere).cloned().collect();
181        names.sort();
182        names
183    }
184}
185
186#[derive(Debug, Clone, Default)]
187pub struct Profiles {
188    /// Where DRE looked for the file.
189    pub path: PathBuf,
190    /// Why that directory: `--profiles-dir`, `DRE_PROFILES_DIR`, `the project directory` or `~/.dre`.
191    pub found_by: &'static str,
192    /// The file as shown in diagnostics; `None` when it doesn't exist. Loading problems are
193    /// reported when it's needed.
194    pub file: Option<PathBuf>,
195    /// Every profile each section declares, broken ones too, with its name's line.
196    declared: BTreeMap<Role, BTreeMap<String, usize>>,
197    pub connections: BTreeMap<String, Profile>,
198    pub destinations: BTreeMap<String, Profile>,
199    /// The section connections were read from: `connections`, or 0.1's `sources`.
200    connections_section: &'static str,
201    /// The run's target, which picks each profile's entry (see [`Profiles::target_of`]).
202    pub run: RunTarget,
203}
204
205/// Resolve the profiles directory: `--profiles-dir` > `DRE_PROFILES_DIR` > `~/.dre`.
206/// `dre init` writes here; projects look in their own directory first (see [`locate`]).
207pub fn profiles_dir(cli: Option<&Path>) -> PathBuf {
208    locate(cli, None).0
209}
210
211/// Find the profiles directory for a project, with the reason it was chosen:
212/// `--profiles-dir` > `DRE_PROFILES_DIR` > the project directory (when it holds a
213/// profiles.yml) > `~/.dre`. With the default `--project-dir .` this is dbt's order.
214pub fn locate(cli: Option<&Path>, project: Option<&Path>) -> (PathBuf, &'static str) {
215    if let Some(p) = cli {
216        return (p.to_path_buf(), "--profiles-dir");
217    }
218    if let Some(p) = crate::settings::env(crate::settings::PROFILES_DIR) {
219        return (PathBuf::from(p), "DRE_PROFILES_DIR");
220    }
221    if let Some(p) = project.filter(|p| p.join(PROFILES_FILE).is_file()) {
222        return (p.to_path_buf(), "the project directory");
223    }
224    (crate::dre_home(), "~/.dre")
225}
226
227impl Profiles {
228    pub fn load(dir: &Path, found_by: &'static str, diags: &mut Diagnostics) -> Profiles {
229        let path = crate::slash(&dir.join(PROFILES_FILE));
230        let mut out = Profiles {
231            path: path.clone(),
232            found_by,
233            ..Default::default()
234        };
235        if !path.is_file() {
236            return out;
237        }
238        // Parsed badly or not, the file exists: references to it don't pile up extra errors.
239        out.file = Some(path.clone());
240        let text = match std::fs::read_to_string(&path) {
241            Ok(t) => t,
242            Err(e) => {
243                diags.error(Code::IoError, Some(path), None, format!("cannot read file: {e}"));
244                return out;
245            }
246        };
247        let tree = match node::parse(&text) {
248            Ok(n) => n,
249            Err(e) => {
250                diags.error(
251                    Code::YamlSyntax,
252                    Some(path),
253                    e.line,
254                    format!("invalid YAML: {}", e.message),
255                );
256                return out;
257            }
258        };
259        let file = Some(path.clone());
260        let raw: Loose<ProfilesFile> = match de::from_node(&tree) {
261            Ok(r) => r,
262            Err(e) => {
263                diags.error(Code::InvalidProfiles, file, e.line, e.message);
264                return out;
265            }
266        };
267        let raw = match raw {
268            Loose::Ok(r) => r,
269            Loose::Bad(Found { kind: "nothing", .. }) => return out,
270            Loose::Bad(_) => {
271                diags.error(
272                    Code::InvalidProfiles,
273                    file,
274                    None,
275                    "profiles.yml must be a map with `connections:` and/or `destinations:`",
276                );
277                return out;
278            }
279        };
280        for k in &raw.unknown.0 {
281            diags.error(
282                Code::InvalidProfiles,
283                file.clone(),
284                Some(k.line),
285                format!(
286                    "unknown profiles.yml section `{}`; profiles go under `connections:` or `destinations:`",
287                    k.name
288                ),
289            );
290        }
291        let connections = match (raw.connections, raw.sources) {
292            (Some(c), Some(s)) => {
293                diags.error(
294                    Code::InvalidProfiles,
295                    file.clone(),
296                    s.line(),
297                    "profiles.yml has both `connections:` and `sources:`; `sources:` is the old name of `connections:`, so move its profiles under `connections:`",
298                );
299                Some(("connections", c))
300            }
301            (Some(c), None) => Some(("connections", c)),
302            (None, Some(s)) => {
303                diags.warning(
304                    Code::ProfilesSourcesRenamed,
305                    file.clone(),
306                    s.line(),
307                    "`sources:` in profiles.yml is now `connections:` (DRE 0.2); rename it. `sources:` still works in 0.2.x",
308                );
309                out.connections_section = OLD_CONNECTIONS_SECTION;
310                Some((OLD_CONNECTIONS_SECTION, s))
311            }
312            (None, None) => None,
313        };
314        if let Some((key, section)) = connections {
315            let (parsed, declared) = parse_section(Role::Connection, key, section, &path, diags);
316            out.connections = parsed;
317            out.declared.insert(Role::Connection, declared);
318        }
319        if let Some(section) = raw.destinations {
320            let (parsed, declared) = parse_section(Role::Destination, "destinations", section, &path, diags);
321            out.destinations = parsed;
322            out.declared.insert(Role::Destination, declared);
323        }
324        // Mask what secret fields read through env_var(); a plugin's own declarations are
325        // added when a run asks it (`run::ProfileConnections::view`).
326        for profile in out.connections.values().chain(out.destinations.values()) {
327            for target in profile.targets.values() {
328                crate::secrets::register_secret_fields(&target.fields, &[]);
329            }
330        }
331        out
332    }
333
334    pub fn exists(&self) -> bool {
335        self.file.is_some()
336    }
337
338    fn section(&self, role: Role) -> &BTreeMap<String, Profile> {
339        match role {
340            Role::Connection => &self.connections,
341            Role::Destination => &self.destinations,
342        }
343    }
344
345    /// The section key a role's profiles were read from (`sources` for an 0.1 file).
346    pub fn section_key(&self, role: Role) -> &'static str {
347        match role {
348            Role::Connection if self.connections_section == OLD_CONNECTIONS_SECTION => {
349                OLD_CONNECTIONS_SECTION
350            }
351            r => r.section(),
352        }
353    }
354
355    pub fn get(&self, role: Role, name: &str) -> Option<&Profile> {
356        self.section(role).get(name)
357    }
358
359    /// Whether a (possibly partially broken) profile of this name is declared in its section.
360    pub fn declares(&self, role: Role, name: &str) -> bool {
361        self.declared.get(&role).is_some_and(|s| s.contains_key(name))
362    }
363
364    /// Whether `name` means the built-in local destination (no `local` profile is defined).
365    pub fn is_builtin_local(&self, name: &str) -> bool {
366        name == LOCAL_TYPE && !self.destinations.contains_key(name)
367    }
368
369    /// The line of a profile's name in the file.
370    pub fn line_of(&self, role: Role, name: &str) -> Option<usize> {
371        self.declared.get(&role)?.get(name).copied()
372    }
373
374    /// The entry a profile uses in this run: the run's target when `--target` or `DRE_TARGET`
375    /// chose it, else the profile's own `target:`, else `dev`.
376    pub fn target_of(&self, role: Role, profile: &str) -> String {
377        if self.run.chosen() {
378            return self.run.name.clone();
379        }
380        self.get(role, profile)
381            .and_then(|p| p.target.clone())
382            .unwrap_or_else(|| self.run.name.clone())
383    }
384
385    /// What a profile uses in this run. `profile: local` without a defined `local` profile is
386    /// the built-in local destination, for every target.
387    pub fn entry(&self, role: Role, profile: &str) -> Entry<'_> {
388        if role == Role::Destination && self.is_builtin_local(profile) {
389            return Entry::Use(&BUILTIN_LOCAL);
390        }
391        let Some(p) = self.get(role, profile) else {
392            return Entry::Unknown;
393        };
394        let t = self.target_of(role, profile);
395        match p.targets.get(&t) {
396            Some(o) => Entry::Use(o),
397            None if p.nowhere.contains(&t) => Entry::Nowhere,
398            None => Entry::Missing,
399        }
400    }
401
402    /// A profile's settings for this run, when it has an entry that connects or delivers.
403    pub fn target(&self, role: Role, profile: &str) -> Option<&ProfileTarget> {
404        match self.entry(role, profile) {
405            Entry::Use(o) => Some(o),
406            _ => None,
407        }
408    }
409
410    /// Why a used profile has no entry for this run, with the way to fix it.
411    pub fn missing_entry(&self, role: Role, profile: &str) -> String {
412        let t = self.target_of(role, profile);
413        let has = self
414            .get(role, profile)
415            .map(|p| p.entry_names().join(", "))
416            .unwrap_or_default();
417        let why = match self.run.from {
418            TargetSource::Default if self.get(role, profile).is_some_and(|p| p.target.is_some()) => {
419                "its `target:`".to_string()
420            }
421            TargetSource::Default => "the default".to_string(),
422            from => from.to_string(),
423        };
424        let mut msg = format!(
425            "{} `{profile}` has no `{t}` entry (it has: {has}); `{t}` comes from {why}",
426            role.as_str()
427        );
428        if role == Role::Destination {
429            msg.push_str(&format!(
430                ". To deliver nowhere on `{t}`, add `{t}: {{deliver: false}}` to its `targets:`"
431            ));
432        }
433        msg
434    }
435}
436
437/// A section's profiles, and every profile name it declares with its line.
438fn parse_section<P: Into<RawProfile>>(
439    role: Role,
440    key: &str,
441    section: Section<P>,
442    path: &Path,
443    diags: &mut Diagnostics,
444) -> (BTreeMap<String, Profile>, BTreeMap<String, usize>) {
445    let Loose::Ok(profiles) = section.value else {
446        diags.error(
447            Code::InvalidProfiles,
448            Some(path.to_path_buf()),
449            section.line(),
450            format!("`{key}` must be a map of profile names"),
451        );
452        return Default::default();
453    };
454    let declared = profiles.iter().map(|(k, _)| (k.value.clone(), k.line)).collect();
455    let parsed = profiles
456        .0
457        .into_iter()
458        .filter_map(|(name, p)| {
459            parse_profile(role, &name, p.map(|p| p.map(Into::into)), path, diags).map(|p| (name.value, p))
460        })
461        .collect();
462    (parsed, declared)
463}
464
465fn parse_profile(
466    role: Role,
467    name: &Located<String>,
468    v: Located<Loose<RawProfile>>,
469    path: &Path,
470    diags: &mut Diagnostics,
471) -> Option<Profile> {
472    let file = Some(path.to_path_buf());
473    let line = name.line();
474    let what = format!("{} profile `{}`", role.as_str(), name.value);
475    let Loose::Ok(m) = v.value else {
476        diags.error(
477            Code::InvalidProfile,
478            file,
479            line,
480            format!("{what} must be a map with `targets`"),
481        );
482        return None;
483    };
484    let target_line = m.target.as_ref().and_then(Located::line);
485    let own_target = match m.target {
486        None => None,
487        Some(Located {
488            value: Loose::Ok(t), ..
489        }) if !t.trim().is_empty() => Some(t),
490        Some(t) => {
491            diags.error(
492                Code::InvalidProfile,
493                file.clone(),
494                t.line(),
495                format!("{what}: `target` must be the name of one of its `targets`"),
496            );
497            return None;
498        }
499    };
500    let Some(Loose::Ok(targets)) = m.targets.map(|t| t.value) else {
501        let hint = if m.outputs.is_some() {
502            " (`outputs:` is now `targets:`)"
503        } else {
504            ""
505        };
506        diags.error(
507            Code::InvalidProfile,
508            file,
509            line,
510            format!("{what} needs a map of named `targets`{hint}"),
511        );
512        return None;
513    };
514    let mut parsed = BTreeMap::new();
515    let mut nowhere = std::collections::BTreeSet::new();
516    let mut ok = true;
517    for (tname, o) in targets.0 {
518        let tline = tname.line();
519        let tname = tname.value;
520        let o = o.value;
521        if let Some(d) = o.get("deliver") {
522            let problem = if role == Role::Connection {
523                Some("`deliver: false` is only for destinations; a connection entry needs a `type`")
524            } else if d != &serde_json::Value::Bool(false) {
525                Some("`deliver` can only be `false` (an entry that delivers just has a `type`)")
526            } else if o.as_object().is_some_and(|m| m.len() > 1) {
527                Some("`deliver: false` takes no other settings: the entry delivers nowhere")
528            } else {
529                None
530            };
531            match problem {
532                Some(p) => {
533                    diags.error(
534                        Code::InvalidProfile,
535                        file.clone(),
536                        tline,
537                        format!("target `{tname}` of {what}: {p}"),
538                    );
539                    ok = false;
540                }
541                None => {
542                    nowhere.insert(tname);
543                }
544            }
545            continue;
546        }
547        let Some(kind) = o.get("type").and_then(serde_json::Value::as_str) else {
548            diags.error(
549                Code::InvalidProfile,
550                file.clone(),
551                tline,
552                format!("target `{tname}` of {what} has no `type`"),
553            );
554            ok = false;
555            continue;
556        };
557        let kind = kind.to_string();
558        let fields = match o {
559            serde_json::Value::Object(mut f) => {
560                f.remove("type");
561                f
562            }
563            _ => serde_json::Map::new(),
564        };
565        parsed.insert(tname, ProfileTarget { kind, fields });
566    }
567    if let Some(t) = &own_target
568        && ok
569        && !parsed.contains_key(t)
570        && !nowhere.contains(t)
571    {
572        let mut has: Vec<&String> = parsed.keys().chain(&nowhere).collect();
573        has.sort();
574        diags.error(
575            Code::InvalidProfile,
576            file.clone(),
577            target_line,
578            format!(
579                "{what}: `target: {t}` isn't one of its targets ({})",
580                has.into_iter().cloned().collect::<Vec<_>>().join(", ")
581            ),
582        );
583        return None;
584    }
585    ok.then_some(Profile {
586        target: own_target,
587        targets: parsed,
588        nowhere,
589    })
590}
591
592// The file's shape. These types are what's read, and they generate profiles.schema.json; their
593// doc comments are its descriptions. Values the loader checks itself, to say what's wrong in
594// DRE's words, are `Loose`.
595
596/// A section of profiles by name, each checked by the loader.
597type Section<P> = Located<Loose<Map<Located<Loose<P>>>>>;
598
599/// Connections and destinations, in `profiles.yml`: what reports read from and where outputs go. Kept outside the project (`~/.dre`, `--profiles-dir` or `DRE_PROFILES_DIR`). Every profile lists its targets (environments). Each profile the run uses picks one: `--target`, else `DRE_TARGET` (either sets every profile), else the profile's own `target`, else `dev`; a used profile without that entry is an error.
600#[derive(Deserialize, JsonSchema)]
601#[schemars(title = "DRE profiles", deny_unknown_fields)]
602#[schemars(extend("not" = {"required": ["connections", "sources"]}))]
603pub struct ProfilesFile {
604    /// Database connections, referenced by `default_profile`, `profile:` (report, query, Set, folder `+profile`) and a source's `profile`.
605    connections: Option<Section<ConnectionProfile>>,
606    /// DRE 0.1's name for `connections:`. Still read in 0.2.x, with a warning: rename it to `connections:`.
607    #[schemars(extend("deprecated" = true))]
608    sources: Option<Section<ConnectionProfile>>,
609    /// Delivery targets, referenced by `output.destination.profile`.
610    destinations: Option<Section<DestinationProfile>>,
611    #[serde(rename = "$unknown", default)]
612    #[schemars(skip)]
613    unknown: UnknownKeys,
614}
615
616/// A named connection with one entry per target (environment).
617#[derive(Deserialize, JsonSchema)]
618#[schemars(rename = "connection", deny_unknown_fields)]
619pub struct ConnectionProfile {
620    /// This profile's entry when neither `--target` nor `DRE_TARGET` is set (dbt's key). Default: `dev`. It doesn't change `target.name`, the run's target.
621    target: Option<Located<Loose<String>>>,
622    /// The environments of this connection, by name (e.g. `dev`, `prod`).
623    #[schemars(required, extend("minProperties" = 1))]
624    targets: Option<Located<Loose<Map<Located<ConnectionEntry>>>>>,
625    /// DRE 0.1's name for `targets`, read only to say so.
626    #[schemars(skip)]
627    outputs: Option<serde::de::IgnoredAny>,
628}
629
630/// A named destination with one entry per target (environment).
631#[derive(Deserialize, JsonSchema)]
632#[schemars(rename = "destination", deny_unknown_fields)]
633pub struct DestinationProfile {
634    /// The environments of this destination, by name (e.g. `dev`, `prod`). An entry is a delivery configuration, or `{deliver: false}` to deliver nowhere on that target.
635    #[schemars(required, extend("minProperties" = 1))]
636    targets: Option<Located<Loose<Map<Located<DestinationEntry>>>>>,
637    /// This profile's entry when neither `--target` nor `DRE_TARGET` is set (dbt's key). Default: `dev`. It doesn't change `target.name`, the run's target.
638    target: Option<Located<Loose<String>>>,
639    #[schemars(skip)]
640    outputs: Option<serde::de::IgnoredAny>,
641}
642
643/// Either kind of profile, as the loader checks it.
644pub struct RawProfile {
645    target: Option<Located<Loose<String>>>,
646    targets: Option<Located<Loose<Map<Located<serde_json::Value>>>>>,
647    outputs: Option<serde::de::IgnoredAny>,
648}
649
650fn entries<E: Into<serde_json::Value>>(
651    t: Option<Located<Loose<Map<Located<E>>>>>,
652) -> Option<Located<Loose<Map<Located<serde_json::Value>>>>> {
653    t.map(|t| {
654        t.map(|t| match t {
655            Loose::Ok(m) => Loose::Ok(Map(m
656                .0
657                .into_iter()
658                .map(|(k, v)| (k, v.map(Into::into)))
659                .collect())),
660            Loose::Bad(f) => Loose::Bad(f),
661        })
662    })
663}
664
665impl From<ConnectionProfile> for RawProfile {
666    fn from(p: ConnectionProfile) -> RawProfile {
667        RawProfile {
668            target: p.target,
669            targets: entries(p.targets),
670            outputs: p.outputs,
671        }
672    }
673}
674
675impl From<DestinationProfile> for RawProfile {
676    fn from(p: DestinationProfile) -> RawProfile {
677        RawProfile {
678            target: p.target,
679            targets: entries(p.targets),
680            outputs: p.outputs,
681        }
682    }
683}
684
685/// A connection's target: `type` and the plugin's fields, passed on unchecked.
686#[derive(Deserialize)]
687#[serde(transparent)]
688pub struct ConnectionEntry(serde_json::Value);
689
690/// A destination's target: a delivery configuration, or `{deliver: false}`.
691#[derive(Deserialize)]
692#[serde(transparent)]
693pub struct DestinationEntry(serde_json::Value);
694
695impl From<ConnectionEntry> for serde_json::Value {
696    fn from(e: ConnectionEntry) -> Self {
697        e.0
698    }
699}
700
701impl From<DestinationEntry> for serde_json::Value {
702    fn from(e: DestinationEntry) -> Self {
703        e.0
704    }
705}
706
707impl JsonSchema for ConnectionEntry {
708    fn schema_name() -> std::borrow::Cow<'static, str> {
709        "ConnectionEntry".into()
710    }
711    fn inline_schema() -> bool {
712        true
713    }
714    fn json_schema(g: &mut schemars::SchemaGenerator) -> schemars::Schema {
715        g.subschema_for::<TargetEntry>()
716    }
717}
718
719impl JsonSchema for DestinationEntry {
720    fn schema_name() -> std::borrow::Cow<'static, str> {
721        "DestinationEntry".into()
722    }
723    fn inline_schema() -> bool {
724        true
725    }
726    fn json_schema(g: &mut schemars::SchemaGenerator) -> schemars::Schema {
727        schemars::json_schema!({
728            "oneOf": [g.subschema_for::<TargetEntry>(), g.subschema_for::<NoDelivery>()]
729        })
730    }
731}
732
733/// One target of a profile: a connection or delivery configuration. `type` names the plugin; every other key is a field of that plugin (see the plugins reference), and secrets belong in `{{ env_var('NAME') }}`.
734#[derive(JsonSchema)]
735#[schemars(rename = "target", extend("additionalProperties" = true, "not" = {"required": ["deliver"]}))]
736#[allow(dead_code)]
737struct TargetEntry {
738    /// The plugin type of the connection, e.g. `duckdb`, `postgres`, `databricks`, `sftp`, `s3`. `local` needs no plugin.
739    #[serde(rename = "type")]
740    kind: String,
741    /// Connections only: how many Bindings may run on this entry at once in a `dre run` (default 1: one after another). A DuckDB file is always 1. `--threads` and `DRE_THREADS` cap the whole run. Concurrent Bindings must not write the same permanent table, and peak memory grows with it.
742    #[schemars(range(min = 1))]
743    threads: Option<u64>,
744}
745
746/// A destination entry is a delivery configuration (`type` and the plugin's fields, as for a connection), or `{deliver: false}`: it deliberately delivers nowhere. The output stays in the target path, the run logs it, and `run_results.json` records the delivery as `not_delivered`. Destinations only.
747#[derive(JsonSchema)]
748#[schemars(rename = "no_delivery", deny_unknown_fields)]
749#[allow(dead_code)]
750struct NoDelivery {
751    /// `false`: this target delivers nowhere. Takes no other keys.
752    #[schemars(schema_with = "deliver_false")]
753    deliver: bool,
754}
755
756fn deliver_false(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
757    schemars::json_schema!({"const": false, "x-doc-type": "`false`"})
758}
759
760#[cfg(test)]
761mod tests {
762    use super::*;
763
764    fn load(yaml: &str) -> (Profiles, Diagnostics) {
765        let dir = tempfile::tempdir().unwrap();
766        std::fs::write(dir.path().join(PROFILES_FILE), yaml).unwrap();
767        let mut diags = Diagnostics::default();
768        let p = Profiles::load(dir.path(), "--profiles-dir", &mut diags);
769        (p, diags)
770    }
771
772    fn errors(d: &Diagnostics) -> Vec<(Option<usize>, String)> {
773        d.sorted()
774            .into_iter()
775            .map(|d| (d.line, d.message.clone()))
776            .collect()
777    }
778
779    #[test]
780    fn a_name_used_twice_gets_the_line_of_the_one_meant() {
781        // Searching the text for `wat:` finds the profile on line 2 first.
782        let (_, d) = load("connections:\n  wat:\n    targets:\n      dev: {type: duckdb}\nwat: {}\n");
783        assert_eq!(
784            errors(&d),
785            [(
786                Some(5),
787                "unknown profiles.yml section `wat`; profiles go under `connections:` or `destinations:`"
788                    .into()
789            )]
790        );
791    }
792
793    #[test]
794    fn a_target_named_like_a_key_gets_its_own_line() {
795        let yaml = "connections:\n  w:\n    target: target\n    targets:\n      dev: {type: duckdb}\n      target: {path: x}\n";
796        let (_, d) = load(yaml);
797        assert_eq!(
798            errors(&d),
799            [(
800                Some(6),
801                "target `target` of connection profile `w` has no `type`".into()
802            )]
803        );
804    }
805
806    #[test]
807    fn jinja_values_and_yaml_1_1_words_are_strings_passed_on() {
808        let yaml = "connections:\n  w:\n    targets:\n      dev:\n        type: postgres\n        password: \"{{ env_var('PG_PASSWORD') }}\"\n        sslmode: no\n";
809        let (p, d) = load(yaml);
810        assert!(errors(&d).is_empty(), "{:?}", errors(&d));
811        let t = p.get(Role::Connection, "w").unwrap().targets.get("dev").unwrap();
812        assert_eq!(t.kind, "postgres");
813        assert_eq!(t.fields["password"], "{{ env_var('PG_PASSWORD') }}");
814        assert_eq!(t.fields["sslmode"], "no");
815        assert_eq!(p.line_of(Role::Connection, "w"), Some(2));
816    }
817
818    #[test]
819    fn syntax_errors_have_their_line() {
820        let (p, d) = load("connections:\n  w: [\n");
821        assert!(p.exists());
822        let e = errors(&d);
823        assert_eq!(e.len(), 1);
824        assert!(e[0].1.starts_with("invalid YAML: "), "{e:?}");
825        assert!(e[0].0.is_some(), "{e:?}");
826    }
827
828    #[test]
829    fn merge_keys_share_settings_between_targets() {
830        let yaml = "connections:\n  w:\n    targets:\n      dev: &dev {type: duckdb, path: dev.duckdb}\n      ci: {<<: *dev, path: ci.duckdb}\n";
831        let (p, d) = load(yaml);
832        assert!(errors(&d).is_empty());
833        let ci = p.get(Role::Connection, "w").unwrap().targets.get("ci").unwrap();
834        assert_eq!(
835            (ci.kind.as_str(), &ci.fields["path"]),
836            ("duckdb", &serde_json::json!("ci.duckdb"))
837        );
838    }
839}
dre_core/profiles.rs

dre_core/
profiles.rs
