dre_core/config/
project.rs

1//! The shape of `dre_project.yml` and its folder config (`reports:`).
2//!
3//! These types are what's read, and they generate `project.schema.json`; their doc comments are
4//! its descriptions. Values the loader checks itself, to say what's wrong in DRE's words, are
5//! [`Loose`]. Outputs (`default_output`, `+output`) and the sections other files share (`plugins`,
6//! `sources`) are read where they're used.
7
8use schemars::JsonSchema;
9use serde::Deserialize;
10use serde::de::IgnoredAny;
11
12use super::de::{Located, Loose, Map, UnknownKeys};
13use crate::dates::{WeekNumbering, WeekStart};
14
15type JsonMap = serde_json::Map<String, serde_json::Value>;
16/// `format_options:`: options by format name.
17pub type FormatOptions = Located<Loose<Option<Map<Loose<JsonMap>>>>>;
18/// `dispatch:`: macro namespaces and their search order.
19pub type DispatchList = Located<Loose<Vec<Loose<Dispatch>>>>;
20
21/// The project file, `dre_project.yml`, at the project root.
22#[derive(Deserialize, JsonSchema)]
23#[schemars(title = "DRE project", deny_unknown_fields)]
24pub struct ProjectFile {
25    /// The project's name. Required.
26    #[schemars(required, length(min = 1))]
27    pub name: Option<Located<Loose<String>>>,
28    /// Metadata for external tools, preserved as the first report metadata layer.
29    pub meta: Option<Located<Loose<JsonMap>>>,
30    /// The connection (in `profiles.yml`) reports use when nothing else names one. May use Jinja with `var()`, `env_var()`, `run.*` and `target.name`.
31    pub default_profile: Option<Located<Loose<String>>>,
32    /// The output every report starts from (the built-in default is `format: csv`). A report's own `output` is merged on top.
33    #[schemars(schema_with = "output_ref")]
34    #[serde(default)]
35    pub default_output: Option<IgnoredAny>,
36    /// Default options per output format, under every output of that format. A report's own keys win.
37    #[schemars(schema_with = "format_options")]
38    #[serde(default)]
39    pub format_options: Option<FormatOptions>,
40    /// The Set used when a report has Sets and none is chosen.
41    pub default_set: Option<Located<Loose<String>>>,
42    /// The lowest level of `var()`: folders, reports, Sets, schedules and `--var` override these.
43    #[schemars(schema_with = "any_map")]
44    #[serde(default)]
45    pub vars: Option<Located<Loose<Option<JsonMap>>>>,
46    /// `run_query()` refuses results larger than this.
47    #[schemars(range(min = 1), extend("default" = 10000))]
48    pub run_query_max_rows: Option<Located<Loose<u64>>>,
49    /// Lookups over this many rows are loaded into a temp table instead of inlined into the SQL.
50    #[schemars(range(min = 0), extend("default" = 200))]
51    pub lookup_inline_max_rows: Option<Located<Loose<u64>>>,
52    /// Which macro packages `dispatch()` searches, and in what order.
53    pub dispatch: Option<DispatchList>,
54    /// Whether `DRE_SECRET_*` values are masked as `*****` in the console, logs, JSON events, `run_results.json` and `target/compiled/`.
55    #[schemars(extend("default" = true))]
56    pub mask_secrets: Option<Located<Loose<bool>>>,
57    /// The timezone `run.date` and `run.now` use, an IANA name such as `Australia/Sydney`. Default: UTC.
58    pub timezone: Option<Located<Loose<String>>>,
59    /// The locale the number filters (`number`, `percent`, `currency`, `compact`, `signed`) format for, such as `de-DE` or `fr`: decimal and group separators, and where the currency symbol and percent sign go. Default: `en`.
60    pub locale: Option<Located<Loose<String>>>,
61    /// The first day of the week for `run.date` week arithmetic.
62    #[schemars(extend("default" = "monday"))]
63    pub week_start: Option<Located<Loose<WeekStart>>>,
64    /// How weeks are numbered: ISO 8601 or US style.
65    #[schemars(extend("default" = "iso"))]
66    pub week_numbering: Option<Located<Loose<WeekNumbering>>>,
67    /// Folder config: settings for the report folders, by folder name, nested to match the folders under `reports/`.
68    pub reports: Option<Located<Loose<Folder>>>,
69    /// How DRE itself behaves, as in dbt's `flags:`. Each flag has a `DRE_` environment variable that wins over it.
70    pub flags: Option<Located<Loose<Flags>>>,
71    /// Where DRE writes its generated files (compiled SQL, run outputs, the manifest). Default: `target/` in the project. `--target-path` and `DRE_TARGET_PATH` override it.
72    pub target_path: Option<Located<Loose<String>>>,
73    /// The plugin packages this project uses. DRE installs them on demand into `dre_deps/` and pins them in `dre.lock`. May be written in any project YAML file; `dependencies.yml` is the usual place.
74    #[schemars(schema_with = "super::dependencies::plugins")]
75    #[serde(default)]
76    pub plugins: Option<IgnoredAny>,
77    /// dbt-style source declarations (see the sources schema). May be written in any project YAML file.
78    #[schemars(schema_with = "sources_ref")]
79    #[serde(default)]
80    pub sources: Option<IgnoredAny>,
81    /// Removed in DRE 0.2.1; read only to say so.
82    #[schemars(skip)]
83    pub target: Option<Located<IgnoredAny>>,
84    /// Where plugins were declared before packages; read only to say so.
85    #[schemars(skip)]
86    pub destinations: Option<Located<IgnoredAny>>,
87    #[schemars(skip)]
88    pub formats: Option<Located<IgnoredAny>>,
89    #[serde(rename = "$unknown", default)]
90    #[schemars(skip)]
91    pub unknown: UnknownKeys,
92}
93
94/// `flags:`: how DRE itself behaves. Each has a `DRE_` environment variable that wins over it.
95#[derive(Deserialize, JsonSchema)]
96#[schemars(deny_unknown_fields)]
97pub struct Flags {
98    /// Seconds a download (the plugin registry, a plugin package, a DRE update) may receive nothing before it's tried again (3 tries in all). `DRE_HTTP_TIMEOUT` overrides it.
99    #[schemars(range(min = 1), extend("default" = 60))]
100    pub http_timeout: Option<Located<Loose<u64>>>,
101    /// How many runs of each report and Set to keep in `target/run/` (any status; the current run always stays). Runs beyond it are removed after each run. Recommended on servers and anything scheduled (an audit trail of what was sent), mindful of file sizes. `dre run --keep-runs` and `DRE_KEEP_RUNS` override it.
102    #[schemars(range(min = 1), extend("default" = 1))]
103    pub keep_runs: Option<Located<Loose<u64>>>,
104    /// How long a `dre run` may take before it's stopped (as for a termination signal; its Bindings are recorded as `timed_out` and `dre` exits 124): a duration such as `2h` or `90m`, or seconds. Off by default. `dre run --timeout` and `DRE_RUN_TIMEOUT` override it.
105    #[schemars(schema_with = "duration")]
106    #[serde(default)]
107    pub run_timeout: Option<Located<Loose<serde_json::Value>>>,
108    #[serde(rename = "$unknown", default)]
109    #[schemars(skip)]
110    pub unknown: UnknownKeys,
111}
112
113/// One macro namespace.
114#[derive(Deserialize, JsonSchema)]
115#[schemars(deny_unknown_fields, inline)]
116pub struct Dispatch {
117    /// The macro package the dispatching macro belongs to, e.g. `dre_utils`.
118    #[schemars(required)]
119    pub macro_namespace: Option<Loose<String>>,
120    /// Packages searched for the variant, first match wins: this project's name and installed packages.
121    #[schemars(required, length(min = 1))]
122    pub search_order: Option<Loose<Vec<Loose<String>>>>,
123}
124
125/// Settings for every report in this folder and the folders below it. Keys starting with `+` are settings; any other key is a subfolder.
126#[derive(Deserialize, JsonSchema)]
127#[schemars(rename = "folder", extend("additionalProperties" = {"$ref": "#/$defs/folder"}))]
128pub struct Folder {
129    /// Metadata for external tools, preserved as a layer with its configuration origin.
130    #[serde(rename = "+meta")]
131    pub meta: Option<Located<Loose<JsonMap>>>,
132    /// Tags added to every report in the folder.
133    #[serde(rename = "+tags")]
134    pub tags: Option<Located<Loose<Vec<String>>>>,
135    /// Output settings every report in the folder starts from; a report's own keys win.
136    #[serde(rename = "+output")]
137    #[serde(default)]
138    #[schemars(schema_with = "output_ref")]
139    pub output: Option<Located<Loose<JsonMap>>>,
140    /// The connection for reports in the folder. May use Jinja with `var()`, `env_var()`, `run.*` and `target.name`.
141    #[serde(rename = "+profile")]
142    pub profile: Option<Located<Loose<String>>>,
143    /// Removed: schedules live in schedules.yml. Read only to say so.
144    #[serde(rename = "+schedule")]
145    #[schemars(skip)]
146    pub schedule: Option<Located<IgnoredAny>>,
147    /// Variables, read in SQL and YAML with `var('name')`. Values can be strings, numbers, booleans, lists or maps.
148    #[serde(rename = "+vars")]
149    #[serde(default)]
150    #[schemars(schema_with = "any_map")]
151    pub vars: Option<Located<Loose<JsonMap>>>,
152    /// The timezone `run.date` and `run.now` use, an IANA name such as `Australia/Sydney`. Default: UTC.
153    #[serde(rename = "+timezone")]
154    pub timezone: Option<Located<Loose<String>>>,
155    /// The `locale:` for reports in this folder; a report's own wins. See `locale` in dre_project.yml.
156    #[serde(rename = "+locale")]
157    pub locale: Option<Located<Loose<String>>>,
158    /// Subfolders by name, and `+` keys that aren't settings.
159    #[serde(rename = "$unknown", default)]
160    #[schemars(skip)]
161    pub rest: Map<Located<Loose<Folder>>>,
162}
163
164impl Folder {
165    /// Whether it sets anything itself (any `+` key, known or not), rather than only holding
166    /// subfolders.
167    pub fn sets_anything(&self) -> bool {
168        self.tags.is_some()
169            || self.meta.is_some()
170            || self.output.is_some()
171            || self.profile.is_some()
172            || self.schedule.is_some()
173            || self.vars.is_some()
174            || self.timezone.is_some()
175            || self.locale.is_some()
176            || self.rest.iter().any(|(k, _)| k.value.starts_with('+'))
177    }
178}
179
180fn output_ref(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
181    schemars::json_schema!({"$ref": "report.schema.json#/$defs/output"})
182}
183
184pub(super) fn sources_ref(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
185    schemars::json_schema!({"$ref": "sources.schema.json#/properties/sources"})
186}
187
188fn duration(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
189    schemars::json_schema!({
190        "oneOf": [
191            {"type": "string", "pattern": "^\\s*[0-9]+(\\.[0-9]+)?\\s*(ms|s|m|h)?\\s*$"},
192            {"type": "number", "minimum": 0}
193        ]
194    })
195}
196
197fn any_map(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
198    schemars::json_schema!({"type": "object", "additionalProperties": true})
199}
200
201fn format_options(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
202    schemars::json_schema!({"type": "object", "additionalProperties": {"type": "object"}})
203}
dre_core/config/project.rs

dre_core/config/
project.rs
