dre_core/config/project.rs
1//! The shape of `dre_project.yml` and its folder config (`reports:`).
2//!
3//! These types are what's read, and they generate `project.schema.json`; their doc comments are
4//! its descriptions. Values the loader checks itself, to say what's wrong in DRE's words, are
5//! [`Loose`]. Outputs (`default_output`, `+output`) and the sections other files share (`plugins`,
6//! `sources`) are read where they're used.
7
8use schemars::JsonSchema;
9use serde::Deserialize;
10use serde::de::IgnoredAny;
11
12use super::de::{Located, Loose, Map, UnknownKeys};
13use crate::dates::{WeekNumbering, WeekStart};
14
15type JsonMap = serde_json::Map<String, serde_json::Value>;
16/// `format_options:`: options by format name.
17pub type FormatOptions = Located<Loose<Option<Map<Loose<JsonMap>>>>>;
18/// `dispatch:`: macro namespaces and their search order.
19pub type DispatchList = Located<Loose<Vec<Loose<Dispatch>>>>;
20
21/// The project file, `dre_project.yml`, at the project root.
22#[derive(Deserialize, JsonSchema)]
23#[schemars(title = "DRE project", deny_unknown_fields)]
24pub struct ProjectFile {
25 /// The project's name. Required.
26 #[schemars(required, length(min = 1))]
27 pub name: Option<Located<Loose<String>>>,
28 /// Metadata for external tools, preserved as the first report metadata layer.
29 pub meta: Option<Located<Loose<JsonMap>>>,
30 /// The connection (in `profiles.yml`) reports use when nothing else names one. May use Jinja with `var()`, `env_var()`, `run.*` and `target.name`.
31 pub default_profile: Option<Located<Loose<String>>>,
32 /// The output every report starts from (the built-in default is `format: csv`). A report's own `output` is merged on top.
33 #[schemars(schema_with = "output_ref")]
34 #[serde(default)]
35 pub default_output: Option<IgnoredAny>,
36 /// Default options per output format, under every output of that format. A report's own keys win.
37 #[schemars(schema_with = "format_options")]
38 #[serde(default)]
39 pub format_options: Option<FormatOptions>,
40 /// The Set used when a report has Sets and none is chosen.
41 pub default_set: Option<Located<Loose<String>>>,
42 /// The lowest level of `var()`: folders, reports, Sets, schedules and `--var` override these.
43 #[schemars(schema_with = "any_map")]
44 #[serde(default)]
45 pub vars: Option<Located<Loose<Option<JsonMap>>>>,
46 /// `run_query()` refuses results larger than this.
47 #[schemars(range(min = 1), extend("default" = 10000))]
48 pub run_query_max_rows: Option<Located<Loose<u64>>>,
49 /// Lookups over this many rows are loaded into a temp table instead of inlined into the SQL.
50 #[schemars(range(min = 0), extend("default" = 200))]
51 pub lookup_inline_max_rows: Option<Located<Loose<u64>>>,
52 /// Which macro packages `dispatch()` searches, and in what order.
53 pub dispatch: Option<DispatchList>,
54 /// Whether `DRE_SECRET_*` values are masked as `*****` in the console, logs, JSON events, `run_results.json` and `target/compiled/`.
55 #[schemars(extend("default" = true))]
56 pub mask_secrets: Option<Located<Loose<bool>>>,
57 /// Whether a per-row output's destination values rendered from `each.*` (addresses, paths) are masked in the console, logs, `run_results.json` and `rows.jsonl`. A `for_each` block's own `mask_row_values` wins.
58 #[schemars(extend("default" = true))]
59 pub mask_row_values: Option<Located<Loose<bool>>>,
60 /// The timezone `run.date` and `run.now` use, an IANA name such as `Australia/Sydney`. Default: UTC.
61 pub timezone: Option<Located<Loose<String>>>,
62 /// The locale the number filters (`number`, `percent`, `currency`, `compact`, `signed`) format for, such as `de-DE` or `fr`: decimal and group separators, and where the currency symbol and percent sign go. Default: `en`.
63 pub locale: Option<Located<Loose<String>>>,
64 /// The first day of the week for `run.date` week arithmetic.
65 #[schemars(extend("default" = "monday"))]
66 pub week_start: Option<Located<Loose<WeekStart>>>,
67 /// How weeks are numbered: ISO 8601 or US style.
68 #[schemars(extend("default" = "iso"))]
69 pub week_numbering: Option<Located<Loose<WeekNumbering>>>,
70 /// Folder config: settings for the report folders, by folder name, nested to match the folders under `reports/`.
71 pub reports: Option<Located<Loose<Folder>>>,
72 /// How DRE itself behaves, as in dbt's `flags:`. Each flag has a `DRE_` environment variable that wins over it.
73 pub flags: Option<Located<Loose<Flags>>>,
74 /// Where DRE writes its generated files (compiled SQL, run outputs, the manifest). Default: `target/` in the project. `--target-path` and `DRE_TARGET_PATH` override it.
75 pub target_path: Option<Located<Loose<String>>>,
76 /// The plugin packages this project uses. DRE installs them on demand into `dre_deps/` and pins them in `dre.lock`. May be written in any project YAML file; `dependencies.yml` is the usual place.
77 #[schemars(schema_with = "super::dependencies::plugins")]
78 #[serde(default)]
79 pub plugins: Option<IgnoredAny>,
80 /// dbt-style source declarations (see the sources schema). May be written in any project YAML file.
81 #[schemars(schema_with = "sources_ref")]
82 #[serde(default)]
83 pub sources: Option<IgnoredAny>,
84 /// Removed in DRE 0.2.1; read only to say so.
85 #[schemars(skip)]
86 pub target: Option<Located<IgnoredAny>>,
87 /// Where plugins were declared before packages; read only to say so.
88 #[schemars(skip)]
89 pub destinations: Option<Located<IgnoredAny>>,
90 #[schemars(skip)]
91 pub formats: Option<Located<IgnoredAny>>,
92 #[serde(rename = "$unknown", default)]
93 #[schemars(skip)]
94 pub unknown: UnknownKeys,
95}
96
97/// `flags:`: how DRE itself behaves. Each has a `DRE_` environment variable that wins over it.
98#[derive(Deserialize, JsonSchema)]
99#[schemars(deny_unknown_fields)]
100pub struct Flags {
101 /// Seconds a download (the plugin registry, a plugin package, a DRE update) may receive nothing before it's tried again (3 tries in all). `DRE_HTTP_TIMEOUT` overrides it.
102 #[schemars(range(min = 1), extend("default" = 60))]
103 pub http_timeout: Option<Located<Loose<u64>>>,
104 /// How many runs of each report and Set to keep in `target/run/` (any status; the current run always stays). Runs beyond it are removed after each run. Recommended on servers and anything scheduled (an audit trail of what was sent), mindful of file sizes. `dre run --keep-runs` and `DRE_KEEP_RUNS` override it.
105 #[schemars(range(min = 1), extend("default" = 1))]
106 pub keep_runs: Option<Located<Loose<u64>>>,
107 /// How long a `dre run` may take before it's stopped (as for a termination signal; its Bindings are recorded as `timed_out` and `dre` exits 124): a duration such as `2h` or `90m`, or seconds. Off by default. `dre run --timeout` and `DRE_RUN_TIMEOUT` override it.
108 #[schemars(schema_with = "duration")]
109 #[serde(default)]
110 pub run_timeout: Option<Located<Loose<serde_json::Value>>>,
111 #[serde(rename = "$unknown", default)]
112 #[schemars(skip)]
113 pub unknown: UnknownKeys,
114}
115
116/// One macro namespace.
117#[derive(Deserialize, JsonSchema)]
118#[schemars(deny_unknown_fields, inline)]
119pub struct Dispatch {
120 /// The macro package the dispatching macro belongs to, e.g. `dre_utils`.
121 #[schemars(required)]
122 pub macro_namespace: Option<Loose<String>>,
123 /// Packages searched for the variant, first match wins: this project's name and installed packages.
124 #[schemars(required, length(min = 1))]
125 pub search_order: Option<Loose<Vec<Loose<String>>>>,
126}
127
128/// Settings for every report in this folder and the folders below it. Keys starting with `+` are settings; any other key is a subfolder.
129#[derive(Deserialize, JsonSchema)]
130#[schemars(rename = "folder", extend("additionalProperties" = {"$ref": "#/$defs/folder"}))]
131pub struct Folder {
132 /// Metadata for external tools, preserved as a layer with its configuration origin.
133 #[serde(rename = "+meta")]
134 pub meta: Option<Located<Loose<JsonMap>>>,
135 /// Tags added to every report in the folder.
136 #[serde(rename = "+tags")]
137 pub tags: Option<Located<Loose<Vec<String>>>>,
138 /// Output settings every report in the folder starts from; a report's own keys win.
139 #[serde(rename = "+output")]
140 #[serde(default)]
141 #[schemars(schema_with = "output_ref")]
142 pub output: Option<Located<Loose<JsonMap>>>,
143 /// The connection for reports in the folder. May use Jinja with `var()`, `env_var()`, `run.*` and `target.name`.
144 #[serde(rename = "+profile")]
145 pub profile: Option<Located<Loose<String>>>,
146 /// Removed: schedules live in schedules.yml. Read only to say so.
147 #[serde(rename = "+schedule")]
148 #[schemars(skip)]
149 pub schedule: Option<Located<IgnoredAny>>,
150 /// Variables, read in SQL and YAML with `var('name')`. Values can be strings, numbers, booleans, lists or maps.
151 #[serde(rename = "+vars")]
152 #[serde(default)]
153 #[schemars(schema_with = "any_map")]
154 pub vars: Option<Located<Loose<JsonMap>>>,
155 /// The timezone `run.date` and `run.now` use, an IANA name such as `Australia/Sydney`. Default: UTC.
156 #[serde(rename = "+timezone")]
157 pub timezone: Option<Located<Loose<String>>>,
158 /// The `locale:` for reports in this folder; a report's own wins. See `locale` in dre_project.yml.
159 #[serde(rename = "+locale")]
160 pub locale: Option<Located<Loose<String>>>,
161 /// Subfolders by name, and `+` keys that aren't settings.
162 #[serde(rename = "$unknown", default)]
163 #[schemars(skip)]
164 pub rest: Map<Located<Loose<Folder>>>,
165}
166
167impl Folder {
168 /// Whether it sets anything itself (any `+` key, known or not), rather than only holding
169 /// subfolders.
170 pub fn sets_anything(&self) -> bool {
171 self.tags.is_some()
172 || self.meta.is_some()
173 || self.output.is_some()
174 || self.profile.is_some()
175 || self.schedule.is_some()
176 || self.vars.is_some()
177 || self.timezone.is_some()
178 || self.locale.is_some()
179 || self.rest.iter().any(|(k, _)| k.value.starts_with('+'))
180 }
181}
182
183fn output_ref(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
184 schemars::json_schema!({"$ref": "report.schema.json#/$defs/output"})
185}
186
187pub(super) fn sources_ref(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
188 schemars::json_schema!({"$ref": "sources.schema.json#/properties/sources"})
189}
190
191fn duration(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
192 schemars::json_schema!({
193 "oneOf": [
194 {"type": "string", "pattern": "^\\s*[0-9]+(\\.[0-9]+)?\\s*(ms|s|m|h)?\\s*$"},
195 {"type": "number", "minimum": 0}
196 ]
197 })
198}
199
200fn any_map(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
201 schemars::json_schema!({"type": "object", "additionalProperties": true})
202}
203
204fn format_options(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
205 schemars::json_schema!({"type": "object", "additionalProperties": {"type": "object"}})
206}