dre_core/config/
project.rs

1//! The shape of `dre_project.yml` and its folder config (`reports:`).
2//!
3//! These types are what's read, and they generate `project.schema.json`; their doc comments are
4//! its descriptions. Values the loader checks itself, to say what's wrong in DRE's words, are
5//! [`Loose`]. Outputs (`default_output`, `+output`) and the sections other files share (`plugins`,
6//! `sources`) are read where they're used.
7
8use schemars::JsonSchema;
9use serde::Deserialize;
10use serde::de::IgnoredAny;
11
12use super::de::{Located, Loose, Map, UnknownKeys};
13use crate::dates::{WeekNumbering, WeekStart};
14
15type JsonMap = serde_json::Map<String, serde_json::Value>;
16/// `format_options:`: options by format name.
17pub type FormatOptions = Located<Loose<Option<Map<Loose<JsonMap>>>>>;
18/// `dispatch:`: macro namespaces and their search order.
19pub type DispatchList = Located<Loose<Vec<Loose<Dispatch>>>>;
20
21/// The project file, `dre_project.yml`, at the project root.
22#[derive(Deserialize, JsonSchema)]
23#[schemars(title = "DRE project", deny_unknown_fields)]
24pub struct ProjectFile {
25    /// The project's name. Required.
26    #[schemars(required, length(min = 1))]
27    pub name: Option<Located<Loose<String>>>,
28    /// The connection (in `profiles.yml`) reports use when nothing else names one. May use Jinja with `var()`, `env_var()`, `run.*` and `target.name`.
29    pub default_profile: Option<Located<Loose<String>>>,
30    /// The output every report starts from (the built-in default is `format: csv`). A report's own `output` is merged on top.
31    #[schemars(schema_with = "output_ref")]
32    #[serde(default)]
33    pub default_output: Option<IgnoredAny>,
34    /// Default options per output format, under every output of that format. A report's own keys win.
35    #[schemars(schema_with = "format_options")]
36    #[serde(default)]
37    pub format_options: Option<FormatOptions>,
38    /// The Set used when a report has Sets and none is chosen.
39    pub default_set: Option<Located<Loose<String>>>,
40    /// The lowest level of `var()`: folders, reports, Sets, schedules and `--var` override these.
41    #[schemars(schema_with = "any_map")]
42    #[serde(default)]
43    pub vars: Option<Located<Loose<Option<JsonMap>>>>,
44    /// `run_query()` refuses results larger than this.
45    #[schemars(range(min = 1), extend("default" = 10000))]
46    pub run_query_max_rows: Option<Located<Loose<u64>>>,
47    /// Lookups over this many rows are loaded into a temp table instead of inlined into the SQL.
48    #[schemars(range(min = 0), extend("default" = 200))]
49    pub lookup_inline_max_rows: Option<Located<Loose<u64>>>,
50    /// Which macro packages `dispatch()` searches, and in what order.
51    pub dispatch: Option<DispatchList>,
52    /// Whether `DRE_SECRET_*` values are masked as `*****` in the console, logs, JSON events, `run_results.json` and `target/compiled/`.
53    #[schemars(extend("default" = true))]
54    pub mask_secrets: Option<Located<Loose<bool>>>,
55    /// The timezone `run.date` and `run.now` use, an IANA name such as `Australia/Sydney`. Default: UTC.
56    pub timezone: Option<Located<Loose<String>>>,
57    /// The locale the number filters (`number`, `percent`, `currency`, `compact`, `signed`) format for, such as `de-DE` or `fr`: decimal and group separators, and where the currency symbol and percent sign go. Default: `en`.
58    pub locale: Option<Located<Loose<String>>>,
59    /// The first day of the week for `run.date` week arithmetic.
60    #[schemars(extend("default" = "monday"))]
61    pub week_start: Option<Located<Loose<WeekStart>>>,
62    /// How weeks are numbered: ISO 8601 or US style.
63    #[schemars(extend("default" = "iso"))]
64    pub week_numbering: Option<Located<Loose<WeekNumbering>>>,
65    /// Folder config: settings for the report folders, by folder name, nested to match the folders under `reports/`.
66    pub reports: Option<Located<Loose<Folder>>>,
67    /// How DRE itself behaves, as in dbt's `flags:`. Each flag has a `DRE_` environment variable that wins over it.
68    pub flags: Option<Located<Loose<Flags>>>,
69    /// Where DRE writes its generated files (compiled SQL, run outputs, the manifest). Default: `target/` in the project. `--target-path` and `DRE_TARGET_PATH` override it.
70    pub target_path: Option<Located<Loose<String>>>,
71    /// The plugin packages this project uses. DRE installs them on demand into `dre_deps/` and pins them in `dre.lock`. May be written in any project YAML file; `dependencies.yml` is the usual place.
72    #[schemars(schema_with = "super::dependencies::plugins")]
73    #[serde(default)]
74    pub plugins: Option<IgnoredAny>,
75    /// dbt-style source declarations (see the sources schema). May be written in any project YAML file.
76    #[schemars(schema_with = "sources_ref")]
77    #[serde(default)]
78    pub sources: Option<IgnoredAny>,
79    /// Removed in DRE 0.2.1; read only to say so.
80    #[schemars(skip)]
81    pub target: Option<Located<IgnoredAny>>,
82    /// Where plugins were declared before packages; read only to say so.
83    #[schemars(skip)]
84    pub destinations: Option<Located<IgnoredAny>>,
85    #[schemars(skip)]
86    pub formats: Option<Located<IgnoredAny>>,
87    #[serde(rename = "$unknown", default)]
88    #[schemars(skip)]
89    pub unknown: UnknownKeys,
90}
91
92/// `flags:`: how DRE itself behaves. Each has a `DRE_` environment variable that wins over it.
93#[derive(Deserialize, JsonSchema)]
94#[schemars(deny_unknown_fields)]
95pub struct Flags {
96    /// Seconds a download (the plugin registry, a plugin package, a DRE update) may receive nothing before it's tried again (3 tries in all). `DRE_HTTP_TIMEOUT` overrides it.
97    #[schemars(range(min = 1), extend("default" = 60))]
98    pub http_timeout: Option<Located<Loose<u64>>>,
99    /// How many runs of each report and Set to keep in `target/run/` (any status; the current run always stays). Runs beyond it are removed after each run. Recommended on servers and anything scheduled (an audit trail of what was sent), mindful of file sizes. `dre run --keep-runs` and `DRE_KEEP_RUNS` override it.
100    #[schemars(range(min = 1), extend("default" = 1))]
101    pub keep_runs: Option<Located<Loose<u64>>>,
102    /// How long a `dre run` may take before it's stopped (as for a termination signal; its Bindings are recorded as `timed_out` and `dre` exits 124): a duration such as `2h` or `90m`, or seconds. Off by default. `dre run --timeout` and `DRE_RUN_TIMEOUT` override it.
103    #[schemars(schema_with = "duration")]
104    #[serde(default)]
105    pub run_timeout: Option<Located<Loose<serde_json::Value>>>,
106    #[serde(rename = "$unknown", default)]
107    #[schemars(skip)]
108    pub unknown: UnknownKeys,
109}
110
111/// One macro namespace.
112#[derive(Deserialize, JsonSchema)]
113#[schemars(deny_unknown_fields, inline)]
114pub struct Dispatch {
115    /// The macro package the dispatching macro belongs to, e.g. `dre_utils`.
116    #[schemars(required)]
117    pub macro_namespace: Option<Loose<String>>,
118    /// Packages searched for the variant, first match wins: this project's name and installed packages.
119    #[schemars(required, length(min = 1))]
120    pub search_order: Option<Loose<Vec<Loose<String>>>>,
121}
122
123/// Settings for every report in this folder and the folders below it. Keys starting with `+` are settings; any other key is a subfolder.
124#[derive(Deserialize, JsonSchema)]
125#[schemars(rename = "folder", extend("additionalProperties" = {"$ref": "#/$defs/folder"}))]
126pub struct Folder {
127    /// Tags added to every report in the folder.
128    #[serde(rename = "+tags")]
129    pub tags: Option<Located<Loose<Vec<String>>>>,
130    /// Output settings every report in the folder starts from; a report's own keys win.
131    #[serde(rename = "+output")]
132    #[serde(default)]
133    #[schemars(schema_with = "output_ref")]
134    pub output: Option<Located<Loose<JsonMap>>>,
135    /// The connection for reports in the folder. May use Jinja with `var()`, `env_var()`, `run.*` and `target.name`.
136    #[serde(rename = "+profile")]
137    pub profile: Option<Located<Loose<String>>>,
138    /// Removed: schedules live in schedules.yml. Read only to say so.
139    #[serde(rename = "+schedule")]
140    #[schemars(skip)]
141    pub schedule: Option<Located<IgnoredAny>>,
142    /// Variables, read in SQL and YAML with `var('name')`. Values can be strings, numbers, booleans, lists or maps.
143    #[serde(rename = "+vars")]
144    #[serde(default)]
145    #[schemars(schema_with = "any_map")]
146    pub vars: Option<Located<Loose<JsonMap>>>,
147    /// The timezone `run.date` and `run.now` use, an IANA name such as `Australia/Sydney`. Default: UTC.
148    #[serde(rename = "+timezone")]
149    pub timezone: Option<Located<Loose<String>>>,
150    /// The `locale:` for reports in this folder; a report's own wins. See `locale` in dre_project.yml.
151    #[serde(rename = "+locale")]
152    pub locale: Option<Located<Loose<String>>>,
153    /// Subfolders by name, and `+` keys that aren't settings.
154    #[serde(rename = "$unknown", default)]
155    #[schemars(skip)]
156    pub rest: Map<Located<Loose<Folder>>>,
157}
158
159impl Folder {
160    /// Whether it sets anything itself (any `+` key, known or not), rather than only holding
161    /// subfolders.
162    pub fn sets_anything(&self) -> bool {
163        self.tags.is_some()
164            || self.output.is_some()
165            || self.profile.is_some()
166            || self.schedule.is_some()
167            || self.vars.is_some()
168            || self.timezone.is_some()
169            || self.locale.is_some()
170            || self.rest.iter().any(|(k, _)| k.value.starts_with('+'))
171    }
172}
173
174fn output_ref(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
175    schemars::json_schema!({"$ref": "report.schema.json#/$defs/output"})
176}
177
178pub(super) fn sources_ref(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
179    schemars::json_schema!({"$ref": "sources.schema.json#/properties/sources"})
180}
181
182fn duration(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
183    schemars::json_schema!({
184        "oneOf": [
185            {"type": "string", "pattern": "^\\s*[0-9]+(\\.[0-9]+)?\\s*(ms|s|m|h)?\\s*$"},
186            {"type": "number", "minimum": 0}
187        ]
188    })
189}
190
191fn any_map(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
192    schemars::json_schema!({"type": "object", "additionalProperties": true})
193}
194
195fn format_options(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
196    schemars::json_schema!({"type": "object", "additionalProperties": {"type": "object"}})
197}
dre_core/config/project.rs

dre_core/config/
project.rs
