dre_core/config/
sources.rs

1//! The shape of dbt-style source declarations: a top-level `sources:` in any project YAML file.
2
3use schemars::JsonSchema;
4use serde::Deserialize;
5use serde::de::IgnoredAny;
6
7use super::de::{Located, Loose, Map, OneOf, Text, UnknownKeys};
8
9type JsonMap = serde_json::Map<String, serde_json::Value>;
10
11/// dbt-style source declarations: the tables a project reads, with DRE's `profile:` for the connection they live on. In any project YAML file under a top-level `sources:` key (`sources/` is the conventional folder); dbt's `version: 2` is accepted beside it. Use a table in SQL with `{{ source('<source>', '<table>') }}`.
12#[derive(Deserialize, JsonSchema)]
13#[schemars(title = "DRE sources", deny_unknown_fields)]
14pub struct SourcesFile {
15    /// The sources this file declares.
16    #[schemars(required)]
17    pub sources: Option<Located<Loose<Option<Vec<SourceItem>>>>>,
18    /// dbt's file version (`2`); accepted and ignored.
19    #[serde(default)]
20    #[schemars(schema_with = "anything")]
21    pub version: Option<IgnoredAny>,
22}
23
24/// One source of `sources:`.
25pub type SourceItem = Located<Loose<Source>>;
26/// One table of a source's `tables:`.
27pub type TableItem = Located<Loose<Table>>;
28
29/// A dbt key DRE accepts but doesn't use yet.
30type Dbt = Option<IgnoredAny>;
31
32/// A source: tables in one schema of one system.
33#[derive(Deserialize, JsonSchema)]
34#[schemars(rename = "source", deny_unknown_fields)]
35pub struct Source {
36    /// The source's name, the first argument of `source()`. Unique in the project.
37    #[schemars(required, with = "String")]
38    pub name: Option<Loose<String>>,
39    /// What the source is.
40    pub description: Option<Loose<Text>>,
41    /// The database (catalog). Set: `source()` renders `database.schema.table`; unset: `schema.table`. May use Jinja with `var()`, `env_var()`, `run.*` and `target.name`.
42    pub database: Option<Loose<Text>>,
43    /// The schema; the source's name unless set. May use Jinja with `var()`, `env_var()`, `run.*` and `target.name`.
44    pub schema: Option<Loose<Text>>,
45    /// DRE's addition: the connection (in `profiles.yml`) the source lives on. A query using the source runs there; it overrides the report's, Set's, folder's and project's default. Unset: the source runs wherever its query runs. May use Jinja with `var()`, `env_var()`, `run.*` and `target.name`.
46    pub profile: Option<Loose<Text>>,
47    pub quoting: Option<Loose<Quoting>>,
48    pub tags: Option<Loose<Tags>>,
49    pub meta: Option<Loose<Meta>>,
50    /// The source's tables.
51    pub tables: Option<Loose<Option<Vec<TableItem>>>>,
52    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
53    #[serde(default)]
54    #[schemars(schema_with = "anything")]
55    pub loader: Dbt,
56    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
57    #[serde(default)]
58    #[schemars(schema_with = "anything")]
59    pub loaded_at_field: Dbt,
60    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
61    #[serde(default)]
62    #[schemars(schema_with = "anything")]
63    pub loaded_at_query: Dbt,
64    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
65    #[serde(default)]
66    #[schemars(schema_with = "anything")]
67    pub config: Dbt,
68    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
69    #[serde(default)]
70    #[schemars(schema_with = "anything")]
71    pub overrides: Dbt,
72    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
73    #[serde(default)]
74    #[schemars(schema_with = "anything")]
75    pub freshness: Dbt,
76    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
77    #[serde(default)]
78    #[schemars(schema_with = "anything")]
79    pub docs: Dbt,
80    #[serde(rename = "$unknown", default)]
81    #[schemars(skip)]
82    pub unknown: UnknownKeys,
83}
84
85/// One table of the source.
86#[derive(Deserialize, JsonSchema)]
87#[schemars(rename = "table", deny_unknown_fields)]
88pub struct Table {
89    /// The name `source()` uses.
90    #[schemars(required, with = "String")]
91    pub name: Option<Loose<String>>,
92    /// The real table name, when it differs from `name`. May use Jinja with `var()`, `env_var()`, `run.*` and `target.name`.
93    pub identifier: Option<Loose<Text>>,
94    /// What the table holds.
95    pub description: Option<Loose<Text>>,
96    pub quoting: Option<Loose<Quoting>>,
97    pub tags: Option<Loose<Tags>>,
98    pub meta: Option<Loose<Meta>>,
99    /// Declared columns, checked by `dre validate --live`.
100    pub columns: Option<Loose<Option<Vec<Loose<Column>>>>>,
101    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
102    #[serde(default)]
103    #[schemars(schema_with = "anything")]
104    pub loaded_at_field: Dbt,
105    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
106    #[serde(default)]
107    #[schemars(schema_with = "anything")]
108    pub loaded_at_query: Dbt,
109    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
110    #[serde(default)]
111    #[schemars(schema_with = "anything")]
112    pub tests: Dbt,
113    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
114    #[serde(default)]
115    #[schemars(schema_with = "anything")]
116    pub data_tests: Dbt,
117    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
118    #[serde(default)]
119    #[schemars(schema_with = "anything")]
120    pub freshness: Dbt,
121    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
122    #[serde(default)]
123    #[schemars(schema_with = "anything")]
124    pub external: Dbt,
125    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
126    #[serde(default)]
127    #[schemars(schema_with = "anything")]
128    pub config: Dbt,
129    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
130    #[serde(default)]
131    #[schemars(schema_with = "anything")]
132    pub docs: Dbt,
133    #[serde(rename = "$unknown", default)]
134    #[schemars(skip)]
135    pub unknown: UnknownKeys,
136}
137
138/// A column of the table. `dre validate --live` checks that it exists and, with `data_type`, that its type matches loosely.
139#[derive(Deserialize, JsonSchema)]
140#[schemars(rename = "column", deny_unknown_fields)]
141pub struct Column {
142    /// The column's name (compared case-insensitively).
143    #[schemars(required, with = "String")]
144    pub name: Option<Loose<String>>,
145    /// What the column holds.
146    pub description: Option<Loose<Text>>,
147    /// The column's SQL type, e.g. `bigint`, `varchar`, `timestamp`; compared loosely with what the database returns.
148    pub data_type: Option<Loose<Text>>,
149    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
150    #[serde(default)]
151    #[schemars(schema_with = "anything")]
152    pub meta: Dbt,
153    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
154    #[serde(default)]
155    #[schemars(schema_with = "anything")]
156    pub tags: Dbt,
157    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
158    #[serde(default)]
159    #[schemars(schema_with = "anything")]
160    pub quote: Dbt,
161    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
162    #[serde(default)]
163    #[schemars(schema_with = "anything")]
164    pub tests: Dbt,
165    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
166    #[serde(default)]
167    #[schemars(schema_with = "anything")]
168    pub data_tests: Dbt,
169    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
170    #[serde(default)]
171    #[schemars(schema_with = "anything")]
172    pub constraints: Dbt,
173    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
174    #[serde(default)]
175    #[schemars(schema_with = "anything")]
176    pub config: Dbt,
177    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
178    #[serde(default)]
179    #[schemars(schema_with = "anything")]
180    pub docs: Dbt,
181    /// A dbt key DRE accepts but doesn't use yet (noted by `dre validate`).
182    #[serde(default)]
183    #[schemars(schema_with = "anything")]
184    pub granularity: Dbt,
185    #[serde(rename = "$unknown", default)]
186    #[schemars(skip)]
187    pub unknown: UnknownKeys,
188}
189
190/// Which parts `source()` quotes, with the connection's identifier quote character. Unset parts inherit (table from source), else aren't quoted.
191#[derive(Deserialize, JsonSchema)]
192#[schemars(rename = "quoting", deny_unknown_fields)]
193pub struct Quoting {
194    /// Quote the database.
195    #[serde(default)]
196    #[schemars(with = "bool")]
197    pub database: Option<Loose<bool>>,
198    /// Quote the schema.
199    #[serde(default)]
200    #[schemars(with = "bool")]
201    pub schema: Option<Loose<bool>>,
202    /// Quote the table name.
203    #[serde(default)]
204    #[schemars(with = "bool")]
205    pub identifier: Option<Loose<bool>>,
206    #[serde(rename = "$unknown", default)]
207    #[schemars(skip)]
208    pub unknown: Map<serde_json::Value>,
209}
210
211/// Tags, for documentation.
212#[derive(Deserialize, JsonSchema)]
213#[serde(transparent)]
214#[schemars(rename = "tags")]
215pub struct Tags(pub OneOf<String, Vec<String>>);
216
217/// Free-form metadata, kept in the manifest.
218#[derive(Deserialize, JsonSchema)]
219#[serde(transparent)]
220#[schemars(rename = "meta")]
221pub struct Meta(#[schemars(schema_with = "any_map")] pub JsonMap);
222
223fn anything(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
224    schemars::json_schema!({})
225}
226
227fn any_map(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
228    schemars::json_schema!({"type": "object", "additionalProperties": true})
229}
230
231impl Source {
232    /// The dbt keys it sets that DRE doesn't use yet.
233    pub fn dbt_keys(&self) -> Vec<&'static str> {
234        [
235            ("loader", &self.loader),
236            ("loaded_at_field", &self.loaded_at_field),
237            ("loaded_at_query", &self.loaded_at_query),
238            ("config", &self.config),
239            ("overrides", &self.overrides),
240            ("freshness", &self.freshness),
241            ("docs", &self.docs),
242        ]
243        .into_iter()
244        .filter_map(|(k, v)| v.is_some().then_some(k))
245        .collect()
246    }
247}
248
249impl Table {
250    /// The dbt keys it sets that DRE doesn't use yet.
251    pub fn dbt_keys(&self) -> Vec<&'static str> {
252        [
253            ("loaded_at_field", &self.loaded_at_field),
254            ("loaded_at_query", &self.loaded_at_query),
255            ("tests", &self.tests),
256            ("data_tests", &self.data_tests),
257            ("freshness", &self.freshness),
258            ("external", &self.external),
259            ("config", &self.config),
260            ("docs", &self.docs),
261        ]
262        .into_iter()
263        .filter_map(|(k, v)| v.is_some().then_some(k))
264        .collect()
265    }
266}
267
268impl Column {
269    /// The dbt keys it sets that DRE doesn't use yet.
270    pub fn dbt_keys(&self) -> Vec<&'static str> {
271        [
272            ("meta", &self.meta),
273            ("tags", &self.tags),
274            ("quote", &self.quote),
275            ("tests", &self.tests),
276            ("data_tests", &self.data_tests),
277            ("constraints", &self.constraints),
278            ("config", &self.config),
279            ("docs", &self.docs),
280            ("granularity", &self.granularity),
281        ]
282        .into_iter()
283        .filter_map(|(k, v)| v.is_some().then_some(k))
284        .collect()
285    }
286}
dre_core/config/sources.rs

dre_core/config/
sources.rs
