cognite/dto/data_ingestion/
extpipes.rs

1use std::collections::HashMap;
2
3use serde::{Deserialize, Serialize};
4use serde_with::skip_serializing_none;
5
6use crate::{
7    Identity, IntoPatch, IntoPatchItem, Patch, Range, UpdateList, UpdateMap, UpdateSet,
8    UpsertOptions,
9};
10
11#[derive(Serialize, Deserialize, Clone, Default, Debug)]
12#[serde(rename_all = "camelCase")]
13/// Reference to a raw table in an extraction pipeline
14pub struct ExtPipeRawTable {
15    /// Raw database name
16    pub db_name: String,
17    /// Raw table name
18    pub table_name: String,
19}
20
21#[skip_serializing_none]
22#[derive(Serialize, Deserialize, Clone, Default, Debug)]
23#[serde(rename_all = "camelCase")]
24/// Contact person for an extraction pipeline
25pub struct ExtPipeContact {
26    /// Name of contact.
27    pub name: Option<String>,
28    /// Contact e-mail.
29    pub email: Option<String>,
30    /// Contact role.
31    pub role: Option<String>,
32    /// Whether to send a notification to this person when
33    /// the status of an extraciton pipeline changes.
34    pub send_notification: Option<bool>,
35}
36
37#[skip_serializing_none]
38#[derive(Debug, Serialize, Deserialize, Clone)]
39#[serde(rename_all = "camelCase")]
40/// An extraction pipeline.
41pub struct ExtPipe {
42    /// Internal ID
43    pub id: i64,
44    /// Extraction pipeline external ID. Must be unique accross all extraction pipelines in the project.
45    pub external_id: String,
46    /// Extraction pipeline name.
47    pub name: String,
48    /// Short description.
49    pub description: Option<String>,
50    /// Data set this extraction pipeline belongs to.
51    pub data_set_id: i64,
52    /// List of raw tables the extractor this extraction pipeline represents is documented
53    /// to write to.
54    pub raw_tables: Option<Vec<ExtPipeRawTable>>,
55    /// Documented schedule.
56    pub schedule: Option<String>,
57    /// List of contacts.
58    pub contacts: Option<Vec<ExtPipeContact>>,
59    /// Application specific metadata.
60    pub metadata: Option<HashMap<String, String>>,
61    /// User provided source.
62    pub source: Option<String>,
63    /// Long form documentation.
64    pub documentation: Option<String>,
65    /// Timestamp of last success, in milliseconds since epoch.
66    pub last_success: Option<i64>,
67    /// Timestamp of last failure, in milliseconds since epoch.
68    pub last_failure: Option<i64>,
69    /// Last message received.
70    pub last_message: Option<String>,
71    /// Timestamp of last event, in milliseconds since epoch.
72    pub last_seen: Option<i64>,
73    /// Time this extraction pipeline was created, in milliseconds since epoch.
74    pub created_time: i64,
75    /// Time this extraction pipeline was last updated, in milliseconds since epoch.
76    pub last_updated_time: i64,
77    /// The user that created this extraction pipeline, if available.
78    pub created_by: Option<String>,
79}
80
81#[skip_serializing_none]
82#[derive(Debug, Serialize, Deserialize, Default, Clone)]
83#[serde(rename_all = "camelCase")]
84/// Create an extraction pipeline.
85pub struct AddExtPipe {
86    /// Extraction pipeline external ID. Must be unique accross all extraction pipelines in the project.
87    pub external_id: String,
88    /// Extraction pipeline name.
89    pub name: String,
90    /// Short description.
91    pub description: Option<String>,
92    /// Data set this extraction pipeline belongs to.
93    pub data_set_id: i64,
94    /// List of raw tables the extractor this extraction pipeline represents is documented
95    /// to write to.
96    pub raw_tables: Option<Vec<ExtPipeRawTable>>,
97    /// Documented schedule.
98    pub schedule: Option<String>,
99    /// List of contacts.
100    pub contacts: Option<Vec<ExtPipeContact>>,
101    /// Application specific metadata.
102    pub metadata: Option<HashMap<String, String>>,
103    /// User provided source.
104    pub source: Option<String>,
105    /// Long form documentation.
106    pub documentation: Option<String>,
107}
108
109impl From<ExtPipe> for AddExtPipe {
110    fn from(pipe: ExtPipe) -> Self {
111        AddExtPipe {
112            external_id: pipe.external_id,
113            name: pipe.name,
114            description: pipe.description,
115            data_set_id: pipe.data_set_id,
116            raw_tables: pipe.raw_tables,
117            schedule: pipe.schedule,
118            contacts: pipe.contacts,
119            metadata: pipe.metadata,
120            source: pipe.source,
121            documentation: pipe.documentation,
122        }
123    }
124}
125
126#[skip_serializing_none]
127#[derive(Serialize, Deserialize, Default, Clone)]
128#[serde(rename_all = "camelCase")]
129/// Update an extraction pipeline
130pub struct PatchExtPipe {
131    /// Extraction pipeline external ID. Must be unique accross all extraction pipelines in the project.
132    pub external_id: Option<UpdateSet<String>>,
133    /// Extraction pipeline name.
134    pub name: Option<UpdateSet<String>>,
135    /// Short description.
136    pub description: Option<UpdateSet<Option<String>>>,
137    /// Data set this extraction pipeline belongs to.
138    pub data_set_id: Option<UpdateSet<i64>>,
139    /// Documented schedule.
140    pub schedule: Option<UpdateSet<Option<String>>>,
141    /// List of raw tables the extractor this extraction pipeline represents is documented
142    /// to write to.
143    pub raw_tables: Option<UpdateList<ExtPipeRawTable, ExtPipeRawTable>>,
144    /// List of contacts.
145    pub contacts: Option<UpdateList<ExtPipeContact, ExtPipeContact>>,
146    /// Application specific metadata.
147    pub metadata: Option<UpdateMap<String, String>>,
148    /// User provided source.
149    pub source: Option<UpdateSet<Option<String>>>,
150    /// Long form documentation.
151    pub documentation: Option<UpdateSet<Option<String>>>,
152}
153
154impl IntoPatch<Patch<PatchExtPipe>> for ExtPipe {
155    fn patch(self, options: &UpsertOptions) -> Patch<PatchExtPipe> {
156        Patch::<PatchExtPipe> {
157            id: Identity::ExternalId {
158                external_id: self.external_id,
159            },
160            update: PatchExtPipe {
161                external_id: None,
162                name: self.name.patch(options),
163                description: self.description.patch(options),
164                data_set_id: self.data_set_id.patch(options),
165                schedule: self.schedule.patch(options),
166                raw_tables: self.raw_tables.patch(options),
167                contacts: self.contacts.patch(options),
168                metadata: self.metadata.patch(options),
169                source: self.source.patch(options),
170                documentation: self.documentation.patch(options),
171            },
172        }
173    }
174}
175
176impl IntoPatch<PatchExtPipe> for AddExtPipe {
177    fn patch(self, options: &UpsertOptions) -> PatchExtPipe {
178        PatchExtPipe {
179            external_id: self.external_id.patch(options),
180            name: self.name.patch(options),
181            description: self.description.patch(options),
182            data_set_id: self.data_set_id.patch(options),
183            schedule: self.schedule.patch(options),
184            raw_tables: self.raw_tables.patch(options),
185            contacts: self.contacts.patch(options),
186            metadata: self.metadata.patch(options),
187            source: self.source.patch(options),
188            documentation: self.documentation.patch(options),
189        }
190    }
191}
192
193impl From<ExtPipe> for Patch<PatchExtPipe> {
194    fn from(sequence: ExtPipe) -> Self {
195        IntoPatch::<Patch<PatchExtPipe>>::patch(sequence, &Default::default())
196    }
197}
198
199#[derive(Serialize, Deserialize, Clone, Default)]
200#[serde(rename_all = "camelCase")]
201/// Status of an extraction pipeline run.
202pub enum ExtPipeRunStatus {
203    /// Success, the run completed succesfully.
204    Success,
205    /// Failure, the run failed.
206    Failure,
207    /// Seen, the run is a heartbeat.
208    #[default]
209    Seen,
210}
211
212#[skip_serializing_none]
213#[derive(Serialize, Deserialize, Default, Clone)]
214#[serde(rename_all = "camelCase")]
215/// Filter extraction pipelines.
216pub struct ExtPipeFilter {
217    /// Filter using this (case-sensitive) prefix on external ID.
218    pub external_id_prefix: Option<String>,
219    /// Include extraction pipelines with this name.
220    pub name: Option<String>,
221    /// Include extraction pipelines with this description.
222    pub description: Option<String>,
223    /// Include extraction pipelines which belongs to one of these data set ids.
224    pub data_set_ids: Option<Vec<Identity>>,
225    /// Include extraction pipelines with this schedule.
226    pub schedule: Option<String>,
227    /// Include extraction pipelines with these contacts.
228    pub contacts: Option<Vec<ExtPipeContact>>,
229    /// Include extraction pipelines with these raw tables.
230    pub raw_tables: Option<Vec<ExtPipeRawTable>>,
231    /// Include extraction pipelines with this metadata.
232    pub metadata: Option<HashMap<String, String>>,
233    /// Include extraction pipelines with this source.
234    pub source: Option<String>,
235    /// Include extraction pipelines with this documentation.
236    pub documentation: Option<String>,
237    /// Include extraction pipelines with this creator.
238    pub created_by: Option<String>,
239    /// Range of timestamps for `created_time`.
240    pub created_time: Option<Range<i64>>,
241    /// Range of timestamps for `last_updated_time`.
242    pub last_updated_time: Option<Range<i64>>,
243}
244
245#[skip_serializing_none]
246#[derive(Serialize, Deserialize)]
247#[serde(rename_all = "camelCase")]
248/// A single completed run or heartbeat.
249pub struct ExtPipeRun {
250    /// Run ID.
251    pub id: i64,
252    /// Run status
253    pub status: ExtPipeRunStatus,
254    /// Optional message.
255    pub message: Option<String>,
256    /// Time this run happened, in milliseconds since epoch.
257    pub created_time: i64,
258    /// Extraction pipeline external ID.
259    pub external_id: Option<String>,
260}
261
262#[skip_serializing_none]
263#[derive(Serialize, Deserialize, Default)]
264#[serde(rename_all = "camelCase")]
265/// Create a new extraction pipeline run.
266pub struct AddExtPipeRun {
267    /// Run status
268    pub status: ExtPipeRunStatus,
269    /// Optional message.
270    pub message: Option<String>,
271    /// Time this run happened, in milliseconds since epoch.
272    pub created_time: Option<i64>,
273    /// Extraction pipeline external ID.
274    pub external_id: String,
275}
276
277#[derive(Serialize, Deserialize, Default, Clone)]
278#[serde(rename_all = "camelCase")]
279/// Filter on a string in extraction pipeline runs.
280pub struct ExtPipeStringFilter {
281    /// Match on a substring of the filtered value.
282    pub substring: String,
283}
284
285#[skip_serializing_none]
286#[derive(Serialize, Deserialize, Default, Clone)]
287#[serde(rename_all = "camelCase")]
288/// Filter extraction pipeline runs.
289pub struct ExtPipeRunFilter {
290    /// Extraction pipeline external ID.
291    pub external_id: String,
292    /// Include runs with one of these statuses.
293    pub statuses: Option<Vec<ExtPipeRunStatus>>,
294    /// Include runs within this range.
295    pub created_time: Option<Range<i64>>,
296    /// Include runs with messages matching this filter.
297    pub message: Option<ExtPipeStringFilter>,
298}
cognite/dto/data_ingestion/extpipes.rs

cognite/dto/data_ingestion/
extpipes.rs
