nominal_api_conjure/conjure/endpoints/ingest/api/
ingest_service.rs

1use conjure_http::endpoint;
2/// The Ingest Service handles the data ingestion into Nominal/Clickhouse.
3#[conjure_http::conjure_endpoints(
4    name = "IngestService",
5    use_legacy_error_serialization
6)]
7pub trait IngestService {
8    /// Triggers an ingest job, allowing either creating a new dataset or uploading to an
9    /// existing one. This endpoint is meant to supersede all other ingestion endpoints as their functionality
10    /// gets migrated to this one.
11    #[endpoint(
12        method = POST,
13        path = "/ingest/v1/ingest",
14        name = "ingest",
15        produces = conjure_http::server::StdResponseSerializer
16    )]
17    fn ingest(
18        &self,
19        #[auth]
20        auth_: conjure_object::BearerToken,
21        #[body(
22            deserializer = conjure_http::server::StdRequestDeserializer,
23            log_as = "triggerIngest"
24        )]
25        trigger_ingest: super::super::super::super::objects::ingest::api::IngestRequest,
26    ) -> Result<
27        super::super::super::super::objects::ingest::api::IngestResponse,
28        conjure_http::private::Error,
29    >;
30    /// Triggers an ingest job using an existing ingest job RID.
31    /// Returns the same response format as the /ingest endpoint.
32    ///
33    /// Only a job that has finished can be re-run: re-running one still in flight races or duplicates
34    /// the run already in progress. Cancel it first.
35    #[endpoint(
36        method = POST,
37        path = "/ingest/v1/re-ingest",
38        name = "rerunIngest",
39        produces = conjure_http::server::StdResponseSerializer
40    )]
41    fn rerun_ingest(
42        &self,
43        #[auth]
44        auth_: conjure_object::BearerToken,
45        #[body(deserializer = conjure_http::server::StdRequestDeserializer, safe)]
46        request: super::super::super::super::objects::ingest::api::RerunIngestRequest,
47    ) -> Result<
48        super::super::super::super::objects::ingest::api::IngestResponse,
49        conjure_http::private::Error,
50    >;
51    /// Creates a run and ingests data sources to be added to the run.
52    #[endpoint(
53        method = POST,
54        path = "/ingest/v1/ingest-run",
55        name = "ingestRun",
56        produces = conjure_http::server::StdResponseSerializer
57    )]
58    fn ingest_run(
59        &self,
60        #[auth]
61        auth_: conjure_object::BearerToken,
62        #[body(deserializer = conjure_http::server::StdRequestDeserializer)]
63        request: super::super::super::super::objects::ingest::api::IngestRunRequest,
64    ) -> Result<
65        super::super::super::super::objects::ingest::api::IngestRunResponse,
66        conjure_http::private::Error,
67    >;
68    /// Ingests video data from a S3 Nominal upload bucket.
69    #[endpoint(
70        method = POST,
71        path = "/ingest/v1/ingest-video",
72        name = "ingestVideo",
73        produces = conjure_http::server::StdResponseSerializer
74    )]
75    fn ingest_video(
76        &self,
77        #[auth]
78        auth_: conjure_object::BearerToken,
79        #[body(
80            deserializer = conjure_http::server::StdRequestDeserializer,
81            log_as = "ingestVideo"
82        )]
83        ingest_video: super::super::super::super::objects::ingest::api::IngestVideoRequest,
84    ) -> Result<
85        super::super::super::super::objects::ingest::api::IngestVideoResponse,
86        conjure_http::private::Error,
87    >;
88    /// Re-ingests data from provided source datasets into either an existing target dataset, or a new one.
89    /// Only supported for CSV and Parquet dataset files.
90    /// Will only reingest dataset files and will drop streaming data from datasets.
91    #[endpoint(
92        method = POST,
93        path = "/ingest/v1/reingest-dataset-files",
94        name = "reingestFromDatasets",
95        produces = conjure_http::server::StdResponseSerializer
96    )]
97    fn reingest_from_datasets(
98        &self,
99        #[auth]
100        auth_: conjure_object::BearerToken,
101        #[body(deserializer = conjure_http::server::StdRequestDeserializer)]
102        request: super::super::super::super::objects::ingest::api::ReingestDatasetsRequest,
103    ) -> Result<
104        super::super::super::super::objects::ingest::api::ReingestDatasetsResponse,
105        conjure_http::private::Error,
106    >;
107    /// This is a best effort deletion of the file's data based on the ingestedAt timestamp. This is an unreversible
108    /// action. Only v2 dataset file deletion is supported.
109    /// !!!WARNING!!!
110    /// It's possible that the file has overwritten points, in which case, those older points will not be recovered.
111    /// Only use this endpoint if this is acceptable, the data across files are disjoint, or you're willing to
112    /// re-ingest files to manually recover older points.
113    #[endpoint(
114        method = DELETE,
115        path = "/ingest/v1/delete-file/{datasetRid}/file/{fileId}",
116        name = "deleteFile"
117    )]
118    fn delete_file(
119        &self,
120        #[auth]
121        auth_: conjure_object::BearerToken,
122        #[path(
123            name = "datasetRid",
124            decoder = conjure_http::server::conjure::FromPlainDecoder,
125            log_as = "datasetRid",
126            safe
127        )]
128        dataset_rid: super::super::super::super::objects::api::rids::DatasetRid,
129        #[path(
130            name = "fileId",
131            decoder = conjure_http::server::conjure::FromPlainDecoder,
132            log_as = "fileId"
133        )]
134        file_id: conjure_object::Uuid,
135    ) -> Result<(), conjure_http::private::Error>;
136}
137/// The Ingest Service handles the data ingestion into Nominal/Clickhouse.
138#[conjure_http::conjure_endpoints(
139    name = "IngestService",
140    use_legacy_error_serialization
141)]
142pub trait AsyncIngestService {
143    /// Triggers an ingest job, allowing either creating a new dataset or uploading to an
144    /// existing one. This endpoint is meant to supersede all other ingestion endpoints as their functionality
145    /// gets migrated to this one.
146    #[endpoint(
147        method = POST,
148        path = "/ingest/v1/ingest",
149        name = "ingest",
150        produces = conjure_http::server::StdResponseSerializer
151    )]
152    async fn ingest(
153        &self,
154        #[auth]
155        auth_: conjure_object::BearerToken,
156        #[body(
157            deserializer = conjure_http::server::StdRequestDeserializer,
158            log_as = "triggerIngest"
159        )]
160        trigger_ingest: super::super::super::super::objects::ingest::api::IngestRequest,
161    ) -> Result<
162        super::super::super::super::objects::ingest::api::IngestResponse,
163        conjure_http::private::Error,
164    >;
165    /// Triggers an ingest job using an existing ingest job RID.
166    /// Returns the same response format as the /ingest endpoint.
167    ///
168    /// Only a job that has finished can be re-run: re-running one still in flight races or duplicates
169    /// the run already in progress. Cancel it first.
170    #[endpoint(
171        method = POST,
172        path = "/ingest/v1/re-ingest",
173        name = "rerunIngest",
174        produces = conjure_http::server::StdResponseSerializer
175    )]
176    async fn rerun_ingest(
177        &self,
178        #[auth]
179        auth_: conjure_object::BearerToken,
180        #[body(deserializer = conjure_http::server::StdRequestDeserializer, safe)]
181        request: super::super::super::super::objects::ingest::api::RerunIngestRequest,
182    ) -> Result<
183        super::super::super::super::objects::ingest::api::IngestResponse,
184        conjure_http::private::Error,
185    >;
186    /// Creates a run and ingests data sources to be added to the run.
187    #[endpoint(
188        method = POST,
189        path = "/ingest/v1/ingest-run",
190        name = "ingestRun",
191        produces = conjure_http::server::StdResponseSerializer
192    )]
193    async fn ingest_run(
194        &self,
195        #[auth]
196        auth_: conjure_object::BearerToken,
197        #[body(deserializer = conjure_http::server::StdRequestDeserializer)]
198        request: super::super::super::super::objects::ingest::api::IngestRunRequest,
199    ) -> Result<
200        super::super::super::super::objects::ingest::api::IngestRunResponse,
201        conjure_http::private::Error,
202    >;
203    /// Ingests video data from a S3 Nominal upload bucket.
204    #[endpoint(
205        method = POST,
206        path = "/ingest/v1/ingest-video",
207        name = "ingestVideo",
208        produces = conjure_http::server::StdResponseSerializer
209    )]
210    async fn ingest_video(
211        &self,
212        #[auth]
213        auth_: conjure_object::BearerToken,
214        #[body(
215            deserializer = conjure_http::server::StdRequestDeserializer,
216            log_as = "ingestVideo"
217        )]
218        ingest_video: super::super::super::super::objects::ingest::api::IngestVideoRequest,
219    ) -> Result<
220        super::super::super::super::objects::ingest::api::IngestVideoResponse,
221        conjure_http::private::Error,
222    >;
223    /// Re-ingests data from provided source datasets into either an existing target dataset, or a new one.
224    /// Only supported for CSV and Parquet dataset files.
225    /// Will only reingest dataset files and will drop streaming data from datasets.
226    #[endpoint(
227        method = POST,
228        path = "/ingest/v1/reingest-dataset-files",
229        name = "reingestFromDatasets",
230        produces = conjure_http::server::StdResponseSerializer
231    )]
232    async fn reingest_from_datasets(
233        &self,
234        #[auth]
235        auth_: conjure_object::BearerToken,
236        #[body(deserializer = conjure_http::server::StdRequestDeserializer)]
237        request: super::super::super::super::objects::ingest::api::ReingestDatasetsRequest,
238    ) -> Result<
239        super::super::super::super::objects::ingest::api::ReingestDatasetsResponse,
240        conjure_http::private::Error,
241    >;
242    /// This is a best effort deletion of the file's data based on the ingestedAt timestamp. This is an unreversible
243    /// action. Only v2 dataset file deletion is supported.
244    /// !!!WARNING!!!
245    /// It's possible that the file has overwritten points, in which case, those older points will not be recovered.
246    /// Only use this endpoint if this is acceptable, the data across files are disjoint, or you're willing to
247    /// re-ingest files to manually recover older points.
248    #[endpoint(
249        method = DELETE,
250        path = "/ingest/v1/delete-file/{datasetRid}/file/{fileId}",
251        name = "deleteFile"
252    )]
253    async fn delete_file(
254        &self,
255        #[auth]
256        auth_: conjure_object::BearerToken,
257        #[path(
258            name = "datasetRid",
259            decoder = conjure_http::server::conjure::FromPlainDecoder,
260            log_as = "datasetRid",
261            safe
262        )]
263        dataset_rid: super::super::super::super::objects::api::rids::DatasetRid,
264        #[path(
265            name = "fileId",
266            decoder = conjure_http::server::conjure::FromPlainDecoder,
267            log_as = "fileId"
268        )]
269        file_id: conjure_object::Uuid,
270    ) -> Result<(), conjure_http::private::Error>;
271}
272/// The Ingest Service handles the data ingestion into Nominal/Clickhouse.
273#[conjure_http::conjure_endpoints(
274    name = "IngestService",
275    use_legacy_error_serialization,
276    local
277)]
278pub trait LocalAsyncIngestService {
279    /// Triggers an ingest job, allowing either creating a new dataset or uploading to an
280    /// existing one. This endpoint is meant to supersede all other ingestion endpoints as their functionality
281    /// gets migrated to this one.
282    #[endpoint(
283        method = POST,
284        path = "/ingest/v1/ingest",
285        name = "ingest",
286        produces = conjure_http::server::StdResponseSerializer
287    )]
288    async fn ingest(
289        &self,
290        #[auth]
291        auth_: conjure_object::BearerToken,
292        #[body(
293            deserializer = conjure_http::server::StdRequestDeserializer,
294            log_as = "triggerIngest"
295        )]
296        trigger_ingest: super::super::super::super::objects::ingest::api::IngestRequest,
297    ) -> Result<
298        super::super::super::super::objects::ingest::api::IngestResponse,
299        conjure_http::private::Error,
300    >;
301    /// Triggers an ingest job using an existing ingest job RID.
302    /// Returns the same response format as the /ingest endpoint.
303    ///
304    /// Only a job that has finished can be re-run: re-running one still in flight races or duplicates
305    /// the run already in progress. Cancel it first.
306    #[endpoint(
307        method = POST,
308        path = "/ingest/v1/re-ingest",
309        name = "rerunIngest",
310        produces = conjure_http::server::StdResponseSerializer
311    )]
312    async fn rerun_ingest(
313        &self,
314        #[auth]
315        auth_: conjure_object::BearerToken,
316        #[body(deserializer = conjure_http::server::StdRequestDeserializer, safe)]
317        request: super::super::super::super::objects::ingest::api::RerunIngestRequest,
318    ) -> Result<
319        super::super::super::super::objects::ingest::api::IngestResponse,
320        conjure_http::private::Error,
321    >;
322    /// Creates a run and ingests data sources to be added to the run.
323    #[endpoint(
324        method = POST,
325        path = "/ingest/v1/ingest-run",
326        name = "ingestRun",
327        produces = conjure_http::server::StdResponseSerializer
328    )]
329    async fn ingest_run(
330        &self,
331        #[auth]
332        auth_: conjure_object::BearerToken,
333        #[body(deserializer = conjure_http::server::StdRequestDeserializer)]
334        request: super::super::super::super::objects::ingest::api::IngestRunRequest,
335    ) -> Result<
336        super::super::super::super::objects::ingest::api::IngestRunResponse,
337        conjure_http::private::Error,
338    >;
339    /// Ingests video data from a S3 Nominal upload bucket.
340    #[endpoint(
341        method = POST,
342        path = "/ingest/v1/ingest-video",
343        name = "ingestVideo",
344        produces = conjure_http::server::StdResponseSerializer
345    )]
346    async fn ingest_video(
347        &self,
348        #[auth]
349        auth_: conjure_object::BearerToken,
350        #[body(
351            deserializer = conjure_http::server::StdRequestDeserializer,
352            log_as = "ingestVideo"
353        )]
354        ingest_video: super::super::super::super::objects::ingest::api::IngestVideoRequest,
355    ) -> Result<
356        super::super::super::super::objects::ingest::api::IngestVideoResponse,
357        conjure_http::private::Error,
358    >;
359    /// Re-ingests data from provided source datasets into either an existing target dataset, or a new one.
360    /// Only supported for CSV and Parquet dataset files.
361    /// Will only reingest dataset files and will drop streaming data from datasets.
362    #[endpoint(
363        method = POST,
364        path = "/ingest/v1/reingest-dataset-files",
365        name = "reingestFromDatasets",
366        produces = conjure_http::server::StdResponseSerializer
367    )]
368    async fn reingest_from_datasets(
369        &self,
370        #[auth]
371        auth_: conjure_object::BearerToken,
372        #[body(deserializer = conjure_http::server::StdRequestDeserializer)]
373        request: super::super::super::super::objects::ingest::api::ReingestDatasetsRequest,
374    ) -> Result<
375        super::super::super::super::objects::ingest::api::ReingestDatasetsResponse,
376        conjure_http::private::Error,
377    >;
378    /// This is a best effort deletion of the file's data based on the ingestedAt timestamp. This is an unreversible
379    /// action. Only v2 dataset file deletion is supported.
380    /// !!!WARNING!!!
381    /// It's possible that the file has overwritten points, in which case, those older points will not be recovered.
382    /// Only use this endpoint if this is acceptable, the data across files are disjoint, or you're willing to
383    /// re-ingest files to manually recover older points.
384    #[endpoint(
385        method = DELETE,
386        path = "/ingest/v1/delete-file/{datasetRid}/file/{fileId}",
387        name = "deleteFile"
388    )]
389    async fn delete_file(
390        &self,
391        #[auth]
392        auth_: conjure_object::BearerToken,
393        #[path(
394            name = "datasetRid",
395            decoder = conjure_http::server::conjure::FromPlainDecoder,
396            log_as = "datasetRid",
397            safe
398        )]
399        dataset_rid: super::super::super::super::objects::api::rids::DatasetRid,
400        #[path(
401            name = "fileId",
402            decoder = conjure_http::server::conjure::FromPlainDecoder,
403            log_as = "fileId"
404        )]
405        file_id: conjure_object::Uuid,
406    ) -> Result<(), conjure_http::private::Error>;
407}
nominal_api_conjure/conjure/endpoints/ingest/api/ingest_service.rs

nominal_api_conjure/conjure/endpoints/ingest/api/
ingest_service.rs
