nominal_api/conjure/endpoints/ingest/api/
ingest_service.rs

1use conjure_http::endpoint;
2/// The Ingest Service handles the data ingestion into Nominal/Clickhouse.
3#[conjure_http::conjure_endpoints(
4    name = "IngestService",
5    use_legacy_error_serialization
6)]
7pub trait IngestService {
8    /// Triggers an ingest job, allowing either creating a new dataset or uploading to an
9    /// existing one. This endpoint is meant to supersede all other ingestion endpoints as their functionality
10    /// gets migrated to this one.
11    #[endpoint(
12        method = POST,
13        path = "/ingest/v1/ingest",
14        name = "ingest",
15        produces = conjure_http::server::StdResponseSerializer
16    )]
17    fn ingest(
18        &self,
19        #[auth]
20        auth_: conjure_object::BearerToken,
21        #[body(
22            deserializer = conjure_http::server::StdRequestDeserializer,
23            log_as = "triggerIngest"
24        )]
25        trigger_ingest: super::super::super::super::objects::ingest::api::IngestRequest,
26    ) -> Result<
27        super::super::super::super::objects::ingest::api::IngestResponse,
28        conjure_http::private::Error,
29    >;
30    /// Triggers an ingest job using an existing ingest job RID.
31    /// Returns the same response format as the /ingest endpoint.
32    #[endpoint(
33        method = POST,
34        path = "/ingest/v1/re-ingest",
35        name = "rerunIngest",
36        produces = conjure_http::server::StdResponseSerializer
37    )]
38    fn rerun_ingest(
39        &self,
40        #[auth]
41        auth_: conjure_object::BearerToken,
42        #[body(deserializer = conjure_http::server::StdRequestDeserializer)]
43        request: super::super::super::super::objects::ingest::api::RerunIngestRequest,
44    ) -> Result<
45        super::super::super::super::objects::ingest::api::IngestResponse,
46        conjure_http::private::Error,
47    >;
48    /// Creates a run and ingests data sources to be added to the run.
49    #[endpoint(
50        method = POST,
51        path = "/ingest/v1/ingest-run",
52        name = "ingestRun",
53        produces = conjure_http::server::StdResponseSerializer
54    )]
55    fn ingest_run(
56        &self,
57        #[auth]
58        auth_: conjure_object::BearerToken,
59        #[body(deserializer = conjure_http::server::StdRequestDeserializer)]
60        request: super::super::super::super::objects::ingest::api::IngestRunRequest,
61    ) -> Result<
62        super::super::super::super::objects::ingest::api::IngestRunResponse,
63        conjure_http::private::Error,
64    >;
65    /// Ingests video data from a S3 Nominal upload bucket.
66    #[endpoint(
67        method = POST,
68        path = "/ingest/v1/ingest-video",
69        name = "ingestVideo",
70        produces = conjure_http::server::StdResponseSerializer
71    )]
72    fn ingest_video(
73        &self,
74        #[auth]
75        auth_: conjure_object::BearerToken,
76        #[body(
77            deserializer = conjure_http::server::StdRequestDeserializer,
78            log_as = "ingestVideo"
79        )]
80        ingest_video: super::super::super::super::objects::ingest::api::IngestVideoRequest,
81    ) -> Result<
82        super::super::super::super::objects::ingest::api::IngestVideoResponse,
83        conjure_http::private::Error,
84    >;
85    /// Re-ingests data from provided source datasets into either an existing target dataset, or a new one.
86    /// Only supported for CSV and Parquet dataset files.
87    /// Will only reingest dataset files and will drop streaming data from datasets.
88    #[endpoint(
89        method = POST,
90        path = "/ingest/v1/reingest-dataset-files",
91        name = "reingestFromDatasets",
92        produces = conjure_http::server::StdResponseSerializer
93    )]
94    fn reingest_from_datasets(
95        &self,
96        #[auth]
97        auth_: conjure_object::BearerToken,
98        #[body(deserializer = conjure_http::server::StdRequestDeserializer)]
99        request: super::super::super::super::objects::ingest::api::ReingestDatasetsRequest,
100    ) -> Result<
101        super::super::super::super::objects::ingest::api::ReingestDatasetsResponse,
102        conjure_http::private::Error,
103    >;
104    /// This is a best effort deletion of the file's data based on the ingestedAt timestamp. This is an unreversible
105    /// action. Only v2 dataset file deletion is supported.
106    /// !!!WARNING!!!
107    /// It's possible that the file has overwritten points, in which case, those older points will not be recovered.
108    /// Only use this endpoint if this is acceptable, the data across files are disjoint, or you're willing to
109    /// re-ingest files to manually recover older points.
110    #[endpoint(
111        method = DELETE,
112        path = "/ingest/v1/delete-file/{datasetRid}/file/{fileId}",
113        name = "deleteFile"
114    )]
115    fn delete_file(
116        &self,
117        #[auth]
118        auth_: conjure_object::BearerToken,
119        #[path(
120            name = "datasetRid",
121            decoder = conjure_http::server::conjure::FromPlainDecoder,
122            log_as = "datasetRid"
123        )]
124        dataset_rid: conjure_object::ResourceIdentifier,
125        #[path(
126            name = "fileId",
127            decoder = conjure_http::server::conjure::FromPlainDecoder,
128            log_as = "fileId"
129        )]
130        file_id: conjure_object::Uuid,
131    ) -> Result<(), conjure_http::private::Error>;
132}
133/// The Ingest Service handles the data ingestion into Nominal/Clickhouse.
134#[conjure_http::conjure_endpoints(
135    name = "IngestService",
136    use_legacy_error_serialization
137)]
138pub trait AsyncIngestService {
139    /// Triggers an ingest job, allowing either creating a new dataset or uploading to an
140    /// existing one. This endpoint is meant to supersede all other ingestion endpoints as their functionality
141    /// gets migrated to this one.
142    #[endpoint(
143        method = POST,
144        path = "/ingest/v1/ingest",
145        name = "ingest",
146        produces = conjure_http::server::StdResponseSerializer
147    )]
148    async fn ingest(
149        &self,
150        #[auth]
151        auth_: conjure_object::BearerToken,
152        #[body(
153            deserializer = conjure_http::server::StdRequestDeserializer,
154            log_as = "triggerIngest"
155        )]
156        trigger_ingest: super::super::super::super::objects::ingest::api::IngestRequest,
157    ) -> Result<
158        super::super::super::super::objects::ingest::api::IngestResponse,
159        conjure_http::private::Error,
160    >;
161    /// Triggers an ingest job using an existing ingest job RID.
162    /// Returns the same response format as the /ingest endpoint.
163    #[endpoint(
164        method = POST,
165        path = "/ingest/v1/re-ingest",
166        name = "rerunIngest",
167        produces = conjure_http::server::StdResponseSerializer
168    )]
169    async fn rerun_ingest(
170        &self,
171        #[auth]
172        auth_: conjure_object::BearerToken,
173        #[body(deserializer = conjure_http::server::StdRequestDeserializer)]
174        request: super::super::super::super::objects::ingest::api::RerunIngestRequest,
175    ) -> Result<
176        super::super::super::super::objects::ingest::api::IngestResponse,
177        conjure_http::private::Error,
178    >;
179    /// Creates a run and ingests data sources to be added to the run.
180    #[endpoint(
181        method = POST,
182        path = "/ingest/v1/ingest-run",
183        name = "ingestRun",
184        produces = conjure_http::server::StdResponseSerializer
185    )]
186    async fn ingest_run(
187        &self,
188        #[auth]
189        auth_: conjure_object::BearerToken,
190        #[body(deserializer = conjure_http::server::StdRequestDeserializer)]
191        request: super::super::super::super::objects::ingest::api::IngestRunRequest,
192    ) -> Result<
193        super::super::super::super::objects::ingest::api::IngestRunResponse,
194        conjure_http::private::Error,
195    >;
196    /// Ingests video data from a S3 Nominal upload bucket.
197    #[endpoint(
198        method = POST,
199        path = "/ingest/v1/ingest-video",
200        name = "ingestVideo",
201        produces = conjure_http::server::StdResponseSerializer
202    )]
203    async fn ingest_video(
204        &self,
205        #[auth]
206        auth_: conjure_object::BearerToken,
207        #[body(
208            deserializer = conjure_http::server::StdRequestDeserializer,
209            log_as = "ingestVideo"
210        )]
211        ingest_video: super::super::super::super::objects::ingest::api::IngestVideoRequest,
212    ) -> Result<
213        super::super::super::super::objects::ingest::api::IngestVideoResponse,
214        conjure_http::private::Error,
215    >;
216    /// Re-ingests data from provided source datasets into either an existing target dataset, or a new one.
217    /// Only supported for CSV and Parquet dataset files.
218    /// Will only reingest dataset files and will drop streaming data from datasets.
219    #[endpoint(
220        method = POST,
221        path = "/ingest/v1/reingest-dataset-files",
222        name = "reingestFromDatasets",
223        produces = conjure_http::server::StdResponseSerializer
224    )]
225    async fn reingest_from_datasets(
226        &self,
227        #[auth]
228        auth_: conjure_object::BearerToken,
229        #[body(deserializer = conjure_http::server::StdRequestDeserializer)]
230        request: super::super::super::super::objects::ingest::api::ReingestDatasetsRequest,
231    ) -> Result<
232        super::super::super::super::objects::ingest::api::ReingestDatasetsResponse,
233        conjure_http::private::Error,
234    >;
235    /// This is a best effort deletion of the file's data based on the ingestedAt timestamp. This is an unreversible
236    /// action. Only v2 dataset file deletion is supported.
237    /// !!!WARNING!!!
238    /// It's possible that the file has overwritten points, in which case, those older points will not be recovered.
239    /// Only use this endpoint if this is acceptable, the data across files are disjoint, or you're willing to
240    /// re-ingest files to manually recover older points.
241    #[endpoint(
242        method = DELETE,
243        path = "/ingest/v1/delete-file/{datasetRid}/file/{fileId}",
244        name = "deleteFile"
245    )]
246    async fn delete_file(
247        &self,
248        #[auth]
249        auth_: conjure_object::BearerToken,
250        #[path(
251            name = "datasetRid",
252            decoder = conjure_http::server::conjure::FromPlainDecoder,
253            log_as = "datasetRid"
254        )]
255        dataset_rid: conjure_object::ResourceIdentifier,
256        #[path(
257            name = "fileId",
258            decoder = conjure_http::server::conjure::FromPlainDecoder,
259            log_as = "fileId"
260        )]
261        file_id: conjure_object::Uuid,
262    ) -> Result<(), conjure_http::private::Error>;
263}
264/// The Ingest Service handles the data ingestion into Nominal/Clickhouse.
265#[conjure_http::conjure_endpoints(
266    name = "IngestService",
267    use_legacy_error_serialization,
268    local
269)]
270pub trait LocalAsyncIngestService {
271    /// Triggers an ingest job, allowing either creating a new dataset or uploading to an
272    /// existing one. This endpoint is meant to supersede all other ingestion endpoints as their functionality
273    /// gets migrated to this one.
274    #[endpoint(
275        method = POST,
276        path = "/ingest/v1/ingest",
277        name = "ingest",
278        produces = conjure_http::server::StdResponseSerializer
279    )]
280    async fn ingest(
281        &self,
282        #[auth]
283        auth_: conjure_object::BearerToken,
284        #[body(
285            deserializer = conjure_http::server::StdRequestDeserializer,
286            log_as = "triggerIngest"
287        )]
288        trigger_ingest: super::super::super::super::objects::ingest::api::IngestRequest,
289    ) -> Result<
290        super::super::super::super::objects::ingest::api::IngestResponse,
291        conjure_http::private::Error,
292    >;
293    /// Triggers an ingest job using an existing ingest job RID.
294    /// Returns the same response format as the /ingest endpoint.
295    #[endpoint(
296        method = POST,
297        path = "/ingest/v1/re-ingest",
298        name = "rerunIngest",
299        produces = conjure_http::server::StdResponseSerializer
300    )]
301    async fn rerun_ingest(
302        &self,
303        #[auth]
304        auth_: conjure_object::BearerToken,
305        #[body(deserializer = conjure_http::server::StdRequestDeserializer)]
306        request: super::super::super::super::objects::ingest::api::RerunIngestRequest,
307    ) -> Result<
308        super::super::super::super::objects::ingest::api::IngestResponse,
309        conjure_http::private::Error,
310    >;
311    /// Creates a run and ingests data sources to be added to the run.
312    #[endpoint(
313        method = POST,
314        path = "/ingest/v1/ingest-run",
315        name = "ingestRun",
316        produces = conjure_http::server::StdResponseSerializer
317    )]
318    async fn ingest_run(
319        &self,
320        #[auth]
321        auth_: conjure_object::BearerToken,
322        #[body(deserializer = conjure_http::server::StdRequestDeserializer)]
323        request: super::super::super::super::objects::ingest::api::IngestRunRequest,
324    ) -> Result<
325        super::super::super::super::objects::ingest::api::IngestRunResponse,
326        conjure_http::private::Error,
327    >;
328    /// Ingests video data from a S3 Nominal upload bucket.
329    #[endpoint(
330        method = POST,
331        path = "/ingest/v1/ingest-video",
332        name = "ingestVideo",
333        produces = conjure_http::server::StdResponseSerializer
334    )]
335    async fn ingest_video(
336        &self,
337        #[auth]
338        auth_: conjure_object::BearerToken,
339        #[body(
340            deserializer = conjure_http::server::StdRequestDeserializer,
341            log_as = "ingestVideo"
342        )]
343        ingest_video: super::super::super::super::objects::ingest::api::IngestVideoRequest,
344    ) -> Result<
345        super::super::super::super::objects::ingest::api::IngestVideoResponse,
346        conjure_http::private::Error,
347    >;
348    /// Re-ingests data from provided source datasets into either an existing target dataset, or a new one.
349    /// Only supported for CSV and Parquet dataset files.
350    /// Will only reingest dataset files and will drop streaming data from datasets.
351    #[endpoint(
352        method = POST,
353        path = "/ingest/v1/reingest-dataset-files",
354        name = "reingestFromDatasets",
355        produces = conjure_http::server::StdResponseSerializer
356    )]
357    async fn reingest_from_datasets(
358        &self,
359        #[auth]
360        auth_: conjure_object::BearerToken,
361        #[body(deserializer = conjure_http::server::StdRequestDeserializer)]
362        request: super::super::super::super::objects::ingest::api::ReingestDatasetsRequest,
363    ) -> Result<
364        super::super::super::super::objects::ingest::api::ReingestDatasetsResponse,
365        conjure_http::private::Error,
366    >;
367    /// This is a best effort deletion of the file's data based on the ingestedAt timestamp. This is an unreversible
368    /// action. Only v2 dataset file deletion is supported.
369    /// !!!WARNING!!!
370    /// It's possible that the file has overwritten points, in which case, those older points will not be recovered.
371    /// Only use this endpoint if this is acceptable, the data across files are disjoint, or you're willing to
372    /// re-ingest files to manually recover older points.
373    #[endpoint(
374        method = DELETE,
375        path = "/ingest/v1/delete-file/{datasetRid}/file/{fileId}",
376        name = "deleteFile"
377    )]
378    async fn delete_file(
379        &self,
380        #[auth]
381        auth_: conjure_object::BearerToken,
382        #[path(
383            name = "datasetRid",
384            decoder = conjure_http::server::conjure::FromPlainDecoder,
385            log_as = "datasetRid"
386        )]
387        dataset_rid: conjure_object::ResourceIdentifier,
388        #[path(
389            name = "fileId",
390            decoder = conjure_http::server::conjure::FromPlainDecoder,
391            log_as = "fileId"
392        )]
393        file_id: conjure_object::Uuid,
394    ) -> Result<(), conjure_http::private::Error>;
395}
nominal_api/conjure/endpoints/ingest/api/ingest_service.rs

nominal_api/conjure/endpoints/ingest/api/
ingest_service.rs
