marple_db/
models.rs

1use crate::{NoopProgress, ProgressReporter};
2use serde::{Deserialize, Serialize};
3use serde_json::{Map, Value};
4use std::fmt;
5use std::sync::Arc;
6
7/// JSON object used for user-defined stream or dataset metadata.
8///
9/// This is an insertion-preserving `serde_json::Map<String, Value>`.
10pub type Metadata = Map<String, Value>;
11
12/// Health response returned by the MarpleDB API.
13#[non_exhaustive]
14#[derive(Clone, Debug, Serialize, Deserialize)]
15pub struct HealthResponse {
16    /// Service health status.
17    pub status: String,
18}
19
20/// MarpleDB stream metadata.
21#[non_exhaustive]
22#[derive(Clone, Debug, Serialize, Deserialize)]
23pub struct Stream {
24    /// Stream id.
25    pub id: i32,
26    /// Stream name.
27    pub name: String,
28    /// Stream type.
29    #[serde(rename = "type")]
30    pub stream_type: StreamType,
31    /// Owning datapool.
32    pub datapool: String,
33    /// Stream description.
34    #[serde(default, deserialize_with = "deserialize_default_string")]
35    pub description: String,
36    /// Number of datasets, if known.
37    #[serde(default)]
38    pub n_datasets: Option<u64>,
39    /// Number of datapoints, if known.
40    #[serde(default)]
41    pub n_datapoints: Option<u64>,
42    /// Cold-storage byte size, if known.
43    #[serde(default)]
44    pub cold_bytes: Option<u64>,
45    /// Hot-storage byte size, if known.
46    #[serde(default)]
47    pub hot_bytes: Option<u64>,
48    /// Import plugin name for file streams.
49    #[serde(default)]
50    pub plugin: Option<String>,
51    /// Import plugin arguments for file streams.
52    #[serde(default)]
53    pub plugin_args: Option<String>,
54    /// Additional stream fields returned by the API.
55    #[serde(flatten)]
56    pub extra: Value,
57}
58
59/// MarpleDB stream type.
60#[non_exhaustive]
61#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
62#[serde(rename_all = "lowercase")]
63pub enum StreamType {
64    Files,
65    Realtime,
66}
67
68fn deserialize_default_string<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
69where
70    D: serde::Deserializer<'de>,
71{
72    Ok(Option::<String>::deserialize(deserializer)?.unwrap_or_default())
73}
74
75/// Dataset import lifecycle status.
76///
77/// Serialized values match the MarpleDB API and Python SDK enum names.
78#[non_exhaustive]
79#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
80#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
81pub enum ImportStatus {
82    /// File upload is still in progress.
83    Uploading,
84    /// Dataset is waiting to be imported.
85    Waiting,
86    /// Dataset import is running.
87    Importing,
88    /// Dataset post-processing is running.
89    Postprocessing,
90    /// Dataset post-processing failed.
91    PostprocessingFailed,
92    /// Dataset import finished successfully.
93    Finished,
94    /// Dataset is a live dataset.
95    Live,
96    /// Dataset import failed.
97    Failed,
98}
99
100/// Dataset metadata returned by the MarpleDB API.
101#[non_exhaustive]
102#[derive(Clone, Debug, Serialize, Deserialize)]
103pub struct Dataset {
104    /// Dataset id.
105    pub id: i32,
106    /// Owning stream id.
107    pub datastream_id: i32,
108    /// Owning stream version.
109    pub datastream_version: Option<i32>,
110    /// Creation timestamp as epoch seconds.
111    pub created_at: f64,
112    /// User that created the dataset, if available.
113    pub created_by: Option<String>,
114    /// Current import status.
115    pub import_status: ImportStatus,
116    /// Current import progress, if available.
117    pub import_progress: Option<f64>,
118    /// Import status message, if available.
119    pub import_message: Option<String>,
120    /// Import duration, if available.
121    pub import_time: Option<f64>,
122    /// Original dataset path or filename.
123    pub path: String,
124    /// User-defined dataset metadata.
125    pub metadata: Metadata,
126    /// Cold-storage path.
127    pub cold_path: Option<String>,
128    /// Cold-storage byte size.
129    pub cold_bytes: Option<u64>,
130    /// Hot-storage byte size.
131    pub hot_bytes: Option<u64>,
132    /// Backup path, if available.
133    pub backup_path: Option<String>,
134    /// Backup byte size, if available.
135    pub backup_size: Option<u64>,
136    /// Import plugin name.
137    pub plugin: Option<String>,
138    /// Import plugin arguments.
139    pub plugin_args: Option<String>,
140    /// Number of datapoints, if known.
141    pub n_datapoints: Option<u64>,
142    /// Number of signals, if known.
143    pub n_signals: Option<u64>,
144    /// Dataset start timestamp, if known.
145    pub timestamp_start: Option<f64>,
146    /// Dataset stop timestamp, if known.
147    pub timestamp_stop: Option<f64>,
148    /// Import speed, if known.
149    pub import_speed: Option<f64>,
150}
151
152/// Upload mode preference for `MarpleDB::push_file`.
153#[non_exhaustive]
154#[derive(Clone, Copy, Debug)]
155pub enum UploadModeOverride {
156    /// Let the server choose the upload mode.
157    Auto,
158    /// Force upload through the API server.
159    Server,
160}
161
162/// Options for uploading a file.
163#[non_exhaustive]
164pub struct PushFileOptions {
165    pub(crate) metadata: Metadata,
166    pub(crate) concurrency: usize,
167    pub(crate) upload_mode: UploadModeOverride,
168    pub(crate) progress: Arc<dyn ProgressReporter>,
169}
170
171impl fmt::Debug for PushFileOptions {
172    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
173        f.debug_struct("PushFileOptions")
174            .field("metadata", &self.metadata)
175            .field("concurrency", &self.concurrency)
176            .field("upload_mode", &self.upload_mode)
177            .finish_non_exhaustive()
178    }
179}
180
181impl PushFileOptions {
182    /// Creates a builder for upload options.
183    pub fn builder() -> PushFileOptionsBuilder {
184        PushFileOptionsBuilder::default()
185    }
186}
187
188impl Default for PushFileOptions {
189    fn default() -> Self {
190        Self {
191            metadata: Default::default(),
192            concurrency: 4,
193            upload_mode: UploadModeOverride::Auto,
194            progress: Arc::new(NoopProgress),
195        }
196    }
197}
198
199/// Builder for `PushFileOptions`.
200#[non_exhaustive]
201#[derive(Clone)]
202pub struct PushFileOptionsBuilder {
203    metadata: Metadata,
204    concurrency: usize,
205    upload_mode: UploadModeOverride,
206    progress: Arc<dyn ProgressReporter>,
207}
208
209impl fmt::Debug for PushFileOptionsBuilder {
210    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
211        f.debug_struct("PushFileOptionsBuilder")
212            .field("metadata", &self.metadata)
213            .field("concurrency", &self.concurrency)
214            .field("upload_mode", &self.upload_mode)
215            .finish_non_exhaustive()
216    }
217}
218
219impl Default for PushFileOptionsBuilder {
220    fn default() -> Self {
221        let options = PushFileOptions::default();
222        Self {
223            metadata: options.metadata,
224            concurrency: options.concurrency,
225            upload_mode: options.upload_mode,
226            progress: options.progress,
227        }
228    }
229}
230
231impl PushFileOptionsBuilder {
232    /// Sets dataset metadata for the upload.
233    ///
234    /// ```
235    /// use marple_db::PushFileOptions;
236    /// use serde_json::json;
237    ///
238    /// let options = PushFileOptions::builder()
239    ///     .metadata([
240    ///         ("driver", json!("Mbaerto")),
241    ///         ("run", json!(42)),
242    ///     ])
243    ///     .build();
244    /// ```
245    pub fn metadata<I, K, V>(mut self, entries: I) -> Self
246    where
247        I: IntoIterator<Item = (K, V)>,
248        K: Into<String>,
249        V: Into<Value>,
250    {
251        self.metadata = entries
252            .into_iter()
253            .map(|(key, value)| (key.into(), value.into()))
254            .collect();
255        self
256    }
257
258    /// Sets max concurrent part uploads for multipart modes.
259    ///
260    /// Higher values can improve throughput for large direct-storage uploads,
261    /// but also increase memory use and the number of active storage requests.
262    pub fn concurrency(mut self, concurrency: usize) -> Self {
263        self.concurrency = concurrency;
264        self
265    }
266
267    /// Sets the upload mode preference.
268    pub fn upload_mode(mut self, upload_mode: UploadModeOverride) -> Self {
269        self.upload_mode = upload_mode;
270        self
271    }
272
273    /// Sets the progress reporter.
274    pub fn progress(mut self, progress: Arc<dyn ProgressReporter>) -> Self {
275        self.progress = progress;
276        self
277    }
278
279    /// Builds upload options.
280    pub fn build(self) -> PushFileOptions {
281        PushFileOptions {
282            metadata: self.metadata,
283            concurrency: self.concurrency,
284            upload_mode: self.upload_mode,
285            progress: self.progress,
286        }
287    }
288}
289
290#[derive(Debug, Serialize, Deserialize)]
291pub(crate) struct StreamsResponse {
292    pub(crate) streams: Vec<Stream>,
293}
294
295#[derive(Debug, Deserialize)]
296#[serde(rename_all = "lowercase")]
297pub(crate) enum UploadMode {
298    Server,
299    Azure,
300    Single,
301    Multipart,
302}
303
304#[derive(Debug, Deserialize)]
305pub(crate) struct IngestionInit {
306    pub(crate) dataset_id: i32,
307    pub(crate) ingestion_id: i32,
308    pub(crate) mode: UploadMode,
309    pub(crate) presigned_url: Option<String>,
310    pub(crate) part_size: Option<u64>,
311    #[serde(rename = "expires_in")]
312    pub(crate) _expires_in: u64,
313}
314
315#[derive(Debug, Deserialize)]
316pub(crate) struct PartUrl {
317    pub(crate) part_number: u32,
318    pub(crate) url: String,
319}
320
321#[derive(Debug, Deserialize)]
322pub(crate) struct PartUrlsResponse {
323    pub(crate) parts: Vec<PartUrl>,
324    #[serde(rename = "expires_in")]
325    pub(crate) _expires_in: u64,
326    pub(crate) next_part: Option<u32>,
327}
marple_db/models.rs

marple_db/
models.rs
