1#[derive(
3 Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
4)]
5#[serde(rename_all = "lowercase")]
6#[non_exhaustive]
7pub enum FileFormat {
8 Csv,
9 Json,
10 Jsonl,
11 Parquet,
12 Arrow,
16 #[serde(untagged)]
17 Unknown(String),
18}
19
20impl std::fmt::Display for FileFormat {
21 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
22 match self {
23 Self::Csv => write!(f, "csv"),
24 Self::Json => write!(f, "json"),
25 Self::Jsonl => write!(f, "jsonl"),
26 Self::Parquet => write!(f, "parquet"),
27 Self::Arrow => write!(f, "arrow"),
28 Self::Unknown(s) => write!(f, "{}", s),
29 }
30 }
31}
32
33#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)]
38#[serde(rename_all = "lowercase")]
39pub enum JsonErrorPolicy {
40 #[default]
43 Skip,
44 Strict,
47}
48
49#[derive(
51 Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
52)]
53#[serde(rename_all = "lowercase")]
54pub enum QueryEngine {
55 Postgres,
56 MySql,
57 Sqlite,
58 Snowflake,
59 BigQuery,
60 #[serde(untagged)]
61 Custom(String),
62}
63
64impl std::fmt::Display for QueryEngine {
65 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
66 match self {
67 Self::Postgres => write!(f, "postgres"),
68 Self::MySql => write!(f, "mysql"),
69 Self::Sqlite => write!(f, "sqlite"),
70 Self::Snowflake => write!(f, "snowflake"),
71 Self::BigQuery => write!(f, "bigquery"),
72 Self::Custom(s) => write!(f, "{}", s),
73 }
74 }
75}
76
77#[derive(
79 Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
80)]
81#[serde(rename_all = "lowercase")]
82pub enum DataFrameLibrary {
83 Pandas,
84 Polars,
85 PyArrow,
86 #[serde(untagged)]
87 Custom(String),
88}
89
90impl std::fmt::Display for DataFrameLibrary {
91 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
92 match self {
93 Self::Pandas => write!(f, "pandas"),
94 Self::Polars => write!(f, "polars"),
95 Self::PyArrow => write!(f, "pyarrow"),
96 Self::Custom(s) => write!(f, "{}", s),
97 }
98 }
99}
100
101#[derive(
103 Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
104)]
105#[serde(rename_all = "lowercase")]
106pub enum StreamSourceSystem {
107 Kafka,
108 Kinesis,
109 Pulsar,
110 Http,
111 WebSocket,
112 #[serde(rename = "object_store")]
113 ObjectStore,
114 #[serde(rename = "message_queue")]
115 MessageQueue,
116 Database,
117 #[serde(untagged)]
118 Custom(String),
119}
120
121impl std::fmt::Display for StreamSourceSystem {
122 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
123 match self {
124 Self::Kafka => write!(f, "kafka"),
125 Self::Kinesis => write!(f, "kinesis"),
126 Self::Pulsar => write!(f, "pulsar"),
127 Self::Http => write!(f, "http"),
128 Self::WebSocket => write!(f, "websocket"),
129 Self::ObjectStore => write!(f, "object_store"),
130 Self::MessageQueue => write!(f, "message_queue"),
131 Self::Database => write!(f, "database"),
132 Self::Custom(s) => write!(f, "{}", s),
133 }
134 }
135}
136
137#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
139pub struct ParquetMetadata {
140 pub num_row_groups: usize,
142 pub compression: String,
144 pub version: i32,
146 pub schema_summary: String,
148 pub compressed_size_bytes: u64,
150 pub uncompressed_size_bytes: Option<u64>,
152}
153
154#[derive(
156 Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
157)]
158#[non_exhaustive]
159pub struct CsvSourceMetadata {
160 pub delimiter: String,
162 #[serde(default, skip_serializing_if = "Option::is_none")]
165 pub delimiter_detection: Option<DelimiterDetection>,
166}
167
168impl CsvSourceMetadata {
169 pub fn configured(delimiter: u8) -> Self {
171 Self {
172 delimiter: char::from(delimiter).to_string(),
173 delimiter_detection: None,
174 }
175 }
176
177 pub fn configured_char(delimiter: char) -> Self {
181 Self {
182 delimiter: delimiter.to_string(),
183 delimiter_detection: None,
184 }
185 }
186
187 pub fn detected(detection: DelimiterDetection) -> Self {
189 Self {
190 delimiter: detection.delimiter.clone(),
191 delimiter_detection: Some(detection),
192 }
193 }
194}
195
196#[derive(
202 Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
203)]
204#[non_exhaustive]
205pub struct DelimiterDetection {
206 pub delimiter: String,
208 pub header_fields: usize,
211 #[serde(default, skip_serializing_if = "Option::is_none")]
215 pub runner_up: Option<String>,
216 #[serde(default, skip_serializing_if = "Option::is_none")]
221 pub unlisted_delimiter: Option<String>,
222}
223
224impl DelimiterDetection {
225 pub fn new(delimiter: u8, header_fields: usize, runner_up: Option<u8>) -> Self {
227 Self {
228 delimiter: char::from(delimiter).to_string(),
229 header_fields,
230 runner_up: runner_up.map(|byte| char::from(byte).to_string()),
231 unlisted_delimiter: None,
232 }
233 }
234
235 pub fn with_unlisted_delimiter(mut self, unlisted: Option<u8>) -> Self {
237 self.unlisted_delimiter = unlisted.map(|byte| char::from(byte).to_string());
238 self
239 }
240}
241
242#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
244#[serde(tag = "type", rename_all = "snake_case")]
245pub enum DataSource {
246 File {
248 path: String,
250 format: FileFormat,
252 size_bytes: u64,
254 #[serde(skip_serializing_if = "Option::is_none")]
256 modified_at: Option<String>,
257 #[serde(skip_serializing_if = "Option::is_none")]
259 parquet_metadata: Option<ParquetMetadata>,
260 #[serde(default, skip_serializing_if = "Option::is_none")]
262 csv_metadata: Option<CsvSourceMetadata>,
263 },
264 Query {
266 engine: QueryEngine,
268 statement: String,
270 #[serde(skip_serializing_if = "Option::is_none")]
272 database: Option<String>,
273 #[serde(skip_serializing_if = "Option::is_none")]
275 execution_id: Option<String>,
276 },
277 #[serde(rename = "dataframe")]
279 DataFrame {
280 name: String,
282 source_library: DataFrameLibrary,
284 row_count: usize,
286 column_count: usize,
288 #[serde(skip_serializing_if = "Option::is_none")]
290 memory_bytes: Option<u64>,
291 },
292 Bytes {
297 name: String,
299 format: FileFormat,
301 size_bytes: u64,
303 #[serde(default, skip_serializing_if = "Option::is_none")]
305 csv_metadata: Option<CsvSourceMetadata>,
306 },
307 Stream {
309 topic: String,
311 batch_id: String,
313 #[serde(skip_serializing_if = "Option::is_none")]
315 partition: Option<u32>,
316 #[serde(skip_serializing_if = "Option::is_none")]
318 consumer_group: Option<String>,
319 #[serde(default, skip_serializing_if = "Option::is_none")]
323 source_system: Option<StreamSourceSystem>,
324 #[serde(default, skip_serializing_if = "Option::is_none")]
327 format: Option<FileFormat>,
328 #[serde(skip_serializing_if = "Option::is_none")]
330 session_id: Option<String>,
331 #[serde(skip_serializing_if = "Option::is_none")]
333 first_record_at: Option<String>,
334 #[serde(skip_serializing_if = "Option::is_none")]
336 last_record_at: Option<String>,
337 #[serde(default, skip_serializing_if = "Option::is_none")]
339 csv_metadata: Option<CsvSourceMetadata>,
340 },
341}
342
343impl DataSource {
344 pub fn identifier(&self) -> String {
346 match self {
347 Self::File { path, .. } => path.clone(),
348 Self::Query {
349 engine, statement, ..
350 } => {
351 let truncated = if statement.chars().count() > 50 {
352 let mut prefix: String = statement.chars().take(47).collect();
353 prefix.push_str("...");
354 prefix
355 } else {
356 statement.clone()
357 };
358 format!("{}: {}", engine, truncated)
359 }
360 Self::DataFrame {
361 name,
362 source_library,
363 ..
364 } => format!("{}[{}]", source_library, name),
365 Self::Bytes { name, .. } => format!("bytes[{name}]"),
366 Self::Stream {
367 source_system,
368 topic,
369 batch_id,
370 ..
371 } => match source_system {
372 Some(system) => format!("{system}[{topic}]-batch:{batch_id}"),
373 None => format!("stream[{topic}]-batch:{batch_id}"),
374 },
375 }
376 }
377
378 pub fn size_mb(&self) -> Option<f64> {
380 match self {
381 Self::File { size_bytes, .. } => Some(*size_bytes as f64 / 1_048_576.0),
382 Self::DataFrame { memory_bytes, .. } => memory_bytes.map(|b| b as f64 / 1_048_576.0),
383 Self::Bytes { size_bytes, .. } => Some(*size_bytes as f64 / 1_048_576.0),
384 Self::Query { .. } | Self::Stream { .. } => None,
385 }
386 }
387
388 pub fn is_file(&self) -> bool {
390 matches!(self, Self::File { .. })
391 }
392
393 pub fn is_query(&self) -> bool {
395 matches!(self, Self::Query { .. })
396 }
397
398 pub fn is_dataframe(&self) -> bool {
400 matches!(self, Self::DataFrame { .. })
401 }
402
403 pub fn is_stream(&self) -> bool {
405 matches!(self, Self::Stream { .. })
406 }
407
408 pub fn is_bytes(&self) -> bool {
410 matches!(self, Self::Bytes { .. })
411 }
412
413 pub fn file_path(&self) -> Option<&str> {
415 match self {
416 Self::File { path, .. } => Some(path),
417 _ => None,
418 }
419 }
420
421 pub fn format(&self) -> Option<&FileFormat> {
424 match self {
425 Self::File { format, .. } | Self::Bytes { format, .. } => Some(format),
426 Self::Stream { format, .. } => format.as_ref(),
427 Self::Query { .. } | Self::DataFrame { .. } => None,
428 }
429 }
430
431 pub fn csv_metadata(&self) -> Option<&CsvSourceMetadata> {
433 match self {
434 Self::File { csv_metadata, .. }
435 | Self::Bytes { csv_metadata, .. }
436 | Self::Stream { csv_metadata, .. } => csv_metadata.as_ref(),
437 Self::Query { .. } | Self::DataFrame { .. } => None,
438 }
439 }
440
441 pub fn with_csv_metadata(mut self, metadata: CsvSourceMetadata) -> Self {
444 match &mut self {
445 Self::File { csv_metadata, .. }
446 | Self::Bytes { csv_metadata, .. }
447 | Self::Stream { csv_metadata, .. } => *csv_metadata = Some(metadata),
448 Self::Query { .. } | Self::DataFrame { .. } => {}
449 }
450 self
451 }
452
453 pub fn stream_topic(&self) -> Option<&str> {
455 match self {
456 Self::Stream { topic, .. } => Some(topic),
457 _ => None,
458 }
459 }
460
461 pub fn batch_id(&self) -> Option<&str> {
463 match self {
464 Self::Stream { batch_id, .. } => Some(batch_id),
465 _ => None,
466 }
467 }
468}
469
470#[cfg(test)]
471mod tests {
472 use super::*;
473
474 #[test]
475 fn test_data_source_file_identifier() {
476 let ds = DataSource::File {
477 path: "/path/to/data.csv".to_string(),
478 format: FileFormat::Csv,
479 size_bytes: 0,
480 modified_at: None,
481 parquet_metadata: None,
482 csv_metadata: None,
483 };
484
485 assert_eq!(ds.identifier(), "/path/to/data.csv");
486 assert!(ds.is_file());
487 assert!(!ds.is_query());
488 assert!(!ds.is_dataframe());
489 assert!(!ds.is_stream());
490 }
491
492 #[test]
493 fn test_data_source_bytes_identifier_and_helpers() {
494 let ds = DataSource::Bytes {
495 name: "csv_bytes".to_string(),
496 format: FileFormat::Csv,
497 size_bytes: 42,
498 csv_metadata: None,
499 };
500
501 assert_eq!(ds.identifier(), "bytes[csv_bytes]");
502 assert!(ds.is_bytes());
503 assert!(!ds.is_file());
504 assert!(!ds.is_dataframe());
505 assert!(!ds.is_stream());
506 assert_eq!(ds.size_mb(), Some(42.0 / 1_048_576.0));
507 }
508
509 #[test]
510 fn test_data_source_stream_identifier_and_helpers() {
511 let ds = DataSource::Stream {
512 topic: "events".to_string(),
513 batch_id: "b1".to_string(),
514 partition: Some(0),
515 consumer_group: None,
516 source_system: Some(StreamSourceSystem::Kafka),
517 format: None,
518 session_id: None,
519 first_record_at: None,
520 last_record_at: None,
521 csv_metadata: None,
522 };
523
524 assert_eq!(ds.identifier(), "kafka[events]-batch:b1");
525 assert!(ds.is_stream());
526 assert_eq!(ds.stream_topic(), Some("events"));
527 assert_eq!(ds.batch_id(), Some("b1"));
528 assert!(!ds.is_file());
529 assert!(!ds.is_query());
530 assert!(ds.size_mb().is_none());
531 }
532
533 #[test]
534 fn test_stream_without_a_declared_system_records_none() {
535 let ds = DataSource::Stream {
536 topic: "events".to_string(),
537 batch_id: "b1".to_string(),
538 partition: None,
539 consumer_group: None,
540 source_system: None,
541 format: None,
542 session_id: None,
543 first_record_at: None,
544 last_record_at: None,
545 csv_metadata: None,
546 };
547
548 assert_eq!(ds.identifier(), "stream[events]-batch:b1");
549 let json = serde_json::to_value(&ds).unwrap();
550 assert!(json.get("source_system").is_none(), "{json}");
551 let reloaded: DataSource = serde_json::from_value(json.clone()).unwrap();
552 assert_eq!(serde_json::to_value(&reloaded).unwrap(), json);
553 }
554
555 #[test]
556 fn test_stream_json_serialization() {
557 let ds = DataSource::Stream {
558 topic: "sensor-data".to_string(),
559 batch_id: "batch-789".to_string(),
560 partition: Some(2),
561 consumer_group: Some("processing-group".to_string()),
562 source_system: Some(StreamSourceSystem::Kinesis),
563 format: None,
564 session_id: Some("session-1".to_string()),
565 first_record_at: Some("2023-01-01T10:00:00Z".to_string()),
566 last_record_at: Some("2023-01-01T10:05:00Z".to_string()),
567 csv_metadata: None,
568 };
569
570 let json = serde_json::to_string(&ds).unwrap();
571 assert!(json.contains(r#""type":"stream""#));
572 assert!(json.contains(r#""source_system":"kinesis""#));
573 assert!(json.contains(r#""topic":"sensor-data""#));
574
575 let deserialized: DataSource = serde_json::from_str(&json).unwrap();
576 assert!(deserialized.is_stream());
577 assert_eq!(deserialized.stream_topic(), Some("sensor-data"));
578 }
579
580 #[test]
581 fn test_stream_source_system_serialization_names() {
582 let object_store = serde_json::to_string(&StreamSourceSystem::ObjectStore).unwrap();
583 let message_queue = serde_json::to_string(&StreamSourceSystem::MessageQueue).unwrap();
584 let database = serde_json::to_string(&StreamSourceSystem::Database).unwrap();
585
586 assert_eq!(object_store, r#""object_store""#);
587 assert_eq!(message_queue, r#""message_queue""#);
588 assert_eq!(database, r#""database""#);
589
590 let object_store: StreamSourceSystem = serde_json::from_str(r#""object_store""#).unwrap();
591 let message_queue: StreamSourceSystem = serde_json::from_str(r#""message_queue""#).unwrap();
592 let database: StreamSourceSystem = serde_json::from_str(r#""database""#).unwrap();
593
594 assert_eq!(object_store, StreamSourceSystem::ObjectStore);
595 assert_eq!(message_queue, StreamSourceSystem::MessageQueue);
596 assert_eq!(database, StreamSourceSystem::Database);
597 }
598}