Skip to main content

dataprof_core/
source.rs

1/// Supported file formats for data profiling
2#[derive(
3    Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
4)]
5#[serde(rename_all = "lowercase")]
6#[non_exhaustive]
7pub enum FileFormat {
8    Csv,
9    Json,
10    Jsonl,
11    Parquet,
12    /// An Arrow C stream, whose schema lists every column before the first
13    /// batch. Only the Arrow stream route records it: no file, bytes or URL
14    /// route detects or reads this format.
15    Arrow,
16    #[serde(untagged)]
17    Unknown(String),
18}
19
20impl std::fmt::Display for FileFormat {
21    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
22        match self {
23            Self::Csv => write!(f, "csv"),
24            Self::Json => write!(f, "json"),
25            Self::Jsonl => write!(f, "jsonl"),
26            Self::Parquet => write!(f, "parquet"),
27            Self::Arrow => write!(f, "arrow"),
28            Self::Unknown(s) => write!(f, "{}", s),
29        }
30    }
31}
32
33/// How JSON/JSONL parsing reacts to a malformed record.
34///
35/// Shared across the file, in-memory, and async byte input paths so callers get
36/// the same recovery behavior regardless of transport.
37#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)]
38#[serde(rename_all = "lowercase")]
39pub enum JsonErrorPolicy {
40    /// Skip the malformed record, count it, and keep scanning. The profile is
41    /// reported as partial via `execution.error_count`.
42    #[default]
43    Skip,
44    /// Abort on the first malformed record with a `JsonParsingError` carrying
45    /// line/column context (never the record contents).
46    Strict,
47}
48
49/// Supported query engines for SQL-based profiling
50#[derive(
51    Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
52)]
53#[serde(rename_all = "lowercase")]
54pub enum QueryEngine {
55    Postgres,
56    MySql,
57    Sqlite,
58    Snowflake,
59    BigQuery,
60    #[serde(untagged)]
61    Custom(String),
62}
63
64impl std::fmt::Display for QueryEngine {
65    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
66        match self {
67            Self::Postgres => write!(f, "postgres"),
68            Self::MySql => write!(f, "mysql"),
69            Self::Sqlite => write!(f, "sqlite"),
70            Self::Snowflake => write!(f, "snowflake"),
71            Self::BigQuery => write!(f, "bigquery"),
72            Self::Custom(s) => write!(f, "{}", s),
73        }
74    }
75}
76
77/// Source library for in-memory DataFrame profiling
78#[derive(
79    Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
80)]
81#[serde(rename_all = "lowercase")]
82pub enum DataFrameLibrary {
83    Pandas,
84    Polars,
85    PyArrow,
86    #[serde(untagged)]
87    Custom(String),
88}
89
90impl std::fmt::Display for DataFrameLibrary {
91    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
92        match self {
93            Self::Pandas => write!(f, "pandas"),
94            Self::Polars => write!(f, "polars"),
95            Self::PyArrow => write!(f, "pyarrow"),
96            Self::Custom(s) => write!(f, "{}", s),
97        }
98    }
99}
100
101/// Supported stream source systems
102#[derive(
103    Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
104)]
105#[serde(rename_all = "lowercase")]
106pub enum StreamSourceSystem {
107    Kafka,
108    Kinesis,
109    Pulsar,
110    Http,
111    WebSocket,
112    #[serde(rename = "object_store")]
113    ObjectStore,
114    #[serde(rename = "message_queue")]
115    MessageQueue,
116    Database,
117    #[serde(untagged)]
118    Custom(String),
119}
120
121impl std::fmt::Display for StreamSourceSystem {
122    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
123        match self {
124            Self::Kafka => write!(f, "kafka"),
125            Self::Kinesis => write!(f, "kinesis"),
126            Self::Pulsar => write!(f, "pulsar"),
127            Self::Http => write!(f, "http"),
128            Self::WebSocket => write!(f, "websocket"),
129            Self::ObjectStore => write!(f, "object_store"),
130            Self::MessageQueue => write!(f, "message_queue"),
131            Self::Database => write!(f, "database"),
132            Self::Custom(s) => write!(f, "{}", s),
133        }
134    }
135}
136
137/// Metadata specific to Parquet files
138#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
139pub struct ParquetMetadata {
140    /// Number of row groups in the Parquet file
141    pub num_row_groups: usize,
142    /// Compression codec used (e.g., "SNAPPY", "GZIP", "ZSTD", "UNCOMPRESSED")
143    pub compression: String,
144    /// Parquet file version (e.g., "1.0", "2.0")
145    pub version: i32,
146    /// Arrow schema as string representation
147    pub schema_summary: String,
148    /// Total compressed size in bytes
149    pub compressed_size_bytes: u64,
150    /// Estimated uncompressed size if available
151    pub uncompressed_size_bytes: Option<u64>,
152}
153
154/// How a CSV source was split into fields.
155#[derive(
156    Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
157)]
158#[non_exhaustive]
159pub struct CsvSourceMetadata {
160    /// The delimiter the reader split records on, as one character.
161    pub delimiter: String,
162    /// How detection chose the delimiter. `None` when the delimiter was set
163    /// explicitly, so detection did not run.
164    #[serde(default, skip_serializing_if = "Option::is_none")]
165    pub delimiter_detection: Option<DelimiterDetection>,
166}
167
168impl CsvSourceMetadata {
169    /// A delimiter the caller set explicitly.
170    pub fn configured(delimiter: u8) -> Self {
171        Self {
172            delimiter: char::from(delimiter).to_string(),
173            delimiter_detection: None,
174        }
175    }
176
177    /// A delimiter character the caller set, for a reader that splits on
178    /// characters rather than bytes: Python's `csv` module, which reads
179    /// synchronous CSV bytes, takes any one character, `ยง` included.
180    pub fn configured_char(delimiter: char) -> Self {
181        Self {
182            delimiter: delimiter.to_string(),
183            delimiter_detection: None,
184        }
185    }
186
187    /// A delimiter detection chose.
188    pub fn detected(detection: DelimiterDetection) -> Self {
189        Self {
190            delimiter: detection.delimiter.clone(),
191            delimiter_detection: Some(detection),
192        }
193    }
194}
195
196/// What delimiter detection saw when it chose a delimiter.
197///
198/// Detection scores comma, semicolon, tab and pipe over the first records of
199/// the source. Some sources read equally well with two of them, and some with
200/// none: the record says so, and `findings()` warns about both.
201#[derive(
202    Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
203)]
204#[non_exhaustive]
205pub struct DelimiterDetection {
206    /// The delimiter detection chose, as one character.
207    pub delimiter: String,
208    /// Fields the chosen delimiter splits the header into: 1 when no
209    /// candidate splits it, 0 when the sampled source holds no header record.
210    pub header_fields: usize,
211    /// Another candidate that splits the header and keeps as many sampled
212    /// records at the header's width as the chosen one, so that structure
213    /// alone does not tell them apart. `None` when the choice was clear.
214    #[serde(default, skip_serializing_if = "Option::is_none")]
215    pub runner_up: Option<String>,
216    /// When no candidate splits the header, a punctuation character outside
217    /// the candidates that occurs as often in the header as in every sampled
218    /// record, so it likely delimits the source. `None` when a candidate
219    /// splits the header, and for an ordinary one-column source.
220    #[serde(default, skip_serializing_if = "Option::is_none")]
221    pub unlisted_delimiter: Option<String>,
222}
223
224impl DelimiterDetection {
225    /// A detection outcome; `runner_up` is the tied candidate, if any.
226    pub fn new(delimiter: u8, header_fields: usize, runner_up: Option<u8>) -> Self {
227        Self {
228            delimiter: char::from(delimiter).to_string(),
229            header_fields,
230            runner_up: runner_up.map(|byte| char::from(byte).to_string()),
231            unlisted_delimiter: None,
232        }
233    }
234
235    /// Record a likely delimiter detection does not try.
236    pub fn with_unlisted_delimiter(mut self, unlisted: Option<u8>) -> Self {
237        self.unlisted_delimiter = unlisted.map(|byte| char::from(byte).to_string());
238        self
239    }
240}
241
242/// Source-agnostic data source metadata.
243#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)]
244#[serde(tag = "type", rename_all = "snake_case")]
245pub enum DataSource {
246    /// File-based data source (CSV, JSON, Parquet, etc.)
247    File {
248        /// Absolute or relative path to the file
249        path: String,
250        /// Detected or specified file format
251        format: FileFormat,
252        /// File size in bytes
253        size_bytes: u64,
254        /// Last modification timestamp (ISO 8601 / RFC 3339)
255        #[serde(skip_serializing_if = "Option::is_none")]
256        modified_at: Option<String>,
257        /// Parquet-specific metadata (only present for Parquet files)
258        #[serde(skip_serializing_if = "Option::is_none")]
259        parquet_metadata: Option<ParquetMetadata>,
260        /// How a CSV source was split into fields (only present for CSV)
261        #[serde(default, skip_serializing_if = "Option::is_none")]
262        csv_metadata: Option<CsvSourceMetadata>,
263    },
264    /// SQL query-based data source
265    Query {
266        /// Database engine used for the query
267        engine: QueryEngine,
268        /// SQL statement executed
269        statement: String,
270        /// Target database name (if applicable)
271        #[serde(skip_serializing_if = "Option::is_none")]
272        database: Option<String>,
273        /// Unique execution identifier for tracing
274        #[serde(skip_serializing_if = "Option::is_none")]
275        execution_id: Option<String>,
276    },
277    /// In-memory DataFrame source (pandas/polars via PyCapsule)
278    #[serde(rename = "dataframe")]
279    DataFrame {
280        /// User-provided name for identification
281        name: String,
282        /// Source library (pandas, polars, pyarrow)
283        source_library: DataFrameLibrary,
284        /// Number of rows at profiling time
285        row_count: usize,
286        /// Number of columns
287        column_count: usize,
288        /// Memory usage in bytes (if available)
289        #[serde(skip_serializing_if = "Option::is_none")]
290        memory_bytes: Option<u64>,
291    },
292    /// In-memory byte buffer profiled without a path behind it. The buffer may
293    /// be text (CSV/JSON/JSONL) or binary (Parquet); `format` says which.
294    /// Distinguished from [`DataSource::DataFrame`]: the caller handed over
295    /// bytes, not a schema-bearing dataframe, so provenance must say so.
296    Bytes {
297        /// Human-readable name for identification
298        name: String,
299        /// Format the buffer was parsed as
300        format: FileFormat,
301        /// Length of the buffer handed over, not of the values decoded from it
302        size_bytes: u64,
303        /// How a CSV source was split into fields (only present for CSV)
304        #[serde(default, skip_serializing_if = "Option::is_none")]
305        csv_metadata: Option<CsvSourceMetadata>,
306    },
307    /// Streaming data source
308    Stream {
309        /// Stream identifier (e.g., Kafka topic, Kinesis stream name)
310        topic: String,
311        /// Batch identifier for ordering and deduplication
312        batch_id: String,
313        /// Partition for parallel processing (optional)
314        #[serde(skip_serializing_if = "Option::is_none")]
315        partition: Option<u32>,
316        /// Consumer group for Kafka-style coordination (optional)
317        #[serde(skip_serializing_if = "Option::is_none")]
318        consumer_group: Option<String>,
319        /// Source system identifier (kafka, kinesis, pulsar, http, etc.), as
320        /// the caller declared it. Absent when the caller declared none: the
321        /// report does not guess a transport (#1219).
322        #[serde(default, skip_serializing_if = "Option::is_none")]
323        source_system: Option<StreamSourceSystem>,
324        /// Format the stream was parsed as: `arrow` for an Arrow C stream
325        /// (#1230). Absent in reports written before 0.14 (#1042).
326        #[serde(default, skip_serializing_if = "Option::is_none")]
327        format: Option<FileFormat>,
328        /// Session ID for multi-tenant scenarios
329        #[serde(skip_serializing_if = "Option::is_none")]
330        session_id: Option<String>,
331        /// Timestamp of first record in batch (ISO 8601)
332        #[serde(skip_serializing_if = "Option::is_none")]
333        first_record_at: Option<String>,
334        /// Timestamp of last record in batch (ISO 8601)
335        #[serde(skip_serializing_if = "Option::is_none")]
336        last_record_at: Option<String>,
337        /// How a CSV source was split into fields (only present for CSV)
338        #[serde(default, skip_serializing_if = "Option::is_none")]
339        csv_metadata: Option<CsvSourceMetadata>,
340    },
341}
342
343impl DataSource {
344    /// Get a human-readable identifier for this data source.
345    pub fn identifier(&self) -> String {
346        match self {
347            Self::File { path, .. } => path.clone(),
348            Self::Query {
349                engine, statement, ..
350            } => {
351                let truncated = if statement.chars().count() > 50 {
352                    let mut prefix: String = statement.chars().take(47).collect();
353                    prefix.push_str("...");
354                    prefix
355                } else {
356                    statement.clone()
357                };
358                format!("{}: {}", engine, truncated)
359            }
360            Self::DataFrame {
361                name,
362                source_library,
363                ..
364            } => format!("{}[{}]", source_library, name),
365            Self::Bytes { name, .. } => format!("bytes[{name}]"),
366            Self::Stream {
367                source_system,
368                topic,
369                batch_id,
370                ..
371            } => match source_system {
372                Some(system) => format!("{system}[{topic}]-batch:{batch_id}"),
373                None => format!("stream[{topic}]-batch:{batch_id}"),
374            },
375        }
376    }
377
378    /// Get file size in megabytes if this is a file-based source or dataframe.
379    pub fn size_mb(&self) -> Option<f64> {
380        match self {
381            Self::File { size_bytes, .. } => Some(*size_bytes as f64 / 1_048_576.0),
382            Self::DataFrame { memory_bytes, .. } => memory_bytes.map(|b| b as f64 / 1_048_576.0),
383            Self::Bytes { size_bytes, .. } => Some(*size_bytes as f64 / 1_048_576.0),
384            Self::Query { .. } | Self::Stream { .. } => None,
385        }
386    }
387
388    /// Check if this is a file-based source.
389    pub fn is_file(&self) -> bool {
390        matches!(self, Self::File { .. })
391    }
392
393    /// Check if this is a query-based source.
394    pub fn is_query(&self) -> bool {
395        matches!(self, Self::Query { .. })
396    }
397
398    /// Check if this is a DataFrame-based source.
399    pub fn is_dataframe(&self) -> bool {
400        matches!(self, Self::DataFrame { .. })
401    }
402
403    /// Check if this is a Stream-based source.
404    pub fn is_stream(&self) -> bool {
405        matches!(self, Self::Stream { .. })
406    }
407
408    /// Check if this is an in-memory byte-buffer source.
409    pub fn is_bytes(&self) -> bool {
410        matches!(self, Self::Bytes { .. })
411    }
412
413    /// Get the file path if this is a file-based source.
414    pub fn file_path(&self) -> Option<&str> {
415        match self {
416            Self::File { path, .. } => Some(path),
417            _ => None,
418        }
419    }
420
421    /// The format a file, bytes or stream source was parsed as. `None` for a
422    /// query or dataframe source, and for a stream that recorded none.
423    pub fn format(&self) -> Option<&FileFormat> {
424        match self {
425            Self::File { format, .. } | Self::Bytes { format, .. } => Some(format),
426            Self::Stream { format, .. } => format.as_ref(),
427            Self::Query { .. } | Self::DataFrame { .. } => None,
428        }
429    }
430
431    /// How a CSV source was split into fields, when the source records it.
432    pub fn csv_metadata(&self) -> Option<&CsvSourceMetadata> {
433        match self {
434            Self::File { csv_metadata, .. }
435            | Self::Bytes { csv_metadata, .. }
436            | Self::Stream { csv_metadata, .. } => csv_metadata.as_ref(),
437            Self::Query { .. } | Self::DataFrame { .. } => None,
438        }
439    }
440
441    /// Record how a CSV source was split into fields. A query or dataframe
442    /// source has no delimiter, so it is returned unchanged.
443    pub fn with_csv_metadata(mut self, metadata: CsvSourceMetadata) -> Self {
444        match &mut self {
445            Self::File { csv_metadata, .. }
446            | Self::Bytes { csv_metadata, .. }
447            | Self::Stream { csv_metadata, .. } => *csv_metadata = Some(metadata),
448            Self::Query { .. } | Self::DataFrame { .. } => {}
449        }
450        self
451    }
452
453    /// Get the stream topic if this is a stream-based source.
454    pub fn stream_topic(&self) -> Option<&str> {
455        match self {
456            Self::Stream { topic, .. } => Some(topic),
457            _ => None,
458        }
459    }
460
461    /// Get the batch ID if this is a stream-based source.
462    pub fn batch_id(&self) -> Option<&str> {
463        match self {
464            Self::Stream { batch_id, .. } => Some(batch_id),
465            _ => None,
466        }
467    }
468}
469
470#[cfg(test)]
471mod tests {
472    use super::*;
473
474    #[test]
475    fn test_data_source_file_identifier() {
476        let ds = DataSource::File {
477            path: "/path/to/data.csv".to_string(),
478            format: FileFormat::Csv,
479            size_bytes: 0,
480            modified_at: None,
481            parquet_metadata: None,
482            csv_metadata: None,
483        };
484
485        assert_eq!(ds.identifier(), "/path/to/data.csv");
486        assert!(ds.is_file());
487        assert!(!ds.is_query());
488        assert!(!ds.is_dataframe());
489        assert!(!ds.is_stream());
490    }
491
492    #[test]
493    fn test_data_source_bytes_identifier_and_helpers() {
494        let ds = DataSource::Bytes {
495            name: "csv_bytes".to_string(),
496            format: FileFormat::Csv,
497            size_bytes: 42,
498            csv_metadata: None,
499        };
500
501        assert_eq!(ds.identifier(), "bytes[csv_bytes]");
502        assert!(ds.is_bytes());
503        assert!(!ds.is_file());
504        assert!(!ds.is_dataframe());
505        assert!(!ds.is_stream());
506        assert_eq!(ds.size_mb(), Some(42.0 / 1_048_576.0));
507    }
508
509    #[test]
510    fn test_data_source_stream_identifier_and_helpers() {
511        let ds = DataSource::Stream {
512            topic: "events".to_string(),
513            batch_id: "b1".to_string(),
514            partition: Some(0),
515            consumer_group: None,
516            source_system: Some(StreamSourceSystem::Kafka),
517            format: None,
518            session_id: None,
519            first_record_at: None,
520            last_record_at: None,
521            csv_metadata: None,
522        };
523
524        assert_eq!(ds.identifier(), "kafka[events]-batch:b1");
525        assert!(ds.is_stream());
526        assert_eq!(ds.stream_topic(), Some("events"));
527        assert_eq!(ds.batch_id(), Some("b1"));
528        assert!(!ds.is_file());
529        assert!(!ds.is_query());
530        assert!(ds.size_mb().is_none());
531    }
532
533    #[test]
534    fn test_stream_without_a_declared_system_records_none() {
535        let ds = DataSource::Stream {
536            topic: "events".to_string(),
537            batch_id: "b1".to_string(),
538            partition: None,
539            consumer_group: None,
540            source_system: None,
541            format: None,
542            session_id: None,
543            first_record_at: None,
544            last_record_at: None,
545            csv_metadata: None,
546        };
547
548        assert_eq!(ds.identifier(), "stream[events]-batch:b1");
549        let json = serde_json::to_value(&ds).unwrap();
550        assert!(json.get("source_system").is_none(), "{json}");
551        let reloaded: DataSource = serde_json::from_value(json.clone()).unwrap();
552        assert_eq!(serde_json::to_value(&reloaded).unwrap(), json);
553    }
554
555    #[test]
556    fn test_stream_json_serialization() {
557        let ds = DataSource::Stream {
558            topic: "sensor-data".to_string(),
559            batch_id: "batch-789".to_string(),
560            partition: Some(2),
561            consumer_group: Some("processing-group".to_string()),
562            source_system: Some(StreamSourceSystem::Kinesis),
563            format: None,
564            session_id: Some("session-1".to_string()),
565            first_record_at: Some("2023-01-01T10:00:00Z".to_string()),
566            last_record_at: Some("2023-01-01T10:05:00Z".to_string()),
567            csv_metadata: None,
568        };
569
570        let json = serde_json::to_string(&ds).unwrap();
571        assert!(json.contains(r#""type":"stream""#));
572        assert!(json.contains(r#""source_system":"kinesis""#));
573        assert!(json.contains(r#""topic":"sensor-data""#));
574
575        let deserialized: DataSource = serde_json::from_str(&json).unwrap();
576        assert!(deserialized.is_stream());
577        assert_eq!(deserialized.stream_topic(), Some("sensor-data"));
578    }
579
580    #[test]
581    fn test_stream_source_system_serialization_names() {
582        let object_store = serde_json::to_string(&StreamSourceSystem::ObjectStore).unwrap();
583        let message_queue = serde_json::to_string(&StreamSourceSystem::MessageQueue).unwrap();
584        let database = serde_json::to_string(&StreamSourceSystem::Database).unwrap();
585
586        assert_eq!(object_store, r#""object_store""#);
587        assert_eq!(message_queue, r#""message_queue""#);
588        assert_eq!(database, r#""database""#);
589
590        let object_store: StreamSourceSystem = serde_json::from_str(r#""object_store""#).unwrap();
591        let message_queue: StreamSourceSystem = serde_json::from_str(r#""message_queue""#).unwrap();
592        let database: StreamSourceSystem = serde_json::from_str(r#""database""#).unwrap();
593
594        assert_eq!(object_store, StreamSourceSystem::ObjectStore);
595        assert_eq!(message_queue, StreamSourceSystem::MessageQueue);
596        assert_eq!(database, StreamSourceSystem::Database);
597    }
598}