dataprof_core/
partial.rs

1use crate::{classification::DataType, source::FileFormat};
2
3/// Result of fast schema inference — column names paired with inferred data types.
4#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
5pub struct SchemaResult {
6    /// Columns with their inferred types, in source order: CSV/Parquet follow
7    /// the header/schema, JSON/JSONL follow the first record's field order with
8    /// later-only fields appended in first-seen order.
9    pub columns: Vec<ColumnSchema>,
10    /// How many rows were sampled to infer the schema. 0 when nothing had to
11    /// be read: a Parquet file whose every column is typed by its encoding
12    /// is answered from the metadata alone.
13    pub rows_sampled: usize,
14    /// Time taken for inference in milliseconds.
15    pub inference_time_ms: u128,
16    /// `true` when the entire file was consumed or the schema was read from
17    /// metadata alone; `false` when inference stopped at the sample-size cap
18    /// and the schema may not have fully stabilized.
19    pub schema_stable: bool,
20}
21
22/// A single column's name and inferred data type.
23#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
24pub struct ColumnSchema {
25    pub name: String,
26    pub data_type: DataType,
27}
28
29/// Lightweight structural summary for a file.
30///
31/// This intentionally sits between `quick_row_count()` / `infer_schema()` and a
32/// full profile: it reports shape and sampled per-column counters, not quality
33/// scores, pattern detection, recommendations, or raw values.
34#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
35pub struct StructureReport {
36    /// Source identifier, currently the file path passed by the caller.
37    pub source: String,
38    /// Detected or explicitly provided file format.
39    pub format: FileFormat,
40    /// Exact or estimated row count for the source.
41    pub row_count: RowCountEstimate,
42    /// Number of rows consumed for the structural column summaries.
43    pub rows_sampled: usize,
44    /// True when the structural pass consumed the whole source.
45    pub source_exhausted: bool,
46    /// True when the structural pass stopped at the row sample cap.
47    pub truncated: bool,
48    /// Human-readable reason for truncation, if any.
49    pub truncation_reason: Option<String>,
50    /// CSV delimiter used for parsing, when applicable.
51    pub delimiter: Option<String>,
52    /// Per-column structural summaries.
53    pub columns: Vec<StructureColumnSummary>,
54    /// Non-fatal caveats, such as estimated row counts.
55    pub warnings: Vec<String>,
56}
57
58/// Sample-derived or metadata-derived structural summary for one column.
59///
60/// The counters and the type answer to different sources, and `provenance`
61/// names the one that decided the *type*. A counter is present only when it
62/// describes the whole source: for CSV/JSON/JSONL that is the bounded
63/// structural sample, disclosed by `rows_sampled` and `truncated`; for Parquet
64/// the counts come from the file footer and cover every row, so they hold even
65/// when the type sample was truncated. `None` keeps its usual meaning — not
66/// analyzed — and is what a counter reports rather than a partial number
67/// (#700).
68#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
69pub struct StructureColumnSummary {
70    pub name: String,
71    pub data_type: DataType,
72    pub total_count: Option<usize>,
73    pub null_count: Option<usize>,
74    pub null_ratio: Option<f64>,
75    pub unique_count: Option<usize>,
76    pub uniqueness_ratio: Option<f64>,
77    pub distinct_count_approximate: Option<bool>,
78    /// Which source governs this column's type: `"metadata"` when the declared
79    /// schema decided it, `"sample"` when only the column's values can. It
80    /// names the source, not the read — a `"sample"` column is one the metadata
81    /// could not type, whether or not the caller's row budget allowed any row
82    /// to be read for it. `rows_sampled` and `truncated` report the read.
83    ///
84    /// CSV/JSON/JSONL are always `"sample"`; a Parquet file carries both, since
85    /// only its text columns need values to be typed (#693).
86    pub provenance: String,
87}
88
89/// Result of a quick row count operation.
90#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
91pub struct RowCountEstimate {
92    /// The estimated or exact row count.
93    pub count: u64,
94    /// Whether the count is exact or an estimate.
95    pub exact: bool,
96    /// How the count was obtained.
97    pub method: CountMethod,
98    /// Time taken in milliseconds.
99    pub count_time_ms: u128,
100    /// Approximate 1-sigma relative standard error of the estimate, expressed
101    /// as a fraction of `count` (e.g. `0.02` ≈ ±2%). Populated only for
102    /// sampled estimates where a variance across sampling windows is available;
103    /// `None` for exact counts and when it cannot be estimated.
104    #[serde(default)]
105    pub relative_error: Option<f64>,
106}
107
108/// Method used to obtain the row count.
109#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
110pub enum CountMethod {
111    /// Read from Parquet file footer metadata (exact, zero row reading).
112    ParquetMetadata,
113    /// Full scan of the file (exact).
114    FullScan,
115    /// Sample-based estimation (approximate).
116    Sampling,
117    /// Full scan of a streaming source (no file metadata available).
118    StreamFullScan,
119}
dataprof_core/partial.rs

dataprof_core/
partial.rs
