1use std::{cmp::Ordering, collections::HashSet, fmt, num::NonZeroU64, str::FromStr};
8
9use arrow::datatypes::SchemaRef;
10use chrono::{DateTime, Utc};
11use serde::{Deserialize, Serialize};
12use snafu::prelude::*;
13
14use crate::metadata::logical_schema::{LogicalSchema, SchemaConvertError};
15
16pub const TABLE_FORMAT_VERSION: u32 = 6;
20
21#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
27pub enum TableKind {
28 TimeSeries(IndexSpec),
30
31 Generic,
34}
35
36#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
41pub struct TableMeta {
42 pub(crate) kind: TableKind,
44
45 pub(crate) logical_schema: Option<LogicalSchema>,
50
51 pub(crate) created_at: DateTime<Utc>,
53
54 pub(crate) format_version: u32,
58}
59
60#[derive(Debug, Snafu)]
62pub enum TableMetaSchemaError {
63 #[snafu(display("table has no canonical logical schema yet (logical_schema is None)"))]
65 MissingCanonicalSchema,
66
67 #[snafu(transparent)]
69 Convert {
70 source: SchemaConvertError,
72 },
73}
74
75impl TableMeta {
76 pub fn kind(&self) -> &TableKind {
78 &self.kind
79 }
80
81 pub fn logical_schema(&self) -> Option<&LogicalSchema> {
83 self.logical_schema.as_ref()
84 }
85
86 pub fn created_at(&self) -> DateTime<Utc> {
88 self.created_at
89 }
90
91 pub fn format_version(&self) -> u32 {
93 self.format_version
94 }
95
96 pub fn new_time_series(index: IndexSpec) -> Self {
103 TableMeta {
104 kind: TableKind::TimeSeries(index),
105 logical_schema: None,
106 created_at: Utc::now(),
107 format_version: TABLE_FORMAT_VERSION,
108 }
109 }
110
111 pub fn new_time_series_with_schema(index: IndexSpec, logical_schema: LogicalSchema) -> Self {
113 TableMeta {
114 kind: TableKind::TimeSeries(index),
115 logical_schema: Some(logical_schema),
116 created_at: Utc::now(),
117 format_version: TABLE_FORMAT_VERSION,
118 }
119 }
120
121 pub fn arrow_schema_ref(&self) -> Result<SchemaRef, TableMetaSchemaError> {
126 let logical = self
127 .logical_schema
128 .as_ref()
129 .ok_or(TableMetaSchemaError::MissingCanonicalSchema)?;
130
131 logical
132 .to_arrow_schema_ref()
133 .map_err(|source| TableMetaSchemaError::Convert { source })
134 }
135}
136
137pub type TableMetaDelta = TableMeta;
143
144#[derive(Debug, Snafu, PartialEq, Eq)]
146pub enum ParseTimeBucketError {
147 #[snafu(display("time bucket spec is empty"))]
149 Empty,
150
151 #[snafu(display("time bucket spec '{spec}' is missing a numeric value"))]
153 MissingNumber {
154 spec: String,
156 },
157
158 #[snafu(display("time bucket spec '{spec}' is missing a unit suffix (expected s|m|h|d)"))]
160 MissingUnit {
161 spec: String,
163 },
164
165 #[snafu(display("invalid bucket value in '{spec}': {source}"))]
167 InvalidNumber {
168 spec: String,
170 source: std::num::ParseIntError,
172 },
173
174 #[snafu(display("bucket value must be > 0 (got {value}) in '{spec}'"))]
176 NonPositive {
177 spec: String,
179 value: u64,
181 },
182
183 #[snafu(display("bucket value too large for u32 (got {value}) in '{spec}'"))]
185 TooLarge {
186 spec: String,
188 value: u64,
190 },
191
192 #[snafu(display("unknown time bucket unit '{unit}' in '{spec}' (expected s|m|h|d)"))]
194 UnknownUnit {
195 spec: String,
197 unit: String,
199 },
200}
201
202#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
207pub enum TimeBucket {
208 Seconds(u32),
210 Minutes(u32),
212 Hours(u32),
214 Days(u32),
216}
217
218impl FromStr for TimeBucket {
219 type Err = ParseTimeBucketError;
220
221 fn from_str(input: &str) -> Result<Self, Self::Err> {
222 let spec = input.trim();
223 if spec.is_empty() {
224 return Err(ParseTimeBucketError::Empty);
225 }
226
227 let unit_start = spec
229 .char_indices()
230 .find(|(_, c)| c.is_ascii_alphabetic())
231 .map(|(i, _)| i);
232
233 let Some(unit_start) = unit_start else {
234 return Err(ParseTimeBucketError::MissingUnit {
235 spec: spec.to_string(),
236 });
237 };
238
239 if unit_start == 0 {
240 return Err(ParseTimeBucketError::MissingNumber {
242 spec: spec.to_string(),
243 });
244 }
245
246 let (num_str, unit_str) = spec.split_at(unit_start);
247 let num_str = num_str.trim();
248 let unit_str = unit_str.trim();
249
250 if unit_str.is_empty() {
251 return Err(ParseTimeBucketError::MissingUnit {
252 spec: spec.to_string(),
253 });
254 }
255
256 let value: u64 = num_str
257 .parse()
258 .map_err(|source| ParseTimeBucketError::InvalidNumber {
259 spec: spec.to_string(),
260 source,
261 })?;
262
263 if value == 0 {
264 return Err(ParseTimeBucketError::NonPositive {
265 spec: spec.to_string(),
266 value,
267 });
268 }
269
270 if value > u32::MAX as u64 {
271 return Err(ParseTimeBucketError::TooLarge {
272 spec: spec.to_string(),
273 value,
274 });
275 }
276
277 let v = value as u32;
278 let unit = unit_str.to_ascii_lowercase();
279
280 match unit.as_str() {
281 "s" | "sec" | "secs" | "second" | "seconds" => Ok(TimeBucket::Seconds(v)),
282 "m" | "min" | "mins" | "minute" | "minutes" => Ok(TimeBucket::Minutes(v)),
283 "h" | "hr" | "hrs" | "hour" | "hours" => Ok(TimeBucket::Hours(v)),
284 "d" | "day" | "days" => Ok(TimeBucket::Days(v)),
285 _ => Err(ParseTimeBucketError::UnknownUnit {
286 spec: spec.to_string(),
287 unit: unit_str.to_string(),
288 }),
289 }
290 }
291}
292
293impl TimeBucket {
294 pub fn parse(spec: &str) -> Result<Self, ParseTimeBucketError> {
304 spec.parse()
305 }
306}
307
308#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
310#[serde(deny_unknown_fields)]
311pub struct IndexSpec {
312 pub column: String,
314
315 #[serde(default)]
317 pub entity_columns: Vec<String>,
318
319 pub kind: IndexKind,
321}
322
323impl IndexSpec {
324 pub fn validate(&self) -> Result<(), IndexSpecError> {
331 if self.column.is_empty() {
332 return Err(IndexSpecError::EmptyColumn);
333 }
334
335 let mut seen = HashSet::with_capacity(self.entity_columns.len());
336 for (position, column) in self.entity_columns.iter().enumerate() {
337 if column.is_empty() {
338 return Err(IndexSpecError::EmptyEntityColumn { position });
339 }
340 if column == &self.column {
341 return Err(IndexSpecError::EntityColumnMatchesIndex {
342 column: column.clone(),
343 });
344 }
345 if !seen.insert(column) {
346 return Err(IndexSpecError::DuplicateEntityColumn {
347 column: column.clone(),
348 });
349 }
350 }
351
352 self.kind.validate()
353 }
354}
355
356#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
358#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
359pub enum IndexKind {
360 Timestamp {
362 bucket: TimeBucket,
364 #[serde(default, skip_serializing_if = "Option::is_none")]
366 timezone: Option<String>,
367 },
368 Int64 {
370 bucket_width: NonZeroU64,
372 },
373 UInt64 {
375 bucket_width: NonZeroU64,
377 },
378}
379
380impl IndexKind {
381 pub fn name(&self) -> &'static str {
383 match self {
384 Self::Timestamp { .. } => "timestamp",
385 Self::Int64 { .. } => "int64",
386 Self::UInt64 { .. } => "uint64",
387 }
388 }
389
390 pub fn validate(&self) -> Result<(), IndexSpecError> {
392 if let Self::Timestamp { bucket, .. } = self {
393 let width = match bucket {
394 TimeBucket::Seconds(width)
395 | TimeBucket::Minutes(width)
396 | TimeBucket::Hours(width)
397 | TimeBucket::Days(width) => *width,
398 };
399 if width == 0 {
400 return Err(IndexSpecError::ZeroTimeBucket);
401 }
402 }
403 Ok(())
404 }
405}
406
407#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
409#[serde(
410 tag = "type",
411 content = "value",
412 rename_all = "snake_case",
413 deny_unknown_fields
414)]
415pub enum IndexValue {
416 Timestamp(DateTime<Utc>),
418 Int64(i64),
420 UInt64(u64),
422}
423
424impl IndexValue {
425 pub fn kind_name(&self) -> &'static str {
427 match self {
428 Self::Timestamp(_) => "timestamp",
429 Self::Int64(_) => "int64",
430 Self::UInt64(_) => "uint64",
431 }
432 }
433
434 pub fn compare(&self, other: &Self) -> Result<Ordering, IndexValueError> {
439 match (self, other) {
440 (Self::Timestamp(left), Self::Timestamp(right)) => Ok(left.cmp(right)),
441 (Self::Int64(left), Self::Int64(right)) => Ok(left.cmp(right)),
442 (Self::UInt64(left), Self::UInt64(right)) => Ok(left.cmp(right)),
443 _ => Err(IndexValueError::DomainMismatch {
444 left: self.kind_name(),
445 right: other.kind_name(),
446 }),
447 }
448 }
449
450 pub fn validate_kind(&self, kind: &IndexKind) -> Result<(), IndexValueError> {
455 if self.kind_name() == kind.name() {
456 Ok(())
457 } else {
458 Err(IndexValueError::KindMismatch {
459 expected: kind.name(),
460 actual: self.kind_name(),
461 })
462 }
463 }
464}
465
466impl fmt::Display for IndexValue {
467 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
468 match self {
469 Self::Timestamp(value) => write!(f, "timestamp({value})"),
470 Self::Int64(value) => write!(f, "int64({value})"),
471 Self::UInt64(value) => write!(f, "uint64({value})"),
472 }
473 }
474}
475
476impl From<DateTime<Utc>> for IndexValue {
477 fn from(value: DateTime<Utc>) -> Self {
478 Self::Timestamp(value)
479 }
480}
481
482impl From<i64> for IndexValue {
483 fn from(value: i64) -> Self {
484 Self::Int64(value)
485 }
486}
487
488impl From<u64> for IndexValue {
489 fn from(value: u64) -> Self {
490 Self::UInt64(value)
491 }
492}
493
494pub fn validate_index_range(
500 kind: &IndexKind,
501 start: &IndexValue,
502 end: &IndexValue,
503) -> Result<(), IndexValueError> {
504 start.validate_kind(kind)?;
505 end.validate_kind(kind)?;
506 if start.compare(end)? != Ordering::Less {
507 return Err(IndexValueError::InvalidRange {
508 start: start.clone(),
509 end: end.clone(),
510 });
511 }
512 Ok(())
513}
514
515#[derive(Debug, Snafu, PartialEq, Eq)]
517pub enum IndexSpecError {
518 #[snafu(display("ordered index column is empty"))]
520 EmptyColumn,
521 #[snafu(display("entity column at position {position} is empty"))]
523 EmptyEntityColumn {
524 position: usize,
526 },
527 #[snafu(display("duplicate entity column: {column}"))]
529 DuplicateEntityColumn {
530 column: String,
532 },
533 #[snafu(display("entity column cannot also be the ordered index column: {column}"))]
535 EntityColumnMatchesIndex {
536 column: String,
538 },
539 #[snafu(display("timestamp bucket width must be nonzero"))]
541 ZeroTimeBucket,
542}
543
544#[derive(Debug, Snafu, PartialEq, Eq)]
546pub enum IndexValueError {
547 #[snafu(display("ordered index domain mismatch: left={left}, right={right}"))]
549 DomainMismatch {
550 left: &'static str,
552 right: &'static str,
554 },
555 #[snafu(display("ordered index kind mismatch: expected {expected}, found {actual}"))]
557 KindMismatch {
558 expected: &'static str,
560 actual: &'static str,
562 },
563 #[snafu(display(
565 "invalid ordered index range: start={start}, end={end} (expected start < end)"
566 ))]
567 InvalidRange {
568 start: IndexValue,
570 end: IndexValue,
572 },
573 #[snafu(display("invalid ordered index bounds: min={min}, max={max} (expected min <= max)"))]
575 InvalidBounds {
576 min: IndexValue,
578 max: IndexValue,
580 },
581}
582
583#[cfg(test)]
584mod tests {
585 use crate::metadata::logical_schema::{LogicalDataType, LogicalField};
586
587 use super::*;
588 use chrono::TimeZone;
589
590 fn sample_time_index_spec() -> IndexSpec {
591 IndexSpec {
592 column: "ts".to_string(),
593 entity_columns: vec!["symbol".to_string()],
594 kind: IndexKind::Timestamp {
595 bucket: TimeBucket::Minutes(1),
596 timezone: None,
597 },
598 }
599 }
600
601 #[test]
602 fn index_spec_json_roundtrips_all_domains() {
603 let specs = [
604 sample_time_index_spec(),
605 IndexSpec {
606 column: "sequence".to_string(),
607 entity_columns: Vec::new(),
608 kind: IndexKind::Int64 {
609 bucket_width: NonZeroU64::new(u64::MAX).unwrap(),
610 },
611 },
612 IndexSpec {
613 column: "offset".to_string(),
614 entity_columns: vec!["source".to_string()],
615 kind: IndexKind::UInt64 {
616 bucket_width: NonZeroU64::new(7).unwrap(),
617 },
618 },
619 ];
620
621 for spec in specs {
622 let json = serde_json::to_string(&spec).unwrap();
623 let restored: IndexSpec = serde_json::from_str(&json).unwrap();
624 assert_eq!(restored, spec);
625 }
626 }
627
628 #[test]
629 fn index_spec_json_rejects_impossible_field_combinations() {
630 let timestamp_with_integer_width = r#"{
631 "column":"ts",
632 "kind":{"type":"timestamp","bucket":{"Seconds":1},"bucket_width":1}
633 }"#;
634 let integer_with_timestamp_bucket = r#"{
635 "column":"id",
636 "kind":{"type":"int64","bucket_width":1,"bucket":{"Seconds":1}}
637 }"#;
638 let zero_integer_width = r#"{"column":"id","kind":{"type":"uint64","bucket_width":0}}"#;
639
640 assert!(serde_json::from_str::<IndexSpec>(timestamp_with_integer_width).is_err());
641 assert!(serde_json::from_str::<IndexSpec>(integer_with_timestamp_bucket).is_err());
642 assert!(serde_json::from_str::<IndexSpec>(zero_integer_width).is_err());
643 }
644
645 #[test]
646 fn index_spec_validation_rejects_invalid_structure_and_time_bucket() {
647 let mut spec = sample_time_index_spec();
648 spec.column.clear();
649 assert_eq!(spec.validate(), Err(IndexSpecError::EmptyColumn));
650
651 let mut spec = sample_time_index_spec();
652 spec.entity_columns.push("symbol".to_string());
653 assert!(matches!(
654 spec.validate(),
655 Err(IndexSpecError::DuplicateEntityColumn { .. })
656 ));
657
658 let mut spec = sample_time_index_spec();
659 spec.entity_columns = vec![spec.column.clone()];
660 assert_eq!(
661 spec.validate(),
662 Err(IndexSpecError::EntityColumnMatchesIndex {
663 column: "ts".to_string(),
664 })
665 );
666
667 let mut spec = sample_time_index_spec();
668 spec.kind = IndexKind::Timestamp {
669 bucket: TimeBucket::Seconds(0),
670 timezone: None,
671 };
672 assert_eq!(spec.validate(), Err(IndexSpecError::ZeroTimeBucket));
673 }
674
675 #[test]
676 fn index_value_roundtrips_and_compares_integer_extremes() {
677 let timestamp = Utc.timestamp_opt(1, 987_654_321).single().unwrap();
678 let values = [
679 IndexValue::Timestamp(timestamp),
680 IndexValue::Int64(i64::MIN),
681 IndexValue::Int64(i64::MAX),
682 IndexValue::UInt64(0),
683 IndexValue::UInt64(u64::MAX),
684 ];
685
686 for value in values {
687 let json = serde_json::to_string(&value).unwrap();
688 assert_eq!(serde_json::from_str::<IndexValue>(&json).unwrap(), value);
689 assert_eq!(value.compare(&value).unwrap(), Ordering::Equal);
690 }
691 assert_eq!(
692 IndexValue::Int64(i64::MIN)
693 .compare(&IndexValue::Int64(i64::MAX))
694 .unwrap(),
695 Ordering::Less
696 );
697 assert_eq!(
698 IndexValue::UInt64(u64::MAX)
699 .compare(&IndexValue::UInt64(0))
700 .unwrap(),
701 Ordering::Greater
702 );
703 }
704
705 #[test]
706 fn index_value_cross_domain_comparison_and_ranges_are_typed_errors() {
707 assert_eq!(
708 IndexValue::Int64(0).compare(&IndexValue::UInt64(0)),
709 Err(IndexValueError::DomainMismatch {
710 left: "int64",
711 right: "uint64"
712 })
713 );
714
715 let kind = IndexKind::UInt64 {
716 bucket_width: NonZeroU64::new(1).unwrap(),
717 };
718 assert!(matches!(
719 validate_index_range(&kind, &IndexValue::Int64(0), &IndexValue::Int64(1)),
720 Err(IndexValueError::KindMismatch { .. })
721 ));
722 assert!(matches!(
723 validate_index_range(&kind, &IndexValue::UInt64(1), &IndexValue::UInt64(1)),
724 Err(IndexValueError::InvalidRange { .. })
725 ));
726 }
727
728 #[test]
729 fn table_meta_arrow_schema_ref_requires_logical_schema() {
730 let meta = TableMeta::new_time_series(sample_time_index_spec());
731 let err = meta.arrow_schema_ref().unwrap_err();
732 assert!(matches!(err, TableMetaSchemaError::MissingCanonicalSchema));
733 }
734
735 #[test]
736 fn table_meta_arrow_schema_ref_propagates_convert_error() {
737 let logical = LogicalSchema::new(vec![LogicalField {
738 name: "legacy_ts".to_string(),
739 data_type: LogicalDataType::Int96,
740 nullable: false,
741 }])
742 .expect("valid schema structure");
743 let meta = TableMeta::new_time_series_with_schema(sample_time_index_spec(), logical);
744
745 let err = meta.arrow_schema_ref().unwrap_err();
746 assert!(
747 matches!(
748 &err,
749 TableMetaSchemaError::Convert {
750 source: SchemaConvertError::Int96Unsupported { column }
751 } if column == "legacy_ts"
752 ),
753 "unexpected error: {err:?}"
754 );
755 }
756
757 #[test]
758 fn time_bucket_parse_accepts_basic_units() {
759 let cases = [
760 ("1s", TimeBucket::Seconds(1)),
761 ("2m", TimeBucket::Minutes(2)),
762 ("3h", TimeBucket::Hours(3)),
763 ("4d", TimeBucket::Days(4)),
764 ];
765
766 for (input, expected) in cases {
767 assert_eq!(input.parse::<TimeBucket>().unwrap(), expected);
768 }
769 }
770
771 #[test]
772 fn time_bucket_parse_accepts_aliases_case_and_whitespace() {
773 let cases = [
774 ("1sec", TimeBucket::Seconds(1)),
775 ("1secs", TimeBucket::Seconds(1)),
776 ("1second", TimeBucket::Seconds(1)),
777 ("1seconds", TimeBucket::Seconds(1)),
778 ("1min", TimeBucket::Minutes(1)),
779 ("1mins", TimeBucket::Minutes(1)),
780 ("1minute", TimeBucket::Minutes(1)),
781 ("1minutes", TimeBucket::Minutes(1)),
782 ("1hr", TimeBucket::Hours(1)),
783 ("1hrs", TimeBucket::Hours(1)),
784 ("1hour", TimeBucket::Hours(1)),
785 ("1hours", TimeBucket::Hours(1)),
786 ("1day", TimeBucket::Days(1)),
787 ("1days", TimeBucket::Days(1)),
788 ("1H", TimeBucket::Hours(1)),
789 ("1MiN", TimeBucket::Minutes(1)),
790 (" 2h", TimeBucket::Hours(2)),
791 ("3d ", TimeBucket::Days(3)),
792 (" 4m ", TimeBucket::Minutes(4)),
793 ("1 h", TimeBucket::Hours(1)),
794 ];
795
796 for (input, expected) in cases {
797 assert_eq!(input.parse::<TimeBucket>().unwrap(), expected);
798 }
799 }
800
801 #[test]
802 fn time_bucket_parse_rejects_empty_or_whitespace() {
803 let cases = ["", " ", "\n\t"];
804 for input in cases {
805 let err = input.parse::<TimeBucket>().unwrap_err();
806 assert!(matches!(err, ParseTimeBucketError::Empty));
807 }
808 }
809
810 #[test]
811 fn time_bucket_parse_rejects_missing_number() {
812 let cases = ["h", " hr", "day", "abcmin"];
813 for input in cases {
814 let err = input.parse::<TimeBucket>().unwrap_err();
815 assert!(
816 matches!(err, ParseTimeBucketError::MissingNumber { .. }),
817 "expected MissingNumber for {input:?}, got {err:?}"
818 );
819 }
820 }
821
822 #[test]
823 fn time_bucket_parse_rejects_missing_unit() {
824 let cases = ["1", " 42 "];
825 for input in cases {
826 let err = input.parse::<TimeBucket>().unwrap_err();
827 assert!(
828 matches!(err, ParseTimeBucketError::MissingUnit { .. }),
829 "expected MissingUnit for {input:?}, got {err:?}"
830 );
831 }
832 }
833
834 #[test]
835 fn time_bucket_parse_rejects_invalid_number() {
836 let cases = ["1.5h", "1_000s"];
837 for input in cases {
838 let err = input.parse::<TimeBucket>().unwrap_err();
839 assert!(
840 matches!(err, ParseTimeBucketError::InvalidNumber { .. }),
841 "expected InvalidNumber for {input:?}, got {err:?}"
842 );
843 }
844 }
845
846 #[test]
847 fn time_bucket_parse_rejects_non_positive() {
848 let cases = ["0s", "0m"];
849 for input in cases {
850 let err = input.parse::<TimeBucket>().unwrap_err();
851 assert!(
852 matches!(err, ParseTimeBucketError::NonPositive { value: 0, .. }),
853 "expected NonPositive for {input:?}, got {err:?}"
854 );
855 }
856 }
857
858 #[test]
859 fn time_bucket_parse_rejects_too_large() {
860 let too_large = (u32::MAX as u64 + 1).to_string();
861 let input = format!("{too_large}h");
862 let err = input.parse::<TimeBucket>().unwrap_err();
863 assert!(
864 matches!(err, ParseTimeBucketError::TooLarge { value, .. } if value == u32::MAX as u64 + 1),
865 "expected TooLarge for {input:?}, got {err:?}"
866 );
867 }
868
869 #[test]
870 fn time_bucket_parse_rejects_unknown_units() {
871 let cases = ["1w", "1ms", "1mo", "10msec"];
872 for input in cases {
873 let err = input.parse::<TimeBucket>().unwrap_err();
874 assert!(
875 matches!(err, ParseTimeBucketError::UnknownUnit { .. }),
876 "expected UnknownUnit for {input:?}, got {err:?}"
877 );
878 }
879 }
880
881 #[test]
882 fn time_bucket_parse_matches_from_str() {
883 let via_method = TimeBucket::parse("5m").unwrap();
884 let via_trait: TimeBucket = "5m".parse().unwrap();
885 assert_eq!(via_method, via_trait);
886 }
887}