Skip to main content

vortex_array/extension/datetime/
timestamp.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! Temporal extension data types.
5
6use std::fmt;
7use std::sync::Arc;
8
9use jiff::fmt::temporal::DateTimeParser;
10use jiff::tz::TimeZone;
11use vortex_error::VortexExpect;
12use vortex_error::VortexResult;
13use vortex_error::vortex_bail;
14use vortex_error::vortex_ensure;
15use vortex_error::vortex_err;
16use vortex_error::vortex_panic;
17use vortex_session::registry::CachedId;
18
19use crate::dtype::DType;
20use crate::dtype::Nullability;
21use crate::dtype::PType;
22use crate::dtype::extension::ExtDType;
23use crate::dtype::extension::ExtId;
24use crate::dtype::extension::ExtVTable;
25use crate::extension::datetime::TimeUnit;
26use crate::scalar::ScalarValue;
27
28/// A timestamp stored as an `i64` count of seconds, milliseconds, microseconds, or nanoseconds.
29///
30/// Every storage value is valid. Formatting falls back to the raw count when jiff cannot format it.
31#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
32pub struct Timestamp;
33
34impl Timestamp {
35    /// Creates a timestamp dtype with the given time unit and nullability.
36    ///
37    /// # Panics
38    ///
39    /// Panics if `time_unit` is [`TimeUnit::Days`].
40    pub fn new(time_unit: TimeUnit, nullability: Nullability) -> ExtDType<Self> {
41        Self::new_with_tz(time_unit, None, nullability)
42    }
43
44    /// Creates a timestamp dtype with the given time unit, time zone, and nullability.
45    ///
46    /// # Errors
47    ///
48    /// Returns an error for [`TimeUnit::Days`] or an unresolved time zone. IANA names and fixed
49    /// UTC offsets are accepted.
50    pub fn try_new_with_tz(
51        time_unit: TimeUnit,
52        timezone: Option<Arc<str>>,
53        nullability: Nullability,
54    ) -> VortexResult<ExtDType<Self>> {
55        ExtDType::try_new(
56            TimestampOptions {
57                unit: time_unit,
58                tz: timezone,
59            },
60            DType::Primitive(PType::I64, nullability),
61        )
62    }
63
64    /// Creates a timestamp dtype with the given time unit, time zone, and nullability.
65    ///
66    /// # Panics
67    ///
68    /// Panics if [`Self::try_new_with_tz`] rejects the unit or time zone.
69    pub fn new_with_tz(
70        time_unit: TimeUnit,
71        timezone: Option<Arc<str>>,
72        nullability: Nullability,
73    ) -> ExtDType<Self> {
74        Self::try_new_with_tz(time_unit, timezone, nullability)
75            .vortex_expect("timestamp unit and time zone must be valid")
76    }
77
78    /// Creates a timestamp dtype with the given options and nullability.
79    ///
80    /// # Panics
81    ///
82    /// Panics if [`Self::try_new_with_tz`] rejects the unit or time zone.
83    pub fn new_with_options(options: TimestampOptions, nullability: Nullability) -> ExtDType<Self> {
84        Self::new_with_tz(options.unit, options.tz, nullability)
85    }
86}
87
88/// Options for the Timestamp DType.
89#[derive(Clone, Debug, PartialEq, Eq, Hash)]
90pub struct TimestampOptions {
91    /// The time unit of the timestamp.
92    pub unit: TimeUnit,
93    /// The timezone of the timestamp, if any.
94    pub tz: Option<Arc<str>>,
95}
96
97impl fmt::Display for TimestampOptions {
98    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
99        match &self.tz {
100            Some(tz) => write!(f, "{}, tz={}", self.unit, tz),
101            None => write!(f, "{}", self.unit),
102        }
103    }
104}
105
106/// Resolve an Arrow timestamp timezone, accepting IANA names and fixed offsets.
107pub(crate) fn resolve_time_zone(name: Option<&str>) -> VortexResult<TimeZone> {
108    Ok(match name {
109        Some(name) if name.starts_with('+') || name.starts_with('-') => {
110            DateTimeParser::new().parse_time_zone(name)?
111        }
112        Some(name) => TimeZone::get(name)?,
113        None => TimeZone::UTC,
114    })
115}
116
117/// Unpacked value of a [`Timestamp`] extension scalar.
118///
119/// Each variant carries the raw storage value and an optional timezone.
120pub enum TimestampValue<'a> {
121    /// Seconds since the Unix epoch.
122    Seconds(i64, Option<&'a Arc<str>>),
123    /// Milliseconds since the Unix epoch.
124    Milliseconds(i64, Option<&'a Arc<str>>),
125    /// Microseconds since the Unix epoch.
126    Microseconds(i64, Option<&'a Arc<str>>),
127    /// Nanoseconds since the Unix epoch.
128    Nanoseconds(i64, Option<&'a Arc<str>>),
129}
130
131impl fmt::Display for TimestampValue<'_> {
132    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
133        let (unit, value, tz) = match *self {
134            TimestampValue::Seconds(v, tz) => (TimeUnit::Seconds, v, tz),
135            TimestampValue::Milliseconds(v, tz) => (TimeUnit::Milliseconds, v, tz),
136            TimestampValue::Microseconds(v, tz) => (TimeUnit::Microseconds, v, tz),
137            TimestampValue::Nanoseconds(v, tz) => (TimeUnit::Nanoseconds, v, tz),
138        };
139
140        // jiff cannot format every storage value, including the last 26 hours of year 9999.
141        let Ok(ts) = unit
142            .to_jiff_span(value)
143            .and_then(|span| Ok(jiff::Timestamp::UNIX_EPOCH.checked_add(span)?))
144        else {
145            return write!(f, "{value}{unit}");
146        };
147
148        match tz {
149            None => write!(f, "{ts}"),
150            Some(tz) => {
151                let zone = resolve_time_zone(Some(tz)).vortex_expect("unknown timezone");
152                let adjusted_ts = ts.to_zoned(zone);
153                write!(f, "{adjusted_ts}",)
154            }
155        }
156    }
157}
158
159impl ExtVTable for Timestamp {
160    type Metadata = TimestampOptions;
161
162    type NativeValue<'a> = TimestampValue<'a>;
163
164    fn id(&self) -> ExtId {
165        static ID: CachedId = CachedId::new("vortex.timestamp");
166        *ID
167    }
168
169    // NOTE(ngates): unfortunately we're stuck with this hand-rolled serialization format for
170    //  backwards compatibility.
171    fn serialize_metadata(&self, metadata: &Self::Metadata) -> VortexResult<Vec<u8>> {
172        let mut bytes = Vec::with_capacity(4);
173        let unit_tag: u8 = metadata.unit.into();
174
175        bytes.push(unit_tag);
176
177        // Encode time_zone as u16 length followed by utf8 bytes.
178        match &metadata.tz {
179            None => bytes.extend_from_slice(0u16.to_le_bytes().as_slice()),
180            Some(tz) => {
181                let tz_bytes = tz.as_bytes();
182                let tz_len = u16::try_from(tz_bytes.len())
183                    .unwrap_or_else(|err| vortex_panic!("tz did not fit in u16: {}", err));
184                bytes.extend_from_slice(tz_len.to_le_bytes().as_slice());
185                bytes.extend_from_slice(tz_bytes);
186            }
187        }
188
189        Ok(bytes)
190    }
191
192    fn deserialize_metadata(&self, data: &[u8]) -> VortexResult<Self::Metadata> {
193        vortex_ensure!(
194            data.len() >= 3,
195            "Timestamp metadata must have at least 3 bytes, got {}",
196            data.len()
197        );
198
199        let tag = data[0];
200        let time_unit = TimeUnit::try_from(tag)?;
201        let tz_len_bytes: [u8; 2] = data[1..3]
202            .try_into()
203            .ok()
204            .vortex_expect("Verified to have two bytes");
205        let tz_len = u16::from_le_bytes(tz_len_bytes) as usize;
206        if tz_len == 0 {
207            return Ok(TimestampOptions {
208                unit: time_unit,
209                tz: None,
210            });
211        }
212
213        // Attempt to load from len-prefixed bytes
214        vortex_ensure!(
215            data.len() >= 3 + tz_len,
216            "Timestamp metadata is truncated: declared timezone length {} but only {} bytes available",
217            tz_len,
218            data.len() - 3
219        );
220        let tz_bytes = &data[3..3 + tz_len];
221        let tz: Arc<str> = str::from_utf8(tz_bytes)
222            .map_err(|e| vortex_err!("timezone is not valid utf8 string: {e}"))?
223            .to_string()
224            .into();
225        Ok(TimestampOptions {
226            unit: time_unit,
227            tz: Some(tz),
228        })
229    }
230
231    fn validate_dtype(ext_dtype: &ExtDType<Self>) -> VortexResult<()> {
232        vortex_ensure!(
233            matches!(ext_dtype.storage_dtype(), DType::Primitive(PType::I64, _)),
234            "Timestamp storage dtype must be i64"
235        );
236        vortex_ensure!(
237            ext_dtype.metadata().unit != TimeUnit::Days,
238            "Timestamp time unit must be finer than days, got {:?}",
239            ext_dtype.metadata().unit
240        );
241        resolve_time_zone(ext_dtype.metadata().tz.as_deref())?;
242
243        Ok(())
244    }
245
246    fn unpack_native<'a>(
247        ext_dtype: &'a ExtDType<Self>,
248        storage_value: &'a ScalarValue,
249    ) -> VortexResult<Self::NativeValue<'a>> {
250        let metadata = ext_dtype.metadata();
251        let ts_value = storage_value.as_primitive().cast::<i64>()?;
252        let tz = metadata.tz.as_ref();
253
254        Ok(match metadata.unit {
255            TimeUnit::Nanoseconds => TimestampValue::Nanoseconds(ts_value, tz),
256            TimeUnit::Microseconds => TimestampValue::Microseconds(ts_value, tz),
257            TimeUnit::Milliseconds => TimestampValue::Milliseconds(ts_value, tz),
258            TimeUnit::Seconds => TimestampValue::Seconds(ts_value, tz),
259            TimeUnit::Days => vortex_bail!("Timestamp does not support Days time unit"),
260        })
261    }
262}
263
264#[cfg(test)]
265mod tests {
266    use std::sync::Arc;
267
268    use rstest::rstest;
269    use vortex_error::VortexResult;
270
271    use crate::dtype::DType;
272    use crate::dtype::Nullability::Nullable;
273    use crate::extension::datetime::TimeUnit;
274    use crate::extension::datetime::Timestamp;
275    use crate::scalar::PValue;
276    use crate::scalar::Scalar;
277    use crate::scalar::ScalarValue;
278
279    #[test]
280    fn validate_timestamp_scalar() -> VortexResult<()> {
281        let dtype = DType::Extension(Timestamp::new(TimeUnit::Seconds, Nullable).erased());
282        Scalar::try_new(dtype, Some(ScalarValue::Primitive(PValue::I64(0))))?;
283
284        Ok(())
285    }
286
287    #[cfg_attr(miri, ignore)]
288    #[rstest]
289    #[case::end_of_9999(
290        TimeUnit::Microseconds,
291        253_402_214_400_000_000,
292        "253402214400000000µs"
293    )]
294    #[case::start_of_neg_9999(TimeUnit::Seconds, -377_705_116_800, "-377705116800s")]
295    #[case::max(TimeUnit::Microseconds, i64::MAX, "9223372036854775807µs")]
296    fn test_accept_timestamp_beyond_jiff_range(
297        #[case] unit: TimeUnit,
298        #[case] value: i64,
299        #[case] expected: &str,
300        #[values(None, Some(Arc::from("UTC")))] timezone: Option<Arc<str>>,
301    ) -> VortexResult<()> {
302        let dtype = DType::Extension(Timestamp::new_with_tz(unit, timezone, Nullable).erased());
303        let scalar = Scalar::try_new(dtype, Some(ScalarValue::from(value)))?;
304        assert_eq!(scalar.as_extension().to_string(), expected);
305
306        Ok(())
307    }
308
309    #[cfg_attr(miri, ignore)]
310    #[rstest]
311    #[case("+01:00", "1970-01-01T01:00:00+01:00[+01:00]")]
312    #[case("-05:30", "1969-12-31T18:30:00-05:30[-05:30]")]
313    fn display_fixed_offset_timestamp(
314        #[case] timezone: &str,
315        #[case] expected: &str,
316    ) -> VortexResult<()> {
317        let dtype = DType::Extension(
318            Timestamp::new_with_tz(TimeUnit::Seconds, Some(timezone.into()), Nullable).erased(),
319        );
320        let scalar = Scalar::try_new(dtype, Some(ScalarValue::Primitive(PValue::I64(0))))?;
321        assert_eq!(format!("{}", scalar.as_extension()), expected);
322        Ok(())
323    }
324
325    #[cfg_attr(miri, ignore)]
326    #[test]
327    fn reject_timestamp_with_invalid_timezone() {
328        assert!(
329            Timestamp::try_new_with_tz(
330                TimeUnit::Seconds,
331                Some(Arc::from("Not/A/Timezone")),
332                Nullable,
333            )
334            .is_err()
335        );
336    }
337
338    #[test]
339    fn test_reject_timestamp_days() {
340        assert!(Timestamp::try_new_with_tz(TimeUnit::Days, None, Nullable).is_err());
341    }
342
343    #[rstest]
344    #[case::seconds(TimeUnit::Seconds, i64::MAX)]
345    #[case::milliseconds(TimeUnit::Milliseconds, i64::MAX)]
346    #[case::microseconds(TimeUnit::Microseconds, i64::MAX)]
347    #[case::nanoseconds(TimeUnit::Nanoseconds, i64::MIN)]
348    fn test_accept_timestamp_storage_extremes(
349        #[case] unit: TimeUnit,
350        #[case] value: i64,
351    ) -> VortexResult<()> {
352        let dtype = DType::Extension(Timestamp::new(unit, Nullable).erased());
353        Scalar::try_new(dtype, Some(ScalarValue::from(value)))?;
354
355        Ok(())
356    }
357
358    #[cfg_attr(miri, ignore)]
359    #[test]
360    fn display_timestamp_scalar() {
361        // Local (no timezone) timestamp.
362        let local_dtype = DType::Extension(Timestamp::new(TimeUnit::Seconds, Nullable).erased());
363        let scalar = Scalar::new(local_dtype, Some(ScalarValue::Primitive(PValue::I64(0))));
364        assert_eq!(format!("{}", scalar.as_extension()), "1970-01-01T00:00:00Z");
365
366        // Zoned timestamp.
367        let zoned_dtype = DType::Extension(
368            Timestamp::new_with_tz(
369                TimeUnit::Seconds,
370                Some(Arc::from("America/New_York")),
371                Nullable,
372            )
373            .erased(),
374        );
375        let scalar = Scalar::new(zoned_dtype, Some(ScalarValue::Primitive(PValue::I64(0))));
376        assert_eq!(
377            format!("{}", scalar.as_extension()),
378            "1969-12-31T19:00:00-05:00[America/New_York]"
379        );
380    }
381
382    #[test]
383    fn deserialize_empty_metadata_returns_error() {
384        use crate::dtype::extension::ExtVTable;
385
386        let vtable = Timestamp;
387        assert!(vtable.deserialize_metadata(&[]).is_err());
388    }
389
390    #[test]
391    fn deserialize_too_short_metadata_returns_error() {
392        use crate::dtype::extension::ExtVTable;
393
394        let vtable = Timestamp;
395        // Only 2 bytes - too short for the required 3-byte header.
396        assert!(vtable.deserialize_metadata(&[0x00, 0x01]).is_err());
397    }
398
399    #[test]
400    fn deserialize_truncated_timezone_returns_error() {
401        use crate::dtype::extension::ExtVTable;
402
403        let vtable = Timestamp;
404        // Valid tag (0x00 = Nanoseconds), tz_len = 10 (little-endian: [0x0A, 0x00]),
405        // but only 3 bytes of timezone data instead of the declared 10.
406        let data = [0x00u8, 0x0A, 0x00, b'U', b'T', b'C'];
407        assert!(vtable.deserialize_metadata(&data).is_err());
408    }
409}