nominal_api/conjure/objects/ingest/api/
parquet_opts.rs

1/// Options for ingesting parquet files.
2/// Supported file formats include .parquet, .parquet.gz
3/// and archives such as .tar, .tar.gz, and .zip (must set the isArchive flag).
4#[derive(
5    Debug,
6    Clone,
7    conjure_object::serde::Serialize,
8    conjure_object::serde::Deserialize,
9    PartialEq,
10    Eq,
11    PartialOrd,
12    Ord,
13    Hash
14)]
15#[serde(crate = "conjure_object::serde")]
16#[conjure_object::private::staged_builder::staged_builder]
17#[builder(crate = conjure_object::private::staged_builder, update, inline)]
18pub struct ParquetOpts {
19    #[builder(custom(type = super::IngestSource, convert = Box::new))]
20    #[serde(rename = "source")]
21    source: Box<super::IngestSource>,
22    #[builder(custom(type = super::DatasetIngestTarget, convert = Box::new))]
23    #[serde(rename = "target")]
24    target: Box<super::DatasetIngestTarget>,
25    #[builder(custom(type = super::TimestampMetadata, convert = Box::new))]
26    #[serde(rename = "timestampMetadata")]
27    timestamp_metadata: Box<super::TimestampMetadata>,
28    #[builder(default)]
29    #[serde(rename = "channelPrefix", skip_serializing_if = "Option::is_none", default)]
30    channel_prefix: super::ChannelPrefix,
31    #[builder(default, into)]
32    #[serde(
33        rename = "tagKeysFromColumns",
34        skip_serializing_if = "Option::is_none",
35        default
36    )]
37    tag_keys_from_columns: Option<
38        std::collections::BTreeSet<super::super::super::api::TagName>,
39    >,
40    #[builder(default, into)]
41    #[serde(rename = "tagColumns", skip_serializing_if = "Option::is_none", default)]
42    tag_columns: Option<
43        std::collections::BTreeMap<
44            super::super::super::api::TagName,
45            super::super::super::api::ColumnName,
46        >,
47    >,
48    #[builder(default, into)]
49    #[serde(
50        rename = "additionalFileTags",
51        skip_serializing_if = "Option::is_none",
52        default
53    )]
54    additional_file_tags: Option<
55        std::collections::BTreeMap<
56            super::super::super::api::TagName,
57            super::super::super::api::TagValue,
58        >,
59    >,
60    #[builder(default, into)]
61    #[serde(rename = "isArchive", skip_serializing_if = "Option::is_none", default)]
62    is_archive: Option<bool>,
63    #[builder(default, set(item(type = super::super::super::api::ColumnName)))]
64    #[serde(
65        rename = "excludeColumns",
66        skip_serializing_if = "std::collections::BTreeSet::is_empty",
67        default
68    )]
69    exclude_columns: std::collections::BTreeSet<super::super::super::api::ColumnName>,
70}
71impl ParquetOpts {
72    /// Constructs a new instance of the type.
73    #[inline]
74    pub fn new(
75        source: super::IngestSource,
76        target: super::DatasetIngestTarget,
77        timestamp_metadata: super::TimestampMetadata,
78    ) -> Self {
79        Self::builder()
80            .source(source)
81            .target(target)
82            .timestamp_metadata(timestamp_metadata)
83            .build()
84    }
85    #[inline]
86    pub fn source(&self) -> &super::IngestSource {
87        &*self.source
88    }
89    #[inline]
90    pub fn target(&self) -> &super::DatasetIngestTarget {
91        &*self.target
92    }
93    #[inline]
94    pub fn timestamp_metadata(&self) -> &super::TimestampMetadata {
95        &*self.timestamp_metadata
96    }
97    #[inline]
98    pub fn channel_prefix(&self) -> &super::ChannelPrefix {
99        &self.channel_prefix
100    }
101    #[deprecated(note = "Deprecated in favor of tagColumns.")]
102    #[inline]
103    pub fn tag_keys_from_columns(
104        &self,
105    ) -> Option<&std::collections::BTreeSet<super::super::super::api::TagName>> {
106        self.tag_keys_from_columns.as_ref().map(|o| &*o)
107    }
108    /// A map of tag names to column names to derive the tag values from.
109    #[inline]
110    pub fn tag_columns(
111        &self,
112    ) -> Option<
113        &std::collections::BTreeMap<
114            super::super::super::api::TagName,
115            super::super::super::api::ColumnName,
116        >,
117    > {
118        self.tag_columns.as_ref().map(|o| &*o)
119    }
120    /// Specifies a tag set to apply to all data in the file.
121    #[inline]
122    pub fn additional_file_tags(
123        &self,
124    ) -> Option<
125        &std::collections::BTreeMap<
126            super::super::super::api::TagName,
127            super::super::super::api::TagValue,
128        >,
129    > {
130        self.additional_file_tags.as_ref().map(|o| &*o)
131    }
132    /// If true, the file is an archive. Supported archive formats include
133    /// .tar, .tar.gz, and .zip. Only files ending in .parquet
134    /// within the archive will be ingested. If field not provided, defaults to false.
135    #[inline]
136    pub fn is_archive(&self) -> Option<bool> {
137        self.is_archive.as_ref().map(|o| *o)
138    }
139    /// A set of column names to exclude from ingestion. These columns will not be
140    /// ingested as channels. Useful for excluding columns that contain unsupported
141    /// data types like multidimensional arrays.
142    #[inline]
143    pub fn exclude_columns(
144        &self,
145    ) -> &std::collections::BTreeSet<super::super::super::api::ColumnName> {
146        &self.exclude_columns
147    }
148}
nominal_api/conjure/objects/ingest/api/parquet_opts.rs

nominal_api/conjure/objects/ingest/api/
parquet_opts.rs
