nominal_api_conjure/conjure/objects/ingest/api/
parquet_opts.rs

1/// Options for ingesting parquet files.
2/// Supported file formats include .parquet, .parquet.gz
3/// and archives such as .tar, .tar.gz, and .zip (must set the isArchive flag).
4#[derive(
5    Debug,
6    Clone,
7    conjure_object::serde::Serialize,
8    conjure_object::serde::Deserialize,
9    PartialEq,
10    Eq,
11    PartialOrd,
12    Ord,
13    Hash
14)]
15#[serde(crate = "conjure_object::serde")]
16#[conjure_object::private::staged_builder::staged_builder]
17#[builder(crate = conjure_object::private::staged_builder, update, inline)]
18pub struct ParquetOpts {
19    #[builder(custom(type = super::IngestSource, convert = Box::new))]
20    #[serde(rename = "source")]
21    source: Box<super::IngestSource>,
22    #[builder(custom(type = super::DatasetIngestTarget, convert = Box::new))]
23    #[serde(rename = "target")]
24    target: Box<super::DatasetIngestTarget>,
25    #[builder(custom(type = super::TimestampMetadata, convert = Box::new))]
26    #[serde(rename = "timestampMetadata")]
27    timestamp_metadata: Box<super::TimestampMetadata>,
28    #[builder(default)]
29    #[serde(rename = "channelPrefix", skip_serializing_if = "Option::is_none", default)]
30    channel_prefix: super::ChannelPrefix,
31    #[builder(
32        default,
33        map(
34            key(type = super::super::super::api::ColumnName),
35            value(type = super::super::super::api::Channel)
36        )
37    )]
38    #[serde(
39        rename = "channelNameOverrides",
40        skip_serializing_if = "std::collections::BTreeMap::is_empty",
41        default
42    )]
43    channel_name_overrides: std::collections::BTreeMap<
44        super::super::super::api::ColumnName,
45        super::super::super::api::Channel,
46    >,
47    #[builder(default, into)]
48    #[serde(
49        rename = "tagKeysFromColumns",
50        skip_serializing_if = "Option::is_none",
51        default
52    )]
53    tag_keys_from_columns: Option<
54        std::collections::BTreeSet<super::super::super::api::TagName>,
55    >,
56    #[builder(default, into)]
57    #[serde(rename = "tagColumns", skip_serializing_if = "Option::is_none", default)]
58    tag_columns: Option<
59        std::collections::BTreeMap<
60            super::super::super::api::TagName,
61            super::super::super::api::ColumnName,
62        >,
63    >,
64    #[builder(default, into)]
65    #[serde(
66        rename = "additionalFileTags",
67        skip_serializing_if = "Option::is_none",
68        default
69    )]
70    additional_file_tags: Option<
71        std::collections::BTreeMap<
72            super::super::super::api::TagName,
73            super::super::super::api::TagValue,
74        >,
75    >,
76    #[builder(default, into)]
77    #[serde(rename = "isArchive", skip_serializing_if = "Option::is_none", default)]
78    is_archive: Option<bool>,
79    #[builder(default, set(item(type = super::super::super::api::ColumnName)))]
80    #[serde(
81        rename = "excludeColumns",
82        skip_serializing_if = "std::collections::BTreeSet::is_empty",
83        default
84    )]
85    exclude_columns: std::collections::BTreeSet<super::super::super::api::ColumnName>,
86}
87impl ParquetOpts {
88    /// Constructs a new instance of the type.
89    #[inline]
90    pub fn new(
91        source: super::IngestSource,
92        target: super::DatasetIngestTarget,
93        timestamp_metadata: super::TimestampMetadata,
94    ) -> Self {
95        Self::builder()
96            .source(source)
97            .target(target)
98            .timestamp_metadata(timestamp_metadata)
99            .build()
100    }
101    #[inline]
102    pub fn source(&self) -> &super::IngestSource {
103        &*self.source
104    }
105    #[inline]
106    pub fn target(&self) -> &super::DatasetIngestTarget {
107        &*self.target
108    }
109    #[inline]
110    pub fn timestamp_metadata(&self) -> &super::TimestampMetadata {
111        &*self.timestamp_metadata
112    }
113    #[inline]
114    pub fn channel_prefix(&self) -> &super::ChannelPrefix {
115        &self.channel_prefix
116    }
117    #[inline]
118    pub fn channel_name_overrides(
119        &self,
120    ) -> &std::collections::BTreeMap<
121        super::super::super::api::ColumnName,
122        super::super::super::api::Channel,
123    > {
124        &self.channel_name_overrides
125    }
126    #[deprecated(note = "Deprecated in favor of tagColumns.")]
127    #[inline]
128    pub fn tag_keys_from_columns(
129        &self,
130    ) -> Option<&std::collections::BTreeSet<super::super::super::api::TagName>> {
131        self.tag_keys_from_columns.as_ref().map(|o| &*o)
132    }
133    /// A map of tag names to column names to derive the tag values from.
134    #[inline]
135    pub fn tag_columns(
136        &self,
137    ) -> Option<
138        &std::collections::BTreeMap<
139            super::super::super::api::TagName,
140            super::super::super::api::ColumnName,
141        >,
142    > {
143        self.tag_columns.as_ref().map(|o| &*o)
144    }
145    /// Specifies a tag set to apply to all data in the file.
146    #[inline]
147    pub fn additional_file_tags(
148        &self,
149    ) -> Option<
150        &std::collections::BTreeMap<
151            super::super::super::api::TagName,
152            super::super::super::api::TagValue,
153        >,
154    > {
155        self.additional_file_tags.as_ref().map(|o| &*o)
156    }
157    /// If true, the file is an archive. Supported archive formats include
158    /// .tar, .tar.gz, and .zip. Only files ending in .parquet
159    /// within the archive will be ingested. If field not provided, defaults to false.
160    #[inline]
161    pub fn is_archive(&self) -> Option<bool> {
162        self.is_archive.as_ref().map(|o| *o)
163    }
164    /// A set of column names to exclude from ingestion. These columns will not be
165    /// ingested as channels. Useful for excluding columns that contain unsupported
166    /// data types like multidimensional arrays.
167    #[inline]
168    pub fn exclude_columns(
169        &self,
170    ) -> &std::collections::BTreeSet<super::super::super::api::ColumnName> {
171        &self.exclude_columns
172    }
173}
nominal_api_conjure/conjure/objects/ingest/api/parquet_opts.rs

nominal_api_conjure/conjure/objects/ingest/api/
parquet_opts.rs
