nominal_api_conjure/conjure/objects/ingest/api/
parquet_opts.rs

1/// Options for ingesting parquet files.
2/// Supported file formats include .parquet, .parquet.gz
3/// and archives such as .tar, .tar.gz, and .zip (must set the isArchive flag).
4#[derive(
5    Debug,
6    Clone,
7    conjure_object::serde::Serialize,
8    conjure_object::serde::Deserialize,
9    PartialEq,
10    Eq,
11    PartialOrd,
12    Ord,
13    Hash
14)]
15#[serde(crate = "conjure_object::serde")]
16#[conjure_object::private::staged_builder::staged_builder]
17#[builder(crate = conjure_object::private::staged_builder, update, inline)]
18pub struct ParquetOpts {
19    #[builder(custom(type = super::IngestSource, convert = Box::new))]
20    #[serde(rename = "source")]
21    source: Box<super::IngestSource>,
22    #[builder(custom(type = super::DatasetIngestTarget, convert = Box::new))]
23    #[serde(rename = "target")]
24    target: Box<super::DatasetIngestTarget>,
25    #[builder(custom(type = super::TimestampMetadata, convert = Box::new))]
26    #[serde(rename = "timestampMetadata")]
27    timestamp_metadata: Box<super::TimestampMetadata>,
28    #[builder(default)]
29    #[serde(rename = "channelPrefix", skip_serializing_if = "Option::is_none", default)]
30    channel_prefix: super::ChannelPrefix,
31    #[builder(default, map(key(type = String, into), value(type = String, into)))]
32    #[serde(
33        rename = "channelNameOverrides",
34        skip_serializing_if = "std::collections::BTreeMap::is_empty",
35        default
36    )]
37    channel_name_overrides: std::collections::BTreeMap<String, String>,
38    #[builder(default, into)]
39    #[serde(
40        rename = "tagKeysFromColumns",
41        skip_serializing_if = "Option::is_none",
42        default
43    )]
44    tag_keys_from_columns: Option<std::collections::BTreeSet<String>>,
45    #[builder(default, into)]
46    #[serde(rename = "tagColumns", skip_serializing_if = "Option::is_none", default)]
47    tag_columns: Option<std::collections::BTreeMap<String, String>>,
48    #[builder(default, into)]
49    #[serde(
50        rename = "additionalFileTags",
51        skip_serializing_if = "Option::is_none",
52        default
53    )]
54    additional_file_tags: Option<std::collections::BTreeMap<String, String>>,
55    #[builder(default, into)]
56    #[serde(rename = "isArchive", skip_serializing_if = "Option::is_none", default)]
57    is_archive: Option<bool>,
58    #[builder(default, set(item(type = String, into)))]
59    #[serde(
60        rename = "excludeColumns",
61        skip_serializing_if = "std::collections::BTreeSet::is_empty",
62        default
63    )]
64    exclude_columns: std::collections::BTreeSet<String>,
65}
66impl ParquetOpts {
67    /// Constructs a new instance of the type.
68    #[inline]
69    pub fn new(
70        source: super::IngestSource,
71        target: super::DatasetIngestTarget,
72        timestamp_metadata: super::TimestampMetadata,
73    ) -> Self {
74        Self::builder()
75            .source(source)
76            .target(target)
77            .timestamp_metadata(timestamp_metadata)
78            .build()
79    }
80    #[inline]
81    pub fn source(&self) -> &super::IngestSource {
82        &*self.source
83    }
84    #[inline]
85    pub fn target(&self) -> &super::DatasetIngestTarget {
86        &*self.target
87    }
88    #[inline]
89    pub fn timestamp_metadata(&self) -> &super::TimestampMetadata {
90        &*self.timestamp_metadata
91    }
92    #[inline]
93    pub fn channel_prefix(&self) -> &super::ChannelPrefix {
94        &self.channel_prefix
95    }
96    #[inline]
97    pub fn channel_name_overrides(&self) -> &std::collections::BTreeMap<String, String> {
98        &self.channel_name_overrides
99    }
100    #[deprecated(note = "Deprecated in favor of tagColumns.")]
101    #[inline]
102    pub fn tag_keys_from_columns(&self) -> Option<&std::collections::BTreeSet<String>> {
103        self.tag_keys_from_columns.as_ref().map(|o| &*o)
104    }
105    /// A map of tag names to column names to derive the tag values from.
106    #[inline]
107    pub fn tag_columns(&self) -> Option<&std::collections::BTreeMap<String, String>> {
108        self.tag_columns.as_ref().map(|o| &*o)
109    }
110    /// Specifies a tag set to apply to all data in the file.
111    #[inline]
112    pub fn additional_file_tags(
113        &self,
114    ) -> Option<&std::collections::BTreeMap<String, String>> {
115        self.additional_file_tags.as_ref().map(|o| &*o)
116    }
117    /// If true, the file is an archive. Supported archive formats include
118    /// .tar, .tar.gz, and .zip. Only files ending in .parquet
119    /// within the archive will be ingested. If field not provided, defaults to false.
120    #[inline]
121    pub fn is_archive(&self) -> Option<bool> {
122        self.is_archive.as_ref().map(|o| *o)
123    }
124    /// A set of column names to exclude from ingestion. These columns will not be
125    /// ingested as channels. Useful for excluding columns that contain unsupported
126    /// data types like multidimensional arrays.
127    #[inline]
128    pub fn exclude_columns(&self) -> &std::collections::BTreeSet<String> {
129        &self.exclude_columns
130    }
131}
nominal_api_conjure/conjure/objects/ingest/api/parquet_opts.rs

nominal_api_conjure/conjure/objects/ingest/api/
parquet_opts.rs
