1use faucet_common_azure::{AzureConnection, AzureCredentials};
4use faucet_core::DEFAULT_BATCH_SIZE;
5use schemars::JsonSchema;
6use serde::{Deserialize, Serialize};
7
8#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
10#[serde(rename_all = "snake_case")]
11pub enum AzureFileFormat {
12 #[default]
14 JsonLines,
15 JsonArray,
17 RawText,
19 #[cfg(feature = "file-format-csv")]
24 Csv,
25 #[cfg(feature = "file-format-xml")]
29 Xml,
30 #[cfg(feature = "file-format-excel")]
35 Xlsx,
36 #[cfg(feature = "file-format-avro")]
42 Avro,
43 #[cfg(feature = "file-format-orc")]
47 Orc,
48 #[cfg(feature = "arrow")]
52 Parquet,
53}
54
55impl AzureFileFormat {
56 #[cfg(any(
59 feature = "file-format-csv",
60 feature = "file-format-xml",
61 feature = "file-format-excel",
62 feature = "file-format-avro",
63 feature = "file-format-orc"
64 ))]
65 pub(crate) fn shared(&self) -> Option<faucet_core::FileFormat> {
66 match self {
67 Self::JsonLines => Some(faucet_core::FileFormat::JsonLines),
68 Self::JsonArray => Some(faucet_core::FileFormat::JsonArray),
69 Self::RawText => None,
70 #[cfg(feature = "file-format-csv")]
71 Self::Csv => Some(faucet_core::FileFormat::Csv),
72 #[cfg(feature = "file-format-xml")]
73 Self::Xml => Some(faucet_core::FileFormat::Xml),
74 #[cfg(feature = "file-format-excel")]
75 Self::Xlsx => Some(faucet_core::FileFormat::Xlsx),
76 #[cfg(feature = "file-format-avro")]
77 Self::Avro => Some(faucet_core::FileFormat::Avro),
78 #[cfg(feature = "file-format-orc")]
79 Self::Orc => Some(faucet_core::FileFormat::Orc),
80 #[cfg(feature = "arrow")]
81 Self::Parquet => Some(faucet_core::FileFormat::Parquet),
82 }
83 }
84}
85
86#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
88pub struct AzureBlobSourceConfig {
89 #[serde(flatten)]
91 pub connection: AzureConnection,
92 pub prefix: Option<String>,
94 #[serde(default)]
100 pub include: Option<String>,
101 pub object_keys: Option<Vec<String>>,
104 #[serde(default)]
106 pub file_format: AzureFileFormat,
107 pub max_objects: Option<usize>,
109 #[serde(default = "default_concurrency")]
111 pub concurrency: usize,
112 #[serde(default = "default_batch_size")]
115 pub batch_size: usize,
116 #[serde(default = "default_true")]
127 pub verify_length: bool,
128 #[serde(default)]
133 pub verify_checksum: bool,
134 #[serde(default = "default_max_object_bytes")]
140 pub max_object_bytes: u64,
141 #[serde(default, skip_serializing_if = "Option::is_none")]
147 pub max_decoded_bytes: Option<u64>,
148 #[cfg(feature = "compression")]
154 #[serde(default)]
155 pub compression: faucet_core::CompressionConfig,
156 #[serde(default)]
158 pub csv: faucet_core::CsvOptions,
159 #[serde(default)]
161 pub excel: faucet_core::ExcelOptions,
162 #[serde(default)]
164 pub xml: faucet_core::XmlOptions,
165 #[serde(default)]
167 pub avro: faucet_core::AvroOptions,
168 #[serde(default)]
170 pub orc: faucet_core::OrcOptions,
171 #[serde(default)]
173 pub parquet: faucet_core::ParquetReadOptions,
174}
175
176fn default_true() -> bool {
178 true
179}
180
181fn default_batch_size() -> usize {
182 DEFAULT_BATCH_SIZE
183}
184fn default_max_object_bytes() -> u64 {
185 faucet_core::file_format::DEFAULT_MAX_OBJECT_BYTES
186}
187
188fn default_concurrency() -> usize {
189 10
190}
191
192impl AzureBlobSourceConfig {
193 pub fn new(container: impl Into<String>) -> Self {
195 Self {
196 connection: AzureConnection::new(container),
197 prefix: None,
198 include: None,
199 object_keys: None,
200 file_format: AzureFileFormat::default(),
201 max_objects: None,
202 concurrency: default_concurrency(),
203 batch_size: default_batch_size(),
204 verify_length: true,
205 verify_checksum: false,
206 max_object_bytes: default_max_object_bytes(),
207 max_decoded_bytes: None,
208 #[cfg(feature = "compression")]
209 compression: faucet_core::CompressionConfig::default(),
210 csv: faucet_core::CsvOptions::default(),
211 excel: faucet_core::ExcelOptions::default(),
212 xml: faucet_core::XmlOptions::default(),
213 avro: faucet_core::AvroOptions::default(),
214 orc: faucet_core::OrcOptions::default(),
215 parquet: faucet_core::ParquetReadOptions::default(),
216 }
217 }
218
219 #[cfg_attr(
221 not(any(feature = "file-format-avro", feature = "file-format-excel")),
222 allow(dead_code)
223 )]
224 pub(crate) fn decoded_bytes_limit(&self) -> u64 {
225 faucet_core::file_format::budget::effective_max_decoded_bytes(
226 self.max_decoded_bytes,
227 self.max_object_bytes,
228 )
229 }
230
231 #[cfg(any(
234 feature = "file-format-csv",
235 feature = "file-format-xml",
236 feature = "file-format-excel",
237 feature = "file-format-avro",
238 feature = "file-format-orc"
239 ))]
240 pub(crate) fn format_options(&self) -> faucet_core::FormatOptions {
241 faucet_core::FormatOptions {
242 csv: self.csv.clone(),
243 excel: self.excel.clone(),
244 xml: self.xml.clone(),
245 avro: self.avro.clone(),
246 orc: self.orc.clone(),
247 }
248 }
249
250 pub fn account(mut self, account: impl Into<String>) -> Self {
252 self.connection = self.connection.account(account);
253 self
254 }
255
256 pub fn auth(mut self, creds: AzureCredentials) -> Self {
258 self.connection = self.connection.auth(creds);
259 self
260 }
261
262 pub fn endpoint(mut self, endpoint: impl Into<String>) -> Self {
264 self.connection = self.connection.endpoint(endpoint);
265 self
266 }
267
268 pub fn allow_http(mut self, allow: bool) -> Self {
270 self.connection = self.connection.allow_http(allow);
271 self
272 }
273
274 pub fn use_emulator(mut self, use_emulator: bool) -> Self {
276 self.connection = self.connection.use_emulator(use_emulator);
277 self
278 }
279
280 pub fn prefix(mut self, prefix: impl Into<String>) -> Self {
282 self.prefix = Some(prefix.into());
283 self
284 }
285
286 pub fn object_keys(mut self, keys: Vec<String>) -> Self {
288 self.object_keys = Some(keys);
289 self
290 }
291
292 pub fn file_format(mut self, format: AzureFileFormat) -> Self {
294 self.file_format = format;
295 self
296 }
297
298 pub fn max_objects(mut self, max: usize) -> Self {
300 self.max_objects = Some(max);
301 self
302 }
303
304 pub fn concurrency(mut self, concurrency: usize) -> Self {
306 self.concurrency = concurrency;
307 self
308 }
309
310 pub fn with_batch_size(mut self, batch_size: usize) -> Self {
312 self.batch_size = batch_size;
313 self
314 }
315
316 pub fn verify_length(mut self, verify: bool) -> Self {
319 self.verify_length = verify;
320 self
321 }
322
323 pub fn validate(&self) -> Result<(), faucet_core::FaucetError> {
329 if self.connection.container.trim().is_empty() {
330 return Err(faucet_core::FaucetError::Config(
331 "azure-blob source requires a non-empty `container`".into(),
332 ));
333 }
334 if self.verify_checksum {
335 return Err(faucet_core::FaucetError::Config(
336 "azure-blob source: `verify_checksum` is not supported — Azure Blob does not \
337 expose a body checksum through the object-store read API. Leave it unset and \
338 rely on `verify_length`, which is enforced against the reported blob size."
339 .into(),
340 ));
341 }
342 faucet_core::validate_batch_size(self.batch_size)?;
343 self.parquet.validate()?;
344 Ok(())
345 }
346
347 #[cfg(feature = "compression")]
349 pub fn compression(mut self, c: faucet_core::CompressionConfig) -> Self {
350 self.compression = c;
351 self
352 }
353
354 pub fn container(&self) -> &str {
356 &self.connection.container
357 }
358}
359
360#[cfg(test)]
361mod tests {
362 use super::*;
363
364 #[test]
365 fn decoded_bytes_default_to_a_multiple_of_the_object_limit() {
366 let mut c = AzureBlobSourceConfig::new("c");
367 c.max_object_bytes = 1024;
368 assert_eq!(c.decoded_bytes_limit(), 64 * 1024);
369 c.max_object_bytes = faucet_core::file_format::DEFAULT_MAX_OBJECT_BYTES;
370 assert_eq!(
371 c.decoded_bytes_limit(),
372 faucet_core::file_format::budget::MAX_DEFAULT_DECODED_BYTES
373 );
374 c.max_decoded_bytes = Some(10);
375 assert_eq!(c.decoded_bytes_limit(), 10);
376 }
377
378 #[test]
379 fn default_config() {
380 let config = AzureBlobSourceConfig::new("my-container");
381 assert_eq!(config.container(), "my-container");
382 assert!(config.prefix.is_none());
383 assert!(config.object_keys.is_none());
384 assert_eq!(config.connection.auth, AzureCredentials::Default);
385 assert_eq!(config.file_format, AzureFileFormat::JsonLines);
386 assert!(config.max_objects.is_none());
387 assert_eq!(config.concurrency, 10);
388 assert_eq!(config.batch_size, faucet_core::DEFAULT_BATCH_SIZE);
389 }
390
391 #[test]
392 fn builder_methods() {
393 let config = AzureBlobSourceConfig::new("c")
394 .account("acct")
395 .prefix("data/")
396 .file_format(AzureFileFormat::JsonArray)
397 .max_objects(5)
398 .concurrency(20)
399 .with_batch_size(250)
400 .auth(AzureCredentials::AccountKey {
401 account_key: "k".into(),
402 });
403
404 assert_eq!(config.container(), "c");
405 assert_eq!(config.connection.account.as_deref(), Some("acct"));
406 assert_eq!(config.prefix.as_deref(), Some("data/"));
407 assert_eq!(config.file_format, AzureFileFormat::JsonArray);
408 assert_eq!(config.max_objects, Some(5));
409 assert_eq!(config.concurrency, 20);
410 assert_eq!(config.batch_size, 250);
411 assert!(matches!(
412 config.connection.auth,
413 AzureCredentials::AccountKey { .. }
414 ));
415 }
416
417 #[test]
418 fn file_format_default_is_json_lines() {
419 assert_eq!(AzureFileFormat::default(), AzureFileFormat::JsonLines);
420 }
421
422 #[test]
423 fn deserializes_flattened_connection_and_auth() {
424 let json = r#"{
425 "container": "c",
426 "account": "acct",
427 "auth": { "type": "account_key", "config": { "account_key": "k" } },
428 "prefix": "data/",
429 "file_format": "json_array"
430 }"#;
431 let config: AzureBlobSourceConfig = serde_json::from_str(json).unwrap();
432 assert_eq!(config.container(), "c");
433 assert_eq!(config.connection.account.as_deref(), Some("acct"));
434 assert_eq!(config.file_format, AzureFileFormat::JsonArray);
435 assert!(matches!(
436 config.connection.auth,
437 AzureCredentials::AccountKey { .. }
438 ));
439 assert_eq!(config.batch_size, faucet_core::DEFAULT_BATCH_SIZE);
441 assert_eq!(config.concurrency, 10);
442 }
443
444 #[test]
445 fn auth_defaults_to_default_when_absent_from_json() {
446 let json = r#"{ "container": "c" }"#;
447 let config: AzureBlobSourceConfig = serde_json::from_str(json).unwrap();
448 assert_eq!(config.connection.auth, AzureCredentials::Default);
449 }
450
451 #[test]
452 fn batch_size_zero_is_accepted_as_no_batching_sentinel() {
453 let config = AzureBlobSourceConfig::new("c").with_batch_size(0);
454 assert_eq!(config.batch_size, 0);
455 assert!(faucet_core::validate_batch_size(config.batch_size).is_ok());
456 }
457
458 #[test]
459 fn batch_size_above_max_is_rejected() {
460 let config =
461 AzureBlobSourceConfig::new("c").with_batch_size(faucet_core::MAX_BATCH_SIZE + 1);
462 assert!(faucet_core::validate_batch_size(config.batch_size).is_err());
463 }
464
465 #[test]
466 fn schema_generates_without_panicking() {
467 let _ = faucet_core::schema_for!(AzureBlobSourceConfig);
468 }
469
470 #[test]
471 fn verify_defaults_length_on_checksum_off() {
472 let cfg = AzureBlobSourceConfig::new("c");
473 assert!(cfg.verify_length, "length verification defaults on");
474 assert!(!cfg.verify_checksum);
475 }
476
477 #[test]
478 fn verify_length_builder_overrides() {
479 let cfg = AzureBlobSourceConfig::new("c").verify_length(false);
480 assert!(!cfg.verify_length);
481 }
482
483 #[test]
486 fn verify_keys_stay_top_level_on_the_wire() {
487 let json = r#"{ "container": "c", "verify_length": false }"#;
488 let config: AzureBlobSourceConfig = serde_json::from_str(json).unwrap();
489 assert!(!config.verify_length);
490
491 let out = serde_json::to_value(&config).unwrap();
492 assert_eq!(out["verify_length"], serde_json::json!(false));
493 assert_eq!(out["verify_checksum"], serde_json::json!(false));
494 assert!(out.get("verify").is_none(), "no nested block: {out}");
495 }
496
497 #[test]
498 fn verify_defaults_when_absent_from_json() {
499 let config: AzureBlobSourceConfig =
500 serde_json::from_str(r#"{ "container": "c" }"#).unwrap();
501 assert!(config.verify_length);
502 assert!(!config.verify_checksum);
503 }
504
505 #[test]
506 fn validate_accepts_a_default_config() {
507 assert!(AzureBlobSourceConfig::new("c").validate().is_ok());
508 }
509
510 #[test]
511 fn validate_rejects_verify_checksum() {
512 let mut cfg = AzureBlobSourceConfig::new("c");
513 cfg.verify_checksum = true;
514 match cfg.validate() {
515 Err(faucet_core::FaucetError::Config(m)) => {
516 assert!(m.contains("verify_checksum"), "got: {m}")
517 }
518 other => panic!("expected a verify_checksum Config error, got {other:?}"),
519 }
520 }
521
522 #[test]
523 fn validate_rejects_an_empty_parquet_projection() {
524 let mut c = AzureBlobSourceConfig::new("c");
525 c.parquet.columns = Some(vec![]);
526 let e = c.validate().unwrap_err().to_string();
527 assert!(e.contains("parquet.columns"), "{e}");
528 }
529
530 #[test]
531 fn validate_rejects_empty_container() {
532 assert!(matches!(
533 AzureBlobSourceConfig::new(" ").validate(),
534 Err(faucet_core::FaucetError::Config(_))
535 ));
536 }
537
538 #[test]
539 fn validate_rejects_oversized_batch_size() {
540 let cfg = AzureBlobSourceConfig::new("c").with_batch_size(faucet_core::MAX_BATCH_SIZE + 1);
541 assert!(matches!(
542 cfg.validate(),
543 Err(faucet_core::FaucetError::Config(_))
544 ));
545 }
546
547 #[cfg(feature = "compression")]
548 #[test]
549 fn compression_default_is_auto() {
550 let cfg = AzureBlobSourceConfig::new("c");
551 assert_eq!(cfg.compression, faucet_core::CompressionConfig::Auto);
552 }
553
554 #[cfg(any(
561 feature = "file-format-csv",
562 feature = "file-format-xml",
563 feature = "file-format-excel",
564 feature = "file-format-avro",
565 feature = "file-format-orc"
566 ))]
567 #[test]
568 fn formats_map_onto_the_shared_vocabulary_or_opt_out() {
569 assert_eq!(
570 AzureFileFormat::JsonLines.shared(),
571 Some(faucet_core::FileFormat::JsonLines)
572 );
573 assert_eq!(
574 AzureFileFormat::JsonArray.shared(),
575 Some(faucet_core::FileFormat::JsonArray)
576 );
577 assert_eq!(AzureFileFormat::RawText.shared(), None);
578 #[cfg(feature = "file-format-csv")]
579 assert_eq!(
580 AzureFileFormat::Csv.shared(),
581 Some(faucet_core::FileFormat::Csv)
582 );
583 #[cfg(feature = "file-format-xml")]
584 assert_eq!(
585 AzureFileFormat::Xml.shared(),
586 Some(faucet_core::FileFormat::Xml)
587 );
588 #[cfg(feature = "file-format-excel")]
589 assert_eq!(
590 AzureFileFormat::Xlsx.shared(),
591 Some(faucet_core::FileFormat::Xlsx)
592 );
593 #[cfg(feature = "file-format-avro")]
594 assert_eq!(
595 AzureFileFormat::Avro.shared(),
596 Some(faucet_core::FileFormat::Avro)
597 );
598 #[cfg(feature = "file-format-orc")]
599 assert_eq!(
600 AzureFileFormat::Orc.shared(),
601 Some(faucet_core::FileFormat::Orc)
602 );
603 }
604
605 #[cfg(any(
606 feature = "file-format-csv",
607 feature = "file-format-xml",
608 feature = "file-format-excel",
609 feature = "file-format-avro",
610 feature = "file-format-orc"
611 ))]
612 #[test]
613 fn the_format_option_blocks_reach_the_decoder() {
614 let mut cfg = AzureBlobSourceConfig::new("c");
615 cfg.csv = faucet_core::CsvOptions {
616 delimiter: "\\t".into(),
617 has_headers: false,
618 ..Default::default()
619 };
620 cfg.excel = faucet_core::ExcelOptions {
621 sheet: Some("Q3".into()),
622 header_row: 1,
623 };
624 cfg.xml = faucet_core::XmlOptions {
625 record_element: "order".into(),
626 root_element: "orders".into(),
627 };
628 let opts = cfg.format_options();
629 assert_eq!(opts.csv.delimiter_byte().expect("tab"), b'\t');
630 assert!(!opts.csv.has_headers);
631 assert_eq!(opts.excel.sheet.as_deref(), Some("Q3"));
632 assert_eq!(opts.excel.header_row, 1);
633 assert_eq!(opts.xml.record_element, "order");
634 }
635}