1use crate::{
30 CopyMode, CopyOptions, Error, GetOptions, GetResult, ListResult, MultipartId, MultipartUpload,
31 ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result,
32 UploadPart,
33 multipart::{MultipartStore, PartId},
34 path::Path,
35 signer::Signer,
36};
37use async_trait::async_trait;
38use futures_util::stream::{BoxStream, StreamExt, TryStreamExt};
39use http::Method;
40use std::fmt::Debug;
41use std::sync::Arc;
42use std::time::Duration;
43use url::Url;
44
45use crate::client::get::GetClientExt;
46use crate::client::list::{ListClient, ListClientExt};
47use crate::client::{CredentialProvider, crypto_provider};
48use crate::retry::{MultipartRetry, RetryPolicy};
49pub use credential::{AzureAccessKey, AzureAuthorizer, authority_hosts};
50
51mod builder;
52mod client;
53mod credential;
54
55pub type AzureCredentialProvider = Arc<dyn CredentialProvider<Credential = AzureCredential>>;
57use crate::azure::client::AzureClient;
58use crate::client::parts::Parts;
59use crate::list::{PaginatedListOptions, PaginatedListResult, PaginatedListStore};
60pub use builder::{AzureConfigKey, MicrosoftAzureBuilder, split_sas};
61pub use credential::AzureCredential;
62
63const STORE: &str = "MicrosoftAzure";
64
65#[derive(Debug, Clone)]
67pub struct MicrosoftAzure {
68 client: Arc<AzureClient>,
69}
70
71impl MicrosoftAzure {
72 pub fn credentials(&self) -> &AzureCredentialProvider {
74 &self.client.config().credentials
75 }
76
77 fn path_url(&self, path: &Path) -> Url {
79 self.client.config().path_url(path)
80 }
81}
82
83impl std::fmt::Display for MicrosoftAzure {
84 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
85 write!(
86 f,
87 "MicrosoftAzure {{ account: {}, container: {} }}",
88 self.client.config().account,
89 self.client.config().container
90 )
91 }
92}
93
94#[async_trait]
95impl ObjectStore for MicrosoftAzure {
96 async fn put_opts(
97 &self,
98 location: &Path,
99 payload: PutPayload,
100 opts: PutOptions,
101 ) -> Result<PutResult> {
102 self.client.put_blob(location, payload, opts).await
103 }
104
105 async fn put_multipart_opts(
106 &self,
107 location: &Path,
108 opts: PutMultipartOptions,
109 ) -> Result<Box<dyn MultipartUpload>> {
110 let retry_policy = opts.retry_policy();
111 Ok(Box::new(AzureMultiPartUpload {
112 part_idx: 0,
113 opts,
114 state: Arc::new(UploadState {
115 client: Arc::clone(&self.client),
116 location: location.clone(),
117 parts: Default::default(),
118 retry_policy,
119 }),
120 }))
121 }
122
123 async fn get_opts(&self, location: &Path, options: GetOptions) -> Result<GetResult> {
124 self.client.get_opts(location, options).await
125 }
126
127 fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
128 self.client.list(prefix)
129 }
130
131 fn list_with_offset(
132 &self,
133 prefix: Option<&Path>,
134 offset: &Path,
135 ) -> BoxStream<'static, Result<ObjectMeta>> {
136 if self.client.config().is_emulator {
137 let offset = offset.clone();
142 self.list(prefix)
143 .try_filter(move |f| futures_util::future::ready(f.location > offset))
144 .boxed()
145 } else {
146 self.client.list_with_offset(prefix, offset)
147 }
148 }
149
150 fn delete_stream(
151 &self,
152 locations: BoxStream<'static, Result<Path>>,
153 ) -> BoxStream<'static, Result<Path>> {
154 let client = Arc::clone(&self.client);
155 locations
156 .try_chunks(256)
157 .map(move |locations| {
158 let client = Arc::clone(&client);
159 async move {
160 let locations = locations.map_err(|e| e.1)?;
163 client
164 .bulk_delete_request(locations)
165 .await
166 .map(futures_util::stream::iter)
167 }
168 })
169 .buffered(20)
170 .try_flatten()
171 .boxed()
172 }
173
174 async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result<ListResult> {
175 self.client.list_with_delimiter(prefix).await
176 }
177
178 async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> {
179 let CopyOptions {
180 mode,
181 extensions: _,
182 } = options;
183
184 match mode {
185 CopyMode::Overwrite => self.client.copy_request(from, to, true).await,
186 CopyMode::Create => self.client.copy_request(from, to, false).await,
187 }
188 }
189}
190
191#[async_trait]
192impl Signer for MicrosoftAzure {
193 async fn signed_url(&self, method: Method, path: &Path, expires_in: Duration) -> Result<Url> {
231 if self.client.config().encryption_headers.is_enabled() {
232 return Err(crate::Error::NotSupported {
233 source: "Azure signed URLs cannot be used with customer-provided keys because CPK values must be supplied as request headers".into(),
234 });
235 }
236
237 let crypto = crypto_provider(self.client.crypto())?;
238 let mut url = self.path_url(path);
239 let signer = self.client.signer(expires_in).await?;
240 signer.sign(crypto, &method, &mut url)?;
241 Ok(url)
242 }
243
244 async fn signed_urls(
245 &self,
246 method: Method,
247 paths: &[Path],
248 expires_in: Duration,
249 ) -> Result<Vec<Url>> {
250 if self.client.config().encryption_headers.is_enabled() {
251 return Err(crate::Error::NotSupported {
252 source: "Azure signed URLs cannot be used with customer-provided keys because CPK values must be supplied as request headers".into(),
253 });
254 }
255
256 let crypto = crypto_provider(self.client.crypto())?;
257 let mut urls = Vec::with_capacity(paths.len());
258 let signer = self.client.signer(expires_in).await?;
259 for path in paths {
260 let mut url = self.path_url(path);
261 signer.sign(crypto, &method, &mut url)?;
262 urls.push(url);
263 }
264 Ok(urls)
265 }
266}
267
268#[derive(Debug)]
274struct AzureMultiPartUpload {
275 part_idx: usize,
276 state: Arc<UploadState>,
277 opts: PutMultipartOptions,
278}
279
280#[derive(Debug)]
281struct UploadState {
282 location: Path,
283 parts: Parts,
284 client: Arc<AzureClient>,
285 retry_policy: Option<Arc<dyn RetryPolicy>>,
286}
287
288#[async_trait]
289impl MultipartUpload for AzureMultiPartUpload {
290 fn put_part(&mut self, data: PutPayload) -> UploadPart {
291 let idx = self.part_idx;
292 self.part_idx += 1;
293 let state = Arc::clone(&self.state);
294 let content_id = AzureClient::new_block_id();
295 Box::pin(async move {
296 let mut retry = MultipartRetry::new(state.retry_policy.clone());
297 loop {
298 match state
299 .client
300 .put_block(&state.location, &content_id, data.clone())
301 .await
302 {
303 Ok(part) => {
304 state.parts.put(idx, part);
305 return Ok(());
306 }
307 Err(error) if retry.should_retry(&error).await => continue,
308 Err(error) => return Err(error),
309 }
310 }
311 })
312 }
313
314 async fn complete(&mut self) -> Result<PutResult> {
315 let parts = self.state.parts.finish(self.part_idx)?;
316
317 self.state
318 .client
319 .put_block_list(&self.state.location, parts, std::mem::take(&mut self.opts))
320 .await
321 }
322
323 async fn abort(&mut self) -> Result<()> {
324 Ok(())
326 }
327}
328
329#[async_trait]
330impl MultipartStore for MicrosoftAzure {
331 async fn create_multipart(&self, _: &Path) -> Result<MultipartId> {
385 Ok(String::new())
386 }
387
388 async fn create_multipart_opts(
389 &self,
390 path: &Path,
391 opts: PutMultipartOptions,
392 ) -> Result<MultipartId> {
393 let PutMultipartOptions {
394 tags,
395 attributes,
396 extensions: _,
397 } = opts;
398
399 if !tags.is_empty() || !attributes.is_empty() {
400 return Err(Error::NotSupported {
401 source: "`create_multipart_opts` with non-default options is not supported by MicrosoftAzure".into(),
402 });
403 }
404
405 self.create_multipart(path).await
406 }
407
408 async fn put_part(
409 &self,
410 path: &Path,
411 _: &MultipartId,
412 _: usize,
413 data: PutPayload,
414 ) -> Result<PartId> {
415 let content_id = AzureClient::new_block_id();
416 self.client.put_block(path, &content_id, data).await
417 }
418
419 async fn complete_multipart(
420 &self,
421 path: &Path,
422 _: &MultipartId,
423 parts: Vec<PartId>,
424 ) -> Result<PutResult> {
425 self.client
426 .put_block_list(path, parts, Default::default())
427 .await
428 }
429
430 async fn abort_multipart(&self, _: &Path, _: &MultipartId) -> Result<()> {
431 Ok(())
434 }
435}
436
437#[async_trait]
438impl PaginatedListStore for MicrosoftAzure {
439 async fn list_paginated(
440 &self,
441 prefix: Option<&str>,
442 opts: PaginatedListOptions,
443 ) -> Result<PaginatedListResult> {
444 self.client.list_request(prefix, opts).await
445 }
446}
447
448#[cfg(test)]
449mod tests {
450 use super::*;
451 use crate::ObjectStoreExt;
452 use crate::integration::*;
453 use crate::tests::*;
454 use crate::{Attribute, Attributes};
455 use base64::Engine;
456 use base64::prelude::BASE64_STANDARD;
457 use bytes::Bytes;
458 use std::time::{SystemTime, UNIX_EPOCH};
459
460 #[cfg(feature = "reqwest")]
461 #[tokio::test]
462 async fn azure_create_multipart_opts_rejects_attributes() {
463 let integration = MicrosoftAzureBuilder::new()
464 .with_container_name("test")
465 .with_use_emulator(true)
466 .build()
467 .unwrap();
468 let opts = PutMultipartOptions {
469 attributes: Attributes::from_iter([(Attribute::Metadata("key".into()), "value")]),
470 ..Default::default()
471 };
472
473 let err = integration
474 .create_multipart_opts(&Path::from("test"), opts)
475 .await
476 .unwrap_err();
477
478 assert!(matches!(err, Error::NotSupported { .. }));
479 }
480
481 #[tokio::test]
482 async fn azure_blob_test() {
483 maybe_skip_integration!();
484 let integration = MicrosoftAzureBuilder::from_env()
487 .with_http_connector(MarkerHttpConnector::default())
488 .build()
489 .unwrap();
490
491 put_get_delete_list(&integration).await;
492 list_with_offset_exclusivity(&integration).await;
493 get_opts(&integration).await;
494 list_uses_directories_correctly(&integration).await;
495 list_with_delimiter(&integration).await;
496 rename_and_copy(&integration).await;
497 copy_if_not_exists(&integration).await;
498 stream_get(&integration).await;
499 put_opts(&integration, true).await;
500 multipart(&integration, &integration).await;
501 multipart_put_part_out_of_order(&integration, &integration).await;
502 multipart_race_condition(&integration, false).await;
503 multipart_out_of_order(&integration).await;
504 signing(&integration).await;
505 list_paginated(&integration, &integration).await;
506 response_extensions(&integration, true).await;
507
508 let validate = !integration.client.config().disable_tagging;
509 tagging(
510 Arc::new(MicrosoftAzure {
511 client: Arc::clone(&integration.client),
512 }),
513 validate,
514 |p| {
515 let client = Arc::clone(&integration.client);
516 async move { client.get_blob_tagging(&p).await }
517 },
518 )
519 .await;
520
521 if !integration.client.config().is_emulator {
523 put_get_attributes(&integration).await;
524 }
525 }
526
527 #[ignore = "Used for manual testing against a real Workspace Private Link Endpoint."]
528 #[tokio::test]
529 async fn azure_onelake_wspl_test() {
530 maybe_skip_integration!();
531
532 let url =
533 std::env::var("AZURE_ONELAKE_URL").expect("Set AZURE_ONELAKE_URL to a WS-PL FQDN");
534 let parsed = url::Url::parse(&url).unwrap();
535
536 let path = match parsed.scheme() {
537 "abfss" | "abfs" => {
538 let segments: Vec<&str> = parsed.path_segments().unwrap().collect();
541 Path::from(segments.join("/"))
542 }
543 _ => {
544 let segments: Vec<&str> = parsed.path_segments().unwrap().collect();
547 Path::from(segments[1..].join("/"))
548 }
549 };
550
551 let store = MicrosoftAzureBuilder::new()
552 .with_url(&url)
553 .with_bearer_token_authorization(
554 std::env::var("AZURE_STORAGE_TOKEN").expect("Set AZURE_STORAGE_TOKEN"),
555 )
556 .build()
557 .unwrap();
558
559 let data = Bytes::from("Hello OneLake WSPL");
560
561 store.put(&path, data.clone().into()).await.unwrap();
562 let result = store.get(&path).await.unwrap();
563 let loaded = result.bytes().await.unwrap();
564 assert_eq!(data, loaded);
565 store.delete(&path).await.unwrap();
566 }
567
568 #[ignore = "Used for manual testing against a real storage account."]
570 #[tokio::test]
571 async fn azure_blob_cpk_test() {
572 let base = MicrosoftAzureBuilder::from_env();
573 let key = BASE64_STANDARD.encode([7_u8; 32]);
574 let wrong_key = BASE64_STANDARD.encode([9_u8; 32]);
575
576 let encrypted = base.clone().with_encryption_key(&key).build().unwrap();
577 let unencrypted = base.clone().build().unwrap();
578 let wrong = base.with_encryption_key(&wrong_key).build().unwrap();
579
580 let suffix = SystemTime::now()
581 .duration_since(UNIX_EPOCH)
582 .unwrap()
583 .as_nanos();
584 let path = Path::from(format!("cpk-test-{suffix}.txt"));
585 let copy_path = Path::from(format!("cpk-test-copy-{suffix}.txt"));
586 let payload = Bytes::from("customer-provided-key");
587
588 encrypted.put(&path, payload.clone().into()).await.unwrap();
589
590 let loaded = encrypted.get(&path).await.unwrap().bytes().await.unwrap();
591 assert_eq!(loaded, payload);
592
593 let range = encrypted.get_range(&path, 9..17).await.unwrap();
594 assert_eq!(range, Bytes::from("provided"));
595
596 let meta = encrypted.head(&path).await.unwrap();
597 assert_eq!(meta.size, payload.len() as u64);
598
599 encrypted.copy(&path, ©_path).await.unwrap();
600 let copied = encrypted
601 .get(©_path)
602 .await
603 .unwrap()
604 .bytes()
605 .await
606 .unwrap();
607 assert_eq!(copied, payload);
608
609 assert!(unencrypted.get(&path).await.is_err());
610 assert!(wrong.get(&path).await.is_err());
611 assert!(unencrypted.head(&path).await.is_err());
612 assert!(wrong.get_range(&path, 0..8).await.is_err());
613 assert!(unencrypted.get(©_path).await.is_err());
614 assert!(wrong.get(©_path).await.is_err());
615 let meta = encrypted.head(©_path).await.unwrap();
616 assert_eq!(meta.size, payload.len() as u64);
617
618 encrypted.delete(©_path).await.unwrap();
619 encrypted.delete(&path).await.unwrap();
620 }
621
622 #[tokio::test]
623 async fn azure_signed_url_rejects_cpk_configuration() {
624 let store = MicrosoftAzureBuilder::new()
625 .with_account("testaccount")
626 .with_container_name("testcontainer")
627 .with_access_key("Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==")
628 .with_encryption_key(BASE64_STANDARD.encode([7_u8; 32]))
629 .build()
630 .unwrap();
631
632 let err = store
633 .signed_url(
634 Method::GET,
635 &Path::from("file.txt"),
636 Duration::from_secs(60),
637 )
638 .await
639 .unwrap_err();
640 assert!(matches!(err, crate::Error::NotSupported { .. }));
641 assert!(err.to_string().contains("customer-provided keys"));
642
643 let err = store
644 .signed_urls(
645 Method::GET,
646 &[Path::from("file.txt")],
647 Duration::from_secs(60),
648 )
649 .await
650 .unwrap_err();
651 assert!(matches!(err, crate::Error::NotSupported { .. }));
652 assert!(err.to_string().contains("customer-provided keys"));
653 }
654
655 #[ignore = "Used for manual testing against a real storage account."]
656 #[tokio::test]
657 async fn test_user_delegation_key() {
658 let account = std::env::var("AZURE_ACCOUNT_NAME").unwrap();
659 let container = std::env::var("AZURE_CONTAINER_NAME").unwrap();
660 let client_id = std::env::var("AZURE_CLIENT_ID").unwrap();
661 let client_secret = std::env::var("AZURE_CLIENT_SECRET").unwrap();
662 let tenant_id = std::env::var("AZURE_TENANT_ID").unwrap();
663 let integration = MicrosoftAzureBuilder::new()
664 .with_account(account)
665 .with_container_name(container)
666 .with_client_id(client_id)
667 .with_client_secret(client_secret)
668 .with_tenant_id(&tenant_id)
669 .build()
670 .unwrap();
671
672 let data = Bytes::from("hello world");
673 let path = Path::from("file.txt");
674 integration.put(&path, data.clone().into()).await.unwrap();
675
676 let signed = integration
677 .signed_url(Method::GET, &path, Duration::from_secs(60))
678 .await
679 .unwrap();
680
681 let resp = reqwest::get(signed).await.unwrap();
682 let loaded = resp.bytes().await.unwrap();
683
684 assert_eq!(data, loaded);
685 }
686
687 #[test]
688 fn azure_test_config_get_value() {
689 let azure_client_id = "object_store:fake_access_key_id".to_string();
690 let azure_storage_account_name = "object_store:fake_secret_key".to_string();
691 let azure_storage_token = "object_store:fake_default_region".to_string();
692 let builder = MicrosoftAzureBuilder::new()
693 .with_config(AzureConfigKey::ClientId, &azure_client_id)
694 .with_config(AzureConfigKey::AccountName, &azure_storage_account_name)
695 .with_config(AzureConfigKey::Token, &azure_storage_token);
696
697 assert_eq!(
698 builder.get_config_value(&AzureConfigKey::ClientId).unwrap(),
699 azure_client_id
700 );
701 assert_eq!(
702 builder
703 .get_config_value(&AzureConfigKey::AccountName)
704 .unwrap(),
705 azure_storage_account_name
706 );
707 assert_eq!(
708 builder.get_config_value(&AzureConfigKey::Token).unwrap(),
709 azure_storage_token
710 );
711 }
712}