rusty_s3/actions/
list_objects_v2.rs

1use std::borrow::Cow;
2use std::time::Duration;
3
4use instant_xml::FromXml;
5use jiff::Timestamp;
6use percent_encoding::percent_decode_str;
7use url::Url;
8
9use crate::actions::{Method, S3_XML_NS, S3Action};
10use crate::signing::sign;
11use crate::{Bucket, Credentials, Map};
12
13/// List all objects in the bucket.
14///
15/// If `next_continuation_token` is `Some` the response is truncated, and the
16/// rest of the list can be retrieved by reusing the `ListObjectV2` action
17/// but with `continuation-token` set to the value of `next_continuation_token`
18/// received in the previous response.
19///
20/// Find out more about `ListObjectsV2` from the [AWS API Reference][api]
21///
22/// [api]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
23#[allow(clippy::module_name_repetitions)]
24#[derive(Debug, Clone)]
25pub struct ListObjectsV2<'a> {
26    bucket: &'a Bucket,
27    credentials: Option<&'a Credentials>,
28
29    query: Map<'a>,
30    headers: Map<'a>,
31}
32
33#[allow(clippy::module_name_repetitions)]
34#[derive(Debug, Clone, FromXml)]
35#[xml(rename = "ListBucketResult", ns(S3_XML_NS))]
36pub struct ListObjectsV2Response {
37    pub contents: Vec<ListObjectsContent>,
38
39    #[xml(rename = "MaxKeys")]
40    pub max_keys: Option<u16>,
41    pub common_prefixes: Vec<CommonPrefixes>,
42    #[xml(rename = "NextContinuationToken")]
43    pub next_continuation_token: Option<String>,
44    #[xml(rename = "StartAfter")]
45    pub start_after: Option<String>,
46    #[xml(rename = "EncodingType")]
47    pub encoding_type: Option<String>,
48}
49
50#[derive(Debug, Clone, FromXml)]
51#[xml(rename = "Contents", ns(S3_XML_NS))]
52pub struct ListObjectsContent {
53    #[xml(rename = "ETag")]
54    pub etag: String,
55    #[xml(rename = "Key")]
56    pub key: String,
57    #[xml(rename = "LastModified")]
58    pub last_modified: String,
59    pub owner: Option<ListObjectsOwner>,
60    #[xml(rename = "Size")]
61    pub size: u64,
62    #[xml(rename = "StorageClass")]
63    pub storage_class: Option<String>,
64}
65
66#[derive(Debug, Clone, FromXml)]
67#[xml(rename = "Owner", ns(S3_XML_NS))]
68pub struct ListObjectsOwner {
69    #[xml(rename = "ID")]
70    pub id: String,
71    #[xml(rename = "DisplayName")]
72    pub display_name: Option<String>,
73}
74
75#[derive(Debug, Clone, FromXml)]
76#[xml(rename = "CommonPrefixes", ns(S3_XML_NS))]
77pub struct CommonPrefixes {
78    #[xml(rename = "Prefix")]
79    pub prefix: String,
80}
81
82impl<'a> ListObjectsV2<'a> {
83    #[must_use]
84    pub fn new(bucket: &'a Bucket, credentials: Option<&'a Credentials>) -> Self {
85        let mut query = Map::new();
86        query.insert("list-type", "2");
87        query.insert("encoding-type", "url");
88
89        Self {
90            bucket,
91            credentials,
92
93            query,
94            headers: Map::new(),
95        }
96    }
97
98    /// Limits the response to keys that begin with the specified prefix.
99    ///
100    /// See <https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html#API_ListObjectsV2_RequestSyntax> for more infos.
101    /// # Example
102    /// ```
103    /// # let bucket = rusty_s3::Bucket::new(url::Url::parse("http://rusty_s3/").unwrap(), rusty_s3::UrlStyle::Path, "doggo", "doggoland").unwrap();
104    /// let mut list = bucket.list_objects_v2(None);
105    /// list.with_prefix("tamo");
106    /// ```
107    pub fn with_prefix(&mut self, prefix: impl Into<Cow<'a, str>>) {
108        self.query_mut().insert("prefix", prefix);
109    }
110
111    /// A delimiter is a character that you use to group keys.
112    ///
113    /// See <https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html#API_ListObjectsV2_RequestSyntax> for more infos.
114    /// # Example
115    /// ```
116    /// # let bucket = rusty_s3::Bucket::new(url::Url::parse("http://rusty_s3/").unwrap(), rusty_s3::UrlStyle::Path, "doggo", "doggoland").unwrap();
117    /// let mut list = bucket.list_objects_v2(None);
118    /// list.with_delimiter("/");
119    /// ```
120    pub fn with_delimiter(&mut self, delimiter: impl Into<Cow<'a, str>>) {
121        self.query_mut().insert("delimiter", delimiter);
122    }
123
124    /// `StartAfter` is where you want Amazon S3 to start listing from.
125    /// Amazon S3 starts listing after this specified key.
126    /// `StartAfter` can be any key in the bucket.
127    ///
128    /// See <https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html#API_ListObjectsV2_RequestSyntax> for more infos.
129    /// # Example
130    /// ```
131    /// # let bucket = rusty_s3::Bucket::new(url::Url::parse("http://rusty_s3/").unwrap(), rusty_s3::UrlStyle::Path, "doggo", "doggoland").unwrap();
132    /// let mut list = bucket.list_objects_v2(None);
133    /// list.with_start_after("tamo"); // <- This token should come from a previous call to the list API.
134    /// ```
135    pub fn with_start_after(&mut self, start_after: impl Into<Cow<'a, str>>) {
136        self.query_mut().insert("start-after", start_after);
137    }
138
139    /// `ContinuationToken` indicates to Amazon S3 that the list is being continued on this bucket with a token.
140    /// `ContinuationToken` is obfuscated and is not a real key.
141    ///
142    /// See <https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html#API_ListObjectsV2_RequestSyntax> for more infos.
143    /// # Example
144    /// ```
145    /// # let bucket = rusty_s3::Bucket::new(url::Url::parse("http://rusty_s3/").unwrap(), rusty_s3::UrlStyle::Path, "doggo", "doggoland").unwrap();
146    /// let mut list = bucket.list_objects_v2(None);
147    /// list.with_continuation_token("tamo"); // <- This token should come from a previous call to the list API.
148    /// ```
149    pub fn with_continuation_token(&mut self, continuation_token: impl Into<Cow<'a, str>>) {
150        self.query_mut()
151            .insert("continuation-token", continuation_token);
152    }
153
154    /// Sets the maximum number of keys returned in the response.
155    /// By default, the action returns up to 1,000 key names.
156    /// The response might contain fewer keys but will never contain more.
157    ///
158    /// See <https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html#API_ListObjectsV2_RequestSyntax> for more infos.
159    /// # Example
160    /// ```
161    /// # let bucket = rusty_s3::Bucket::new(url::Url::parse("http://rusty_s3/").unwrap(), rusty_s3::UrlStyle::Path, "doggo", "doggoland").unwrap();
162    /// let mut list = bucket.list_objects_v2(None);
163    /// list.with_continuation_token("tamo"); // <- This token should come from a previous call to the list API.
164    /// ```
165    pub fn with_max_keys(&mut self, max_keys: usize) {
166        self.query_mut().insert("max-keys", max_keys.to_string());
167    }
168
169    /// Parse the XML response from S3 into a struct.
170    ///
171    /// # Errors
172    ///
173    /// Returns an error if the XML response could not be parsed.
174    pub fn parse_response(s: &str) -> Result<ListObjectsV2Response, instant_xml::Error> {
175        let mut parsed: ListObjectsV2Response = instant_xml::from_str(s)?;
176
177        // When `encoding-type=url` is requested (which `new` always does), S3
178        // percent-encodes the key-related fields in the response. Some
179        // implementations (e.g. DigitalOcean Spaces) encode characters such as
180        // `/` and `%`, so the values must be decoded to recover the real keys.
181        let url_encoded = parsed.encoding_type.as_deref() == Some("url");
182
183        // S3 returns an Owner with an empty DisplayName and ID when fetch-owner is disabled
184        for content in &mut parsed.contents {
185            if let Some(owner) = &content.owner {
186                if owner.id.is_empty() && owner.display_name.as_deref().is_none_or(str::is_empty) {
187                    content.owner = None;
188                }
189            }
190
191            if url_encoded {
192                percent_decode_in_place(&mut content.key);
193            }
194        }
195
196        if url_encoded {
197            for prefix in &mut parsed.common_prefixes {
198                percent_decode_in_place(&mut prefix.prefix);
199            }
200            if let Some(start_after) = &mut parsed.start_after {
201                percent_decode_in_place(start_after);
202            }
203        }
204
205        Ok(parsed)
206    }
207}
208
209/// Percent-decode `value` in place, leaving it untouched if it is not valid
210/// percent-encoded UTF-8.
211fn percent_decode_in_place(value: &mut String) {
212    if let Ok(Cow::Owned(decoded)) = percent_decode_str(value).decode_utf8() {
213        *value = decoded;
214    }
215}
216
217impl<'a> S3Action<'a> for ListObjectsV2<'a> {
218    const METHOD: Method = Method::Get;
219
220    fn query_mut(&mut self) -> &mut Map<'a> {
221        &mut self.query
222    }
223
224    fn headers_mut(&mut self) -> &mut Map<'a> {
225        &mut self.headers
226    }
227
228    fn sign_with_time(&self, expires_in: Duration, time: &Timestamp) -> Url {
229        let url = self.bucket.base_url().clone();
230
231        match self.credentials {
232            Some(credentials) => sign(
233                time,
234                Self::METHOD,
235                url,
236                credentials.key(),
237                credentials.secret(),
238                credentials.token(),
239                self.bucket.region(),
240                expires_in.as_secs(),
241                self.query.iter(),
242                self.headers.iter(),
243            ),
244            None => crate::signing::util::add_query_params(url, self.query.iter()),
245        }
246    }
247}
248
249#[cfg(test)]
250mod tests {
251    use pretty_assertions::assert_eq;
252
253    use super::*;
254    use crate::{Bucket, Credentials, UrlStyle};
255
256    #[test]
257    fn aws_example() {
258        // Fri, 24 May 2013 00:00:00 GMT
259        let date = Timestamp::from_second(1369353600).unwrap();
260        let expires_in = Duration::from_secs(86400);
261
262        let endpoint = "https://s3.amazonaws.com".parse().unwrap();
263        let bucket = Bucket::new(
264            endpoint,
265            UrlStyle::VirtualHost,
266            "examplebucket",
267            "us-east-1",
268        )
269        .unwrap();
270        let credentials = Credentials::new(
271            "AKIAIOSFODNN7EXAMPLE",
272            "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
273        );
274
275        let action = ListObjectsV2::new(&bucket, Some(&credentials));
276
277        let url = action.sign_with_time(expires_in, &date);
278        let expected = "https://examplebucket.s3.amazonaws.com/?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20130524T000000Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&encoding-type=url&list-type=2&X-Amz-Signature=58e7f65928710f045f6a7e1f7a32b3426b4895900fad799db66faa3ff8b18bd5";
279
280        assert_eq!(expected, url.as_str());
281    }
282
283    #[test]
284    fn anonymous_custom_query() {
285        let expires_in = Duration::from_secs(86400);
286
287        let endpoint = "https://s3.amazonaws.com".parse().unwrap();
288        let bucket = Bucket::new(
289            endpoint,
290            UrlStyle::VirtualHost,
291            "examplebucket",
292            "us-east-1",
293        )
294        .unwrap();
295
296        let mut action = ListObjectsV2::new(&bucket, None);
297        action.query_mut().insert("continuation-token", "duck");
298
299        let url = action.sign(expires_in);
300        let expected = "https://examplebucket.s3.amazonaws.com/?continuation-token=duck&encoding-type=url&list-type=2";
301
302        assert_eq!(expected, url.as_str());
303    }
304
305    #[test]
306    fn anonymous_prefix_with_space() {
307        let expires_in = Duration::from_secs(86400);
308
309        let endpoint = "https://s3.amazonaws.com".parse().unwrap();
310        let bucket = Bucket::new(
311            endpoint,
312            UrlStyle::VirtualHost,
313            "examplebucket",
314            "us-east-1",
315        )
316        .unwrap();
317
318        let mut action = ListObjectsV2::new(&bucket, None);
319        action.with_prefix("my folder/");
320
321        let url = action.sign(expires_in);
322        let expected = "https://examplebucket.s3.amazonaws.com/?encoding-type=url&list-type=2&prefix=my%20folder%2F";
323
324        // The unsigned URL must percent-encode the space as `%20`, never `+`, so
325        // S3 interprets the prefix value correctly.
326        assert_eq!(expected, url.as_str());
327    }
328
329    #[test]
330    fn parse() {
331        let input = r#"<?xml version="1.0" encoding="UTF-8"?>
332        <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
333            <Name>test</Name>
334            <Prefix></Prefix>
335            <KeyCount>3</KeyCount>
336            <MaxKeys>4500</MaxKeys>
337            <Delimiter></Delimiter>
338            <IsTruncated>false</IsTruncated>
339            <Contents>
340                <Key>duck.jpg</Key>
341                <LastModified>2020-12-01T20:43:11.794Z</LastModified>
342                <ETag>"bfd537a51d15208163231b0711e0b1f3"</ETag>
343                <Size>4274</Size>
344                <Owner>
345                    <ID></ID>
346                    <DisplayName></DisplayName>
347                </Owner>
348                <StorageClass>STANDARD</StorageClass>
349            </Contents>
350            <Contents>
351                <Key>idk.txt</Key>
352                <LastModified>2020-12-05T08:23:52.215Z</LastModified>
353                <ETag>"5927c5d64d94a5786f90003aa26d0159-1"</ETag>
354                <Size>9</Size>
355                <Owner>
356                    <ID></ID>
357                    <DisplayName></DisplayName>
358                </Owner>
359                <StorageClass>STANDARD</StorageClass>
360            </Contents>
361            <Contents>
362                <Key>img.jpg</Key>
363                <LastModified>2020-11-26T20:21:35.858Z</LastModified>
364                <ETag>"f7dbec93a0932ccb4d0f4e512eb1a443"</ETag>
365                <Size>41259</Size>
366                <Owner>
367                    <ID></ID>
368                    <DisplayName></DisplayName>
369                </Owner>
370                <StorageClass>STANDARD</StorageClass>
371            </Contents>
372            <EncodingType>url</EncodingType>
373        </ListBucketResult>
374        "#;
375
376        let parsed = ListObjectsV2::parse_response(input).unwrap();
377        assert_eq!(parsed.contents.len(), 3);
378
379        let item_1 = &parsed.contents[0];
380        assert_eq!(item_1.etag, "\"bfd537a51d15208163231b0711e0b1f3\"");
381        assert_eq!(item_1.key, "duck.jpg");
382        assert_eq!(item_1.last_modified, "2020-12-01T20:43:11.794Z");
383        assert!(item_1.owner.is_none());
384        assert_eq!(item_1.size, 4274);
385        assert_eq!(item_1.storage_class, Some("STANDARD".to_string()));
386
387        let item_2 = &parsed.contents[1];
388        assert_eq!(item_2.etag, "\"5927c5d64d94a5786f90003aa26d0159-1\"");
389        assert_eq!(item_2.key, "idk.txt");
390        assert_eq!(item_2.last_modified, "2020-12-05T08:23:52.215Z");
391        assert!(item_2.owner.is_none());
392        assert_eq!(item_2.size, 9);
393        assert_eq!(item_2.storage_class, Some("STANDARD".to_string()));
394
395        let item_3 = &parsed.contents[2];
396        assert_eq!(item_3.etag, "\"f7dbec93a0932ccb4d0f4e512eb1a443\"");
397        assert_eq!(item_3.key, "img.jpg");
398        assert_eq!(item_3.last_modified, "2020-11-26T20:21:35.858Z");
399        assert!(item_3.owner.is_none());
400        assert_eq!(item_3.size, 41259);
401        assert_eq!(item_3.storage_class, Some("STANDARD".to_string()));
402
403        assert_eq!(parsed.max_keys, Some(4500));
404        assert!(parsed.common_prefixes.is_empty());
405        assert!(parsed.next_continuation_token.is_none());
406        assert!(parsed.start_after.is_none());
407    }
408
409    #[test]
410    fn parse_no_contents() {
411        let input = r#"<?xml version="1.0" encoding="UTF-8"?>
412        <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
413            <Name>test</Name>
414            <Prefix></Prefix>
415            <KeyCount>0</KeyCount>
416            <MaxKeys>4500</MaxKeys>
417            <Delimiter></Delimiter>
418            <IsTruncated>false</IsTruncated>
419            <EncodingType>url</EncodingType>
420        </ListBucketResult>
421        "#;
422
423        let parsed = ListObjectsV2::parse_response(input).unwrap();
424        assert_eq!(parsed.contents.is_empty(), true);
425
426        assert_eq!(parsed.max_keys, Some(4500));
427        assert!(parsed.common_prefixes.is_empty());
428        assert!(parsed.next_continuation_token.is_none());
429        assert!(parsed.start_after.is_none());
430    }
431
432    #[test]
433    fn parse_url_encoded() {
434        let input = r#"<?xml version="1.0" encoding="UTF-8"?>
435        <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
436            <Name>test</Name>
437            <Prefix></Prefix>
438            <KeyCount>1</KeyCount>
439            <MaxKeys>4500</MaxKeys>
440            <Delimiter>/</Delimiter>
441            <IsTruncated>false</IsTruncated>
442            <Contents>
443                <Key>100%25tamo%2Fduck.jpg</Key>
444                <LastModified>2020-12-01T20:43:11.794Z</LastModified>
445                <ETag>"bfd537a51d15208163231b0711e0b1f3"</ETag>
446                <Size>4274</Size>
447                <StorageClass>STANDARD</StorageClass>
448            </Contents>
449            <CommonPrefixes>
450                <Prefix>my%20folder%2F</Prefix>
451            </CommonPrefixes>
452            <StartAfter>start%2Fafter</StartAfter>
453            <EncodingType>url</EncodingType>
454        </ListBucketResult>
455        "#;
456
457        let parsed = ListObjectsV2::parse_response(input).unwrap();
458        assert_eq!(parsed.contents.len(), 1);
459        assert_eq!(parsed.contents[0].key, "100%tamo/duck.jpg");
460
461        assert_eq!(parsed.common_prefixes.len(), 1);
462        assert_eq!(parsed.common_prefixes[0].prefix, "my folder/");
463
464        assert_eq!(parsed.start_after.as_deref(), Some("start/after"));
465        assert_eq!(parsed.encoding_type.as_deref(), Some("url"));
466    }
467
468    #[test]
469    fn parse_not_url_encoded_is_left_untouched() {
470        let input = r#"<?xml version="1.0" encoding="UTF-8"?>
471        <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
472            <Name>test</Name>
473            <Prefix></Prefix>
474            <KeyCount>1</KeyCount>
475            <MaxKeys>4500</MaxKeys>
476            <Delimiter></Delimiter>
477            <IsTruncated>false</IsTruncated>
478            <Contents>
479                <Key>100%25tamo.jpg</Key>
480                <LastModified>2020-12-01T20:43:11.794Z</LastModified>
481                <ETag>"bfd537a51d15208163231b0711e0b1f3"</ETag>
482                <Size>4274</Size>
483                <StorageClass>STANDARD</StorageClass>
484            </Contents>
485        </ListBucketResult>
486        "#;
487
488        let parsed = ListObjectsV2::parse_response(input).unwrap();
489        assert_eq!(parsed.contents.len(), 1);
490        // No `EncodingType` in the response, so the key is left as-is.
491        assert_eq!(parsed.contents[0].key, "100%25tamo.jpg");
492        assert!(parsed.encoding_type.is_none());
493    }
494}
rusty_s3/actions/list_objects_v2.rs

rusty_s3/actions/
list_objects_v2.rs
