cloud-file 0.2.0

Simple reading of cloud files in Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

// For comparison, here is a version of line_count.rs that uses the object_store directly
// rather than the cloud-file wrapper crate.
use futures_util::StreamExt; // Enables `.next()` on streams.
pub use object_store::path::Path as StorePath;
use object_store::{parse_url_opts, ObjectStore};
use std::sync::Arc;
use url::Url;

async fn count_lines(
    object_store: &Arc<Box<dyn ObjectStore>>,
    store_path: StorePath,
) -> Result<usize, anyhow::Error> {
    let mut chunks = object_store.get(&store_path).await?.into_stream();
    let mut newline_count: usize = 0;
    while let Some(chunk) = chunks.next().await {
        let chunk = chunk?;
        newline_count += bytecount::count(&chunk, b'\n');
    }
    Ok(newline_count)
}

#[tokio::main]
async fn main() -> Result<(), anyhow::Error> {
    let url = "https://raw.githubusercontent.com/fastlmm/bed-sample-files/main/toydata.5chrom.fam";
    let options = [("timeout", "10s")];

    let url = Url::parse(url)?;
    let (object_store, store_path) = parse_url_opts(&url, options)?;
    let object_store = Arc::new(object_store); // enables cloning and borrowing
    let line_count = count_lines(&object_store, store_path).await?;
    println!("line_count: {line_count}");
    Ok(())
}