#[path = "local_server.rs"]
mod local_server;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, Ordering};
use std::time::Duration;
use bytes::Bytes;
use dataloader_rs::dataset::{Column, HttpDataset, Retry};
use dataloader_rs::{DataLoader, error::Result};
const TOKEN: &str = "Bearer example-token";
fn main() -> Result<()> {
let item_5_requests = Arc::new(AtomicU32::new(0));
let counter = Arc::clone(&item_5_requests);
let addr = local_server::start(move |request| {
if request.header("Authorization") != Some(TOKEN) {
return (401, b"missing token".to_vec());
}
let parts: Vec<&str> = request.path.split('/').collect();
match parts.as_slice() {
["", "shards", shard, "items", file] => {
let id = file.trim_end_matches(".txt");
if id == "5" && counter.fetch_add(1, Ordering::SeqCst) < 2 {
return (503, b"busy, try again".to_vec());
}
(200, format!("item {id} of shard {shard}").into_bytes())
}
_ => (404, b"not found".to_vec()),
}
});
let shards = Column::from(
vec!["shard a"; 4]
.into_iter()
.chain(vec!["shard b"; 4])
.collect::<Vec<_>>(),
);
let ids = Column::from((0..8).collect::<Vec<u64>>());
let dataset = HttpDataset::builder()
.header("Authorization", TOKEN)
.timeout(Duration::from_secs(5)) .retry(Retry {
max_retries: 3, initial_backoff: Duration::from_millis(10), max_backoff: Duration::from_millis(100),
})
.url_template(
&format!("http://{addr}/shards/{{shard}}/items/{{id}}.txt"),
[("shard", shards), ("id", ids)],
)?;
let mut loader = DataLoader::builder_async(dataset)
.batch_size(4)
.num_workers(2)
.item_concurrency(4)
.transform(|_index: usize, body: Bytes| -> Result<String> {
Ok(String::from_utf8(body.to_vec())?)
})
.build();
for batch in &mut loader {
println!("{:?}", batch?);
}
println!(
"item 5 took {} requests: two 503 answers, then a success",
item_5_requests.load(Ordering::SeqCst)
);
let urls = [
format!("http://{addr}/no/such/item"),
format!("http://{addr}/shards/c/items/0.txt"),
];
let dataset = HttpDataset::builder()
.header("Authorization", TOKEN)
.urls(&urls)?;
let mut loader = DataLoader::builder_async(dataset).batch_size(1).build();
for (url, batch) in urls.iter().zip(&mut loader) {
match batch {
Ok(items) => println!("{url}: {} bytes", items[0].len()),
Err(error) => println!("{url}: error: {error}"),
}
}
Ok(())
}