use std::sync::Arc;
use std::time::Duration;
use bytes::Bytes;
use dataloader_rs::dataset::{Column, FileDataset, ObjectStoreDataset, Retry};
use dataloader_rs::{DataLoader, error::Result};
use object_store::memory::InMemory;
use object_store::path::Path;
use object_store::{ObjectStoreExt, PutPayload};
fn to_text(_index: usize, bytes: Bytes) -> Result<String> {
Ok(String::from_utf8(bytes.to_vec())?)
}
fn main() -> Result<()> {
files()?;
objects()
}
fn files() -> Result<()> {
let root = std::env::temp_dir().join(format!("dataloader-rs-example-{}", std::process::id()));
for (index, folder) in ["cats", "cats", "cats", "dogs", "dogs", "dogs"]
.iter()
.enumerate()
{
std::fs::create_dir_all(root.join(folder))?;
std::fs::write(
root.join(folder).join(format!("{index}.txt")),
format!("{folder} #{index}"),
)?;
}
let root_text = root
.display()
.to_string()
.replace('{', "{{")
.replace('}', "}}");
let folders = Column::from(vec!["cats", "cats", "cats", "dogs", "dogs", "dogs"]);
let dataset = FileDataset::builder()
.timeout(Duration::from_secs(10))
.retry(Retry {
max_retries: 2,
..Retry::default()
})
.path_template(
&format!("{root_text}/{{folder}}/{{index}}.txt"),
[("folder", folders)],
)?;
let mut loader = DataLoader::builder_async(dataset)
.batch_size(3)
.item_concurrency(3) .transform(to_text)
.build();
for batch in &mut loader {
println!("files: {:?}", batch?);
}
std::fs::remove_dir_all(&root)?;
Ok(())
}
fn objects() -> Result<()> {
let store = Arc::new(InMemory::new());
tokio::runtime::Builder::new_current_thread()
.build()?
.block_on(async {
for i in 0..4 {
let payload = PutPayload::from(format!("object number {i}").into_bytes());
store
.put(&Path::from(format!("tiles/{i}.txt")), payload)
.await?;
}
let row = PutPayload::from(b"AAAABBBBCCCCDDDD".to_vec());
store.put(&Path::from("rows/packed.bin"), row).await?;
Ok::<_, object_store::Error>(())
})?;
let dataset =
ObjectStoreDataset::builder(store.clone()).key_template_len(4, "tiles/{index}.txt", [])?;
let mut loader = DataLoader::builder_async(dataset)
.batch_size(2)
.num_workers(2)
.transform(to_text)
.build();
for batch in &mut loader {
println!("objects: {:?}", batch?);
}
let ranges = (0..4u64).map(|i| ("rows/packed.bin", i * 4..i * 4 + 4));
let dataset = ObjectStoreDataset::builder(store).ranges(ranges)?;
let mut loader = DataLoader::builder_async(dataset)
.batch_size(4)
.item_concurrency(4)
.transform(to_text)
.build();
for batch in &mut loader {
println!("ranges: {:?}", batch?);
}
Ok(())
}