use std::collections::BTreeMap;
use crate::batch_plan::constants::{BATCH_BYTES, MAX_BATCH_FILES};
use crate::batch_plan::models::FileGroup;
use crate::files_source::models::DataFile;
pub fn file_groups(files: &[DataFile], by_value: Option<&str>) -> Vec<FileGroup> {
let mut partitions: Vec<(Vec<String>, Vec<usize>)> = Vec::new();
let mut by_key: BTreeMap<Vec<String>, usize> = BTreeMap::new();
for (index, data) in files.iter().enumerate() {
let key = partition_key(data, by_value);
match by_key.get(&key) {
Some(position) => partitions[*position].1.push(index),
None => {
by_key.insert(key.clone(), partitions.len());
partitions.push((key, vec![index]));
}
}
}
let size = |index: &usize| files.get(*index).map_or(0, |data| data.file.size as usize);
let mut groups: Vec<FileGroup> = Vec::new();
let mut current = FileGroup {
files: Vec::new(),
values: Vec::new(),
};
let mut bytes: usize = 0;
for (key, indexes) in partitions {
let whole = by_value.is_some();
let pieces: Vec<Vec<usize>> = match whole {
true => vec![indexes],
false => split_partition(&indexes, &size),
};
for piece in pieces {
let piece_bytes: usize = piece.iter().map(size).sum();
let over = current.files.len() + piece.len() > MAX_BATCH_FILES
|| bytes + piece_bytes > BATCH_BYTES;
if !current.files.is_empty() && over {
groups.push(std::mem::replace(
&mut current,
FileGroup {
files: Vec::new(),
values: Vec::new(),
},
));
bytes = 0;
}
current.files.extend(piece);
bytes += piece_bytes;
}
if whole {
current.values.extend(key);
}
}
if !current.files.is_empty() {
groups.push(current);
}
groups
}
fn partition_key(data: &DataFile, by_value: Option<&str>) -> Vec<String> {
match by_value {
Some(column) => data
.partitions
.iter()
.filter(|(name, _)| name == column)
.map(|(_, value)| value.clone())
.collect(),
None => data
.partitions
.iter()
.map(|(name, value)| format!("{name}={value}"))
.collect(),
}
}
fn split_partition(indexes: &[usize], size: &impl Fn(&usize) -> usize) -> Vec<Vec<usize>> {
let mut pieces: Vec<Vec<usize>> = Vec::new();
let mut piece: Vec<usize> = Vec::new();
let mut bytes: usize = 0;
for index in indexes {
let file_bytes = size(index);
if !piece.is_empty() && (piece.len() == MAX_BATCH_FILES || bytes + file_bytes > BATCH_BYTES)
{
pieces.push(std::mem::take(&mut piece));
bytes = 0;
}
piece.push(*index);
bytes += file_bytes;
}
if !piece.is_empty() {
pieces.push(piece);
}
pieces
}