floe-core 0.3.6

Core library for Floe, a YAML-driven technical ingestion tool.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

use crate::io::format::AcceptedWriteMetrics;

pub const BYTES_PER_MIB: f64 = 1024.0 * 1024.0;
pub const DEFAULT_SMALL_FILE_THRESHOLD_BYTES: u64 = 16 * 1024 * 1024;

pub fn default_small_file_threshold_bytes(target_file_size_bytes: Option<u64>) -> u64 {
    match target_file_size_bytes {
        Some(target) if target > 1 => std::cmp::max(1, target / 2),
        Some(_) => 1,
        None => DEFAULT_SMALL_FILE_THRESHOLD_BYTES,
    }
}

pub fn null_accepted_write_metrics() -> AcceptedWriteMetrics {
    AcceptedWriteMetrics {
        total_bytes_written: None,
        avg_file_size_mb: None,
        small_files_count: None,
    }
}

pub fn summarize_written_file_sizes(
    file_sizes: &[u64],
    files_written: u64,
    small_file_threshold_bytes: u64,
) -> AcceptedWriteMetrics {
    if files_written == 0 {
        return AcceptedWriteMetrics {
            total_bytes_written: Some(0),
            avg_file_size_mb: None,
            small_files_count: Some(0),
        };
    }

    if file_sizes.len() != files_written as usize {
        return null_accepted_write_metrics();
    }

    let total_bytes_written = file_sizes.iter().copied().sum::<u64>();
    let avg_file_size_mb = (total_bytes_written as f64 / files_written as f64) / BYTES_PER_MIB;
    let small_files_count = file_sizes
        .iter()
        .filter(|size| **size < small_file_threshold_bytes)
        .count() as u64;

    AcceptedWriteMetrics {
        total_bytes_written: Some(total_bytes_written),
        avg_file_size_mb: Some(avg_file_size_mb),
        small_files_count: Some(small_files_count),
    }
}