rskit-dataset 0.2.0-alpha.3

Dataset collection framework: source, transform, target, collector
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

//! Collector configuration.

use std::path::PathBuf;

use crate::DatasetLimits;

/// Configuration for the collector.
#[derive(Debug, Clone)]
pub struct CollectorConfig {
    /// Directory where dataset output and manifest files are written.
    pub output_dir: PathBuf,
    /// Maximum number of sources processed concurrently; values below 1 run a single worker.
    pub concurrency: usize,
    /// Per-source timeout in seconds. Non-positive means no timeout.
    pub source_timeout_secs: f64,
    /// Ignore manifest cache and rebuild from sources.
    pub force: bool,
    /// Dataset streaming and materialization limits.
    pub limits: DatasetLimits,
}

impl Default for CollectorConfig {
    fn default() -> Self {
        Self {
            output_dir: PathBuf::from("dataset_build"),
            concurrency: 4,
            source_timeout_secs: 600.0,
            force: false,
            limits: DatasetLimits::default(),
        }
    }
}