burn_p2p_dataloader 0.21.8

Dataset planning and lease-assignment boundary crate for burn_p2p.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

use burn_p2p_core::{DatasetManifest, DatasetView};
use serde::{Deserialize, Serialize};

#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
/// Enumerates the supported upstream adapter values.
pub enum UpstreamAdapter {
    /// Uses the hf variant.
    Hf {
        /// The dataset.
        dataset: String,
        /// The config.
        config: Option<String>,
        /// The split.
        split: String,
        /// The streaming.
        streaming: bool,
        /// The num shards.
        num_shards: Option<u32>,
    },
    /// Uses the HTTP variant.
    Http {
        /// The base URL.
        base_url: String,
    },
    /// Uses the local variant.
    Local {
        /// The root.
        root: String,
    },
    /// Uses the s3 variant.
    S3 {
        /// The bucket.
        bucket: String,
        /// The prefix.
        prefix: String,
    },
    /// Exposes the private setting.
    Private {
        /// The scheme.
        scheme: String,
        /// The locator.
        locator: String,
    },
}

#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
/// Represents a dataset registration.
pub struct DatasetRegistration {
    /// The manifest.
    pub manifest: DatasetManifest,
    /// The view.
    pub view: DatasetView,
    /// The upstream.
    pub upstream: UpstreamAdapter,
}

#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
/// Represents a dataset sizing.
pub struct DatasetSizing {
    /// The total examples.
    pub total_examples: u64,
    /// The total tokens.
    pub total_tokens: u64,
    /// The total bytes.
    pub total_bytes: u64,
}