silvic_core/data/
mod.rs

1//! Binned training data: every feature value replaced by a bin index (a byte), features packed into groups whose
2//! bins fit one GPU threadgroup's shared histogram.
3//!
4//! Bin mappers follow LightGBM 3.3.5 `BinMapper` semantics where they reach the model file: a numerical value goes
5//! to the first bin whose upper bound is `>= value`, and a split "bins `<= t` go left" is written with threshold
6//! `upper_bounds[t]`, so `value <= threshold` at prediction time agrees with training for every value. NaN has its
7//! own bin when the training sample had NaN (`MissingType::NaN`); otherwise NaN bins as 0.0, as LightGBM predicts
8//! with `MissingType::None`. Categorical bin 0 collects negative, NaN, unseen and dropped categories, which LightGBM
9//! sends right.
10
11mod bin_mapper;
12mod binning;
13mod build;
14#[cfg(test)]
15mod parity_tests;
16mod sample;
17
18pub use bin_mapper::BinMapper;
19pub use build::{DatasetInput, DenseData, DenseMatrix, Layout};
20
21/// Bins one feature group may hold: a packed i32 per bin fills 32 KiB, Metal's threadgroup memory limit.
22pub const GROUP_BIN_BUDGET: u32 = 8192;
23
24#[derive(Debug, Clone, PartialEq)]
25pub struct FeatureMeta {
26    pub name: String,
27    pub bins: BinMapper,
28    /// Bins in the histogram, the NaN bin included; 0 for a trivial feature.
29    pub num_bins: u32,
30    /// The feature's group; meaningless for a trivial feature.
31    pub group: u32,
32    /// Byte offset of the feature in its group's row.
33    pub column: u32,
34    /// Index of the feature's bin 0 in the flat histogram.
35    pub hist_offset: u32,
36}
37
38impl FeatureMeta {
39    pub fn is_trivial(&self) -> bool {
40        self.num_bins == 0
41    }
42
43    pub fn is_categorical(&self) -> bool {
44        matches!(self.bins, BinMapper::Categorical { .. })
45    }
46}
47
48#[derive(Debug, Clone, PartialEq)]
49pub struct FeatureGroup {
50    /// Indices into `BinnedDataset::features`, in column order.
51    pub features: Vec<usize>,
52    /// Bytes per row: the feature count rounded up to a multiple of 4, padding bytes zero.
53    pub row_stride: u32,
54    /// Index of the group's first bin in the flat histogram; a group's bins are contiguous there.
55    pub hist_offset: u32,
56    pub total_bins: u32,
57    /// Row-major bins: `data[row * row_stride + column]`.
58    pub data: Vec<u8>,
59}
60
61#[derive(Debug, Clone, PartialEq)]
62pub struct BinnedDataset {
63    pub num_rows: usize,
64    /// Global id of local row 0: rows of lower ranks come first. Keys the random draws of quantization and bagging.
65    pub row_offset: u64,
66    /// Every input feature in input order, trivial ones included.
67    pub features: Vec<FeatureMeta>,
68    pub groups: Vec<FeatureGroup>,
69    pub total_bins: usize,
70    pub label: Vec<f32>,
71    pub weight: Option<Vec<f32>>,
72    /// `num_queries + 1` row boundaries; queries are contiguous runs of rows.
73    pub query_boundaries: Option<Vec<u32>>,
74}
75
76impl BinnedDataset {
77    pub fn num_queries(&self) -> usize {
78        self.query_boundaries.as_ref().map_or(0, |b| b.len() - 1)
79    }
80
81    /// The bin of `feature` at `row`.
82    #[inline]
83    pub fn bin(&self, feature: usize, row: usize) -> u8 {
84        let meta = &self.features[feature];
85        let group = &self.groups[meta.group as usize];
86        group.data[row * group.row_stride as usize + meta.column as usize]
87    }
88}
silvic_core/data/mod.rs

silvic_core/data/
mod.rs
