silvic_core/data/
build.rs

1//! `BinnedDataset` construction from dense matrices: LightGBM 3.3.5 `DatasetLoader::ConstructFromSampleData` (bin
2//! mappers from a sample) and `Dataset::Construct` (feature layout), with the `Metadata` checks on labels, weights
3//! and query sizes.
4//!
5//! Where LightGBM's distributed loader builds each rank's mappers from that rank's rows, silvic allgathers one
6//! sample over all ranks (see `sample.rs`), so every rank holds identical mappers and a model trained on N ranks can
7//! equal the one trained on all rows in one process.
8
9use std::collections::HashSet;
10
11use rayon::prelude::*;
12
13use crate::collective::Collective;
14use crate::config::Params;
15use crate::data::binning::bin_rows;
16use crate::data::sample::gather_sample;
17use crate::data::{BinMapper, BinnedDataset, FeatureGroup, FeatureMeta, GROUP_BIN_BUDGET};
18use crate::error::{Result, SilvicError};
19
20/// Characters `Common::CheckAllowedJSON` rejects in feature names.
21const JSON_SPECIAL: &[char] = &['"', ',', ':', '[', ']', '{', '}'];
22
23#[derive(Debug, Clone, Copy, PartialEq, Eq)]
24pub enum Layout {
25    RowMajor,
26    ColumnMajor,
27}
28
29#[derive(Debug, Clone, Copy)]
30pub enum DenseData<'a> {
31    F32(&'a [f32]),
32    F64(&'a [f64]),
33}
34
35/// A borrowed dense feature matrix (numpy arrays cross without copies).
36#[derive(Debug, Clone, Copy)]
37pub struct DenseMatrix<'a> {
38    pub data: DenseData<'a>,
39    pub rows: usize,
40    pub cols: usize,
41    pub layout: Layout,
42}
43
44impl DenseMatrix<'_> {
45    #[inline]
46    pub fn get(&self, row: usize, col: usize) -> f64 {
47        let index = match self.layout {
48            Layout::RowMajor => row * self.cols + col,
49            Layout::ColumnMajor => col * self.rows + row,
50        };
51        match self.data {
52            DenseData::F32(values) => f64::from(values[index]),
53            DenseData::F64(values) => values[index],
54        }
55    }
56
57    fn len(&self) -> usize {
58        match self.data {
59            DenseData::F32(values) => values.len(),
60            DenseData::F64(values) => values.len(),
61        }
62    }
63}
64
65/// One rank's rows and their metadata.
66#[derive(Debug, Clone)]
67pub struct DatasetInput<'a> {
68    pub features: DenseMatrix<'a>,
69    pub label: &'a [f32],
70    pub weight: Option<&'a [f32]>,
71    /// Query sizes (LightGBM `group`): consecutive runs of rows.
72    pub group: Option<&'a [u32]>,
73    /// Defaults to `Column_<i>`, as LightGBM names unnamed columns.
74    pub feature_names: Option<Vec<String>>,
75    pub categorical_features: Vec<usize>,
76}
77
78impl BinnedDataset {
79    /// Bins a training set. Bin mappers come from a sample of `bin_construct_sample_cnt` rows over every rank
80    /// (allgathered), so every rank builds identical mappers; `row_offset` is the rows of lower ranks.
81    pub fn construct(input: &DatasetInput<'_>, params: &Params, collective: &mut dyn Collective) -> Result<Self> {
82        let checked = check_max_bin(params)
83            .and_then(|()| validate(input))
84            .and_then(|()| feature_names(input));
85        let row_counts = gather_row_counts(input.features.rows, checked.as_ref().err(), collective)?;
86        let names = checked?;
87        let row_offset = row_counts[..collective.rank()].iter().sum();
88        let sample = gather_sample(
89            &input.features,
90            &row_counts,
91            params.bin_construct_sample_cnt,
92            params.data_random_seed,
93            collective,
94        )?;
95        let categorical: HashSet<usize> = input.categorical_features.iter().copied().collect();
96        let mappers: Vec<BinMapper> = sample
97            .values
98            .par_iter()
99            .enumerate()
100            .map(|(feature, values)| {
101                if categorical.contains(&feature) {
102                    BinMapper::categorical(values, sample.rows, params)
103                } else {
104                    BinMapper::numerical(values, sample.rows, params)
105                }
106            })
107            .collect();
108        drop(sample);
109
110        let (features, mut groups, total_bins) = layout(names, mappers);
111        bin_rows(&input.features, &features, &mut groups);
112        tracing::debug!(
113            rows = input.features.rows,
114            groups = groups.len(),
115            total_bins,
116            "binned training data"
117        );
118        Ok(BinnedDataset {
119            num_rows: input.features.rows,
120            row_offset,
121            features,
122            groups,
123            total_bins,
124            label: input.label.to_vec(),
125            weight: input.weight.map(<[f32]>::to_vec),
126            query_boundaries: query_boundaries(input.group),
127        })
128    }
129
130    /// Bins a validation set with `reference`'s mappers and group layout; feature names and categorical
131    /// declarations come from `reference`.
132    pub fn construct_with_reference(
133        input: &DatasetInput<'_>,
134        reference: &BinnedDataset,
135        collective: &mut dyn Collective,
136    ) -> Result<Self> {
137        let checked = validate(input).and_then(|()| {
138            if input.features.cols == reference.features.len() {
139                return Ok(());
140            }
141            Err(SilvicError::data(format!(
142                "{} features, the reference dataset has {}",
143                input.features.cols,
144                reference.features.len()
145            )))
146        });
147        let row_counts = gather_row_counts(input.features.rows, checked.as_ref().err(), collective)?;
148        checked?;
149        let row_offset = row_counts[..collective.rank()].iter().sum();
150        let mut groups: Vec<FeatureGroup> = reference
151            .groups
152            .iter()
153            .map(|group| FeatureGroup {
154                features: group.features.clone(),
155                row_stride: group.row_stride,
156                hist_offset: group.hist_offset,
157                total_bins: group.total_bins,
158                data: Vec::new(),
159            })
160            .collect();
161        bin_rows(&input.features, &reference.features, &mut groups);
162        Ok(BinnedDataset {
163            num_rows: input.features.rows,
164            row_offset,
165            features: reference.features.clone(),
166            groups,
167            total_bins: reference.total_bins,
168            label: input.label.to_vec(),
169            weight: input.weight.map(<[f32]>::to_vec),
170            query_boundaries: query_boundaries(input.group),
171        })
172    }
173}
174
175fn validate(input: &DatasetInput<'_>) -> Result<()> {
176    let matrix = &input.features;
177    if matrix.rows.checked_mul(matrix.cols) != Some(matrix.len()) {
178        return Err(SilvicError::data(format!(
179            "feature matrix holds {} values, not {} rows x {} columns",
180            matrix.len(),
181            matrix.rows,
182            matrix.cols
183        )));
184    }
185    validate_label(input.label, matrix.rows)?;
186    if let Some(weight) = input.weight {
187        validate_weight(weight, matrix.rows)?;
188    }
189    if let Some(group) = input.group {
190        validate_group(group, matrix.rows)?;
191    }
192    if let Some(&index) = input.categorical_features.iter().find(|&&index| index >= matrix.cols) {
193        return Err(SilvicError::data(format!(
194            "categorical feature {index} is out of range for {} features",
195            matrix.cols
196        )));
197    }
198    Ok(())
199}
200
201fn validate_label(label: &[f32], rows: usize) -> Result<()> {
202    if label.len() != rows {
203        return Err(SilvicError::data(format!("{} labels for {rows} rows", label.len())));
204    }
205    if let Some((row, value)) = label.iter().enumerate().find(|(_, v)| !v.is_finite()) {
206        return Err(SilvicError::data(format!(
207            "label of row {row} is {value}; labels must be finite"
208        )));
209    }
210    Ok(())
211}
212
213fn validate_weight(weight: &[f32], rows: usize) -> Result<()> {
214    if weight.len() != rows {
215        return Err(SilvicError::data(format!("{} weights for {rows} rows", weight.len())));
216    }
217    if let Some((row, value)) = weight.iter().enumerate().find(|(_, v)| !(v.is_finite() && **v >= 0.0)) {
218        return Err(SilvicError::data(format!(
219            "weight of row {row} is {value}; weights must be finite and non-negative"
220        )));
221    }
222    Ok(())
223}
224
225fn validate_group(group: &[u32], rows: usize) -> Result<()> {
226    let total: u64 = group.iter().map(|&size| u64::from(size)).sum();
227    if total != rows as u64 {
228        return Err(SilvicError::data(format!(
229            "query sizes sum to {total}, not the {rows} rows"
230        )));
231    }
232    if total > u64::from(u32::MAX) {
233        return Err(SilvicError::data(format!(
234            "{rows} rows with queries; query boundaries are 32-bit"
235        )));
236    }
237    Ok(())
238}
239
240/// The input's names or `Column_<i>`, with spaces (and `=`, which the model format also splits on) replaced by `_`
241/// as `Dataset::set_feature_names` does; JSON special characters and duplicates are errors there and here.
242fn feature_names(input: &DatasetInput<'_>) -> Result<Vec<String>> {
243    let cols = input.features.cols;
244    let Some(names) = &input.feature_names else {
245        return Ok((0..cols).map(|i| format!("Column_{i}")).collect());
246    };
247    if names.len() != cols {
248        return Err(SilvicError::data(format!(
249            "{} feature names for {cols} features",
250            names.len()
251        )));
252    }
253    let mut seen = HashSet::with_capacity(cols);
254    let mut out = Vec::with_capacity(cols);
255    for name in names {
256        if name.contains(JSON_SPECIAL) {
257            return Err(SilvicError::data(format!(
258                "feature name {name:?} contains a JSON special character"
259            )));
260        }
261        let name = name.replace([' ', '='], "_");
262        if !seen.insert(name.clone()) {
263            return Err(SilvicError::data(format!(
264                "feature name {name:?} appears more than once"
265            )));
266        }
267        out.push(name);
268    }
269    Ok(out)
270}
271
272fn check_max_bin(params: &Params) -> Result<()> {
273    if !(2..=255).contains(&params.max_bin) {
274        return Err(SilvicError::config(format!(
275            "max_bin must be in 2..=255, got {}",
276            params.max_bin
277        )));
278    }
279    Ok(())
280}
281
282/// Allgathers every rank's row count with its validation outcome (`[u64 rows][error text, empty when valid]`), so
283/// a rank with bad input fails every rank at the first collective instead of leaving the others waiting in a later
284/// one. All ranks return the lowest failing rank's error.
285fn gather_row_counts(
286    rows: usize,
287    local_error: Option<&SilvicError>,
288    collective: &mut dyn Collective,
289) -> Result<Vec<u64>> {
290    let mut status = (rows as u64).to_le_bytes().to_vec();
291    if let Some(error) = local_error {
292        let reason = match error {
293            SilvicError::Data(message) | SilvicError::Config(message) => message.to_string(),
294            other => other.to_string(),
295        };
296        // An empty text would read as success.
297        status.extend(
298            if reason.is_empty() {
299                "invalid input".to_string()
300            } else {
301                reason
302            }
303            .into_bytes(),
304        );
305    }
306    let gathered = collective.allgather_bytes(&status)?;
307    let mut counts = Vec::with_capacity(gathered.len());
308    for (rank, bytes) in gathered.iter().enumerate() {
309        let malformed = || SilvicError::collective(format!("malformed row count from rank {rank}"));
310        let (count, reason) = bytes.split_first_chunk::<8>().ok_or_else(malformed)?;
311        if !reason.is_empty() {
312            return Err(SilvicError::data(format!(
313                "rank {rank}: {}",
314                String::from_utf8_lossy(reason)
315            )));
316        }
317        counts.push(u64::from_le_bytes(*count));
318    }
319    Ok(counts)
320}
321
322fn query_boundaries(group: Option<&[u32]>) -> Option<Vec<u32>> {
323    let group = group?;
324    let mut boundaries = Vec::with_capacity(group.len() + 1);
325    let mut end = 0u32;
326    boundaries.push(end);
327    for &size in group {
328        end += size;
329        boundaries.push(end);
330    }
331    Some(boundaries)
332}
333
334/// Features with their histogram offsets, and the groups they pack into: non-trivial features in input order,
335/// a new group whenever the next feature's bins would take the current one over `GROUP_BIN_BUDGET`.
336fn layout(names: Vec<String>, mappers: Vec<BinMapper>) -> (Vec<FeatureMeta>, Vec<FeatureGroup>, usize) {
337    let mut features = Vec::with_capacity(mappers.len());
338    let mut groups: Vec<FeatureGroup> = Vec::new();
339    let mut offset = 0u32;
340    for (index, (name, bins)) in names.into_iter().zip(mappers).enumerate() {
341        let num_bins = bins.num_bins();
342        if num_bins == 0 {
343            features.push(FeatureMeta {
344                name,
345                bins,
346                num_bins,
347                group: 0,
348                column: 0,
349                hist_offset: offset,
350            });
351            continue;
352        }
353        if groups
354            .last()
355            .is_none_or(|group| group.total_bins + num_bins > GROUP_BIN_BUDGET)
356        {
357            groups.push(FeatureGroup {
358                features: Vec::new(),
359                row_stride: 0,
360                hist_offset: offset,
361                total_bins: 0,
362                data: Vec::new(),
363            });
364        }
365        let group_index = groups.len() as u32 - 1;
366        let group = groups.last_mut().expect("a group");
367        features.push(FeatureMeta {
368            name,
369            bins,
370            num_bins,
371            group: group_index,
372            column: group.features.len() as u32,
373            hist_offset: offset,
374        });
375        group.features.push(index);
376        group.total_bins += num_bins;
377        offset += num_bins;
378    }
379    for group in &mut groups {
380        group.row_stride = group.features.len().next_multiple_of(4) as u32;
381    }
382    (features, groups, offset as usize)
383}
384
385#[cfg(test)]
386mod tests {
387    use super::*;
388    use crate::collective::LocalCollective;
389    use crate::quantize::mix32;
390    use crate::test_util::run_ranks;
391
392    /// A row-major matrix of `rows x cols` with a mix of feature shapes: continuous, with NaN, integer, categorical.
393    fn matrix_values(rows: usize, cols: usize, row_offset: usize) -> Vec<f64> {
394        let mut out = Vec::with_capacity(rows * cols);
395        for row in row_offset..row_offset + rows {
396            for col in 0..cols {
397                let unit = f64::from(mix32(mix32(row as u32) ^ mix32(col as u32 + 1)) >> 8) / 16_777_216.0;
398                out.push(match col % 4 {
399                    0 => (unit - 0.3) * 10.0,
400                    1 if unit < 0.1 => f64::NAN,
401                    1 => unit * unit,
402                    2 => (unit * 7.0).floor(),
403                    _ => (unit * unit * 9.0).floor() - 1.0,
404                });
405            }
406        }
407        out
408    }
409
410    fn input<'a>(values: &'a [f64], label: &'a [f32], cols: usize) -> DatasetInput<'a> {
411        DatasetInput {
412            features: DenseMatrix {
413                data: DenseData::F64(values),
414                rows: label.len(),
415                cols,
416                layout: Layout::RowMajor,
417            },
418            label,
419            weight: None,
420            group: None,
421            feature_names: None,
422            categorical_features: (3..cols).step_by(4).collect(),
423        }
424    }
425
426    fn check_bins(dataset: &BinnedDataset, matrix: &DenseMatrix<'_>, case: &str) {
427        for (feature, meta) in dataset.features.iter().enumerate() {
428            if meta.is_trivial() {
429                continue;
430            }
431            for row in 0..matrix.rows {
432                let expected = meta.bins.value_to_bin(matrix.get(row, feature));
433                assert_eq!(
434                    dataset.bin(feature, row),
435                    expected,
436                    "{case}: feature {feature} row {row}"
437                );
438            }
439        }
440    }
441
442    #[test]
443    fn construct_bins_every_row() {
444        let (rows, cols) = (10_000, 9);
445        let values = matrix_values(rows, cols, 0);
446        let label = vec![0.5f32; rows];
447        let group = [4000u32, 0, 6000];
448        let mut data = input(&values, &label, cols);
449        data.group = Some(&group);
450        data.feature_names = Some((0..cols).map(|i| format!("f {i}=x")).collect());
451        let dataset = BinnedDataset::construct(&data, &Params::default(), &mut LocalCollective).expect("construct");
452        assert_eq!(dataset.row_offset, 0);
453        assert_eq!(dataset.query_boundaries, Some(vec![0, 4000, 4000, 10_000]));
454        assert_eq!(dataset.features[0].name, "f_0_x");
455        assert!(dataset.features[3].is_categorical());
456        assert!(dataset.features[1].bins.value_to_bin(f64::NAN) as u32 == dataset.features[1].num_bins - 1);
457        check_bins(&dataset, &data.features, "row-major f64");
458
459        let column_major: Vec<f32> = (0..cols)
460            .flat_map(|c| (0..rows).map(move |r| (r, c)))
461            .map(|(r, c)| values[r * cols + c] as f32)
462            .collect();
463        let mut data32 = data.clone();
464        data32.features = DenseMatrix {
465            data: DenseData::F32(&column_major),
466            rows,
467            cols,
468            layout: Layout::ColumnMajor,
469        };
470        let dataset32 = BinnedDataset::construct(&data32, &Params::default(), &mut LocalCollective).expect("f32");
471        check_bins(&dataset32, &data32.features, "column-major f32");
472    }
473
474    #[test]
475    fn grouping_respects_budget_and_stride() {
476        let wide = BinMapper::Numerical {
477            upper_bounds: (0..254).map(f64::from).chain([f64::INFINITY]).collect(),
478            nan_bin: true,
479            min: 0.0,
480            max: 1.0,
481        };
482        let narrow = BinMapper::Categorical { categories: vec![1, 2] };
483        let mappers: Vec<BinMapper> = (0..70)
484            .map(|i| match i % 7 {
485                0 => BinMapper::Trivial,
486                1 => narrow.clone(),
487                _ => wide.clone(),
488            })
489            .collect();
490        let names = (0..70).map(|i| format!("Column_{i}")).collect();
491        let (features, groups, total_bins) = layout(names, mappers);
492        assert!(groups.len() > 1);
493        let mut offset = 0;
494        let mut grouped: Vec<usize> = Vec::new();
495        for (index, group) in groups.iter().enumerate() {
496            assert!(group.total_bins <= GROUP_BIN_BUDGET, "group {index}: budget");
497            assert_eq!(group.row_stride % 4, 0, "group {index}: stride");
498            assert!(
499                group.row_stride as usize >= group.features.len()
500                    && group.row_stride as usize - group.features.len() < 4
501            );
502            assert_eq!(group.hist_offset, offset, "group {index}: offset");
503            for (column, &feature) in group.features.iter().enumerate() {
504                let meta = &features[feature];
505                assert_eq!(
506                    (meta.group, meta.column, meta.hist_offset),
507                    (index as u32, column as u32, offset)
508                );
509                offset += meta.num_bins;
510            }
511            assert_eq!(offset - group.hist_offset, group.total_bins);
512            if let Some(next) = groups.get(index + 1) {
513                let next_bins = features[next.features[0]].num_bins;
514                assert!(group.total_bins + next_bins > GROUP_BIN_BUDGET, "group {index}: greedy");
515            }
516            grouped.extend(&group.features);
517        }
518        assert_eq!(offset as usize, total_bins);
519        let expected: Vec<usize> = (0..70).filter(|i| i % 7 != 0).collect();
520        assert_eq!(grouped, expected, "non-trivial features in input order");
521    }
522
523    #[test]
524    fn two_ranks_share_mappers() {
525        let cols = 9;
526        for (case, rows, sample_cnt) in [
527            ("all rows sampled", [3000, 5000], 200_000),
528            ("sampled", [3000, 5000], 2000),
529        ] {
530            let total = rows[0] + rows[1];
531            let values = matrix_values(total, cols, 0);
532            let label = vec![1.0f32; total];
533            let params = Params {
534                bin_construct_sample_cnt: sample_cnt,
535                ..Params::default()
536            };
537            let built = run_ranks(2, |rank, collective| {
538                let start = if rank == 0 { 0 } else { rows[0] };
539                let end = start + rows[rank];
540                let local = input(&values[start * cols..end * cols], &label[start..end], cols);
541                let dataset = BinnedDataset::construct(&local, &params, collective).expect("construct");
542                check_bins(&dataset, &local.features, case);
543                dataset
544            });
545            assert_eq!(built[0].row_offset, 0, "{case}");
546            assert_eq!(built[1].row_offset, rows[0] as u64, "{case}");
547            assert_eq!(built[0].features, built[1].features, "{case}: mappers");
548            if sample_cnt >= total {
549                let whole = input(&values, &label, cols);
550                let single = BinnedDataset::construct(&whole, &params, &mut LocalCollective).expect("one rank");
551                assert_eq!(single.features, built[0].features, "{case}: one rank");
552            }
553        }
554    }
555
556    #[test]
557    fn reference_reproduces_layout() {
558        let cols = 9;
559        let values = matrix_values(4000, cols, 0);
560        let label = vec![0.0f32; 4000];
561        let train = BinnedDataset::construct(&input(&values, &label, cols), &Params::default(), &mut LocalCollective)
562            .expect("train");
563        let valid_values = matrix_values(1500, cols, 77_777);
564        let valid_label = vec![2.0f32; 1500];
565        let valid_input = input(&valid_values, &valid_label, cols);
566        let valid =
567            BinnedDataset::construct_with_reference(&valid_input, &train, &mut LocalCollective).expect("reference");
568        assert_eq!(valid.features, train.features);
569        assert_eq!(valid.total_bins, train.total_bins);
570        for (a, b) in valid.groups.iter().zip(&train.groups) {
571            assert_eq!(
572                (&a.features, a.row_stride, a.hist_offset, a.total_bins),
573                (&b.features, b.row_stride, b.hist_offset, b.total_bins)
574            );
575            assert_eq!(a.data.len(), 1500 * a.row_stride as usize);
576        }
577        check_bins(&valid, &valid_input.features, "reference");
578
579        let narrow = input(&valid_values[..1500 * 3], &valid_label, 3);
580        assert!(BinnedDataset::construct_with_reference(&narrow, &train, &mut LocalCollective).is_err());
581    }
582
583    #[test]
584    fn invalid_rank_fails_every_rank() {
585        let cols = 4;
586        let values = matrix_values(20, cols, 0);
587        let label = vec![1.0f32; 20];
588        let train = BinnedDataset::construct(&input(&values, &label, cols), &Params::default(), &mut LocalCollective)
589            .expect("train");
590        let expected = SilvicError::data("rank 1: 9 labels for 10 rows");
591        for reference in [None, Some(&train)] {
592            let results = run_ranks(2, |rank, collective| {
593                let rows = &values[rank * 10 * cols..(rank + 1) * 10 * cols];
594                let mut local = input(rows, &label[..10 - rank], cols);
595                local.features.rows = 10;
596                match reference {
597                    None => BinnedDataset::construct(&local, &Params::default(), collective),
598                    Some(reference) => BinnedDataset::construct_with_reference(&local, reference, collective),
599                }
600            });
601            for (rank, result) in results.into_iter().enumerate() {
602                let reference = reference.is_some();
603                assert_eq!(
604                    result.err(),
605                    Some(expected.clone()),
606                    "rank {rank}, reference {reference}"
607                );
608            }
609        }
610    }
611
612    #[test]
613    fn rejects_bad_input() {
614        let cols = 4;
615        let values = matrix_values(10, cols, 0);
616        let label = vec![1.0f32; 10];
617        let params = Params::default();
618        let construct = |data: &DatasetInput<'_>| BinnedDataset::construct(data, &params, &mut LocalCollective);
619        assert!(construct(&input(&values, &label, cols)).is_ok());
620
621        let short_label = vec![1.0f32; 9];
622        let mut bad = input(&values, &short_label, cols);
623        bad.features.rows = 10;
624        assert!(construct(&bad).is_err(), "label length");
625        let nan_label = [vec![1.0f32; 9], vec![f32::NAN]].concat();
626        assert!(construct(&input(&values, &nan_label, cols)).is_err(), "label NaN");
627        let negative = [vec![1.0f32; 9], vec![-1.0]].concat();
628        let mut bad = input(&values, &label, cols);
629        bad.weight = Some(&negative);
630        assert!(construct(&bad).is_err(), "negative weight");
631        bad.weight = Some(&label[..5]);
632        assert!(construct(&bad).is_err(), "weight length");
633        let mut bad = input(&values, &label, cols);
634        bad.group = Some(&[3, 3]);
635        assert!(construct(&bad).is_err(), "group sum");
636        let mut bad = input(&values, &label, cols);
637        bad.categorical_features = vec![4];
638        assert!(construct(&bad).is_err(), "categorical index");
639        let mut bad = input(&values, &label, cols);
640        bad.feature_names = Some(vec!["a".into(), "b".into()]);
641        assert!(construct(&bad).is_err(), "name count");
642        bad.feature_names = Some(vec!["a".into(), "b".into(), "c".into(), "a".into()]);
643        assert!(construct(&bad).is_err(), "duplicate name");
644        bad.feature_names = Some(vec!["a".into(), "b".into(), "c".into(), "d:e".into()]);
645        assert!(construct(&bad).is_err(), "json character");
646        let mut bad = input(&values[..39], &label, cols);
647        bad.features.rows = 10;
648        assert!(construct(&bad).is_err(), "matrix length");
649    }
650
651    /// Binning throughput on 10M x 140 f32 rows: `cargo test --release -p silvic-core --lib
652    /// data::build::tests::binning_throughput -- --ignored --nocapture`.
653    #[test]
654    #[ignore = "benchmark: allocates 7 GB"]
655    fn binning_throughput() {
656        let (rows, cols) = (10_000_000usize, 140usize);
657        let mut values = vec![0f32; rows * cols];
658        values.par_chunks_mut(cols).enumerate().for_each(|(row, out)| {
659            for (col, value) in out.iter_mut().enumerate() {
660                let hash = mix32(mix32(row as u32) ^ mix32(col as u32 + 1));
661                *value = if col % 10 == 9 {
662                    (hash % 12) as f32
663                } else {
664                    (hash >> 8) as f32 / 16_777_216.0 - 0.5
665                };
666            }
667        });
668        let label = vec![0f32; rows];
669        let data = DatasetInput {
670            features: DenseMatrix {
671                data: DenseData::F32(&values),
672                rows,
673                cols,
674                layout: Layout::RowMajor,
675            },
676            label: &label,
677            weight: None,
678            group: None,
679            feature_names: None,
680            categorical_features: (9..cols).step_by(10).collect(),
681        };
682        let start = std::time::Instant::now();
683        let dataset = BinnedDataset::construct(&data, &Params::default(), &mut LocalCollective).expect("construct");
684        let construct_seconds = start.elapsed().as_secs_f64();
685        let start = std::time::Instant::now();
686        let rebinned = BinnedDataset::construct_with_reference(&data, &dataset, &mut LocalCollective).expect("rebin");
687        let bin_seconds = start.elapsed().as_secs_f64();
688        assert_eq!(rebinned.groups, dataset.groups);
689        let values_count = (rows * cols) as f64;
690        println!(
691            "{rows} x {cols} f32 row-major, {} groups, {} bins: construct {construct_seconds:.2} s ({:.3e} values/s), \
692             binning alone {bin_seconds:.2} s ({:.3e} values/s)",
693            dataset.groups.len(),
694            dataset.total_bins,
695            values_count / construct_seconds,
696            values_count / bin_seconds,
697        );
698    }
699}
silvic_core/data/build.rs

silvic_core/data/
build.rs
