use csv::ReaderBuilder;
use dataset_core::{Dataset, DatasetError, acquire_dataset, download_to, gunzip};
use ndarray::{Array1, Array2};
use std::fs::File;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Kddcup99Subset {
Percent10,
Full,
}
impl Kddcup99Subset {
const fn url(self) -> &'static str {
match self {
Kddcup99Subset::Percent10 => "https://ndownloader.figshare.com/files/5976042",
Kddcup99Subset::Full => "https://ndownloader.figshare.com/files/5976045",
}
}
const fn gz_filename(self) -> &'static str {
match self {
Kddcup99Subset::Percent10 => "kddcup99_10_percent.data.gz",
Kddcup99Subset::Full => "kddcup99.data.gz",
}
}
const fn filename(self) -> &'static str {
match self {
Kddcup99Subset::Percent10 => "kddcup99_10_percent.csv",
Kddcup99Subset::Full => "kddcup99.csv",
}
}
const fn sha256(self) -> &'static str {
match self {
Kddcup99Subset::Percent10 => {
"f8c8267ebcd9c0ed1fd7d6277fe5bfff8732e9b7db8e61b873542b2a534b6f9a"
}
Kddcup99Subset::Full => {
"3ec2301a9a5d81b40937ba155b4713a77b60e85b89f0423257e58d566aa979fb"
}
}
}
const fn n_samples(self) -> usize {
match self {
Kddcup99Subset::Percent10 => 494_021,
Kddcup99Subset::Full => 4_898_431,
}
}
}
const KDDCUP99_DATASET_NAME: &str = "kddcup99";
const N_STRING_FEATURES: usize = 3;
const N_NUMERIC_FEATURES: usize = 38;
const N_COLUMNS: usize = N_STRING_FEATURES + N_NUMERIC_FEATURES + 1;
const STRING_COLUMNS: [usize; N_STRING_FEATURES] = [1, 2, 3];
const LABEL_COLUMN: usize = 41;
const NUMERIC_COLUMNS: [(usize, &str); N_NUMERIC_FEATURES] = [
(0, "duration"),
(4, "src_bytes"),
(5, "dst_bytes"),
(6, "land"),
(7, "wrong_fragment"),
(8, "urgent"),
(9, "hot"),
(10, "num_failed_logins"),
(11, "logged_in"),
(12, "num_compromised"),
(13, "root_shell"),
(14, "su_attempted"),
(15, "num_root"),
(16, "num_file_creations"),
(17, "num_shells"),
(18, "num_access_files"),
(19, "num_outbound_cmds"),
(20, "is_host_login"),
(21, "is_guest_login"),
(22, "count"),
(23, "srv_count"),
(24, "serror_rate"),
(25, "srv_serror_rate"),
(26, "rerror_rate"),
(27, "srv_rerror_rate"),
(28, "same_srv_rate"),
(29, "diff_srv_rate"),
(30, "srv_diff_host_rate"),
(31, "dst_host_count"),
(32, "dst_host_srv_count"),
(33, "dst_host_same_srv_rate"),
(34, "dst_host_diff_srv_rate"),
(35, "dst_host_same_src_port_rate"),
(36, "dst_host_srv_diff_host_rate"),
(37, "dst_host_serror_rate"),
(38, "dst_host_srv_serror_rate"),
(39, "dst_host_rerror_rate"),
(40, "dst_host_srv_rerror_rate"),
];
type Kddcup99Data = (Array2<String>, Array2<f64>, Array1<String>);
#[derive(Debug)]
pub struct Kddcup99 {
dataset: Dataset<Kddcup99Data, DatasetError>,
}
impl Kddcup99 {
pub fn new(storage_dir: &str) -> Self {
Self::with_subset(storage_dir, Kddcup99Subset::Percent10)
}
pub fn new_full(storage_dir: &str) -> Self {
Self::with_subset(storage_dir, Kddcup99Subset::Full)
}
fn with_subset(storage_dir: &str, subset: Kddcup99Subset) -> Self {
Kddcup99 {
dataset: Dataset::new(storage_dir, move |dir| Self::load_data(dir, subset)),
}
}
fn load_data(dir: &str, subset: Kddcup99Subset) -> Result<Kddcup99Data, DatasetError> {
let gz_filename = subset.gz_filename();
let filename = subset.filename();
let file_path = acquire_dataset(
dir,
filename,
KDDCUP99_DATASET_NAME,
Some(subset.sha256()),
|temp_path| {
download_to(subset.url(), temp_path, Some(gz_filename))?;
let gz_path = temp_path.join(gz_filename);
let csv_path = temp_path.join(filename);
gunzip(&gz_path, &csv_path)?;
Ok(csv_path)
},
)?;
let file = File::open(&file_path)?;
let mut rdr = ReaderBuilder::new().has_headers(false).from_reader(file);
let n_expected = subset.n_samples();
let mut string_features = Vec::with_capacity(n_expected * N_STRING_FEATURES);
let mut numeric_features = Vec::with_capacity(n_expected * N_NUMERIC_FEATURES);
let mut labels = Vec::with_capacity(n_expected);
for (idx, result) in rdr.records().enumerate() {
let record =
result.map_err(|e| DatasetError::csv_read_error(KDDCUP99_DATASET_NAME, e))?;
let line_num = idx + 1;
if record.len() != N_COLUMNS {
return Err(DatasetError::invalid_column_count(
KDDCUP99_DATASET_NAME,
N_COLUMNS,
record.len(),
line_num,
));
}
for &col in STRING_COLUMNS.iter() {
string_features.push(record[col].trim().to_string());
}
for &(col, name) in NUMERIC_COLUMNS.iter() {
let value: f64 = record[col].trim().parse().map_err(|e| {
DatasetError::parse_failed(KDDCUP99_DATASET_NAME, name, line_num, e)
})?;
numeric_features.push(value);
}
let raw_label = record[LABEL_COLUMN].trim();
if raw_label.is_empty() {
return Err(DatasetError::invalid_value(
KDDCUP99_DATASET_NAME,
"label",
raw_label,
line_num,
));
}
labels.push(raw_label.to_string());
}
let n_samples = labels.len();
if n_samples == 0 {
return Err(DatasetError::empty_dataset(KDDCUP99_DATASET_NAME));
}
let string_array = Array2::from_shape_vec((n_samples, N_STRING_FEATURES), string_features)
.map_err(|e| {
DatasetError::array_shape_error(KDDCUP99_DATASET_NAME, "string_features", e)
})?;
let numeric_array =
Array2::from_shape_vec((n_samples, N_NUMERIC_FEATURES), numeric_features).map_err(
|e| DatasetError::array_shape_error(KDDCUP99_DATASET_NAME, "numeric_features", e),
)?;
let labels_array = Array1::from_vec(labels);
Ok((string_array, numeric_array, labels_array))
}
pub fn features(&self) -> Result<(&Array2<String>, &Array2<f64>), DatasetError> {
let data = self.dataset.load()?;
Ok((&data.0, &data.1))
}
pub fn labels(&self) -> Result<&Array1<String>, DatasetError> {
Ok(&self.dataset.load()?.2)
}
pub fn data(&self) -> Result<&Kddcup99Data, DatasetError> {
self.dataset.load()
}
pub fn get_data(&self) -> Option<&Kddcup99Data> {
self.dataset.get()
}
pub fn get_data_mut(&mut self) -> Option<&mut Kddcup99Data> {
self.dataset.get_mut()
}
pub fn into_data(self) -> Result<Kddcup99Data, DatasetError> {
self.dataset.load()?;
Ok(self
.dataset
.into_inner()
.expect("data is present after a successful load"))
}
pub fn take_data(&mut self) -> Result<Kddcup99Data, DatasetError> {
self.dataset.load()?;
Ok(self
.dataset
.take()
.expect("data is present after a successful load"))
}
}