use std::cmp::{max, min};
use std::collections::{HashMap, HashSet};
use std::f64::{INFINITY, NEG_INFINITY};
use std::fs::File;
use std::io::{BufRead, BufReader};
use std::path::PathBuf;
use flate2::read::GzDecoder;
use glob::glob;
use reqwest;
use serde;
use serde::{Deserialize, Serialize};
use serde_aux::prelude::*;
use serde_json;
use serde_with::{serde_as, DefaultOnError};
use titlecase::titlecase;
use url::Url;
use crate::cli;
use crate::error;
use crate::io;
use crate::utils::{max_float, min_float};
pub use cli::PlotOptions;
fn default_accession() -> String {
"draft".to_string()
}
fn default_level() -> String {
"scaffold".to_string()
}
#[derive(Clone, Serialize, Default, Deserialize, Debug)]
pub struct AssemblyMeta {
#[serde(default = "default_accession")]
pub accession: String,
#[serde(default = "default_level")]
pub level: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub prefix: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub alias: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub bioproject: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub biosample: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub file: Option<PathBuf>,
#[serde(rename = "scaffold-count")]
#[serde(skip_serializing_if = "Option::is_none")]
pub scaffold_count: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
pub span: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
pub url: Option<Url>,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[serde(rename_all = "lowercase")]
pub enum Datatype {
Float,
Integer,
Mixed,
String,
}
#[serde_as]
#[derive(Serialize, Default, Deserialize, Debug, Clone)]
pub struct FieldMeta {
pub id: String,
#[serde(rename = "type")]
#[serde(skip_serializing_if = "Option::is_none")]
pub field_type: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub scale: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub datatype: Option<Datatype>,
#[serde(skip_serializing_if = "Option::is_none")]
pub children: Option<Vec<FieldMeta>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub parent: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub data: Option<Vec<FieldMeta>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub count: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
pub range: Option<[f64; 2]>,
#[serde_as(deserialize_as = "DefaultOnError")]
#[serde(default, skip_serializing_if = "Option::is_none")]
pub clamp: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub preload: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub active: Option<bool>,
#[serde(rename = "set")]
#[serde(skip_serializing_if = "Option::is_none")]
pub odb_set: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub file: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub version: Option<String>,
#[serde(rename = "category_slot")]
#[serde(skip_serializing_if = "Option::is_none")]
pub category_slot: Option<u8>,
#[serde(skip_serializing_if = "Option::is_none")]
pub headers: Option<Vec<String>>,
}
#[derive(Serialize, Deserialize, Debug, Default, Clone)]
pub struct PlotMeta {
#[serde(skip_serializing_if = "Option::is_none")]
pub x: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub y: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub z: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cat: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub labels: Option<String>,
}
#[derive(Clone, Serialize, Default, Deserialize, Debug)]
pub struct TaxonMeta {
#[serde(default = "default_taxname")]
pub name: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub class: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub family: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub genus: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub kingdom: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub order: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub phylum: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub superkingdom: Option<String>,
#[serde(
default = "default_taxid",
deserialize_with = "deserialize_string_from_number"
)]
pub taxid: String,
}
fn default_taxname() -> String {
"unnamed".to_string()
}
fn default_taxid() -> String {
"0".to_string()
}
#[derive(Clone, Serialize, Default, Deserialize, Debug)]
pub struct Meta {
pub id: String,
pub name: String,
#[serde(default = "default_level")]
pub record_type: String,
pub records: usize,
#[serde(default = "default_revision")]
pub revision: u8,
#[serde(default = "default_version")]
pub version: u8,
pub assembly: AssemblyMeta,
pub fields: Vec<FieldMeta>,
#[serde(default = "default_plotmeta")]
pub plot: PlotMeta,
pub taxon: TaxonMeta,
#[serde(skip_serializing_if = "Option::is_none")]
pub field_list: Option<HashMap<String, FieldMeta>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub busco_list: Option<Vec<(String, usize, String)>>,
}
fn default_revision() -> u8 {
0
}
fn default_version() -> u8 {
1
}
fn default_plotmeta() -> PlotMeta {
PlotMeta {
..Default::default()
}
}
#[derive(Serialize, Deserialize, Debug)]
pub struct Field<T> {
pub values: Vec<T>,
pub keys: Vec<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub category_slot: Option<u8>,
#[serde(skip_serializing_if = "Option::is_none")]
pub headers: Option<Vec<String>>,
}
impl<T> Field<T> {
pub fn values(&self) -> &Vec<T> {
&self.values
}
}
#[derive(Clone, Debug, Default)]
pub struct Filter {
pub min: Option<f64>,
pub max: Option<f64>,
pub invert: bool,
pub key: Option<Vec<String>>,
}
#[derive(Debug, Clone)]
pub struct BuscoGene {
pub id: String,
pub status: String,
}
pub fn get_path(dir: &PathBuf, prefix: &str) -> Option<String> {
let mut path = dir.clone();
path.push(prefix);
if let Some(e) = glob(&format!("{}*", path.to_string_lossy())).ok()?.next() {
return Some(format!("{}", e.unwrap().to_string_lossy()));
}
None
}
pub fn file_reader(dir: &PathBuf, prefix: &str) -> Option<Box<dyn BufRead>> {
let blobdir = dir.to_str().unwrap();
if blobdir.starts_with("http") {
let mut url = dir.to_str().unwrap().to_string();
if !prefix.starts_with("meta.") {
url = format!(
"{}/{}",
url.replace("/dataset/id/", "/field/"),
prefix.replace(".json", "")
);
}
let response = match reqwest::blocking::get(&url) {
Ok(resp) => resp,
Err(_) => return None,
};
if response.status().is_success() {
Some(Box::new(BufReader::new(response)))
} else {
None
}
} else if blobdir.starts_with("ssh") {
let mut ssh_path = blobdir.to_string();
if !prefix.starts_with("meta.") {
ssh_path = format!("{}/{}", ssh_path, prefix);
} else {
ssh_path = format!("{}/{}", ssh_path, prefix);
}
match io::ssh_file_reader(&ssh_path) {
Ok(reader) => Some(reader),
Err(_) => {
if !ssh_path.ends_with(".gz") {
io::ssh_file_reader(&format!("{}.gz", ssh_path)).ok()
} else {
None
}
}
}
} else {
let path = get_path(dir, prefix)?;
let file = File::open(&path).ok()?;
if path.ends_with(".gz") {
Some(Box::new(BufReader::new(GzDecoder::new(file))))
} else {
Some(Box::new(BufReader::new(file)))
}
}
}
pub fn local_file_reader(dir: &PathBuf, prefix: &str) -> Option<Box<dyn BufRead>> {
let path = get_path(dir, prefix)?;
let file = File::open(&path).ok()?;
if path.ends_with(".gz") {
Some(Box::new(BufReader::new(GzDecoder::new(file))))
} else {
Some(Box::new(BufReader::new(file)))
}
}
#[derive(Deserialize, Debug, Clone, Serialize)]
pub struct Keys {
pub headers: String,
}
pub fn parse_blobdir(blobdir: &PathBuf) -> Result<Meta, error::Error> {
let reader = match file_reader(blobdir, "meta.json") {
Some(r) => r,
None => {
return Err(error::Error::FileNotFound(format!(
"{}/meta.json",
&blobdir.to_str().unwrap()
)))
} };
let mut meta: Meta = match serde_json::from_reader(reader) {
Ok(meta) => meta,
Err(err) => {
return Err(error::Error::SerdeError(format!(
"{}/meta.json {}",
&blobdir.to_str().unwrap(),
err
)))
}
};
let mut fields: HashMap<String, FieldMeta> = HashMap::new();
let mut busco_fields: Vec<(String, usize, String)> = vec![];
fn list_fields(
field_list: &Vec<FieldMeta>,
fields: &mut HashMap<String, FieldMeta>,
busco_fields: &mut Vec<(String, usize, String)>,
busco: bool,
parent: Option<&FieldMeta>,
) {
for f in field_list {
let full = if parent.is_none() {
f.clone()
} else {
let mut tmp = FieldMeta {
id: f.id.clone(),
children: f.children.clone(),
parent: f.parent.clone(),
data: f.data.clone(),
..parent.unwrap().clone()
};
if f.field_type.is_some() {
tmp.field_type = f.field_type.clone()
}
if f.range.is_some() {
tmp.range = f.range
}
if f.preload.is_some() {
tmp.preload = f.preload
}
if f.active.is_some() {
tmp.active = f.active
}
if f.scale.is_some() {
tmp.scale = f.scale.clone()
}
if f.datatype.is_some() {
tmp.datatype = f.datatype.clone()
}
if f.count.is_some() {
tmp.count = f.count
}
if f.odb_set.is_some() {
tmp.odb_set = f.odb_set.clone()
}
tmp
};
let busco_flag = if f.id == "busco" { true } else { busco };
if f.children.is_none() {
fields.insert(f.id.clone(), full.clone());
if busco_flag {
busco_fields.push((
f.id.clone(),
f.count.unwrap_or(1),
f.odb_set.clone().unwrap(),
));
}
} else {
list_fields(
&f.children.clone().unwrap(),
fields,
busco_fields,
busco_flag,
Some(&full),
)
}
if f.data.is_some() {
list_fields(
&f.data.clone().unwrap(),
fields,
busco_fields,
busco_flag,
Some(&full),
)
}
}
}
list_fields(&meta.fields, &mut fields, &mut busco_fields, false, None);
meta.field_list = Some(fields);
meta.busco_list = Some(busco_fields);
if meta.record_type != "scaffold" {
meta.record_type = if titlecase(&meta.assembly.level) == "Contig" {
"contig".to_string()
} else {
"scaffold".to_string()
};
}
Ok(meta)
}
pub fn parse_field_busco(id: String, blobdir: &PathBuf) -> Option<Vec<Vec<BuscoGene>>> {
let reader = file_reader(blobdir, &format!("{}.json", &id))?;
let field: Field<Vec<(String, usize)>> = serde_json::from_reader(reader).ok()?;
let mut values: Vec<Vec<BuscoGene>> = vec![];
let keys = field.keys.clone();
for value in field.values() {
let mut val = vec![];
for v in value {
val.push(BuscoGene {
id: v.0.clone(),
status: keys[v.1].clone(),
});
}
values.push(val);
}
Some(values)
}
pub fn parse_field_cat(
id: String,
blobdir: &PathBuf,
) -> Result<Vec<(String, usize)>, error::Error> {
let reader = match file_reader(blobdir, &format!("{}.json", &id)) {
Some(reader) => reader,
None => {
return Err(error::Error::FileNotFound(format!(
"{}/{}.json",
&blobdir.to_str().unwrap(),
&id
)))
}
};
let field: Field<usize> = serde_json::from_reader(reader)
.map_err(|e| error::Error::SerdeError(format!("Failed to parse field {}: {}", &id, e)))?;
let mut values: Vec<(String, usize)> = vec![];
let keys = field.keys.clone();
for value in field.values() {
values.push((keys[*value].clone(), *value))
}
Ok(values)
}
pub fn parse_field_cat_windows(
id: String,
blobdir: &PathBuf,
wanted_indices: &Vec<usize>,
) -> Result<Vec<Vec<Option<(String, usize)>>>, error::Error> {
let reader = match file_reader(blobdir, &format!("{}.json", &id)) {
Some(reader) => reader,
None => {
return Err(error::Error::FileNotFound(format!(
"{}/{}.json",
&blobdir.to_str().unwrap(),
&id
)))
}
};
let field: Field<Vec<Vec<Option<usize>>>> = serde_json::from_reader(reader)
.map_err(|e| error::Error::SerdeError(format!("Failed to parse field {}: {}", &id, e)))?;
let mut values: Vec<Vec<Option<(String, usize)>>> = vec![];
let keys = field.keys.clone();
let cat_slot;
if let Some(slot) = field.category_slot {
cat_slot = slot as usize;
} else {
return Ok(values);
}
let indices: HashSet<&usize> = HashSet::from_iter(wanted_indices);
for (i, seq) in field.values().iter().enumerate() {
if !indices.contains(&i) {
continue;
}
let mut windows = vec![];
for arr in seq {
let value = arr[cat_slot].map(|v| (keys[v].clone(), v));
windows.push(value);
}
values.push(windows);
}
Ok(values)
}
pub fn parse_field_synonym(
field_name: String,
blobdir: &PathBuf,
) -> Result<Vec<Option<String>>, error::Error> {
let mut id = field_name.clone();
let mut name_header = None;
if field_name.contains(".") {
let (new_id, new_name_header) = field_name.split_once(".").unwrap();
id = new_id.to_string();
name_header = Some(new_name_header.to_string());
}
let reader = match file_reader(blobdir, &format!("{}.json", &id)) {
Some(reader) => reader,
None => {
return Err(error::Error::FileNotFound(format!(
"{}/{}.json",
&blobdir.to_str().unwrap(),
&id
)))
}
};
let field: Field<Vec<String>> = serde_json::from_reader(reader)
.map_err(|e| error::Error::SerdeError(format!("Failed to parse field {}: {}", &id, e)))?;
let mut values: Vec<Option<String>> = vec![];
let mut name_slot = 0;
if let Some(headers) = field.headers.clone() {
if let Some(n_header) = name_header {
if let Some(slot) = headers.iter().position(|x| x == &n_header) {
name_slot = slot;
}
}
}
for value in field.values() {
values.push(if value.len() > name_slot {
Some(value[name_slot].clone())
} else {
None
})
}
Ok(values)
}
pub fn parse_field_float(id: String, blobdir: &PathBuf) -> Result<Vec<f64>, error::Error> {
let reader = match file_reader(blobdir, &format!("{}.json", &id)) {
Some(reader) => reader,
None => {
return Err(error::Error::FileNotFound(format!(
"{}/{}.json",
&blobdir.to_str().unwrap(),
&id
)))
}
};
let field: Field<f64> = serde_json::from_reader(reader)
.map_err(|e| error::Error::SerdeError(format!("Failed to parse field {}: {}", &id, e)))?;
let values = field.values().clone();
Ok(values)
}
pub fn parse_field_float_windows(
id: String,
blobdir: &PathBuf,
wanted_indices: Option<&Vec<usize>>,
) -> Result<(Vec<Vec<f64>>, f64, f64), error::Error> {
let reader = match file_reader(blobdir, &format!("{}.json", &id)) {
Some(reader) => reader,
None => {
return Err(error::Error::FileNotFound(format!(
"{}/{}.json",
&blobdir.to_str().unwrap(),
&id
)))
}
};
let field: Field<Vec<Vec<f64>>> = serde_json::from_reader(reader)
.map_err(|e| error::Error::SerdeError(format!("Failed to parse field {}: {}", &id, e)))?;
let mut values = vec![];
let indices: HashSet<usize> = match wanted_indices {
Some(i) => HashSet::from_iter(i.iter().cloned()),
None => HashSet::from_iter(0..field.values().len()),
};
let mut min_value = INFINITY;
let mut max_value = NEG_INFINITY;
for (i, seq) in field.values().iter().enumerate() {
if !indices.contains(&i) {
continue;
}
let mut windows = vec![];
for arr in seq {
windows.push(arr[0]);
min_value = min_float(min_value, arr[0]);
max_value = max_float(max_value, arr[0]);
}
values.push(windows);
}
Ok((values, min_value, max_value))
}
pub fn parse_field_int(id: String, blobdir: &PathBuf) -> Result<Vec<usize>, error::Error> {
let reader = match file_reader(blobdir, &format!("{}.json", &id)) {
Some(reader) => reader,
None => {
return Err(error::Error::FileNotFound(format!(
"{}/{}.json",
&blobdir.to_str().unwrap(),
&id
)))
}
};
let field: Field<usize> = serde_json::from_reader(reader)
.map_err(|e| error::Error::SerdeError(format!("Failed to parse field {}: {}", &id, e)))?;
let values = field.values().clone();
Ok(values)
}
pub fn parse_field_int_windows(
id: String,
blobdir: &PathBuf,
wanted_indices: Option<&Vec<usize>>,
) -> Result<(Vec<Vec<usize>>, usize, usize), error::Error> {
let reader = match file_reader(blobdir, &format!("{}.json", &id)) {
Some(reader) => reader,
None => {
return Err(error::Error::FileNotFound(format!(
"{}/{}.json",
&blobdir.to_str().unwrap(),
&id
)))
}
};
let field: Field<Vec<Vec<usize>>> = serde_json::from_reader(reader)
.map_err(|e| error::Error::SerdeError(format!("Failed to parse field {}: {}", &id, e)))?;
let mut values = vec![];
let indices: HashSet<usize> = match wanted_indices {
Some(i) => HashSet::from_iter(i.iter().cloned()),
None => HashSet::from_iter(0..field.values().len()),
};
let mut min_value = INFINITY as usize;
let mut max_value = NEG_INFINITY as usize;
for (i, seq) in field.values().iter().enumerate() {
if !indices.contains(&i) {
continue;
}
let mut windows = vec![];
for arr in seq {
windows.push(arr[0]);
min_value = min(min_value, arr[0]);
max_value = max(max_value, arr[0]);
}
values.push(windows);
}
Ok((values, min_value, max_value))
}
pub fn parse_field_string(
id: String,
blobdir: &PathBuf,
) -> Result<(HashMap<String, usize>, Vec<String>), error::Error> {
let reader = match file_reader(blobdir, &format!("{}.json", &id)) {
Some(reader) => reader,
None => {
return Err(error::Error::FileNotFound(format!(
"{}/{}.json",
&blobdir.to_str().unwrap(),
&id
)))
}
};
let field: Field<usize> = serde_json::from_reader(reader)
.map_err(|e| error::Error::SerdeError(format!("Failed to parse field {}: {}", &id, e)))?;
let mut keys = HashMap::new();
for (index, key) in field.keys.iter().enumerate() {
keys.insert(key.clone(), index);
}
let values: Vec<String> = field
.values()
.iter()
.map(|i| field.keys[i.to_owned()].clone())
.collect();
Ok((keys, values))
}
pub fn parse_field_identifiers(id: String, blobdir: &PathBuf) -> Result<Vec<String>, error::Error> {
let reader = match file_reader(blobdir, &format!("{}.json", &id)) {
Some(reader) => reader,
None => {
return Err(error::Error::FileNotFound(format!(
"{}/{}.json",
&blobdir.to_str().unwrap(),
&id
)))
}
};
let field: Field<String> = serde_json::from_reader(reader)
.map_err(|e| error::Error::SerdeError(format!("Failed to parse field {}: {}", &id, e)))?;
Ok(field.values().to_owned())
}
pub fn parse_filters(
options: &cli::PlotOptions,
plot_meta: Option<&HashMap<String, String>>,
) -> HashMap<String, Filter> {
let mut filters = options.filter.clone();
if plot_meta.is_some() && options.x_limit.is_some() {
if let Some((min_value, max_value)) = options.x_limit.clone().unwrap().split_once(",") {
let x_field = plot_meta.unwrap().get("x").unwrap();
if !min_value.is_empty() {
filters.push(format!("{}--Min={}", x_field, min_value))
}
if !max_value.is_empty() {
filters.push(format!("{}--Max={}", x_field, max_value))
}
}
}
if plot_meta.is_some() && options.y_limit.is_some() {
if let Some((min_value, max_value)) = options.y_limit.clone().unwrap().split_once(",") {
let y_field = plot_meta.unwrap().get("y").unwrap();
if !min_value.is_empty() {
filters.push(format!("{}--Min={}", y_field, min_value))
}
if !max_value.is_empty() {
filters.push(format!("{}--Max={}", y_field, max_value))
}
}
}
let mut filter_map = HashMap::new();
for filter in filters.iter() {
if let Some((id, parameter)) = filter.split_once("--").or_else(|| filter.split_once(':')) {
if !filter_map.contains_key(id) {
filter_map.insert(
id.to_string(),
Filter {
..Default::default()
},
);
};
let filter_params = filter_map.get_mut(&id.to_string()).unwrap();
if parameter.eq_ignore_ascii_case("Inv") {
filter_params.invert = true;
continue;
};
if let Some((param, value)) = parameter.split_once("=") {
match param.to_ascii_lowercase().as_str() {
"max" => filter_params.max = Some(value.parse().unwrap()),
"min" => filter_params.min = Some(value.parse().unwrap()),
"keys" | "key" => {
filter_params.key = Some(
value
.split(",")
.map(|x| x.parse::<String>().unwrap())
.collect(),
)
}
"inv" => {
filter_params.key = Some(
value
.split(",")
.map(|x| x.parse::<String>().unwrap())
.collect(),
);
filter_params.invert = true
}
_ => (),
}
}
};
}
filter_map
}
pub fn filter_float_values(values: Vec<f64>, filter: Filter, indices: Vec<usize>) -> Vec<usize> {
let initial: Vec<usize> = if indices.is_empty() {
(0..(values.len() - 1)).collect()
} else {
indices.clone()
};
let mut output = vec![];
for i in initial {
let mut keep = true;
if filter.max.is_some() && values[i] > filter.max.unwrap() {
keep = false;
}
if filter.min.is_some() && values[i] < filter.min.unwrap() {
keep = false;
}
if filter.invert {
keep = !keep;
}
if keep {
output.push(i);
}
}
output
}
pub fn filter_int_values(values: Vec<usize>, filter: Filter, indices: Vec<usize>) -> Vec<usize> {
let initial: Vec<usize> = if indices.is_empty() {
(0..(values.len() - 1)).collect()
} else {
indices.clone()
};
let mut output = vec![];
for i in initial {
let mut keep = true;
if filter.max.is_some() && values[i] as f64 > filter.max.unwrap() {
keep = false;
}
if filter.min.is_some() && (values[i] as f64) < filter.min.unwrap() {
keep = false;
}
if filter.invert {
keep = !keep;
}
if keep {
output.push(i);
}
}
output
}
pub fn filter_string_values(
values: Vec<String>,
keys: HashMap<String, usize>,
filter: Filter,
indices: Vec<usize>,
) -> Vec<usize> {
let initial: Vec<usize> = if indices.is_empty() {
(0..(values.len() - 1)).collect()
} else {
indices.clone()
};
let mut output = vec![];
let ints: Vec<usize> = values
.iter()
.map(|x| match x.parse::<usize>() {
Ok(value) => value,
Err(_) => keys.get(x).copied().unwrap_or(usize::MAX),
})
.collect();
let mut set: HashSet<usize> = HashSet::new();
for x in filter.key.clone().unwrap().iter() {
match x.parse::<usize>() {
Ok(value) => {
set.insert(value);
}
Err(_) => match keys.get(x) {
Some(&value) => {
set.insert(value);
}
None => {
eprintln!(
"Warning: filter key '{}' not found in field keys, skipping",
x
);
}
},
}
}
for i in initial {
let mut keep = true;
if filter.key.is_some() && set.contains(&ints[i]) {
keep = false;
}
if filter.invert {
keep = !keep;
}
if keep {
output.push(i);
}
}
output
}
pub fn set_filters(filters: HashMap<String, Filter>, meta: &Meta, blobdir: &PathBuf) -> Vec<usize> {
let mut indices = vec![];
let field_list = meta.field_list.clone().unwrap();
for (id, filter) in filters {
let field_meta_option = field_list.get(&id);
if let Some(field_meta) = field_meta_option {
let field = field_meta.clone();
match field.datatype {
Some(Datatype::Float) => {
let values = parse_field_float(field_meta.id.clone(), blobdir).unwrap();
indices = filter_float_values(values, filter, indices);
}
Some(Datatype::Integer) => {
let values = parse_field_int(field_meta.id.clone(), blobdir).unwrap();
indices = filter_int_values(values, filter, indices);
}
Some(Datatype::String) => {
let (keys, values) =
parse_field_string(field_meta.id.clone(), blobdir).unwrap();
indices = filter_string_values(values, keys, filter, indices);
}
Some(_) => (),
None => (),
}
};
}
if indices.is_empty() {
indices = (0..meta.records).collect();
}
indices
}
pub fn apply_filter_float(values: &Vec<f64>, indices: &Vec<usize>) -> Vec<f64> {
let mut output = vec![];
for i in indices {
output.push(values[*i])
}
output
}
pub fn apply_filter_int(values: &Vec<usize>, indices: &Vec<usize>) -> Vec<usize> {
let mut output = vec![];
for i in indices {
output.push(values[*i])
}
output
}
pub fn apply_filter_option_int(values: &Vec<Option<usize>>, indices: &Vec<usize>) -> Vec<usize> {
let mut output = vec![];
for i in indices {
output.push(values[*i].unwrap_or_default())
}
output
}
pub fn apply_filter_busco(
values: &Vec<Vec<BuscoGene>>,
indices: &Vec<usize>,
) -> Vec<Vec<BuscoGene>> {
let mut output = vec![];
for i in indices {
output.push(values[*i].clone())
}
output
}
pub fn apply_filter_cat(values: &Vec<(String, usize)>, indices: &Vec<usize>) -> Vec<String> {
let mut output = vec![];
for i in indices {
output.push(values[*i].clone().0)
}
output
}
pub fn apply_filter_cat_tuple(
values: &Vec<(String, usize)>,
indices: &Vec<usize>,
) -> Vec<(String, usize)> {
let mut output = vec![];
for i in indices {
output.push(values[*i].clone())
}
output
}
pub fn apply_filter_string(values: &Vec<String>, indices: &Vec<usize>) -> Vec<String> {
let mut output = vec![];
for i in indices {
output.push(values[*i].clone())
}
output
}
pub fn apply_filter_option_string_with_fallback(
values: &Vec<Option<String>>,
indices: &Vec<usize>,
fallback: &Vec<String>,
) -> Vec<String> {
let mut output = vec![];
for i in indices {
output.push(match &values[*i] {
Some(v) => v.clone(),
_ => fallback[*i].clone(),
})
}
output
}
pub fn get_plot_values(
meta: &Meta,
blobdir: &PathBuf,
plot_map: &HashMap<String, String>,
) -> Result<(HashMap<String, Vec<f64>>, Vec<(String, usize)>), error::Error> {
let mut plot_values = HashMap::new();
let mut cat_values = vec![];
let field_list = meta.field_list.clone().unwrap();
for (axis, id) in plot_map {
let field_meta_option = field_list.get(id);
match field_meta_option {
Some(field_meta) => {
let field = field_meta.clone();
match field.datatype {
Some(Datatype::Float) => {
let values = parse_field_float(field_meta.id.clone(), blobdir)?;
plot_values.insert(axis.clone(), values);
}
Some(Datatype::Integer) => {
let values: Vec<f64> = parse_field_int(field_meta.id.clone(), blobdir)?
.iter()
.map(|x| *x as f64)
.collect();
plot_values.insert(axis.clone(), values);
}
Some(Datatype::String) => {
if field.data.is_some() {
cat_values = parse_field_cat(field_meta.id.clone(), blobdir)?;
}
}
Some(_) => (),
None => (),
}
}
None => {
if axis == "cat" && id == "_" {
cat_values = vec![("blank".to_string(), 0); meta.records]
}
}
};
}
Ok((plot_values, cat_values))
}
pub fn get_window_values(
meta: &Meta,
blobdir: &PathBuf,
plot_map: &HashMap<String, String>,
wanted_indices: &Vec<usize>,
window_size: &Option<String>,
) -> Result<
(
HashMap<String, Vec<Vec<f64>>>,
Vec<Vec<Option<(String, usize)>>>,
HashMap<String, [f64; 2]>,
),
error::Error,
> {
let mut plot_values = HashMap::new();
let mut axis_limits = HashMap::new();
let mut cat_values = vec![];
let field_list = meta.field_list.clone().unwrap();
let axes = vec!["x", "y", "z", "cat"];
for axis in axes {
let default_id = "_".to_string();
let id = plot_map.get(axis).unwrap_or(&default_id);
if id == "_" {
continue;
}
let window_id = match window_size {
Some(ref size) if size != "0.1" => format!("{}_windows_{}", id, size),
_ => format!("{}_windows", id),
};
let field_meta_option = field_list.get(&window_id);
match field_meta_option {
Some(field_meta) => {
let field = field_meta.clone();
match field.datatype {
Some(Datatype::Mixed) => {
let (values, min_value, max_value) = parse_field_float_windows(
field_meta.id.clone(),
blobdir,
Some(wanted_indices),
)?;
plot_values.insert(axis.to_string(), values);
axis_limits.insert(axis.to_string(), [min_value, max_value]);
}
Some(Datatype::String) => {
cat_values = parse_field_cat_windows(
field_meta.id.clone(),
blobdir,
wanted_indices,
)?;
}
Some(_) => (),
None => (),
}
}
None => {
if axis == "cat" && id == "_" {
cat_values = vec![vec![Some(("blank".to_string(), 0)); meta.records]]
}
}
};
if axis == "cat" && (cat_values.is_empty() || cat_values[0].is_empty()) {
let window_counts = plot_values
.get("x")
.unwrap()
.iter()
.map(|x| x.len())
.collect::<Vec<usize>>();
let _single_cat_values = parse_field_cat(id.clone(), blobdir)?;
let mut single_cat_values: Vec<(String, usize)> = vec![];
for i in wanted_indices {
single_cat_values.push(_single_cat_values[*i].clone());
}
for (i, window_count) in window_counts.iter().enumerate() {
if *window_count > 0 {
cat_values.push(vec![Some(single_cat_values[i].clone()); *window_count]);
} else {
cat_values.push(vec![None; *window_count]);
}
}
}
}
Ok((plot_values, cat_values, axis_limits))
}