use std::collections::HashSet;
use std::fs::{File, OpenOptions};
use std::io::{BufRead, BufReader, BufWriter, Write};
use std::str::FromStr;
use clap::ArgMatches;
use ndarray::{Array, Ix1, Ix2, ShapeBuilder};
pub mod matrix_util;
pub mod stats_util;
pub mod timer;
pub fn get_line_count(filepath: &str) -> Result<usize, String> {
let buf = match OpenOptions::new().read(true).open(filepath) {
Err(why) => return Err(format!("failed to open {}: {}", filepath, why)),
Ok(f) => BufReader::new(f)
};
Ok(buf.lines().count())
}
pub fn extract_str_arg(matches: &ArgMatches, arg_name: &str) -> String {
match matches.value_of(arg_name) {
Some(filename) => filename.to_string(),
None => {
eprintln!("the argument {} is required", arg_name);
std::process::exit(1);
}
}
}
pub fn extract_optional_numeric_arg<T: FromStr>(matches: &ArgMatches, arg_name: &str) -> Result<Option<T>, String>
where <T as std::str::FromStr>::Err: std::fmt::Display {
match matches.value_of(arg_name) {
None => Ok(None),
Some(s) => match s.parse::<T>() {
Ok(val) => Ok(Some(val)),
Err(why) => Err(format!("failed to parse {}: {}", arg_name, why))
}
}
}
pub fn extract_numeric_arg<T: FromStr>(matches: &ArgMatches, arg_name: &str) -> Result<T, String>
where <T as std::str::FromStr>::Err: std::fmt::Display {
match extract_optional_numeric_arg(matches, arg_name)? {
None => Err(format!("{} is not provided", arg_name)),
Some(val) => Ok(val)
}
}
pub fn extract_optional_str_arg(matches: &ArgMatches, arg_name: &str) -> Option<String> {
match matches.value_of(arg_name) {
None => None,
Some(v) => Some(v.to_string())
}
}
pub fn extract_optional_str_vec_arg(matches: &ArgMatches, arg_name: &str) -> Option<Vec<String>> {
match matches.values_of(arg_name) {
None => None,
Some(v) => Some(v.map(|s| s.to_string()).collect())
}
}
pub fn extract_str_vec_arg(matches: &ArgMatches, arg_name: &str) -> Result<Vec<String>, String> {
match extract_optional_str_vec_arg(matches, arg_name) {
None => Err(format!("{} is not provided", arg_name)),
Some(v) => Ok(v)
}
}
pub fn get_bed_bim_fam_path(bfile: &str) -> [String; 3] {
[format!("{}.bed", bfile), format!("{}.bim", bfile), format!("{}.fam", bfile)]
}
pub fn load_trace_estimates(load_path: &str) -> Result<Array<f64, Ix2>, String> {
let num_rows = get_line_count(load_path)?;
let buf = match OpenOptions::new().read(true).open(load_path) {
Err(why) => return Err(format!("failed to read the trace estimates from file {}: {}", load_path, why)),
Ok(f) => BufReader::new(f)
};
let trace_vec: Vec<f64> = buf.lines().flat_map(|l|
l.unwrap()
.split_whitespace()
.map(|val| val.parse::<f64>().unwrap())
.collect::<Vec<f64>>()
).collect();
let num_cols = trace_vec.len() / num_rows;
Ok(Array::from_shape_vec((num_rows, num_cols).strides((num_cols, 1)), trace_vec).unwrap())
}
pub fn write_trace_estimates(trace_estimates: &Array<f64, Ix2>, out_path: &str) -> Result<(), String> {
let mut buf = match OpenOptions::new().truncate(true).create(true).write(true).open(out_path) {
Err(why) => return Err(format!("failed to write the trace estimates to file {}: {}", out_path, why)),
Ok(f) => BufWriter::new(f)
};
for row in trace_estimates.genrows() {
for val in row.iter() {
if let Err(why) = buf.write_fmt(format_args!("{} ", val)) {
return Err(format!("failed to write the trace estimates to file {}: {}", out_path, why));
}
}
if let Err(why) = buf.write_fmt(format_args!("\n")) {
return Err(format!("failed to write the trace estimates to file {}: {}", out_path, why));
}
}
Ok(())
}
fn validate_header(header: &str, expected_first_n_tokens: Vec<String>) -> Result<(), String> {
let header_toks: Vec<String> = header.split_whitespace().map(|t| t.to_owned()).collect();
for (i, (actual, expected)) in header_toks.into_iter().zip(expected_first_n_tokens).enumerate() {
if actual != expected {
return Err(format!("expected the header field at position {} to be {}, received {}", i, expected, actual));
}
}
Ok(())
}
fn read_and_validate_plink_header(buf: &mut BufReader<File>) -> Result<String, String> {
let mut header = String::new();
let _ = buf.read_line(&mut header);
header = header.trim_end().to_string();
validate_header(&header, vec!["FID".to_string(), "IID".to_string()])?;
Ok(header)
}
pub fn get_pheno_arr(pheno_path: &str) -> Result<Array<f32, Ix1>, String> {
let mut buf = match OpenOptions::new().read(true).open(pheno_path) {
Err(why) => return Err(format!("failed to open {}: {}", pheno_path, why)),
Ok(f) => BufReader::new(f)
};
let header = read_and_validate_plink_header(&mut buf)?;
println!("\n{} header:\n{}", pheno_path, header);
let pheno_vec = buf.lines().map(|l|
l.unwrap()
.split_whitespace()
.nth(2).unwrap()
.parse::<f32>().unwrap()
).collect();
Ok(Array::from_vec(pheno_vec))
}
pub fn get_plink_pheno_data(pheno_path: &str) -> Result<(String, Vec<String>, Vec<String>, Array<f32, Ix1>), String> {
let mut buf = match OpenOptions::new().read(true).open(pheno_path) {
Err(why) => return Err(format!("failed to open {}: {}", pheno_path, why)),
Ok(f) => BufReader::new(f)
};
let header = read_and_validate_plink_header(&mut buf)?;
println!("\n{} header:\n{}", pheno_path, header);
let mut pheno_vec = Vec::new();
let mut fid_vec = Vec::new();
let mut iid_vec = Vec::new();
for l in buf.lines() {
let toks: Vec<String> = l.unwrap().split_whitespace().map(|t| t.to_string()).collect();
fid_vec.push(toks[0].to_owned());
iid_vec.push(toks[1].to_owned());
pheno_vec.push(toks[2].parse::<f32>().unwrap());
}
Ok((header, fid_vec, iid_vec, Array::from_vec(pheno_vec)))
}
enum PhenoVal<T> {
Missing,
Present(T),
}
pub fn get_plink_pheno_data_replace_missing_with_mean(pheno_path: &str, missing_reps_vec: &Vec<String>)
-> Result<(String, Vec<String>, Vec<String>, Array<f32, Ix1>), String> {
let missing_reps: HashSet<String> = missing_reps_vec.iter().cloned().collect();
let mut buf = match OpenOptions::new().read(true).open(pheno_path) {
Err(why) => return Err(format!("failed to open {}: {}", pheno_path, why)),
Ok(f) => BufReader::new(f)
};
let header = read_and_validate_plink_header(&mut buf)?;
println!("\n{} header:\n{}", pheno_path, header);
let mut pheno = Vec::new();
let mut fid_vec = Vec::new();
let mut iid_vec = Vec::new();
for l in buf.lines() {
let toks: Vec<String> = l.unwrap().split_whitespace().map(|t| t.to_string()).collect();
fid_vec.push(toks[0].to_owned());
iid_vec.push(toks[1].to_owned());
if missing_reps.contains(&toks[2]) {
pheno.push(PhenoVal::Missing);
} else {
pheno.push(PhenoVal::Present(toks[2].parse::<f32>().unwrap()));
}
}
let non_missing_count_sum = pheno.iter().fold((0usize, 0.), |(count, sum), val| {
match val {
PhenoVal::Missing => (count, sum),
PhenoVal::Present(val) => (count + 1, sum + *val)
}
});
let pheno_mean = non_missing_count_sum.1 / non_missing_count_sum.0 as f32;
println!("\n[{}/{}] non-missing phenotype values, with mean: {}", non_missing_count_sum.0, pheno.len(), pheno_mean);
let pheno_vec = pheno.iter().map(|v| match v {
PhenoVal::Missing => pheno_mean,
PhenoVal::Present(v) => *v
}).collect();
Ok((header, fid_vec, iid_vec, Array::from_vec(pheno_vec)))
}
pub fn get_plink_covariate_arr(covariate_path: &str) -> Result<Array<f32, Ix2>, String> {
let num_people = get_line_count(covariate_path)? - 1;
println!("\n{} contains {} people", covariate_path, num_people);
let mut buf = match OpenOptions::new().read(true).open(covariate_path) {
Err(why) => return Err(format!("failed to open {}: {}", covariate_path, why)),
Ok(f) => BufReader::new(f)
};
let header = read_and_validate_plink_header(&mut buf)?;
println!("\n{} header:\n{}", covariate_path, header);
let covariate_vec: Vec<f32> = buf.lines().flat_map(|l| {
l.unwrap()
.split_whitespace()
.skip(2)
.map(|s| s.parse::<f32>().unwrap())
.collect::<Vec<f32>>()
}).collect();
assert_eq!(covariate_vec.len() % num_people, 0,
"total number of elements {} is not divisible by num_people {}", covariate_vec.len(), num_people);
let arr = Array::<f32, Ix2>::from_shape_vec((num_people, covariate_vec.len() / num_people), covariate_vec).unwrap();
Ok(arr)
}
#[cfg(test)]
mod tests {
use std::io::Write;
use ndarray::Array;
use tempfile::NamedTempFile;
use crate::util::{load_trace_estimates, validate_header, write_trace_estimates};
#[test]
fn test_validate_header() {
assert_eq!(Ok(()), validate_header("FID IID", vec!["FID".to_string(), "IID".to_string()]));
assert_eq!(Ok(()), validate_header("FID IID pheno", vec!["FID".to_string(), "IID".to_string()]));
assert!(validate_header("FID WRONG pheno", vec!["FID".to_string(), "IID".to_string()]).is_err());
assert!(validate_header("FID IID", Vec::new()).is_ok());
assert!(validate_header("", Vec::new()).is_ok());
}
#[test]
fn test_load_trace_estimates() {
let mut file = NamedTempFile::new().unwrap();
let arr = vec![vec![2., 123., 0.003, 23., -409.], vec![-0., 1.23, -2.43, 0., -9.]];
for row in arr.iter() {
for val in row.iter() {
write!(file, "{} ", val).unwrap();
}
write!(file, "\n").unwrap();
}
let estimates = load_trace_estimates(
file.path().as_os_str().to_str().unwrap()).unwrap();
let true_estimates = Array::from_shape_vec(
(2, 5),
arr.into_iter().flat_map(|a| a).collect::<Vec<f64>>()).unwrap();
assert_eq!(estimates, true_estimates);
}
#[test]
fn test_write_trace_estimates() {
let file = NamedTempFile::new().unwrap();
let path = file.into_temp_path().to_str().unwrap().to_string();
let estimates = Array::from_shape_vec((2, 5),
vec![2., 123., 0.003, 23., -409.,
-0., 1.23, -2.43, 0., -9.]).unwrap();
write_trace_estimates(&estimates, &path).unwrap();
let loaded_estimates = load_trace_estimates(&path).unwrap();
assert_eq!(loaded_estimates, estimates);
}
}