use crate::sqlite_scratch::ScratchConnection;
use std::cmp::Ordering;
use std::collections::{HashMap, HashSet};
use std::fs::{self, File};
use std::io::{BufRead, BufReader, Write};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering as AtomicOrdering};
use std::sync::mpsc::{Receiver, SyncSender, sync_channel};
use std::thread;
use std::time::{Duration, Instant};
use anyhow::{Context, Result, bail};
use flate2::read::GzDecoder;
use rapidgzip_core::Decoder;
use rayon::prelude::*;
use reqwest::blocking::Client;
use rusqlite::{Connection, params};
use serde_json::Value;
use crate::accession::parse_accessions;
use crate::fasta::CancellationToken;
use crate::taxonomy::{
AccessionLookupOptions, TaxonId, TaxonNode, TaxonomicUtils, assign_rank_codes, canonical_name,
rank_index, taxonomic_order,
};
const TAXDUMP_URL: &str = "https://ftp.ncbi.nih.gov/pub/taxonomy/taxdump.tar.gz";
const TARGETS_URL: &str = "https://web.cs.ucla.edu/~wob/projects/taxutils/targets.json";
const A2T_BASE_URL: &str = "https://ftp.ncbi.nih.gov/pub/taxonomy/accession2taxid";
const GB_FILE: &str = "nucl_gb.accession2taxid.gz";
const WGS_FILE: &str = "nucl_wgs.accession2taxid.gz";
const DB_FILE: &str = "nucl.accession2taxid.db";
const SCHEMA_VERSION: i64 = 2;
const MERGE_BATCH_ROWS: usize = 64 * 1024;
const CANCEL_CHECK_ROWS: u64 = 64 * 1024;
const A2T_SCHEMA_SQL: &str = "\
CREATE TABLE a2t (accession TEXT PRIMARY KEY, taxid INTEGER) WITHOUT ROWID;
CREATE TABLE a2t_meta (key TEXT PRIMARY KEY, value TEXT);
CREATE TABLE a2t_sources (
source TEXT PRIMARY KEY,
status TEXT,
etag TEXT,
last_modified TEXT,
size INTEGER,
row_count INTEGER
);";
const A2T_JOIN_SQL: &str = "SELECT t.accession, a.taxid
FROM tmp_accs t
CROSS JOIN a2t a ON t.accession = a.accession";
const T2A_JOIN_SQL: &str = "SELECT a.accession
FROM tmp_taxa t
CROSS JOIN a2t a INDEXED BY idx_taxid ON a.taxid = t.taxid";
#[derive(Clone, Copy, Debug)]
pub struct AccessionDatabaseOptions {
pub refresh: bool,
pub wgs: bool,
pub threads: Option<usize>,
}
impl Default for AccessionDatabaseOptions {
fn default() -> Self {
Self {
refresh: false,
wgs: false,
threads: None,
}
}
}
#[derive(Clone, Debug)]
pub struct TaxutilsOptions {
pub canonical: bool,
pub accessions: Option<Vec<String>>,
pub low_memory: bool,
pub targets_json: Option<PathBuf>,
pub refresh: bool,
pub wgs: bool,
pub save_folder: PathBuf,
pub threads: Option<usize>,
}
impl Default for TaxutilsOptions {
fn default() -> Self {
Self {
canonical: true,
accessions: None,
low_memory: true,
targets_json: None,
refresh: false,
wgs: false,
save_folder: std::env::var_os("TAXUTILS_GLOBALS")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from("./taxutils/")),
threads: None,
}
}
}
#[derive(Clone, Debug, Default)]
pub struct TaxutilsBuilder {
options: TaxutilsOptions,
}
impl TaxutilsBuilder {
pub fn new() -> Self {
Self::default()
}
pub fn accessions(mut self, accessions: impl IntoIterator<Item = String>) -> Self {
self.options.accessions = Some(accessions.into_iter().collect());
self
}
pub fn low_memory(mut self, value: bool) -> Self {
self.options.low_memory = value;
self
}
pub fn canonical(mut self, value: bool) -> Self {
self.options.canonical = value;
self
}
pub fn targets_json(mut self, value: impl Into<PathBuf>) -> Self {
self.options.targets_json = Some(value.into());
self
}
pub fn refresh(mut self, value: bool) -> Self {
self.options.refresh = value;
self
}
pub fn wgs(mut self, value: bool) -> Self {
self.options.wgs = value;
self
}
pub fn save_folder(mut self, value: impl Into<PathBuf>) -> Self {
self.options.save_folder = value.into();
self
}
pub fn threads(mut self, value: Option<usize>) -> Self {
self.options.threads = value;
self
}
pub fn build(self) -> Result<TaxonomicUtils> {
TaxonomicUtils::new(self.options)
}
}
pub(crate) fn load_taxutils(options: TaxutilsOptions) -> Result<TaxonomicUtils> {
fs::create_dir_all(&options.save_folder)?;
let names_path = options.save_folder.join("names.dmp");
let nodes_path = options.save_folder.join("nodes.dmp");
if options.refresh || !names_path.exists() || !nodes_path.exists() {
download_taxonomy(&options.save_folder, &names_path, &nodes_path)?;
}
let targets_path = if let Some(path) = &options.targets_json {
path.clone()
} else {
let path = options.save_folder.join("targets.json");
if options.refresh || !path.exists() {
download_file(TARGETS_URL, &path)?;
}
path
};
let mut names = build_names(&names_path)?;
let nodes = build_nodes(&nodes_path)?;
let target_taxa = build_target_taxa(&nodes, &names, &targets_path)?;
if options.refresh || !options.low_memory {
ensure_a2t_db(
&options.save_folder,
options.refresh,
options.wgs,
crate::threads::resolve(options.threads)?,
&CancellationToken::default(),
)?;
}
crate::alternatives::prepare_alternative_mappings(
&options.save_folder,
options.canonical,
options.low_memory,
options.wgs,
options.refresh,
options.threads,
&CancellationToken::default(),
)?;
let mut a2t = HashMap::new();
if let Some(accessions) = &options.accessions {
a2t = crate::alternatives::lookup_accession_taxids_with_options(
&options.save_folder,
parse_accessions(accessions, true)
.into_iter()
.filter(|a| a != "NA")
.collect(),
crate::alternatives::AccessionMappingOptions {
canonical: options.canonical,
low_memory: options.low_memory,
wgs: options.wgs,
threads: options.threads,
},
&CancellationToken::default(),
)?;
}
names.insert(2697049, "SARS-CoV-2".to_owned());
names.insert(694009, "SARS-related-CoV".to_owned());
Ok(TaxonomicUtils::from_parts(
names,
nodes,
target_taxa,
a2t,
AccessionLookupOptions {
canonical: options.canonical,
low_memory: options.low_memory,
wgs: options.wgs,
save_folder: options.save_folder,
threads: options.threads,
},
))
}
impl TaxonomicUtils {
pub fn load_a2t<S: AsRef<str> + Sync>(
&mut self,
accessions: &[S],
low_memory: Option<bool>,
extend: bool,
wgs: Option<bool>,
) -> Result<()> {
let mut requested = parse_accessions(accessions, true)
.into_iter()
.filter(|value| value != "NA")
.collect::<HashSet<_>>();
if extend {
requested.retain(|accession| !self.a2t.contains_key(accession));
if requested.is_empty() {
return Ok(());
}
}
let found = crate::alternatives::lookup_accession_taxids_with_options(
&self.save_folder,
requested,
crate::alternatives::AccessionMappingOptions {
canonical: self.canonical,
low_memory: low_memory.unwrap_or(self.low_memory),
wgs: wgs.unwrap_or(self.wgs),
threads: self.threads,
},
&CancellationToken::default(),
)?;
if !extend {
self.a2t.clear();
}
self.a2t.extend(found);
Ok(())
}
pub fn get_t2a(
&self,
taxa: &[TaxonId],
low_memory: Option<bool>,
wgs: Option<bool>,
) -> Result<HashSet<String>> {
crate::alternatives::lookup_taxid_accessions_with_options(
&self.save_folder,
taxa,
crate::alternatives::AccessionMappingOptions {
canonical: self.canonical,
low_memory: low_memory.unwrap_or(self.low_memory),
wgs: wgs.unwrap_or(self.wgs),
threads: self.threads,
},
&CancellationToken::default(),
)
}
}
fn download_file(url: &str, path: &Path) -> Result<()> {
download_file_with_meta(url, path).map(|_| ())
}
fn download_file_with_meta(url: &str, path: &Path) -> Result<SourceMeta> {
let parent = path.parent().unwrap_or_else(|| Path::new("."));
fs::create_dir_all(parent)?;
let mut temporary = tempfile::NamedTempFile::new_in(parent)?;
let mut response = Client::new()
.get(url)
.send()
.with_context(|| format!("failed to download {url}"))?
.error_for_status()?;
let meta = response_meta(&response);
std::io::copy(&mut response, &mut temporary)?;
temporary.flush()?;
temporary
.persist(path)
.map_err(|error| error.error)
.with_context(|| format!("failed to install {}", path.display()))?;
Ok(meta)
}
fn download_taxonomy(save_folder: &Path, names_path: &Path, nodes_path: &Path) -> Result<()> {
let tarball = save_folder.join("taxdump.tar.gz");
download_file(TAXDUMP_URL, &tarball)?;
let file = File::open(&tarball)?;
let mut archive = tar::Archive::new(GzDecoder::new(file));
let mut found = HashSet::new();
for entry in archive.entries()? {
let mut entry = entry?;
let path = entry.path()?;
let Some(filename) = path
.file_name()
.and_then(|value| value.to_str())
.map(str::to_owned)
else {
continue;
};
let destination = match filename.as_str() {
"names.dmp" => names_path,
"nodes.dmp" => nodes_path,
_ => continue,
};
let mut output = File::create(destination)?;
std::io::copy(&mut entry, &mut output)?;
found.insert(filename);
}
let _ = fs::remove_file(&tarball);
if !found.contains("names.dmp") || !found.contains("nodes.dmp") {
bail!("could not find names.dmp and nodes.dmp in taxdump")
}
Ok(())
}
pub(crate) fn build_names(path: &Path) -> Result<HashMap<TaxonId, String>> {
let mut names = HashMap::new();
for line in BufReader::new(File::open(path)?).lines() {
let line = line?;
let mut fields = line.split('|').map(str::trim);
let (Some(taxon), Some(name), Some(_unique_name), Some(name_class)) =
(fields.next(), fields.next(), fields.next(), fields.next())
else {
continue;
};
if name_class == "scientific name" {
names.insert(taxon.parse()?, name.to_owned());
}
}
names.insert(0, "unclassified".to_owned());
Ok(names)
}
pub(crate) fn build_nodes(path: &Path) -> Result<Vec<TaxonNode>> {
let mut raw = Vec::new();
let mut parent = HashMap::new();
let mut ranks = HashMap::new();
for line in BufReader::new(File::open(path)?).lines() {
let line = line?;
let mut fields = line.split('|').map(str::trim);
let (Some(taxon), Some(raw_parent), Some(rank)) =
(fields.next(), fields.next(), fields.next())
else {
continue;
};
let taxon: TaxonId = taxon.parse()?;
let raw_parent: TaxonId = raw_parent.parse()?;
let parent_taxon = Some(raw_parent);
let rank = rank.to_ascii_lowercase();
raw.push((taxon, parent_taxon, rank.clone()));
parent.insert(taxon, parent_taxon);
ranks.insert(taxon, rank);
}
let codes = assign_rank_codes(&parent, &ranks);
Ok(raw
.into_par_iter()
.map(|(taxon, parent, rank)| {
let rank_code = codes[&taxon].clone();
let rank_base = rank_code.chars().next().unwrap_or('U');
TaxonNode {
taxon,
parent,
rank,
rank_code,
rank_base,
rank_idx: rank_index(rank_base),
new_rank: canonical_name(rank_base).to_owned(),
}
})
.collect())
}
fn build_target_taxa(
nodes: &[TaxonNode],
names: &HashMap<TaxonId, String>,
path: &Path,
) -> Result<Vec<TaxonId>> {
let value: Value = serde_json::from_reader(File::open(path)?)?;
let pathogens = value
.get("pathogens")
.and_then(Value::as_object)
.context("targets JSON has no pathogens object")?;
let pathogen_taxa = pathogens
.values()
.map(|value| {
value
.as_i64()
.or_else(|| value.as_str().and_then(|text| text.parse().ok()))
.context("invalid pathogen taxid")
})
.collect::<Result<Vec<_>>>()?;
let mut parent = nodes
.iter()
.map(|node| (node.taxon, node.parent))
.collect::<HashMap<_, _>>();
parent.insert(1, None);
let mut children: HashMap<TaxonId, Vec<TaxonId>> = HashMap::new();
for node in nodes {
if node.taxon != 1
&& let Some(parent) = parent.get(&node.taxon).copied().flatten()
{
children.entry(parent).or_default().push(node.taxon);
}
}
let rank_idx = nodes
.iter()
.map(|node| (node.taxon, node.rank_idx))
.collect::<HashMap<_, _>>();
let ranks = nodes
.iter()
.map(|node| (node.taxon, node.rank_code.clone()))
.collect();
let taxa = pathogen_taxa
.par_iter()
.map(|pathogen| {
let mut local = HashSet::new();
let mut stack = vec![*pathogen];
while let Some(node) = stack.pop() {
local.insert(node);
if let Some(values) = children.get(&node) {
stack.extend(values.iter().rev().copied());
}
}
let mut current = parent.get(pathogen).copied().flatten();
while let Some(node) = current {
if rank_idx.get(&node).copied().unwrap_or(6) < 7 {
break;
}
local.insert(node);
current = parent.get(&node).copied().flatten();
}
local
})
.reduce(HashSet::new, |mut all, local| {
all.extend(local);
all
});
Ok(taxonomic_order(taxa, &parent, &ranks, names))
}
fn a2t_paths(save_folder: &Path, wgs: bool) -> Vec<PathBuf> {
let mut paths = vec![save_folder.join(GB_FILE)];
if wgs {
paths.push(save_folder.join(WGS_FILE));
}
paths
}
fn ensure_a2t_files(save_folder: &Path, wgs: bool) -> Result<Vec<PathBuf>> {
let paths = a2t_paths(save_folder, wgs);
ensure_a2t_paths(&paths)?;
Ok(paths)
}
fn ensure_a2t_paths(paths: &[PathBuf]) -> Result<()> {
paths.par_iter().try_for_each(|path| {
if !path.exists() {
let filename = path.file_name().unwrap().to_string_lossy();
download_file(&format!("{A2T_BASE_URL}/{filename}"), path)?;
}
Ok::<_, anyhow::Error>(())
})?;
Ok(())
}
fn a2t_columns<R: BufRead>(reader: &mut R) -> Result<(usize, usize)> {
let mut header = String::new();
reader.read_line(&mut header)?;
let accession = header
.trim_end()
.split('\t')
.position(|value| value == "accession.version")
.context("missing accession.version column")?;
let taxid = header
.trim_end()
.split('\t')
.position(|value| value == "taxid")
.context("missing taxid column")?;
Ok((accession, taxid))
}
pub(crate) fn lookup_decoder(threads: usize) -> Result<Decoder> {
Ok(Decoder::builder()
.decoder_threads(threads.max(1))
.decoded_chunk_size(1 << 20)
.in_flight_chunks(2)
.build()?)
}
fn build_decoder(threads: usize) -> Result<Decoder> {
Ok(Decoder::builder()
.decoder_threads(threads.max(1))
.decoded_chunk_size(4 << 20)
.in_flight_chunks(4)
.build()?)
}
fn scan_lookup_rows(
path: &Path,
decoder: &Decoder,
cancellation: &CancellationToken,
mut visit: impl FnMut(&str, TaxonId) -> Result<bool>,
) -> Result<()> {
let mut reader = BufReader::with_capacity(1 << 20, decoder.open(path)?);
scan_reader(&mut reader, cancellation, &mut visit)
.with_context(|| format!("failed to scan {}", path.display()))
}
fn scan_reader(
reader: &mut impl BufRead,
cancellation: &CancellationToken,
visit: &mut impl FnMut(&str, TaxonId) -> Result<bool>,
) -> Result<()> {
let (accession_column, taxid_column) = a2t_columns(reader)?;
let mut scanned: u64 = 0;
let mut visit_line = |line: &str| -> Result<bool> {
scanned += 1;
if scanned.is_multiple_of(CANCEL_CHECK_ROWS) && cancellation.is_cancelled() {
bail!("operation cancelled");
}
let mut accession = None;
let mut taxid = None;
for (index, field) in line.split('\t').enumerate() {
if index == accession_column {
accession = Some(field);
}
if index == taxid_column {
taxid = Some(field);
}
if accession.is_some() && taxid.is_some() {
break;
}
}
let (Some(accession), Some(taxid)) = (accession, taxid) else {
return Ok(true);
};
visit(accession, taxid.parse()?)
};
let mut partial = Vec::new();
loop {
let bytes = reader.fill_buf()?;
if bytes.is_empty() {
if !partial.is_empty() {
visit_line(std::str::from_utf8(&partial)?.trim_end_matches('\r'))?;
}
return Ok(());
}
let mut start = 0;
if !partial.is_empty() {
if let Some(end) = memchr::memchr(b'\n', bytes) {
partial.extend_from_slice(&bytes[..end]);
if !visit_line(std::str::from_utf8(&partial)?.trim_end_matches('\r'))? {
return Ok(());
}
partial.clear();
start = end + 1;
} else {
partial.extend_from_slice(bytes);
let consumed = bytes.len();
reader.consume(consumed);
continue;
}
}
if let Some(end) = memchr::memrchr(b'\n', &bytes[start..]) {
let end = start + end + 1;
for line in std::str::from_utf8(&bytes[start..end])?.lines() {
if !visit_line(line)? {
return Ok(());
}
}
start = end;
}
partial.extend_from_slice(&bytes[start..]);
let consumed = bytes.len();
reader.consume(consumed);
}
}
#[cfg(test)]
fn lookup_a2t(
save_folder: &Path,
accessions: &[impl AsRef<str> + Sync],
low_memory: bool,
wgs: bool,
threads: Option<usize>,
cancellation: &CancellationToken,
) -> Result<HashMap<String, TaxonId>> {
let requested = parse_accessions(accessions, true)
.into_iter()
.filter(|value| value != "NA")
.collect::<HashSet<_>>();
lookup_parsed_a2t(
save_folder,
requested,
low_memory,
wgs,
threads,
cancellation,
)
}
fn lookup_parsed_a2t(
save_folder: &Path,
requested: HashSet<String>,
low_memory: bool,
wgs: bool,
threads: Option<usize>,
cancellation: &CancellationToken,
) -> Result<HashMap<String, TaxonId>> {
if requested.is_empty() {
return Ok(HashMap::new());
}
if low_memory {
let paths = ensure_a2t_files(save_folder, wgs)?;
let decoder = lookup_decoder(crate::threads::resolve(threads)?)?;
let partials = paths
.par_iter()
.map(|path| {
let mut found = HashMap::new();
scan_lookup_rows(path, &decoder, cancellation, |accession, taxid| {
if requested.contains(accession) {
found.insert(accession.to_owned(), taxid);
}
Ok(found.len() != requested.len())
})?;
Ok(found)
})
.collect::<Result<Vec<HashMap<String, TaxonId>>>>()?;
return Ok(partials.into_iter().flatten().collect());
}
let mut index =
AccessionTaxidIndex::open(save_folder, wgs, threads, cancellation)?;
index.lookup(requested)
}
pub(crate) struct AccessionTaxidIndex {
connection: ScratchConnection,
}
impl AccessionTaxidIndex {
pub(crate) fn open(
save_folder: impl AsRef<Path>,
wgs: bool,
threads: Option<usize>,
cancellation: &CancellationToken,
) -> Result<Self> {
let save_folder = save_folder.as_ref();
fs::create_dir_all(save_folder)?;
ensure_a2t_db(
save_folder,
false,
wgs,
crate::threads::resolve(threads)?,
cancellation,
)?;
Ok(Self {
connection: ScratchConnection::open(save_folder.join(DB_FILE))?,
})
}
pub(crate) fn lookup(
&mut self,
requested: HashSet<String>,
) -> Result<HashMap<String, TaxonId>> {
lookup_a2t_connection(&mut self.connection, requested)
}
}
fn lookup_a2t_connection(
connection: &mut Connection,
requested: HashSet<String>,
) -> Result<HashMap<String, TaxonId>> {
connection.execute_batch(
"PRAGMA temp_store = MEMORY;
DROP TABLE IF EXISTS temp.tmp_accs;
CREATE TEMP TABLE tmp_accs (accession TEXT PRIMARY KEY) WITHOUT ROWID;",
)?;
let transaction = connection.transaction()?;
{
let mut insert = transaction.prepare("INSERT INTO tmp_accs VALUES (?1)")?;
for accession in requested {
insert.execute([accession])?;
}
}
transaction.commit()?;
let mut statement = connection.prepare(A2T_JOIN_SQL)?;
let rows = statement.query_map([], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, TaxonId>(1)?))
})?;
rows.collect::<rusqlite::Result<HashMap<_, _>>>()
.map_err(Into::into)
}
pub fn lookup_accession_taxids(
save_folder: impl AsRef<Path>,
accessions: HashSet<String>,
low_memory: bool,
wgs: bool,
threads: Option<usize>,
) -> Result<HashMap<String, TaxonId>> {
lookup_accession_taxids_with_cancel(
save_folder,
accessions,
low_memory,
wgs,
threads,
&CancellationToken::default(),
)
}
pub fn lookup_accession_taxids_with_cancel(
save_folder: impl AsRef<Path>,
accessions: HashSet<String>,
low_memory: bool,
wgs: bool,
threads: Option<usize>,
cancellation: &CancellationToken,
) -> Result<HashMap<String, TaxonId>> {
let save_folder = save_folder.as_ref();
fs::create_dir_all(save_folder)?;
lookup_parsed_a2t(
save_folder,
accessions,
low_memory,
wgs,
threads,
cancellation,
)
}
pub fn lookup_taxid_accessions(
save_folder: impl AsRef<Path>,
taxa: &[TaxonId],
low_memory: bool,
wgs: bool,
threads: Option<usize>,
) -> Result<HashSet<String>> {
lookup_taxid_accessions_with_cancel(
save_folder,
taxa,
low_memory,
wgs,
threads,
&CancellationToken::default(),
)
}
pub fn lookup_taxid_accessions_with_cancel(
save_folder: impl AsRef<Path>,
taxa: &[TaxonId],
low_memory: bool,
wgs: bool,
threads: Option<usize>,
cancellation: &CancellationToken,
) -> Result<HashSet<String>> {
let save_folder = save_folder.as_ref();
fs::create_dir_all(save_folder)?;
lookup_t2a(save_folder, taxa, low_memory, wgs, threads, cancellation)
}
fn lookup_t2a(
save_folder: &Path,
taxa: &[TaxonId],
low_memory: bool,
wgs: bool,
threads: Option<usize>,
cancellation: &CancellationToken,
) -> Result<HashSet<String>> {
let requested = taxa.iter().copied().collect::<HashSet<_>>();
if requested.is_empty() {
return Ok(HashSet::new());
}
if low_memory {
let paths = ensure_a2t_files(save_folder, wgs)?;
let decoder = lookup_decoder(crate::threads::resolve(threads)?)?;
let partials = paths
.par_iter()
.map(|path| {
let mut found = HashSet::new();
scan_lookup_rows(path, &decoder, cancellation, |accession, taxid| {
if requested.contains(&taxid) {
found.insert(accession.to_owned());
}
Ok(true)
})?;
Ok(found)
})
.collect::<Result<Vec<HashSet<String>>>>()?;
return Ok(partials.into_iter().flatten().collect());
}
ensure_a2t_db(
save_folder,
false,
wgs,
crate::threads::resolve(threads)?,
cancellation,
)?;
let mut connection = ScratchConnection::open(save_folder.join(DB_FILE))?;
connection.execute_batch(
"DROP TABLE IF EXISTS temp.tmp_taxa;
CREATE TEMP TABLE tmp_taxa (taxid INTEGER PRIMARY KEY);",
)?;
let transaction = connection.transaction()?;
{
let mut insert = transaction.prepare("INSERT OR IGNORE INTO tmp_taxa VALUES (?1)")?;
for taxid in requested {
insert.execute([taxid])?;
}
}
transaction.commit()?;
let mut statement = connection.prepare(T2A_JOIN_SQL)?;
let rows = statement.query_map([], |row| row.get::<_, String>(0))?;
rows.collect::<rusqlite::Result<HashSet<_>>>()
.map_err(Into::into)
}
#[derive(Debug)]
struct AccessionDatabaseState {
loaded_sources: HashSet<String>,
must_rebuild: bool,
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
struct SourceMeta {
etag: Option<String>,
last_modified: Option<String>,
size: Option<i64>,
}
impl SourceMeta {
fn matches(&self, other: &SourceMeta) -> bool {
if self.etag.is_some() && self.etag == other.etag {
return true;
}
self.size.is_some()
&& self.size == other.size
&& self.last_modified.is_some()
&& self.last_modified == other.last_modified
}
}
fn header_value(response: &reqwest::blocking::Response, name: &str) -> Option<String> {
response
.headers()
.get(name)
.and_then(|value| value.to_str().ok())
.map(str::to_owned)
}
fn response_meta(response: &reqwest::blocking::Response) -> SourceMeta {
SourceMeta {
etag: header_value(response, "etag"),
last_modified: header_value(response, "last-modified"),
size: header_value(response, "content-length").and_then(|value| value.parse().ok()),
}
}
fn remote_source_meta(url: &str) -> Result<SourceMeta> {
let response = Client::new().head(url).send()?.error_for_status()?;
Ok(response_meta(&response))
}
fn source_is_stale(connection: &Connection, source: &str) -> Result<bool> {
let stored = stored_source_meta(connection, source)?;
match remote_source_meta(&source_url(source)) {
Err(_) => Ok(false),
Ok(remote) => Ok(!stored.is_some_and(|stored| stored.matches(&remote))),
}
}
fn stored_source_meta(connection: &Connection, source: &str) -> Result<Option<SourceMeta>> {
let mut statement = connection
.prepare("SELECT etag, last_modified, size FROM a2t_sources WHERE source = ?1")?;
let mut rows = statement.query([source])?;
let Some(row) = rows.next()? else {
return Ok(None);
};
Ok(Some(SourceMeta {
etag: row.get(0)?,
last_modified: row.get(1)?,
size: row.get(2)?,
}))
}
fn record_source(
connection: &Connection,
source: &str,
status: &str,
meta: &SourceMeta,
row_count: Option<u64>,
) -> Result<()> {
connection.execute(
"INSERT INTO a2t_sources(source, status, etag, last_modified, size, row_count)
VALUES (?1, ?2, ?3, ?4, ?5, ?6)
ON CONFLICT(source) DO UPDATE SET
status = excluded.status,
etag = excluded.etag,
last_modified = excluded.last_modified,
size = excluded.size,
row_count = excluded.row_count",
params![
source,
status,
meta.etag,
meta.last_modified,
meta.size,
row_count.map(|value| value as i64)
],
)?;
Ok(())
}
fn source_url(source: &str) -> String {
format!("{A2T_BASE_URL}/{source}")
}
fn requested_a2t_sources(wgs: bool) -> Vec<&'static str> {
let mut sources = vec![GB_FILE];
if wgs {
sources.push(WGS_FILE);
}
sources
}
struct SqlInterruptGuard {
stop: Arc<AtomicBool>,
handle: Option<thread::JoinHandle<()>>,
}
impl SqlInterruptGuard {
fn new(connection: &Connection, cancellation: &CancellationToken) -> Self {
let stop = Arc::new(AtomicBool::new(false));
let interrupt = connection.get_interrupt_handle();
let cancellation = cancellation.clone();
let worker_stop = Arc::clone(&stop);
let handle = thread::spawn(move || {
while !worker_stop.load(AtomicOrdering::Relaxed) {
if cancellation.is_cancelled() {
interrupt.interrupt();
return;
}
thread::sleep(Duration::from_millis(50));
}
});
Self {
stop,
handle: Some(handle),
}
}
}
impl Drop for SqlInterruptGuard {
fn drop(&mut self) {
self.stop.store(true, AtomicOrdering::Relaxed);
if let Some(handle) = self.handle.take() {
let _ = handle.join();
}
}
}
#[derive(Default)]
struct RowBatch {
accessions: String,
rows: Vec<(usize, usize, TaxonId)>,
}
impl RowBatch {
fn push(&mut self, accession: &str, taxid: TaxonId) {
let start = self.accessions.len();
self.accessions.push_str(accession);
self.rows.push((start, self.accessions.len(), taxid));
}
fn full(&self) -> bool {
self.rows.len() >= MERGE_BATCH_ROWS || self.accessions.len() >= 4 << 20
}
fn get(&self, index: usize) -> Option<(&str, TaxonId)> {
self.rows.get(index).map(|&(start, end, taxid)| {
(&self.accessions[start..end], taxid)
})
}
}
struct MergeSource {
receiver: Option<Receiver<Result<RowBatch>>>,
handle: Option<thread::JoinHandle<()>>,
batch: RowBatch,
index: usize,
finished: bool,
rows: u64,
}
impl MergeSource {
fn worker(
read: impl FnOnce(&SyncSender<Result<RowBatch>>) -> Result<()> + Send + 'static,
) -> Self {
let (sender, receiver) = sync_channel(2);
let handle = thread::spawn(move || {
if let Err(error) = read(&sender) {
let _ = sender.send(Err(error));
}
});
Self {
receiver: Some(receiver),
handle: Some(handle),
batch: RowBatch::default(),
index: 0,
finished: false,
rows: 0,
}
}
fn spawn(path: &Path, cancellation: &CancellationToken, threads: usize) -> Result<Self> {
let path = path.to_path_buf();
let cancellation = cancellation.clone();
Ok(Self::worker(move |sender| {
read_source_batches(&path, &cancellation, sender, threads)
}))
}
fn database(path: &Path, cancellation: &CancellationToken) -> Self {
let path = path.to_path_buf();
let cancellation = cancellation.clone();
Self::worker(move |sender| {
let connection = Connection::open_with_flags(
path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
)?;
let _guard = SqlInterruptGuard::new(&connection, &cancellation);
connection.execute_batch("BEGIN;")?;
let mut statement =
connection.prepare("SELECT accession, taxid FROM a2t ORDER BY accession")?;
let mut rows = statement.query([])?;
let mut batch = RowBatch::default();
while let Some(row) = rows.next()? {
let accession = row.get_ref(0)?.as_str()?;
batch.push(accession, row.get(1)?);
if batch.full() {
cancellation.check_cancelled()?;
if sender.send(Ok(std::mem::take(&mut batch))).is_err() {
return Ok(());
}
}
}
cancellation.check_cancelled()?;
if !batch.rows.is_empty() {
let _ = sender.send(Ok(batch));
}
Ok(())
})
}
fn ensure_ready(&mut self) -> Result<()> {
while !self.finished && self.index == self.batch.rows.len() {
match self.receiver.as_ref().unwrap().recv() {
Ok(batch) => {
self.batch = batch?;
self.index = 0;
}
Err(_) => {
self.finished = true;
if let Some(handle) = self.handle.take() {
handle.join().map_err(|_| anyhow::anyhow!("accession reader panicked"))?;
}
}
}
}
Ok(())
}
fn current(&self) -> Option<(&str, TaxonId)> {
self.batch.get(self.index)
}
fn advance(&mut self) {
self.index += 1;
self.rows += 1;
}
}
impl Drop for MergeSource {
fn drop(&mut self) {
self.receiver.take();
if let Some(handle) = self.handle.take() {
let _ = handle.join();
}
}
}
fn read_source_batches(
path: &Path,
cancellation: &CancellationToken,
sender: &SyncSender<Result<RowBatch>>,
threads: usize,
) -> Result<()> {
let decoder = build_decoder(threads)?;
let mut reader = BufReader::with_capacity(4 << 20, decoder.open(path)?);
let mut batch = RowBatch::default();
let mut previous = String::new();
let mut have_previous = false;
let mut failed = false;
scan_reader(&mut reader, cancellation, &mut |accession, taxid| {
if have_previous && accession < previous.as_str() {
bail!(
"{} is not sorted by accession ({previous} precedes {accession})",
path.display()
);
}
previous.clear();
previous.push_str(accession);
have_previous = true;
batch.push(accession, taxid);
if batch.full() && sender.send(Ok(std::mem::take(&mut batch))).is_err() {
failed = true;
return Ok(false);
}
Ok(true)
})
.with_context(|| format!("failed to scan {}", path.display()))?;
if !failed && !batch.rows.is_empty() {
let _ = sender.send(Ok(batch));
}
Ok(())
}
struct MergeStream {
sources: Vec<MergeSource>,
}
impl MergeStream {
fn open(paths: &[PathBuf], cancellation: &CancellationToken, threads: usize) -> Result<Self> {
let per_source = (threads / paths.len().max(1)).max(1);
let sources = paths
.iter()
.map(|path| MergeSource::spawn(path, cancellation, per_source))
.collect::<Result<Vec<_>>>()?;
Ok(Self { sources })
}
fn lowest(&mut self) -> Result<Option<usize>> {
for source in &mut self.sources {
source.ensure_ready()?;
}
Ok(self.sources.iter().enumerate()
.filter_map(|(i, source)| source.current().map(|(key, _)| (i, key)))
.min_by(|a, b| a.1.cmp(b.1).then(a.0.cmp(&b.0)))
.map(|(i, _)| i))
}
fn next_into(&mut self, accession: &mut String) -> Result<Option<TaxonId>> {
let Some(index) = self.lowest()? else { return Ok(None); };
let (key, mut taxid) = self.sources[index].current().unwrap();
accession.clear();
accession.push_str(key);
self.sources[index].advance();
while let Some(next) = self.lowest()? {
let (key, next_taxid) = self.sources[next].current().unwrap();
if key != accession.as_str() { break; }
taxid = next_taxid;
self.sources[next].advance();
}
Ok(Some(taxid))
}
fn next_row(&mut self) -> Result<Option<(Box<str>, TaxonId)>> {
let mut accession = String::new();
Ok(self.next_into(&mut accession)?.map(|taxid| (accession.into_boxed_str(), taxid)))
}
fn rows_per_source(&self) -> Vec<u64> {
self.sources.iter().map(|source| source.rows).collect()
}
}
fn schema_version(connection: &Connection) -> Result<Option<i64>> {
let has_meta: bool = connection.query_row(
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'a2t_meta')",
[],
|row| row.get(0),
)?;
if !has_meta {
return Ok(None);
}
let mut statement =
connection.prepare("SELECT value FROM a2t_meta WHERE key = 'schema_version'")?;
let mut rows = statement.query([])?;
let Some(row) = rows.next()? else {
return Ok(None);
};
Ok(row.get::<_, String>(0)?.parse::<i64>().ok())
}
fn create_a2t_schema(connection: &Connection) -> Result<()> {
connection.execute_batch(A2T_SCHEMA_SQL)?;
connection.execute(
"INSERT INTO a2t_meta(key, value) VALUES ('schema_version', ?1)",
[SCHEMA_VERSION.to_string()],
)?;
Ok(())
}
fn inspect_a2t_database(db_path: &Path) -> Result<AccessionDatabaseState> {
let rebuild = AccessionDatabaseState {
loaded_sources: HashSet::new(),
must_rebuild: true,
};
match read_a2t_database_state(db_path) {
Ok(state) => Ok(state),
Err(error) => {
eprintln!(
"taxutils: cannot read {} ({error:#}); rebuilding",
db_path.display()
);
Ok(rebuild)
}
}
}
fn read_a2t_database_state(db_path: &Path) -> Result<AccessionDatabaseState> {
let connection = ScratchConnection::open(db_path)?;
let rebuild = AccessionDatabaseState {
loaded_sources: HashSet::new(),
must_rebuild: true,
};
match schema_version(&connection)? {
Some(version) if version == SCHEMA_VERSION => {}
Some(version) => {
eprintln!(
"taxutils: accession database is schema v{version}, this build needs \
v{SCHEMA_VERSION}; rebuilding once into the compact format"
);
return Ok(rebuild);
}
None => {
eprintln!(
"taxutils: accession database predates schema v{SCHEMA_VERSION}; \
rebuilding once into the compact format"
);
return Ok(rebuild);
}
}
let incomplete: bool = connection.query_row(
"SELECT EXISTS(SELECT 1 FROM a2t_sources WHERE status != 'complete')",
[],
|row| row.get(0),
)?;
let has_rows: bool =
connection.query_row("SELECT EXISTS(SELECT 1 FROM a2t LIMIT 1)", [], |row| {
row.get(0)
})?;
if incomplete || !has_rows {
return Ok(rebuild);
}
let mut loaded_sources = HashSet::new();
{
let mut statement =
connection.prepare("SELECT source FROM a2t_sources WHERE status = 'complete'")?;
let rows = statement.query_map([], |row| row.get::<_, String>(0))?;
for row in rows {
loaded_sources.insert(row?);
}
}
if loaded_sources.is_empty() {
return Ok(rebuild);
}
Ok(AccessionDatabaseState {
loaded_sources,
must_rebuild: false,
})
}
fn configure_bulk_load(connection: &ScratchConnection, threads: usize, _folder: &Path) -> Result<()> {
connection.pragma_update(None, "page_size", 8192_i64)?;
connection.pragma_update(None, "journal_mode", "OFF")?;
connection.pragma_update(None, "synchronous", "OFF")?;
connection.pragma_update(None, "locking_mode", "EXCLUSIVE")?;
connection.pragma_update(None, "cache_size", -524_288_i64)?;
connection.pragma_update(None, "temp_store", 1_i64)?;
connection.pragma_update(None, "threads", threads as i64)?;
Ok(())
}
fn configure_incremental(connection: &ScratchConnection, threads: usize, _folder: &Path) -> Result<()> {
connection.pragma_update(None, "journal_mode", "WAL")?;
connection.pragma_update(None, "synchronous", "NORMAL")?;
connection.pragma_update(None, "cache_size", -262_144_i64)?;
connection.pragma_update(None, "temp_store", 1_i64)?;
connection.pragma_update(None, "threads", threads as i64)?;
Ok(())
}
fn finish_bulk_load(connection: &Connection, cancellation: &CancellationToken) -> Result<()> {
cancellation.check_cancelled()?;
let started = Instant::now();
let workers: i64 = connection.pragma_query_value(None, "threads", |row| row.get(0))?;
eprintln!("taxutils: building taxid index (SQLite workers: {workers})");
{
let _guard = SqlInterruptGuard::new(connection, cancellation);
connection.execute_batch("CREATE INDEX idx_taxid ON a2t(taxid);")?;
}
cancellation.check_cancelled()?;
eprintln!("taxutils: taxid index complete in {:.2}s", started.elapsed().as_secs_f64());
connection.pragma_update(None, "locking_mode", "NORMAL")?;
connection.pragma_update(None, "journal_mode", "DELETE")?;
connection.pragma_update(None, "synchronous", "FULL")?;
Ok(())
}
fn install_temporary_database(temporary: tempfile::TempPath, db_path: &Path) -> Result<()> {
File::open(&temporary)?.sync_all()?;
temporary
.persist(db_path)
.map_err(|error| error.error)
.with_context(|| format!("failed to install {}", db_path.display()))?;
#[cfg(unix)]
File::open(db_path.parent().unwrap_or_else(|| Path::new(".")))?.sync_all()?;
Ok(())
}
fn build_a2t_database_atomic(
sources: &[PathBuf],
metas: &HashMap<String, SourceMeta>,
db_path: &Path,
cancellation: &CancellationToken,
threads: usize,
) -> Result<()> {
let merge = MergeStream::open(sources, cancellation, threads)?;
build_a2t_from_stream(sources, metas, db_path, cancellation, threads, merge, None)
}
fn build_a2t_from_stream(
sources: &[PathBuf],
metas: &HashMap<String, SourceMeta>,
db_path: &Path,
cancellation: &CancellationToken,
threads: usize,
mut merge: MergeStream,
stored_gb_rows: Option<Option<i64>>,
) -> Result<()> {
let started = Instant::now();
let parent = db_path.parent().unwrap_or_else(|| Path::new("."));
let temporary = tempfile::NamedTempFile::new_in(parent)?.into_temp_path();
let mut connection = ScratchConnection::open(&temporary)?;
configure_bulk_load(&connection, threads, parent)?;
create_a2t_schema(&connection)?;
for source in sources {
let name = source_name(source);
record_source(&connection, &name, "loading", &SourceMeta::default(), None)?;
}
let rows_per_source = {
eprintln!("taxutils: loading accession rows");
let transaction = connection.transaction()?;
{
insert_merged_rows(&transaction, &mut merge, cancellation, started)?;
}
transaction.commit()?;
merge.rows_per_source()
};
drop(merge);
cancellation.check_cancelled()?;
for (source, rows) in sources.iter().zip(rows_per_source) {
let name = source_name(source);
let meta = metas.get(&name).cloned().unwrap_or_default();
let rows = if name == GB_FILE {
stored_gb_rows.map(|n| n.map(|n| n as u64)).unwrap_or(Some(rows))
} else { Some(rows) };
record_source(&connection, &name, "complete", &meta, rows)?;
}
finish_bulk_load(&connection, cancellation)?;
connection.close()?;
let install_started = Instant::now();
cancellation.check_cancelled()?;
eprintln!("taxutils: installing accession database");
install_temporary_database(temporary, db_path)?;
eprintln!("taxutils: installed accession database in {:.2}s; build total {:.2}s",
install_started.elapsed().as_secs_f64(), started.elapsed().as_secs_f64());
Ok(())
}
fn insert_merged_rows(
connection: &Connection,
merge: &mut MergeStream,
cancellation: &CancellationToken,
started: Instant,
) -> Result<u64> {
const WIDTH: usize = 64;
let sql = format!("INSERT INTO a2t(accession,taxid) VALUES {}", vec!["(?,?)"; WIDTH].join(","));
let mut insert = connection.prepare(&sql)?;
let mut single = connection.prepare("INSERT INTO a2t(accession,taxid) VALUES (?,?)")?;
let mut accession = String::new();
let mut batch = RowBatch::default();
let mut count = 0_u64;
let mut next_check = MERGE_BATCH_ROWS as u64;
let mut last_report = Instant::now();
loop {
batch.accessions.clear();
batch.rows.clear();
while batch.rows.len() < WIDTH && !batch.full() {
let Some(taxid) = merge.next_into(&mut accession)? else { break; };
batch.push(&accession, taxid);
}
if batch.rows.is_empty() { break; }
if batch.rows.len() == WIDTH {
for (i, &(start, end, taxid)) in batch.rows.iter().enumerate() {
insert.raw_bind_parameter(i * 2 + 1, &batch.accessions[start..end])?;
insert.raw_bind_parameter(i * 2 + 2, taxid)?;
}
insert.raw_execute()?;
} else {
for &(start, end, taxid) in &batch.rows {
single.execute(params![&batch.accessions[start..end], taxid])?;
}
}
count += batch.rows.len() as u64;
if count >= next_check {
next_check = count + MERGE_BATCH_ROWS as u64;
cancellation.check_cancelled()?;
if last_report.elapsed() >= Duration::from_secs(10) {
eprintln!("taxutils: loaded {count} rows ({:.0} rows/s, {:.1}s)",
count as f64 / started.elapsed().as_secs_f64(),
started.elapsed().as_secs_f64());
last_report = Instant::now();
}
}
}
cancellation.check_cancelled()?;
eprintln!("taxutils: loaded {count} rows in {:.2}s ({:.0} rows/s)",
started.elapsed().as_secs_f64(),
count as f64 / started.elapsed().as_secs_f64().max(0.000001));
Ok(count)
}
#[derive(Debug, Default)]
struct RefreshStats {
inserted: u64,
updated: u64,
deleted: u64,
}
fn refresh_a2t_database(
db_path: &Path,
sources: &[PathBuf],
metas: &HashMap<String, SourceMeta>,
cancellation: &CancellationToken,
threads: usize,
) -> Result<RefreshStats> {
let mut connection = ScratchConnection::open(db_path)?;
configure_incremental(
&connection,
threads,
db_path.parent().unwrap_or_else(|| Path::new(".")),
)?;
connection.execute_batch(
"DROP TABLE IF EXISTS temp.a2t_delta;
CREATE TEMP TABLE a2t_delta (accession TEXT PRIMARY KEY, taxid INTEGER, op INTEGER)
WITHOUT ROWID;",
)?;
let mut stats = RefreshStats::default();
let rows_per_source = {
let mut merge = MergeStream::open(sources, cancellation, threads)?;
let mut table =
connection.prepare("SELECT accession, taxid FROM a2t ORDER BY accession")?;
let mut rows = table.query([])?;
let mut existing: Option<(String, TaxonId)> = match rows.next()? {
Some(row) => Some((row.get(0)?, row.get(1)?)),
None => None,
};
let mut incoming = merge.next_row()?;
let delta = connection.unchecked_transaction()?;
{
let mut stage = delta
.prepare("INSERT INTO temp.a2t_delta(accession, taxid, op) VALUES (?1, ?2, ?3)")?;
let mut scanned: u64 = 0;
loop {
scanned += 1;
if scanned.is_multiple_of(CANCEL_CHECK_ROWS) {
cancellation.check_cancelled()?;
}
match (&incoming, &existing) {
(None, None) => break,
(Some((accession, taxid)), None) => {
stage.execute(params![accession.as_ref(), taxid, OP_INSERT])?;
stats.inserted += 1;
incoming = merge.next_row()?;
}
(None, Some((accession, _))) => {
stage.execute(params![accession, None::<TaxonId>, OP_DELETE])?;
stats.deleted += 1;
existing = match rows.next()? {
Some(row) => Some((row.get(0)?, row.get(1)?)),
None => None,
};
}
(Some((new_accession, new_taxid)), Some((old_accession, old_taxid))) => {
match new_accession.as_ref().cmp(old_accession.as_str()) {
Ordering::Less => {
stage.execute(params![
new_accession.as_ref(),
new_taxid,
OP_INSERT
])?;
stats.inserted += 1;
incoming = merge.next_row()?;
}
Ordering::Greater => {
stage.execute(params![
old_accession,
None::<TaxonId>,
OP_DELETE
])?;
stats.deleted += 1;
existing = match rows.next()? {
Some(row) => Some((row.get(0)?, row.get(1)?)),
None => None,
};
}
Ordering::Equal => {
if new_taxid != old_taxid {
stage.execute(params![
new_accession.as_ref(),
new_taxid,
OP_UPDATE
])?;
stats.updated += 1;
}
incoming = merge.next_row()?;
existing = match rows.next()? {
Some(row) => Some((row.get(0)?, row.get(1)?)),
None => None,
};
}
}
}
}
}
}
delta.commit()?;
merge.rows_per_source()
};
cancellation.check_cancelled()?;
let transaction = connection.transaction()?;
{
let _guard = SqlInterruptGuard::new(&transaction, cancellation);
transaction.execute_batch(
"INSERT OR REPLACE INTO a2t(accession, taxid)
SELECT accession, taxid FROM temp.a2t_delta WHERE op IN (0, 1);
DELETE FROM a2t WHERE accession IN
(SELECT accession FROM temp.a2t_delta WHERE op = 2);",
)?;
}
for (source, rows) in sources.iter().zip(rows_per_source) {
let name = source_name(source);
let meta = metas.get(&name).cloned().unwrap_or_default();
record_source(&transaction, &name, "complete", &meta, Some(rows))?;
}
transaction.commit()?;
connection.execute_batch("DROP TABLE IF EXISTS temp.a2t_delta;")?;
connection.pragma_update(None, "journal_mode", "DELETE")?;
connection.pragma_update(None, "synchronous", "FULL")?;
Ok(stats)
}
const OP_INSERT: i64 = 0;
const OP_UPDATE: i64 = 1;
const OP_DELETE: i64 = 2;
fn source_name(path: &Path) -> String {
path.file_name().unwrap().to_string_lossy().into_owned()
}
fn prepare_sources(
save_folder: &Path,
names: &[String],
stale: &HashSet<String>,
previous: &HashMap<String, SourceMeta>,
) -> Result<(Vec<PathBuf>, HashMap<String, SourceMeta>)> {
let paths = names
.iter()
.map(|name| save_folder.join(name))
.collect::<Vec<_>>();
let metas = names
.par_iter()
.map(|name| {
let path = save_folder.join(name);
if stale.contains(name) || !path.exists() {
eprintln!("taxutils: {} {name}",
if stale.contains(name) { "refreshing" } else { "downloading" });
let meta = download_file_with_meta(&source_url(name), &path)?;
return Ok((name.clone(), meta));
}
eprintln!("taxutils: using cached {name}");
Ok((
name.clone(),
previous.get(name).cloned().unwrap_or_default(),
))
})
.collect::<Result<Vec<_>>>()?
.into_iter()
.collect();
Ok((paths, metas))
}
fn ensure_a2t_db(
save_folder: &Path,
refresh: bool,
wgs: bool,
threads: usize,
cancellation: &CancellationToken,
) -> Result<()> {
fs::create_dir_all(save_folder)?;
let db_path = save_folder.join(DB_FILE);
let requested = requested_a2t_sources(wgs)
.into_iter()
.map(str::to_owned)
.collect::<Vec<_>>();
if db_path.exists() {
let state = inspect_a2t_database(&db_path)?;
if !state.must_rebuild {
let connection = ScratchConnection::open(&db_path)?;
let missing = requested
.iter()
.filter(|source| !state.loaded_sources.contains(*source))
.cloned()
.collect::<HashSet<_>>();
let mut merge_names = state.loaded_sources.iter().cloned().collect::<HashSet<_>>();
merge_names.extend(requested.iter().cloned());
let mut merge_names = merge_names.into_iter().collect::<Vec<_>>();
merge_names.sort();
let mut stale = if refresh {
missing.clone()
} else {
HashSet::new()
};
let mut previous = HashMap::new();
for source in &merge_names {
if let Some(meta) = stored_source_meta(&connection, source)? {
previous.insert(source.clone(), meta);
}
if refresh && !stale.contains(source) && source_is_stale(&connection, source)? {
stale.insert(source.clone());
}
}
drop(connection);
if stale.is_empty() && missing.is_empty() {
return Ok(());
}
if !refresh && missing.contains(WGS_FILE)
&& state.loaded_sources == HashSet::from([GB_FILE.to_owned()])
{
let started = Instant::now();
let source_state = if save_folder.join(WGS_FILE).exists() {
"cached source"
} else { "download required" };
eprintln!("taxutils: adding WGS mappings ({source_state}; GB from installed database)");
let (wgs_paths, wgs_metas) = prepare_sources(
save_folder, &[WGS_FILE.to_owned()], &HashSet::new(), &previous,
)?;
previous.extend(wgs_metas);
let old = Connection::open_with_flags(
&db_path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
)?;
let stored_rows: Option<i64> = old.query_row(
"SELECT row_count FROM a2t_sources WHERE source = ?1",
[GB_FILE], |row| row.get(0),
)?;
drop(old);
eprintln!("taxutils: source preparation complete in {:.2}s",
started.elapsed().as_secs_f64());
let merge = MergeStream { sources: vec![
MergeSource::database(&db_path, cancellation),
MergeSource::spawn(&wgs_paths[0], cancellation, threads)?,
] };
build_a2t_from_stream(
&[save_folder.join(GB_FILE), wgs_paths[0].clone()],
&previous, &db_path, cancellation, threads, merge, Some(stored_rows),
)?;
eprintln!("taxutils: finished adding WGS mappings");
return Ok(());
}
let adding_wgs = missing.contains(WGS_FILE);
if adding_wgs {
let source_state = if stale.contains(WGS_FILE) {
"refresh requested; download required"
} else if save_folder.join(WGS_FILE).exists() {
"cached source"
} else { "download required" };
eprintln!(
"taxutils: adding WGS mappings to the accession database \
({source_state}; ordered bulk rebuild)"
);
}
let prepared = Instant::now();
eprintln!("taxutils: preparing gzip sources");
let (paths, metas) = prepare_sources(save_folder, &merge_names, &stale, &previous)?;
eprintln!("taxutils: source preparation complete in {:.2}s",
prepared.elapsed().as_secs_f64());
if !missing.is_empty() {
build_a2t_database_atomic(&paths, &metas, &db_path, cancellation, threads)?;
if adding_wgs {
eprintln!("taxutils: finished adding WGS mappings to the accession database");
}
return Ok(());
}
let stats =
refresh_a2t_database(&db_path, &paths, &metas, cancellation, threads)?;
eprintln!(
"taxutils: refreshed accession database ({} inserted, {} updated, {} deleted)",
stats.inserted, stats.updated, stats.deleted
);
return Ok(());
}
}
let prepared = Instant::now();
eprintln!("taxutils: preparing gzip sources for {} build", if wgs { "GB+WGS" } else { "GB" });
let (paths, metas) =
prepare_sources(save_folder, &requested, &HashSet::new(), &HashMap::new())?;
eprintln!("taxutils: source preparation complete in {:.2}s",
prepared.elapsed().as_secs_f64());
build_a2t_database_atomic(&paths, &metas, &db_path, cancellation, threads)?;
Ok(())
}
pub fn ensure_accession_database(
save_folder: impl AsRef<Path>,
options: AccessionDatabaseOptions,
) -> Result<PathBuf> {
ensure_accession_database_with_cancel(save_folder, options, &CancellationToken::default())
}
pub fn ensure_accession_database_with_cancel(
save_folder: impl AsRef<Path>,
options: AccessionDatabaseOptions,
cancellation: &CancellationToken,
) -> Result<PathBuf> {
let save_folder = save_folder.as_ref();
ensure_a2t_db(
save_folder,
options.refresh,
options.wgs,
crate::threads::resolve(options.threads)?,
cancellation,
)?;
Ok(save_folder.join(DB_FILE))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn reads_small_dump_and_corrects_ranks() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("nodes.dmp");
fs::write(
&path,
"1\t|\t1\t|\tno rank\t|\n2\t|\t1\t|\tsuperkingdom\t|\n3\t|\t2\t|\tno rank\t|\n",
)
.unwrap();
let nodes = build_nodes(&path).unwrap();
assert_eq!(nodes[0].rank_code, "R");
assert_eq!(nodes[1].rank_code, "D");
assert_eq!(nodes[2].rank_code, "D2");
}
#[test]
fn full_fixture_matches_python_reference() {
let dir = tempfile::tempdir().unwrap();
fs::write(
dir.path().join("names.dmp"),
concat!(
"1\t|\troot\t|\t\t|\tscientific name\t|\n",
"2\t|\tBacteria\t|\t\t|\tscientific name\t|\n",
"10\t|\tFam\t|\t\t|\tscientific name\t|\n",
"11\t|\tGen\t|\t\t|\tscientific name\t|\n",
"12\t|\tSp A\t|\t\t|\tscientific name\t|\n",
"13\t|\tSp B\t|\t\t|\tscientific name\t|\n",
"14\t|\tStrain\t|\t\t|\tscientific name\t|\n",
),
)
.unwrap();
fs::write(
dir.path().join("nodes.dmp"),
concat!(
"1\t|\t1\t|\tno rank\t|\n",
"2\t|\t1\t|\tsuperkingdom\t|\n",
"10\t|\t2\t|\tfamily\t|\n",
"11\t|\t10\t|\tgenus\t|\n",
"12\t|\t11\t|\tspecies\t|\n",
"13\t|\t11\t|\tspecies\t|\n",
"14\t|\t12\t|\tno rank\t|\n",
),
)
.unwrap();
fs::write(dir.path().join("targets.json"), r#"{"pathogens":{"a":12}}"#).unwrap();
let tu = TaxonomicUtils::new(TaxutilsOptions {
save_folder: dir.path().to_owned(),
..Default::default()
})
.unwrap();
assert_eq!(tu.node(14).unwrap().rank_code, "S2");
assert_eq!(tu.target_taxa, vec![10, 11, 12, 14]);
assert_eq!(tu.get_branch(14), vec![1, 2, 10, 11, 12, 14]);
assert_eq!(tu.get_subtree(11), vec![11, 12, 14, 13]);
assert_eq!(tu.get_lca(14, 13), 11);
assert_eq!(tu.get_distance(14, 13), 3);
assert_eq!(tu.sort_taxa([14, 10, 13, 12]), vec![10, 12, 14, 13]);
let profile = tu.topology(11, None).unwrap();
assert_eq!(
(profile.n_taxa, profile.n_leaves, profile.max_depth),
(4, 2, 2)
);
assert_eq!(profile.mean_depth, 1.0);
assert_eq!((profile.topology_scale, profile.max_children), (1, 2));
assert_eq!(profile.branching_taxa_fraction, 0.5);
assert_eq!(profile.top_child_fraction, 2.0 / 3.0);
}
fn write_a2t_fixture(path: &Path, rows: &[(&str, i64)]) {
let file = File::create(path).unwrap();
let mut encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast());
writeln!(encoder, "accession\taccession.version\ttaxid\tgi").unwrap();
for (accession, taxid) in rows {
writeln!(
encoder,
"{}\t{}\t{}\t0",
accession.split('.').next().unwrap(),
accession,
taxid
)
.unwrap();
}
encoder.finish().unwrap();
}
#[test]
fn buffered_scan_preserves_rows_at_every_boundary() {
let input = b"taxid\tgi\taccession.version\taccession\r\n13\t0\tNC_000001.1\tNC_000001\r\nshort\n15\t0\tNC_000002.1\tNC_000002";
for capacity in 1..=input.len() {
let mut reader = BufReader::with_capacity(capacity, &input[..]);
let mut rows = Vec::new();
scan_reader(
&mut reader,
&CancellationToken::default(),
&mut |accession: &str, taxid| {
rows.push((accession.to_owned(), taxid));
Ok(true)
},
)
.unwrap();
assert_eq!(
rows,
vec![
("NC_000001.1".to_owned(), 13),
("NC_000002.1".to_owned(), 15)
],
"capacity {capacity}"
);
}
}
#[test]
fn empty_lookups_do_not_download_or_build() {
let dir = tempfile::tempdir().unwrap();
for low_memory in [true, false] {
assert!(
lookup_accession_taxids(dir.path(), HashSet::new(), low_memory, true, None)
.unwrap()
.is_empty()
);
assert!(
lookup_taxid_accessions(dir.path(), &[], low_memory, true, None)
.unwrap()
.is_empty()
);
}
assert_eq!(fs::read_dir(dir.path()).unwrap().count(), 0);
}
#[test]
fn lookup_reads_concatenated_members_and_honors_wgs() {
let dir = tempfile::tempdir().unwrap();
let source = dir.path().join(GB_FILE);
write_a2t_fixture(&source, &[("NC_000001.1", 13)]);
let file = fs::OpenOptions::new().append(true).open(&source).unwrap();
let mut encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast());
write!(encoder, "NC_000002\tNC_000002.1\t15\t0").unwrap();
encoder.finish().unwrap();
write_a2t_fixture(
&dir.path().join(WGS_FILE),
&[("ABCD01000001.1", 15), ("NC_000001.1", 99)],
);
let queries = HashSet::from([
"NC_000001.1".to_owned(),
"NC_000002.1".to_owned(),
"ABCD01000001.1".to_owned(),
]);
for threads in [1, 4] {
rayon::ThreadPoolBuilder::new()
.num_threads(threads)
.build()
.unwrap()
.install(|| {
let gb =
lookup_accession_taxids(dir.path(), queries.clone(), true, false, None).unwrap();
assert_eq!(
gb,
HashMap::from([
("NC_000001.1".to_owned(), 13),
("NC_000002.1".to_owned(), 15)
])
);
let both =
lookup_accession_taxids(dir.path(), queries.clone(), true, true, None).unwrap();
assert_eq!(both["NC_000001.1"], 99); assert_eq!(both["ABCD01000001.1"], 15);
assert_eq!(
lookup_taxid_accessions(dir.path(), &[15, 15], true, false, None).unwrap(),
HashSet::from(["NC_000002.1".to_owned()])
);
assert_eq!(
lookup_taxid_accessions(dir.path(), &[15], true, true, None).unwrap(),
HashSet::from(["NC_000002.1".to_owned(), "ABCD01000001.1".to_owned()])
);
});
}
}
#[test]
fn full_scans_reject_corrupt_or_truncated_gzip() {
let dir = tempfile::tempdir().unwrap();
let source = dir.path().join(GB_FILE);
write_a2t_fixture(&source, &[("NC_000001.1", 13)]);
let original = fs::read(&source).unwrap();
let mut corrupt = original.clone();
let checksum = corrupt.len() - 8;
corrupt[checksum] ^= 0xff;
for invalid in [corrupt, original[..original.len() - 4].to_vec()] {
fs::write(&source, invalid).unwrap();
assert!(
lookup_accession_taxids(
dir.path(),
HashSet::from(["missing".to_owned()]),
true,
false
, None)
.is_err()
);
assert!(lookup_taxid_accessions(dir.path(), &[13], true, false, None).is_err());
}
}
#[test]
fn parallel_scan_handles_multiple_buffers_and_early_stop() {
let dir = tempfile::tempdir().unwrap();
let source = dir.path().join(GB_FILE);
let mut encoder = flate2::write::GzEncoder::new(
File::create(&source).unwrap(),
flate2::Compression::fast(),
);
writeln!(encoder, "accession\taccession.version\ttaxid\tgi").unwrap();
for index in 0..100_000 {
writeln!(
encoder,
"NC_{index:09}\tNC_{index:09}.1\t{}\t0",
index % 1000
)
.unwrap();
}
writeln!(encoder, "NC_bad\tNC_bad.1\tinvalid_taxid\t0").unwrap();
encoder.finish().unwrap();
let pool = rayon::ThreadPoolBuilder::new()
.num_threads(4)
.build()
.unwrap();
pool.install(|| {
let found = lookup_accession_taxids(
dir.path(),
HashSet::from(["NC_000000000.1".to_owned(), "NC_000099999.1".to_owned()]),
true,
false,
None,
)
.unwrap();
assert_eq!(found["NC_000000000.1"], 0);
assert_eq!(found["NC_000099999.1"], 999);
assert!(lookup_taxid_accessions(dir.path(), &[999], true, false, None).is_err());
});
}
#[test]
fn parallel_gzip_and_batched_sqlite_lookups_are_equivalent() {
let dir = tempfile::tempdir().unwrap();
write_a2t_fixture(
&dir.path().join(GB_FILE),
&[("NC_000001.1", 10), ("NC_000002.1", 20)],
);
write_a2t_fixture(
&dir.path().join(WGS_FILE),
&[("ABCD01000001.1", 30), ("ABCD01000002.1", 20)],
);
let accessions = vec![
"NC_000001.1".to_owned(),
"ABCD01000001.1".to_owned(),
"missing".to_owned(),
];
let token = CancellationToken::default();
let scanned = lookup_a2t(dir.path(), &accessions, true, true, None, &token).unwrap();
let indexed = lookup_a2t(dir.path(), &accessions, false, true, None, &token).unwrap();
let direct = lookup_accession_taxids(
dir.path(),
accessions.iter().cloned().collect(),
false,
true,
None,
)
.unwrap();
assert_eq!(scanned, indexed);
assert_eq!(direct, indexed);
assert_eq!(scanned["NC_000001.1"], 10);
assert_eq!(scanned["ABCD01000001.1"], 30);
let connection = Connection::open(dir.path().join(DB_FILE)).unwrap();
connection
.execute_batch("CREATE TEMP TABLE tmp_accs (accession TEXT PRIMARY KEY) WITHOUT ROWID;")
.unwrap();
let mut plan = connection
.prepare(&format!("EXPLAIN QUERY PLAN {A2T_JOIN_SQL}"))
.unwrap();
let details = plan
.query_map([], |row| row.get::<_, String>(3))
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap();
assert!(
details
.iter()
.any(|detail| detail.contains("SEARCH a USING PRIMARY KEY")),
"forward lookup did not seek the accession primary key: {details:?}"
);
assert!(
!details
.iter()
.any(|detail| detail == "SCAN a" || detail.starts_with("SCAN a ")),
"forward lookup scanned the accession table: {details:?}"
);
let scanned_reverse = lookup_t2a(dir.path(), &[20, 30], true, true, None, &token).unwrap();
let indexed_reverse = lookup_t2a(dir.path(), &[20, 30], false, true, None, &token).unwrap();
assert_eq!(scanned_reverse, indexed_reverse);
assert_eq!(scanned_reverse.len(), 3);
}
#[test]
fn reverse_lookup_searches_taxid_index_and_preserves_set_semantics() {
let dir = tempfile::tempdir().unwrap();
write_a2t_fixture(
&dir.path().join(GB_FILE),
&[
("NC_000001.1", 12059),
("NC_000002.1", 12059),
("NC_000003.1", 20),
],
);
ensure_accession_database(dir.path(), AccessionDatabaseOptions::default()).unwrap();
let connection = Connection::open(dir.path().join(DB_FILE)).unwrap();
connection
.execute_batch(
"CREATE TEMP TABLE tmp_taxa (taxid INTEGER PRIMARY KEY);
INSERT INTO tmp_taxa VALUES (12059);",
)
.unwrap();
let mut plan = connection
.prepare(&format!("EXPLAIN QUERY PLAN {T2A_JOIN_SQL}"))
.unwrap();
let details = plan
.query_map([], |row| row.get::<_, String>(3))
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap();
assert!(
details.iter().any(|detail| {
detail.contains("SEARCH a USING COVERING INDEX idx_taxid")
&& detail.contains("taxid=?")
}),
"reverse lookup must search idx_taxid as a covering index: {details:?}"
);
assert!(
!details
.iter()
.any(|detail| detail == "SCAN a" || detail.starts_with("SCAN a ")),
"reverse lookup scanned the accession table: {details:?}"
);
for taxa in [
vec![12059],
vec![12059, 12059, 99999],
vec![12059, 20],
vec![99999],
vec![],
] {
let scanned = lookup_taxid_accessions(dir.path(), &taxa, true, false, None).unwrap();
let indexed = lookup_taxid_accessions(dir.path(), &taxa, false, false, None).unwrap();
assert_eq!(indexed, scanned, "reverse lookup mismatch for {taxa:?}");
}
assert_eq!(
lookup_taxid_accessions(dir.path(), &[12059], false, false, None).unwrap(),
HashSet::from(["NC_000001.1".to_owned(), "NC_000002.1".to_owned()])
);
}
#[test]
fn database_build_is_atomic_and_retains_sources() {
let dir = tempfile::tempdir().unwrap();
let source = dir.path().join(GB_FILE);
write_a2t_fixture(&source, &[("NC_000001.1", 10), ("NC_000002.1", 20)]);
let db_path =
ensure_accession_database(dir.path(), AccessionDatabaseOptions::default()).unwrap();
assert!(db_path.exists());
assert!(source.exists());
assert!(
fs::read_dir(dir.path()).unwrap().all(|entry| !entry
.unwrap()
.file_name()
.to_string_lossy()
.contains("tmp"))
);
ensure_accession_database(dir.path(), AccessionDatabaseOptions::default()).unwrap();
let connection = Connection::open(db_path).unwrap();
let rows: i64 = connection
.query_row("SELECT COUNT(*) FROM a2t", [], |row| row.get(0))
.unwrap();
assert_eq!(rows, 2);
let indexes = connection
.prepare("SELECT name FROM sqlite_master WHERE type = 'index' AND name LIKE 'idx_%'")
.unwrap()
.query_map([], |row| row.get::<_, String>(0))
.unwrap()
.collect::<rusqlite::Result<HashSet<_>>>()
.unwrap();
assert_eq!(indexes, HashSet::from(["idx_taxid".to_owned()]));
}
#[test]
fn failed_atomic_build_preserves_existing_database() {
let dir = tempfile::tempdir().unwrap();
let db_path = dir.path().join(DB_FILE);
fs::write(&db_path, b"existing database sentinel").unwrap();
let invalid_source = dir.path().join(GB_FILE);
fs::write(
&invalid_source,
"accession\taccession.version\ttaxid\tgi\nNC_1\tNC_1.1\tnot-a-taxid\t0\n",
)
.unwrap();
assert!(
build_a2t_database_atomic(
&[invalid_source],
&HashMap::new(),
&db_path,
&CancellationToken::default(),
1,
)
.is_err()
);
assert_eq!(fs::read(&db_path).unwrap(), b"existing database sentinel");
}
#[test]
fn empty_database_with_complete_metadata_is_rebuilt() {
let dir = tempfile::tempdir().unwrap();
let db_path = dir.path().join(DB_FILE);
let connection = Connection::open(&db_path).unwrap();
create_a2t_schema(&connection).unwrap();
connection
.execute(
"INSERT INTO a2t_sources(source, status) VALUES (?1, 'complete')",
[GB_FILE],
)
.unwrap();
drop(connection);
write_a2t_fixture(&dir.path().join(GB_FILE), &[("NC_000001.1", 10)]);
ensure_accession_database(dir.path(), AccessionDatabaseOptions::default()).unwrap();
let connection = Connection::open(db_path).unwrap();
let rows: i64 = connection
.query_row("SELECT COUNT(*) FROM a2t", [], |row| row.get(0))
.unwrap();
assert_eq!(rows, 1);
}
#[test]
fn legacy_schema_database_is_rebuilt() {
let dir = tempfile::tempdir().unwrap();
let db_path = dir.path().join(DB_FILE);
let connection = Connection::open(&db_path).unwrap();
connection
.execute_batch(
"CREATE TABLE a2t (accession TEXT, taxid INTEGER);
CREATE TABLE a2t_sources (source TEXT PRIMARY KEY, status TEXT);
INSERT INTO a2t VALUES ('NC_000001.1', 999);
INSERT INTO a2t_sources VALUES ('nucl_gb.accession2taxid.gz', 'complete');
CREATE INDEX idx_accession ON a2t(accession);
CREATE INDEX idx_taxid ON a2t(taxid);",
)
.unwrap();
drop(connection);
assert!(inspect_a2t_database(&db_path).unwrap().must_rebuild);
write_a2t_fixture(&dir.path().join(GB_FILE), &[("NC_000001.1", 10)]);
ensure_accession_database(dir.path(), AccessionDatabaseOptions::default()).unwrap();
let connection = Connection::open(&db_path).unwrap();
assert_eq!(schema_version(&connection).unwrap(), Some(SCHEMA_VERSION));
let taxid: TaxonId = connection
.query_row(
"SELECT taxid FROM a2t WHERE accession = 'NC_000001.1'",
[],
|row| row.get(0),
)
.unwrap();
assert_eq!(taxid, 10, "stale row from the legacy database survived");
let indexes = connection
.prepare("SELECT name FROM sqlite_master WHERE type = 'index' AND name LIKE 'idx_%'")
.unwrap()
.query_map([], |row| row.get::<_, String>(0))
.unwrap()
.collect::<rusqlite::Result<HashSet<_>>>()
.unwrap();
assert_eq!(indexes, HashSet::from(["idx_taxid".to_owned()]));
}
#[test]
fn merge_interleaves_sources_in_key_order_and_collapses_duplicates() {
let dir = tempfile::tempdir().unwrap();
write_a2t_fixture(
&dir.path().join(GB_FILE),
&[("A00001.1", 1), ("AB000001.1", 2), ("NC_000001.1", 3)],
);
write_a2t_fixture(
&dir.path().join(WGS_FILE),
&[
("AAAA01000001.1", 4),
("AB000001.1", 5),
("ZZZZ01000001.1", 6),
],
);
let paths = vec![dir.path().join(GB_FILE), dir.path().join(WGS_FILE)];
let token = CancellationToken::default();
let mut merge = MergeStream::open(&paths, &token, 1).unwrap();
let mut rows = Vec::new();
while let Some((accession, taxid)) = merge.next_row().unwrap() {
rows.push((accession.to_string(), taxid));
}
assert_eq!(
rows,
vec![
("A00001.1".to_owned(), 1),
("AAAA01000001.1".to_owned(), 4),
("AB000001.1".to_owned(), 5),
("NC_000001.1".to_owned(), 3),
("ZZZZ01000001.1".to_owned(), 6),
]
);
}
#[test]
fn unsorted_source_is_rejected() {
let dir = tempfile::tempdir().unwrap();
let source = dir.path().join(GB_FILE);
write_a2t_fixture(&source, &[("NC_000002.1", 10), ("NC_000001.1", 20)]);
let token = CancellationToken::default();
let mut merge = MergeStream::open(&[source], &token, 1).unwrap();
let error = loop {
match merge.next_row() {
Ok(Some(_)) => continue,
Ok(None) => panic!("unsorted source was accepted"),
Err(error) => break error,
}
};
assert!(
format!("{error:#}").contains("not sorted"),
"unexpected error: {error:#}"
);
}
#[test]
fn batched_inserts_cover_tail_duplicates_and_reader_boundaries() {
let dir = tempfile::tempdir().unwrap();
let source = dir.path().join(GB_FILE);
let mut encoder = flate2::write::GzEncoder::new(
File::create(&source).unwrap(), flate2::Compression::fast());
writeln!(encoder, "accession\taccession.version\ttaxid\tgi").unwrap();
let count = MERGE_BATCH_ROWS + 67;
for i in 0..count {
writeln!(encoder, "X{i:09}\tX{i:09}.1\t1\t0").unwrap();
if i == MERGE_BATCH_ROWS - 1 || i == 63 {
writeln!(encoder, "X{i:09}\tX{i:09}.1\t2\t0").unwrap();
}
}
encoder.finish().unwrap();
let db = ensure_accession_database(dir.path(),
AccessionDatabaseOptions { threads: Some(2), ..Default::default() }).unwrap();
let connection = Connection::open(db).unwrap();
let mut stmt = connection.prepare("SELECT accession,taxid FROM a2t ORDER BY accession").unwrap();
let actual = stmt.query_map([], |row| Ok((row.get::<_,String>(0)?, row.get::<_,i64>(1)?)))
.unwrap().collect::<rusqlite::Result<Vec<_>>>().unwrap();
assert_eq!(actual.len(), count);
for (i, (key, taxid)) in actual.into_iter().enumerate() {
assert_eq!(key, format!("X{i:09}.1"));
assert_eq!(taxid, if i == MERGE_BATCH_ROWS - 1 || i == 63 {2} else {1});
}
assert_eq!(connection.query_row("PRAGMA integrity_check",[],|r|r.get::<_,String>(0)).unwrap(), "ok");
}
#[test]
fn interrupted_index_and_failed_install_preserve_destination() {
let dir = tempfile::tempdir().unwrap();
let db = dir.path().join("installed");
fs::create_dir(&db).unwrap(); fs::write(db.join("sentinel"), b"original").unwrap();
let source = dir.path().join(GB_FILE);
write_a2t_fixture(&source, &[("A.1", 1)]);
assert!(build_a2t_database_atomic(&[source], &HashMap::new(),
&db, &CancellationToken::default(), 1).is_err());
assert_eq!(fs::read(db.join("sentinel")).unwrap(), b"original");
let connection = Connection::open_in_memory().unwrap();
create_a2t_schema(&connection).unwrap();
connection.execute_batch(
"WITH RECURSIVE seq(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM seq WHERE x<200000)
INSERT INTO a2t SELECT printf('%012d',x),x%100 FROM seq;").unwrap();
let cancel = CancellationToken::default();
let trigger = cancel.clone();
connection.progress_handler(100, Some(move || { trigger.cancel(); true })).unwrap();
assert!(finish_bulk_load(&connection, &cancel).is_err());
assert!(cancel.is_cancelled());
connection.progress_handler(0, None::<fn() -> bool>).unwrap();
assert_eq!(connection.query_row(
"SELECT count(*) FROM sqlite_master WHERE name='idx_taxid'",[],|r|r.get::<_,i64>(0)).unwrap(),0);
}
#[test]
fn compact_batches_preserve_large_keys_and_boundary_duplicates() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join(GB_FILE);
let key = "A".repeat((4 << 20) + 1);
write_a2t_fixture(&path, &[(&key, 1), (&key, 2), ("Z.1", 3)]);
let mut merge = MergeStream::open(&[path], &CancellationToken::default(), 1).unwrap();
let mut accession = String::new();
assert_eq!(merge.next_into(&mut accession).unwrap(), Some(2));
assert_eq!(accession, key);
assert_eq!(merge.next_into(&mut accession).unwrap(), Some(3));
assert_eq!(accession, "Z.1");
assert_eq!(merge.next_into(&mut accession).unwrap(), None);
}
#[test]
fn dropping_a_full_reader_queue_does_not_deadlock() {
let (done_tx, done_rx) = std::sync::mpsc::channel();
thread::spawn(move || {
let (filled_tx, filled_rx) = std::sync::mpsc::channel();
let source = MergeSource::worker(move |sender| {
sender.send(Ok(RowBatch::default())).unwrap();
sender.send(Ok(RowBatch::default())).unwrap();
filled_tx.send(()).unwrap();
let _ = sender.send(Ok(RowBatch::default()));
Ok(())
});
filled_rx.recv().unwrap();
drop(source);
done_tx.send(()).unwrap();
});
done_rx.recv_timeout(Duration::from_secs(5)).expect("reader teardown deadlocked");
}
#[test]
fn failed_wgs_upgrade_preserves_installed_database() {
let dir = tempfile::tempdir().unwrap();
write_a2t_fixture(&dir.path().join(GB_FILE), &[("A.1", 1), ("C.1", 2)]);
let options = AccessionDatabaseOptions { threads: Some(1), ..Default::default() };
let db = ensure_accession_database(dir.path(), options).unwrap();
let before = fs::read(&db).unwrap();
fs::remove_file(dir.path().join(GB_FILE)).unwrap();
write_a2t_fixture(&dir.path().join(WGS_FILE), &[("Z.1", 3), ("B.1", 4)]);
assert!(ensure_accession_database(dir.path(),
AccessionDatabaseOptions { wgs: true, ..options }).is_err());
assert_eq!(fs::read(&db).unwrap(), before);
assert!(fs::read_dir(dir.path()).unwrap().all(|entry|
!entry.unwrap().file_name().to_string_lossy().starts_with(".tmp")));
}
#[test]
fn adding_wgs_rebuilds_once_and_preserves_source_priority() {
let dir = tempfile::tempdir().unwrap();
write_a2t_fixture(
&dir.path().join(GB_FILE),
&[("AB000001.1", 10), ("NC_000001.1", 20)],
);
let db_path = ensure_accession_database(
dir.path(),
AccessionDatabaseOptions {
threads: Some(2),
..Default::default()
},
)
.unwrap();
let connection = Connection::open(&db_path).unwrap();
connection
.execute_batch("CREATE TABLE upgrade_sentinel (value INTEGER);")
.unwrap();
drop(connection);
fs::remove_file(dir.path().join(GB_FILE)).unwrap();
write_a2t_fixture(
&dir.path().join(WGS_FILE),
&[("AAAA01000001.1", 30), ("AB000001.1", 40)],
);
ensure_accession_database(
dir.path(),
AccessionDatabaseOptions {
wgs: true,
threads: Some(2),
..Default::default()
},
)
.unwrap();
let connection = Connection::open(db_path).unwrap();
let marker_survived: bool = connection
.query_row(
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name = 'upgrade_sentinel')",
[],
|row| row.get(0),
)
.unwrap();
assert!(!marker_survived, "WGS addition took the incremental refresh path");
let rows = connection
.prepare("SELECT accession, taxid FROM a2t ORDER BY accession")
.unwrap()
.query_map([], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, TaxonId>(1)?))
})
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap();
assert_eq!(
rows,
vec![
("AAAA01000001.1".to_owned(), 30),
("AB000001.1".to_owned(), 40),
("NC_000001.1".to_owned(), 20),
]
);
let sources = connection
.prepare("SELECT source FROM a2t_sources ORDER BY source")
.unwrap()
.query_map([], |row| row.get::<_, String>(0))
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap();
assert_eq!(sources, vec![GB_FILE.to_owned(), WGS_FILE.to_owned()]);
}
#[test]
fn refresh_applies_row_level_difference() {
let dir = tempfile::tempdir().unwrap();
let source = dir.path().join(GB_FILE);
write_a2t_fixture(
&source,
&[
("NC_000001.1", 10),
("NC_000002.1", 20),
("NC_000003.1", 30),
],
);
let db_path =
ensure_accession_database(dir.path(), AccessionDatabaseOptions::default()).unwrap();
let before = fs::metadata(&db_path).unwrap().modified().unwrap();
write_a2t_fixture(
&source,
&[
("NC_000001.1", 10),
("NC_000002.1", 99),
("NC_000004.1", 40),
],
);
let metas = HashMap::new();
let stats = refresh_a2t_database(
&db_path,
&[source],
&metas,
&CancellationToken::default(),
1,
)
.unwrap();
assert_eq!(stats.inserted, 1);
assert_eq!(stats.updated, 1);
assert_eq!(stats.deleted, 1);
let connection = Connection::open(&db_path).unwrap();
let mut statement = connection
.prepare("SELECT accession, taxid FROM a2t ORDER BY accession")
.unwrap();
let rows = statement
.query_map([], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, TaxonId>(1)?))
})
.unwrap()
.collect::<rusqlite::Result<Vec<_>>>()
.unwrap();
assert_eq!(
rows,
vec![
("NC_000001.1".to_owned(), 10),
("NC_000002.1".to_owned(), 99),
("NC_000004.1".to_owned(), 40),
]
);
let _ = before;
}
#[test]
fn cancelled_build_leaves_no_temporary_and_preserves_destination() {
let dir = tempfile::tempdir().unwrap();
let db_path = dir.path().join(DB_FILE);
fs::write(&db_path, b"existing database sentinel").unwrap();
let source = dir.path().join(GB_FILE);
let rows = (0..200_000)
.map(|index| (format!("NC_{index:09}.1"), 10))
.collect::<Vec<_>>();
write_a2t_fixture(
&source,
&rows
.iter()
.map(|(accession, taxid)| (accession.as_str(), *taxid))
.collect::<Vec<_>>(),
);
let cancellation = CancellationToken::default();
cancellation.cancel();
let result = build_a2t_database_atomic(&[source], &HashMap::new(), &db_path, &cancellation, 1);
assert!(result.is_err(), "cancelled build reported success");
assert_eq!(fs::read(&db_path).unwrap(), b"existing database sentinel");
let leftovers = fs::read_dir(dir.path())
.unwrap()
.filter_map(|entry| entry.ok())
.filter(|entry| entry.file_name().to_string_lossy().starts_with(".tmp"))
.count();
assert_eq!(leftovers, 0, "cancelled build left a temporary database");
}
}