use std::collections::HashSet;
use std::io::Write;
use std::process::ExitCode;
use fastx::index::{fai_path, FastaIndex, IndexedFasta};
use fastx::{
Alphabet, BoxedWriter, CompressionLevel, Error, Format, PairedReader, Result, SeqStats,
Sequence, WriterBuilder,
};
const USAGE: &str = "\
fastx — fast FASTA/FASTQ toolkit
USAGE:
fastx <COMMAND> [OPTIONS] [FILES...]
COMMANDS:
stats Summary statistics (records, bases, N50, GC%, Q20/Q30)
convert Convert between FASTA and FASTQ, re-wrap, (de)compress
filter Keep records matching length / quality / name criteria
head Write the first N records
sample Take a random subset, reproducibly
dedup Drop repeated records
rc Reverse complement every record
translate Translate nucleotides to protein (standard genetic code)
faidx Build a .fai index, or extract regions from an indexed FASTA
interleave Merge R1 and R2 into one alternating stream
deinterleave Split an interleaved file back into R1 and R2
help Show this message
COMMON OPTIONS:
-o, --output <PATH> Write to PATH (format and gzip from its extension)
-w, --line-width <N> FASTA line width, 0 for one line per record [60]
-t, --to <FORMAT> Force output format: fasta or fastq
-l, --level <0-9> gzip level [6]; 1 is several times faster and only
slightly larger, which is what pipelines want
-@, --threads <N> BGZF blocks to compress at once [cores x 8]; blocks
are independent, so this never changes the output
--validate Reject records with non-printable residues
-h, --help Show this message
STATS OPTIONS:
--json Emit one JSON object per input instead of a table
SAMPLE OPTIONS:
-n <N> Keep exactly N records (reservoir sampling)
--fraction <F> Keep each record with probability F, streaming
--seed <N> Seed, so a run is reproducible [1]
DEDUP OPTIONS:
--by-seq Compare sequences instead of identifiers
PAIRED OPTIONS:
--out1 <PATH> Where deinterleave writes R1
--out2 <PATH> Where deinterleave writes R2
--no-check-names Do not verify that mate names agree
FILTER OPTIONS:
--min-len <N> Minimum sequence length
--max-len <N> Maximum sequence length
--min-qual <Q> Minimum mean Phred quality (FASTQ only)
--max-errors <F> Maximum expected sequencing errors (FASTQ only)
--max-n <N> Maximum number of ambiguous bases
--name-contains <S> Keep records whose id or description contains S
-v, --invert Keep exactly the records that would be dropped
FILES may be FASTA or FASTQ, plain, gzipped or zstd — detected from the
extension and from the magic bytes. `-` or no file means stdin.
EXAMPLES:
fastx stats reads_R1.fq.gz reads_R2.fq.gz
fastx stats --json reads.fq.gz
fastx convert reads.fq.gz -o reads.fa.gz -w 80
fastx filter --min-len 200 --min-qual 20 reads.fq -o clean.fq
fastx sample -n 10000 --seed 42 reads.fq.gz -o subset.fq.gz
fastx rc contigs.fa | fastx translate --stop-at-stop -o proteins.faa
fastx faidx hg38.fa # writes hg38.fa.fai (and .gzi if BGZF)
fastx faidx hg38.fa chr1:1-60 chr2 # extracts regions
fastx interleave R1.fq.gz R2.fq.gz -o both.fq.gz
fastx deinterleave both.fq.gz --out1 R1.fq.gz --out2 R2.fq.gz
";
fn main() -> ExitCode {
let args: Vec<String> = std::env::args().skip(1).collect();
match run(&args) {
Ok(()) => ExitCode::SUCCESS,
Err(Error::Io(io)) if io.kind() == std::io::ErrorKind::BrokenPipe => {
ExitCode::SUCCESS
}
Err(e) => {
eprintln!("fastx: {e}");
ExitCode::FAILURE
}
}
}
fn run(args: &[String]) -> Result<()> {
let command = match args.first().map(String::as_str) {
None | Some("help") | Some("-h") | Some("--help") => {
print!("{USAGE}");
return Ok(());
}
Some("-V") | Some("--version") | Some("version") => {
println!("fastx {}", env!("CARGO_PKG_VERSION"));
return Ok(());
}
Some(command) => command,
};
let mut args = Args::parse(&args[1..]);
if args.flag("-h") || args.flag("--help") {
print!("{USAGE}");
return Ok(());
}
match command {
"stats" => stats(&mut args),
"convert" => convert(&mut args),
"filter" => filter(&mut args),
"head" => head(&mut args),
"sample" => sample(&mut args),
"dedup" => dedup(&mut args),
"interleave" => interleave(&mut args),
"deinterleave" => deinterleave(&mut args),
"rc" => transform(&mut args, None, |record| Ok(record.reverse_complement())),
"translate" => translate(&mut args),
"faidx" => faidx(&mut args),
other => Err(Error::Other(format!(
"unknown command {other:?}; run `fastx help`"
))),
}
}
fn stats(args: &mut Args) -> Result<()> {
let json = args.flag("--json");
let inputs = args.inputs();
args.finish()?;
let mut out = std::io::stdout();
let mut total = SeqStats::new();
let multiple = inputs.len() > 1;
for (i, input) in inputs.iter().enumerate() {
let mut stats = SeqStats::new();
open_input(input)?.for_each_record(|record| {
stats.push(record);
Ok(())
})?;
if json {
writeln!(
out,
"{{\"file\":{},\"stats\":{}}}",
json_string(display_name(input)),
stats.to_json()
)?;
} else {
if i > 0 {
writeln!(out)?;
}
writeln!(out, "== {} ==", display_name(input))?;
writeln!(out, "{stats}")?;
}
if multiple {
total.merge(&stats);
}
}
if multiple {
if json {
writeln!(out, "{{\"file\":\"total\",\"stats\":{}}}", total.to_json())?;
} else {
writeln!(out, "\n== total ==")?;
writeln!(out, "{total}")?;
}
}
Ok(())
}
fn json_string(text: &str) -> String {
let mut out = String::with_capacity(text.len() + 2);
out.push('"');
for c in text.chars() {
match c {
'"' => out.push_str("\\\""),
'\\' => out.push_str("\\\\"),
'\n' => out.push_str("\\n"),
'\r' => out.push_str("\\r"),
'\t' => out.push_str("\\t"),
c if (c as u32) < 0x20 => out.push_str(&format!("\\u{:04x}", c as u32)),
c => out.push(c),
}
}
out.push('"');
out
}
fn sample(args: &mut Args) -> Result<()> {
let count = args.number::<usize>("-n")?;
let fraction = args.number::<f64>("--fraction")?;
let seed = args.number::<u64>("--seed")?.unwrap_or(1);
let forced = args.format_option()?;
let mut output = Output::from_args(args, forced)?;
let inputs = args.inputs();
args.finish()?;
if count.is_some() == fraction.is_some() {
return Err(Error::Unsupported(
"sample needs exactly one of -n and --fraction",
));
}
if let Some(fraction) = fraction {
if !(0.0..=1.0).contains(&fraction) {
return Err(Error::Unsupported("--fraction must be between 0 and 1"));
}
}
let mut rng = Rng::new(seed);
let mut seen = 0u64;
let mut kept = 0u64;
let mut reservoir: Vec<Sequence> = Vec::new();
for input in &inputs {
let mut reader = open_input(input)?;
let mut record = Sequence::default();
while reader.read_into(&mut record)? {
seen += 1;
match (count, fraction) {
(Some(count), _) => {
if reservoir.len() < count {
reservoir.push(record.clone());
} else {
let victim = rng.below(seen);
if (victim as usize) < count {
reservoir[victim as usize] = record.clone();
}
}
}
(None, Some(fraction)) => {
if rng.unit() < fraction {
output.writer(record.format())?.write_record(&record)?;
kept += 1;
}
}
(None, None) => unreachable!("checked above"),
}
}
}
for record in &reservoir {
output.writer(record.format())?.write_record(record)?;
kept += 1;
}
output.finish()?;
eprintln!("fastx sample: kept {kept} of {seen}");
Ok(())
}
fn dedup(args: &mut Args) -> Result<()> {
let by_sequence = args.flag("--by-seq");
let forced = args.format_option()?;
let mut output = Output::from_args(args, forced)?;
let inputs = args.inputs();
args.finish()?;
let mut seen_ids: HashSet<String> = HashSet::new();
let mut seen_seqs: HashSet<Vec<u8>> = HashSet::new();
let (mut kept, mut dropped) = (0u64, 0u64);
for input in &inputs {
let mut reader = open_input(input)?;
let mut record = Sequence::default();
while reader.read_into(&mut record)? {
let fresh = if by_sequence {
seen_seqs.insert(record.seq.clone())
} else {
seen_ids.insert(record.id.clone())
};
if fresh {
output.writer(record.format())?.write_record(&record)?;
kept += 1;
} else {
dropped += 1;
}
}
}
output.finish()?;
eprintln!("fastx dedup: kept {kept}, dropped {dropped}");
Ok(())
}
fn interleave(args: &mut Args) -> Result<()> {
let check_names = !args.flag("--no-check-names");
let forced = args.format_option()?;
let mut output = Output::from_args(args, forced)?;
let inputs = args.inputs();
args.finish()?;
let (first, second) = match inputs.as_slice() {
[first, second] => (first.clone(), second.clone()),
_ => {
return Err(Error::Unsupported(
"interleave needs exactly two input files",
))
}
};
let mut reader = PairedReader::open(&first, &second)?.check_names(check_names);
let mut pairs = 0u64;
let mut pair = fastx::Pair::default();
while reader.read_into(&mut pair)? {
let format = pair.first.format();
let writer = output.writer(format)?;
writer.write_record(&pair.first)?;
writer.write_record(&pair.second)?;
pairs += 1;
}
output.finish()?;
eprintln!("fastx interleave: {pairs} pairs");
Ok(())
}
fn deinterleave(args: &mut Args) -> Result<()> {
let check_names = !args.flag("--no-check-names");
let out1 = args.value("--out1")?;
let out2 = args.value("--out2")?;
let width = args.line_width()?;
let level = args.compression_level()?;
let inputs = args.inputs();
args.finish()?;
let (out1, out2) = match (out1, out2) {
(Some(a), Some(b)) => (a, b),
_ => {
return Err(Error::Unsupported(
"deinterleave needs --out1 and --out2; there is only one stdout",
))
}
};
let mut reader = match inputs.as_slice() {
[single] if single == "-" => PairedReader::from_interleaved(fastx::from_stdin()?),
[single] => PairedReader::open_interleaved(single)?,
_ => return Err(Error::Unsupported("deinterleave takes exactly one input")),
}
.check_names(check_names);
let mut builder = WriterBuilder::new();
if let Some(width) = width {
builder = builder.line_width(width);
}
if let Some(level) = level {
builder = builder.level(level);
}
let format = reader.format().unwrap_or(Format::Fastq);
let mut writer = fastx::PairedWriter::Split {
first: builder.clone().format(format).create(&out1)?,
second: builder.format(format).create(&out2)?,
};
let mut pairs = 0u64;
reader.for_each_pair(|pair| {
writer.write_pair(pair)?;
pairs += 1;
Ok(())
})?;
writer.finish()?;
eprintln!("fastx deinterleave: {pairs} pairs");
Ok(())
}
struct Rng(u64);
impl Rng {
fn new(seed: u64) -> Rng {
Rng(seed)
}
fn next_u64(&mut self) -> u64 {
self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
let mut z = self.0;
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
z ^ (z >> 31)
}
fn unit(&mut self) -> f64 {
(self.next_u64() >> 11) as f64 / (1u64 << 53) as f64
}
fn below(&mut self, bound: u64) -> u64 {
if bound == 0 {
0
} else {
self.next_u64() % bound
}
}
}
fn convert(args: &mut Args) -> Result<()> {
transform(args, None, |record| Ok(record.clone()))
}
fn filter(args: &mut Args) -> Result<()> {
let min_len = args.number::<usize>("--min-len")?.unwrap_or(0);
let max_len = args.number::<usize>("--max-len")?.unwrap_or(usize::MAX);
let min_qual = args.number::<f64>("--min-qual")?;
let max_errors = args.number::<f64>("--max-errors")?;
let max_ambiguous = args.number::<u64>("--max-n")?;
let name_contains = args.value("--name-contains")?;
let invert = args.flag("-v") || args.flag("--invert");
let forced = args.format_option()?;
let mut output = Output::from_args(args, forced)?;
let inputs = args.inputs();
args.finish()?;
let mut kept = 0u64;
let mut dropped = 0u64;
for input in &inputs {
let mut reader = open_input(input)?;
let mut record = Sequence::default();
while reader.read_into(&mut record)? {
let keep = record.len() >= min_len
&& record.len() <= max_len
&& min_qual.map_or(true, |min| record.mean_quality().is_some_and(|q| q >= min))
&& max_errors.map_or(true, |max| {
record.expected_errors().is_some_and(|e| e <= max)
})
&& max_ambiguous.map_or(true, |max| record.ambiguous_count() <= max)
&& name_contains.as_deref().map_or(true, |needle| {
record.id.contains(needle)
|| record
.description
.as_deref()
.is_some_and(|d| d.contains(needle))
});
if keep != invert {
output.writer(record.format())?.write_record(&record)?;
kept += 1;
} else {
dropped += 1;
}
}
}
output.finish()?;
eprintln!("fastx filter: kept {kept}, dropped {dropped}");
Ok(())
}
fn head(args: &mut Args) -> Result<()> {
let limit = match args.number::<u64>("-n")? {
Some(n) => n,
None => args.number::<u64>("--num")?.unwrap_or(10),
};
let forced = args.format_option()?;
let mut output = Output::from_args(args, forced)?;
let inputs = args.inputs();
args.finish()?;
let mut written = 0u64;
for input in &inputs {
if written >= limit {
break;
}
let mut reader = open_input(input)?;
let mut record = Sequence::default();
while written < limit && reader.read_into(&mut record)? {
output.writer(record.format())?.write_record(&record)?;
written += 1;
}
}
output.finish()
}
fn translate(args: &mut Args) -> Result<()> {
let frame = args.number::<usize>("--frame")?.unwrap_or(0);
if frame > 2 {
return Err(Error::Unsupported("--frame must be 0, 1 or 2"));
}
let stop_at_stop = args.flag("--stop-at-stop");
transform(args, Some(Format::Fasta), move |record| {
Ok(record.translate(frame, stop_at_stop))
})
}
fn transform<F>(args: &mut Args, forced: Option<Format>, f: F) -> Result<()>
where
F: Fn(&Sequence) -> Result<Sequence>,
{
let forced = match forced {
Some(format) => Some(format),
None => args.format_option()?,
};
let mut output = Output::from_args(args, forced)?;
let inputs = args.inputs();
args.finish()?;
for input in &inputs {
let mut reader = open_input(input)?;
let mut record = Sequence::default();
while reader.read_into(&mut record)? {
let transformed = f(&record)?;
output
.writer(transformed.format())?
.write_record(&transformed)?;
}
}
output.finish()
}
fn faidx(args: &mut Args) -> Result<()> {
let width = args.line_width()?;
let positional = args.inputs();
args.finish()?;
let (path, regions) = match positional.split_first() {
Some((path, regions)) if path != "-" => (path.clone(), regions.to_vec()),
_ => return Err(Error::Unsupported("faidx needs a FASTA file, not a stream")),
};
if regions.is_empty() {
let index = FastaIndex::build_from_path(&path)?;
let written = index.write_to_path(&path)?;
eprintln!(
"fastx faidx: indexed {} sequences, {} bases -> {}",
index.len(),
index.total_length(),
written.display()
);
if let Some(gzi) = write_gzi(&path)? {
eprintln!("fastx faidx: block index -> {}", gzi.display());
}
return Ok(());
}
if !fai_path(std::path::Path::new(&path)).exists() {
FastaIndex::build_from_path(&path)?.write_to_path(&path)?;
write_gzi(&path)?;
}
let mut fasta = IndexedFasta::open(&path)?;
let mut writer = WriterBuilder::new()
.format(Format::Fasta)
.line_width(width.unwrap_or(60))
.stdout()?;
for region in ®ions {
writer.write_record(&fasta.fetch_locus(region)?)?;
}
writer.finish()?;
Ok(())
}
#[cfg(feature = "gzip")]
fn write_gzi(path: &str) -> Result<Option<std::path::PathBuf>> {
use fastx::bgzf::{gzi_path, GziIndex};
let mut head = [0u8; 128];
let mut file = std::fs::File::open(path)?;
let read = std::io::Read::read(&mut file, &mut head)?;
if !fastx::bgzf::is_bgzf(&head[..read]) {
return Ok(None);
}
if gzi_path(std::path::Path::new(path)).exists() {
return Ok(None);
}
Ok(Some(GziIndex::build_from_path(path)?.write_to_path(path)?))
}
#[cfg(not(feature = "gzip"))]
fn write_gzi(_path: &str) -> Result<Option<std::path::PathBuf>> {
Ok(None)
}
fn open_input(name: &str) -> Result<fastx::BoxedReader> {
if name == "-" {
fastx::from_stdin()
} else {
fastx::open(name)
}
}
fn display_name(name: &str) -> &str {
if name == "-" {
"<stdin>"
} else {
name
}
}
struct Output {
builder: WriterBuilder,
path: Option<String>,
forced: Option<Format>,
writer: Option<BoxedWriter>,
}
impl Output {
fn from_args(args: &mut Args, forced: Option<Format>) -> Result<Output> {
let path = args.value("-o")?.or(args.value("--output")?);
let mut builder = WriterBuilder::new();
if let Some(width) = args.line_width()? {
builder = builder.line_width(width);
}
if let Some(blocks) = args.blocks_per_batch()? {
builder = builder.blocks_per_batch(blocks);
}
if let Some(level) = args.compression_level()? {
builder = builder.level(level);
}
if args.flag("--validate") {
builder = builder.validate(Alphabet::Any);
}
let forced = match forced {
Some(format) => Some(format),
None => path.as_deref().and_then(Format::from_path),
};
Ok(Output {
builder,
path,
forced,
writer: None,
})
}
fn writer(&mut self, fallback: Format) -> Result<&mut BoxedWriter> {
if self.writer.is_none() {
let builder = self.builder.clone().format(self.forced.unwrap_or(fallback));
self.writer = Some(match &self.path {
Some(path) => builder.create(path)?,
None => builder.stdout()?,
});
}
Ok(self.writer.as_mut().expect("just created"))
}
fn finish(mut self) -> Result<()> {
if self.writer.is_none() && self.path.is_some() {
self.writer(Format::Fasta)?;
}
match self.writer {
Some(writer) => writer.finish().map(|_| ()),
None => Ok(()),
}
}
}
fn parse_format(name: &str) -> Result<Format> {
match name.to_ascii_lowercase().as_str() {
"fa" | "fasta" => Ok(Format::Fasta),
"fq" | "fastq" => Ok(Format::Fastq),
other => Err(Error::UnknownFormat {
hint: format!("--to {other:?}"),
}),
}
}
struct Args {
options: Vec<(String, Option<String>)>,
positional: Vec<String>,
}
impl Args {
fn parse(raw: &[String]) -> Args {
let mut options = Vec::new();
let mut positional = Vec::new();
let mut i = 0;
while i < raw.len() {
let arg = &raw[i];
if arg == "--" {
positional.extend_from_slice(&raw[i + 1..]);
break;
}
if arg.starts_with('-') && arg != "-" {
match arg.split_once('=') {
Some((key, value)) => options.push((key.to_string(), Some(value.to_string()))),
None => {
let value = raw
.get(i + 1)
.filter(|next| !next.starts_with('-') || next.as_str() == "-")
.cloned();
if value.is_some() {
i += 1;
}
options.push((arg.to_string(), value));
}
}
} else {
positional.push(arg.clone());
}
i += 1;
}
Args {
options,
positional,
}
}
fn value(&mut self, key: &str) -> Result<Option<String>> {
match self.options.iter().position(|(k, _)| k == key) {
None => Ok(None),
Some(index) => match self.options.remove(index).1 {
Some(value) => Ok(Some(value)),
None => Err(Error::Other(format!("{key} expects a value"))),
},
}
}
fn number<T: std::str::FromStr>(&mut self, key: &str) -> Result<Option<T>> {
match self.value(key)? {
None => Ok(None),
Some(text) => text
.parse::<T>()
.map(Some)
.map_err(|_| Error::Other(format!("{key} expects a number, got {text:?}"))),
}
}
fn flag(&mut self, key: &str) -> bool {
match self.options.iter().position(|(k, _)| k == key) {
None => false,
Some(index) => {
if let Some(value) = self.options.remove(index).1 {
self.positional.push(value);
}
true
}
}
}
fn line_width(&mut self) -> Result<Option<usize>> {
match self.number::<usize>("-w")? {
Some(width) => Ok(Some(width)),
None => self.number::<usize>("--line-width"),
}
}
fn compression_level(&mut self) -> Result<Option<CompressionLevel>> {
let level = match self.number::<u32>("-l")? {
Some(level) => Some(level),
None => self.number::<u32>("--level")?,
};
match level {
None => Ok(None),
Some(level) if level <= 9 => Ok(Some(CompressionLevel(level))),
Some(level) => Err(Error::Other(format!(
"--level must be between 0 and 9, got {level}"
))),
}
}
fn blocks_per_batch(&mut self) -> Result<Option<usize>> {
match self.number::<usize>("-@")? {
Some(blocks) => Ok(Some(blocks)),
None => self.number::<usize>("--threads"),
}
}
fn format_option(&mut self) -> Result<Option<Format>> {
let name = match self.value("-t")? {
Some(name) => Some(name),
None => self.value("--to")?,
};
match name {
Some(name) => parse_format(&name).map(Some),
None => Ok(None),
}
}
fn inputs(&mut self) -> Vec<String> {
if self.positional.is_empty() {
vec!["-".to_string()]
} else {
std::mem::take(&mut self.positional)
}
}
fn finish(&mut self) -> Result<()> {
match self.options.first() {
None => Ok(()),
Some((key, _)) => Err(Error::Other(format!("unknown option {key:?}"))),
}
}
}