fastx/format.rs
1//! Format and compression detection.
2
3use std::fmt;
4use std::path::Path;
5
6/// A sequence file format.
7///
8/// Deliberately *not* `#[non_exhaustive]`: FASTA and FASTQ are the whole domain
9/// of this crate, so matching on both arms exhaustively is meant to be pleasant
10/// and is a promise we intend to keep.
11#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
12pub enum Format {
13 /// FASTA: `>id description` followed by one or more sequence lines.
14 Fasta,
15 /// FASTQ: `@id description`, sequence, `+`, quality.
16 Fastq,
17}
18
19impl Format {
20 /// Recognised FASTA extensions (without the leading dot).
21 pub const FASTA_EXTENSIONS: &'static [&'static str] = &[
22 "fa", "fasta", "fna", "faa", "ffn", "frn", "fas", "mpfa", "seq",
23 ];
24
25 /// Recognised FASTQ extensions (without the leading dot).
26 pub const FASTQ_EXTENSIONS: &'static [&'static str] = &["fq", "fastq"];
27
28 /// Infer the format from a file extension, ignoring a trailing `.gz`/`.bgz`/`.z`.
29 ///
30 /// ```
31 /// use fastx::Format;
32 /// assert_eq!(Format::from_path("reads.fq.gz"), Some(Format::Fastq));
33 /// assert_eq!(Format::from_path("genome.fna"), Some(Format::Fasta));
34 /// assert_eq!(Format::from_path("notes.txt"), None);
35 /// ```
36 pub fn from_path<P: AsRef<Path>>(path: P) -> Option<Format> {
37 let path = path.as_ref();
38 let ext = path.extension()?.to_str()?.to_ascii_lowercase();
39 if matches!(ext.as_str(), "gz" | "bgz" | "gzip" | "z" | "zst" | "bz2") {
40 let stem = path.file_stem()?;
41 return Format::from_path(Path::new(stem));
42 }
43 Format::from_extension(&ext)
44 }
45
46 /// Infer the format from a bare extension such as `"fasta"`.
47 pub fn from_extension(ext: &str) -> Option<Format> {
48 let ext = ext.trim_start_matches('.').to_ascii_lowercase();
49 if Format::FASTA_EXTENSIONS.contains(&ext.as_str()) {
50 Some(Format::Fasta)
51 } else if Format::FASTQ_EXTENSIONS.contains(&ext.as_str()) {
52 Some(Format::Fastq)
53 } else {
54 None
55 }
56 }
57
58 /// Infer the format from the first meaningful byte of a stream.
59 ///
60 /// ```
61 /// use fastx::Format;
62 /// assert_eq!(Format::from_first_byte(b'>'), Some(Format::Fasta));
63 /// assert_eq!(Format::from_first_byte(b'@'), Some(Format::Fastq));
64 /// ```
65 pub fn from_first_byte(byte: u8) -> Option<Format> {
66 match byte {
67 b'>' | b';' => Some(Format::Fasta),
68 b'@' => Some(Format::Fastq),
69 _ => None,
70 }
71 }
72
73 /// The byte that starts a record header in this format.
74 pub const fn header_byte(self) -> u8 {
75 match self {
76 Format::Fasta => b'>',
77 Format::Fastq => b'@',
78 }
79 }
80
81 /// The canonical extension used when creating files.
82 pub const fn extension(self) -> &'static str {
83 match self {
84 Format::Fasta => "fasta",
85 Format::Fastq => "fastq",
86 }
87 }
88
89 /// Whether records in this format carry per-base quality scores.
90 pub const fn has_quality(self) -> bool {
91 matches!(self, Format::Fastq)
92 }
93}
94
95impl fmt::Display for Format {
96 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
97 match self {
98 Format::Fasta => f.write_str("FASTA"),
99 Format::Fastq => f.write_str("FASTQ"),
100 }
101 }
102}
103
104/// Compression applied to a sequence file.
105///
106/// Marked `#[non_exhaustive]`: match with a `_` arm. New containers keep
107/// appearing — this list has already grown twice — and each one should be a
108/// minor release rather than a breaking one.
109#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
110#[non_exhaustive]
111pub enum Compression {
112 /// Plain, uncompressed bytes.
113 #[default]
114 None,
115 /// One deflate stream, as `gzip` produces. Readable start to finish only.
116 Gzip,
117 /// Block-compressed gzip, as `bgzip` produces.
118 ///
119 /// Valid gzip that any tool can decompress, but split into independent
120 /// members so that a reader with a `.gzi` index can seek into it. Costs a
121 /// percent or two of compression ratio and is what the samtools ecosystem
122 /// expects, so it is the default for compressed output.
123 Bgzf,
124 /// Zstandard (requires the `zstd` feature).
125 ///
126 /// Compresses faster and smaller than gzip, which makes it attractive for
127 /// intermediate files, but the wider bioinformatics toolchain does not read
128 /// it and a plain zstd frame cannot be randomly accessed — so a reference
129 /// genome still wants [`Compression::Bgzf`].
130 Zstd,
131}
132
133impl Compression {
134 /// The first two bytes of a gzip member.
135 pub const GZIP_MAGIC: [u8; 2] = [0x1f, 0x8b];
136
137 /// The four-byte magic number of a Zstandard frame.
138 pub const ZSTD_MAGIC: [u8; 4] = [0x28, 0xb5, 0x2f, 0xfd];
139
140 /// Infer compression from a file extension.
141 ///
142 /// Note that `.gz` maps to [`Compression::Gzip`] rather than
143 /// [`Compression::Bgzf`]: an extension cannot tell the two apart, and only
144 /// the bytes can. Writers pick BGZF for `.gz` deliberately; readers sniff.
145 pub fn from_path<P: AsRef<Path>>(path: P) -> Compression {
146 match path
147 .as_ref()
148 .extension()
149 .and_then(|e| e.to_str())
150 .map(|e| e.to_ascii_lowercase())
151 .as_deref()
152 {
153 Some("gz") | Some("bgz") | Some("gzip") => Compression::Gzip,
154 Some("zst") | Some("zstd") => Compression::Zstd,
155 _ => Compression::None,
156 }
157 }
158
159 /// Infer compression from the leading bytes of a stream.
160 ///
161 /// Does not distinguish plain gzip from BGZF — both report
162 /// [`Compression::Gzip`], since telling them apart means parsing the extra
163 /// field. Use [`crate::bgzf::is_bgzf`] for that.
164 pub fn from_magic(bytes: &[u8]) -> Compression {
165 if bytes.len() >= 2 && bytes[..2] == Compression::GZIP_MAGIC {
166 Compression::Gzip
167 } else if bytes.len() >= 4 && bytes[..4] == Compression::ZSTD_MAGIC {
168 Compression::Zstd
169 } else {
170 Compression::None
171 }
172 }
173}
174
175/// gzip compression level used when writing.
176///
177/// The level dominates the cost of writing compressed output — far more than
178/// parsing does. On an 81 MiB FASTQ, one run took 2.8 s at level 1 and 18.6 s at
179/// level 6, for output of 42.2 MB versus 38.9 MB: **6.5× the time to save 8%**.
180/// [`CompressionLevel::FAST`] is almost always the right choice for files that
181/// another pipeline stage will immediately read back.
182#[derive(Debug, Clone, Copy, PartialEq, Eq)]
183pub struct CompressionLevel(pub u32);
184
185impl CompressionLevel {
186 /// No compression, fastest. Still a valid gzip stream.
187 pub const NONE: CompressionLevel = CompressionLevel(0);
188 /// Fast, slightly larger output — what pipeline intermediates want.
189 pub const FAST: CompressionLevel = CompressionLevel(1);
190 /// gzip's own default, and this crate's, for consistency with other tools.
191 pub const DEFAULT: CompressionLevel = CompressionLevel(6);
192 /// Smallest output, slowest — for data you write once and archive.
193 pub const BEST: CompressionLevel = CompressionLevel(9);
194}
195
196impl Default for CompressionLevel {
197 fn default() -> Self {
198 CompressionLevel::DEFAULT
199 }
200}
201
202#[cfg(test)]
203mod tests {
204 use super::*;
205
206 #[test]
207 fn detects_format_from_extensions() {
208 assert_eq!(Format::from_path("a.fa"), Some(Format::Fasta));
209 assert_eq!(Format::from_path("a.FASTA"), Some(Format::Fasta));
210 assert_eq!(Format::from_path("a.faa"), Some(Format::Fasta));
211 assert_eq!(Format::from_path("/tmp/x/a.fna.gz"), Some(Format::Fasta));
212 assert_eq!(Format::from_path("a.fq"), Some(Format::Fastq));
213 assert_eq!(Format::from_path("a.fastq.gz"), Some(Format::Fastq));
214 assert_eq!(Format::from_path("a.gz"), None);
215 assert_eq!(Format::from_path("a"), None);
216 }
217
218 #[test]
219 fn detects_compression() {
220 assert_eq!(Compression::from_path("a.fq.gz"), Compression::Gzip);
221 assert_eq!(Compression::from_path("a.fq"), Compression::None);
222 assert_eq!(
223 Compression::from_magic(&[0x1f, 0x8b, 0x08]),
224 Compression::Gzip
225 );
226 assert_eq!(Compression::from_magic(b">seq"), Compression::None);
227 assert_eq!(Compression::from_magic(&[0x1f]), Compression::None);
228 }
229
230 #[test]
231 fn detects_zstd() {
232 assert_eq!(Compression::from_path("a.fq.zst"), Compression::Zstd);
233 assert_eq!(Compression::from_path("a.fa.zstd"), Compression::Zstd);
234 // The format still resolves through a zstd suffix.
235 assert_eq!(Format::from_path("reads.fq.zst"), Some(Format::Fastq));
236
237 assert_eq!(
238 Compression::from_magic(&[0x28, 0xb5, 0x2f, 0xfd, 0x00]),
239 Compression::Zstd
240 );
241 // Three bytes of the magic are not enough to claim a match.
242 assert_eq!(
243 Compression::from_magic(&[0x28, 0xb5, 0x2f]),
244 Compression::None
245 );
246 }
247}