dino-seq 0.1.0

Low-allocation FASTQ and FASTA parser core
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127

//! Optional memory-mapped resident visitors.

use std::fs::File;
use std::path::Path;

use memmap2::Mmap;

use crate::{
    FastaStats, FastaVisitRecord, FastqConfig, FastqStats, FastqVisitRecord, Result,
    count_fasta_bytes, count_fastq_bytes, visit_fasta_bytes, visit_fastq_bytes,
};

/// Visit FASTQ records from a memory-mapped file.
///
/// This is intended for already-resident local files where mapping avoids an
/// extra read buffer and lets callers use the resident byte-slice parser.
pub fn visit_fastq_mmap<F>(path: impl AsRef<Path>, config: FastqConfig, visit: F) -> Result<u64>
where
    F: FnMut(FastqVisitRecord<'_>) -> Result<()>,
{
    let map = map_file(path)?;
    visit_fastq_bytes(&map, config, visit)
}

/// Count FASTQ records and bases from a memory-mapped file.
pub fn count_fastq_mmap(path: impl AsRef<Path>, config: FastqConfig) -> Result<FastqStats> {
    let map = map_file(path)?;
    count_fastq_bytes(&map, config)
}

/// Visit FASTA records from a memory-mapped file.
pub fn visit_fasta_mmap<F>(path: impl AsRef<Path>, visit: F) -> Result<u64>
where
    F: FnMut(FastaVisitRecord<'_>) -> Result<()>,
{
    let map = map_file(path)?;
    visit_fasta_bytes(&map, visit)
}

/// Count FASTA records and bases from a memory-mapped file.
pub fn count_fasta_mmap(path: impl AsRef<Path>) -> Result<FastaStats> {
    let map = map_file(path)?;
    count_fasta_bytes(&map)
}

fn map_file(path: impl AsRef<Path>) -> Result<Mmap> {
    let file = File::open(path)?;
    // SAFETY: The map is read-only and kept alive for the duration of parsing.
    // Callers must not mutate the file concurrently through another handle.
    unsafe { Mmap::map(&file) }.map_err(Into::into)
}

#[cfg(test)]
mod tests {
    use std::fs;
    use std::path::PathBuf;

    use super::*;
    use crate::{
        FastqConfig, count_fasta_bytes, count_fastq_bytes, visit_fasta_bytes, visit_fastq_bytes,
    };

    fn temp_file(name: &str, bytes: &[u8]) -> PathBuf {
        let path = std::env::temp_dir().join(format!(
            "dino_seq-mmap-{name}-{}-{}",
            std::process::id(),
            std::time::SystemTime::now()
                .duration_since(std::time::UNIX_EPOCH)
                .unwrap()
                .as_nanos()
        ));
        fs::write(&path, bytes).unwrap();
        path
    }

    #[test]
    fn fastq_mmap_matches_resident_visitor() {
        let input = b"@r1\nACGT\n+\nIIII\n@r2\nTG\n+\n!!\n";
        let path = temp_file("fastq", input);
        let mut mmap_records = Vec::new();
        let mut resident_records = Vec::new();

        let mmap_count = visit_fastq_mmap(&path, FastqConfig::default(), |record| {
            mmap_records.push((record.name().to_vec(), record.seq().to_vec()));
            Ok(())
        })
        .unwrap();
        let resident_count = visit_fastq_bytes(input, FastqConfig::default(), |record| {
            resident_records.push((record.name().to_vec(), record.seq().to_vec()));
            Ok(())
        })
        .unwrap();

        assert_eq!(mmap_count, resident_count);
        assert_eq!(mmap_records, resident_records);
        assert_eq!(
            count_fastq_mmap(&path, FastqConfig::default()).unwrap(),
            count_fastq_bytes(input, FastqConfig::default()).unwrap()
        );
    }

    #[test]
    fn fasta_mmap_matches_resident_visitor_and_counter() {
        let input = b">seq1\nAC\nGT\n>seq2\nTTA\n";
        let path = temp_file("fasta", input);
        let mut mmap_records = Vec::new();
        let mut resident_records = Vec::new();

        let mmap_count = visit_fasta_mmap(&path, |record| {
            mmap_records.push((record.id_token().to_vec(), record.seq().to_vec()));
            Ok(())
        })
        .unwrap();
        let resident_count = visit_fasta_bytes(input, |record| {
            resident_records.push((record.id_token().to_vec(), record.seq().to_vec()));
            Ok(())
        })
        .unwrap();

        assert_eq!(mmap_count, resident_count);
        assert_eq!(mmap_records, resident_records);
        assert_eq!(
            count_fasta_mmap(&path).unwrap(),
            count_fasta_bytes(input).unwrap()
        );
    }
}