sux 0.14.0

A pure Rust implementation of succinct and compressed data structures
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123

/*
 * SPDX-FileCopyrightText: 2024 Sebastiano Vigna
 *
 * SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
 */
use std::{borrow::Borrow, io::BufRead};

use anyhow::{Context, Result, bail};
use clap::Parser;
use dsi_progress_logger::*;
use epserde::ser::Serialize;
use lender::FallibleLender;
use sux::{init_env_logger, prelude::*, utils::DekoBufLineLender};

/// Macro to handle the repeated pattern of checking args.unsorted
/// and calling a function with the appropriate const generic parameter.
/// When args.unsorted is true, we pass false for SORTED (meaning not sorted).
/// When args.unsorted is false, we pass true for SORTED.
macro_rules! call_with_sorted {
    // For compress function with single type parameter
    ($unsorted:expr, compress, $($args:expr),*) => {
        if $unsorted {
            compress::<_, false>($($args),*)
        } else {
            compress::<_, true>($($args),*)
        }
    };
    // For rear_coded_list::store with one type parameter and one const generic
    ($unsorted:expr, rear_coded_list::store_str, $($args:expr),*) => {
        if $unsorted {
            rear_coded_list::store_str::<_,_, false>($($args),*)
        } else {
            rear_coded_list::store_str::<_,_,  true>($($args),*)
        }
    };
}

#[derive(Parser, Debug)]
#[command(about = "Builds a rear-coded list starting from a list of UTF-8 encoded strings.", long_about = None, next_line_help = true, max_term_width = 100)]
struct Args {
    /// A file containing UTF-8 strings, one per line, or - for standard input; it can be compressed with any format supported by the deko crate.​
    source: String,
    /// A name for the ε-serde serialized rear-coded list.​
    dest: String,
    /// Assume the input is not sorted (disables checks and indexing).​
    #[arg(long)]
    unsorted: bool,
    /// The number of strings in a block: higher values provide more compression
    /// at the expense of slower access.​
    #[arg(short = 'r', long, default_value_t = 8)]
    ratio: usize,
    /// Use the slower direct-to-disk construction algorithm, which uses very little memory (cannot be used with stdin input).​
    #[arg(long)]
    low_mem: bool,
}

fn compress<R: BufRead, const SORTED: bool>(
    mut lender: DekoBufLineLender<R>,
    dest: impl Borrow<str>,
    ratio: usize,
) -> Result<()> {
    let mut rclb = RearCodedListBuilder::<str, SORTED>::new(ratio);

    let mut pl = ProgressLogger::default();
    pl.display_memory(true);
    pl.start("Reading the input file...");

    loop {
        match lender.next() {
            Ok(None) => break,
            Ok(Some(line)) => {
                rclb.push(line);
            }
            Err(e) => {
                pl.info(format_args!("Error reading line: {}", e));
                return Result::Err(e.into());
            }
        }
        pl.light_update();
    }

    pl.done();

    rclb.print_stats();

    let rcl = rclb.build();
    let dst_file = std::fs::File::create(dest.borrow())
        .with_context(|| format!("cannot create file '{}'", dest.borrow()))?;
    let mut dst_file = std::io::BufWriter::new(dst_file);
    unsafe {
        rcl.serialize(&mut dst_file)
            .context("cannot serialize rear-coded list")?
    };
    Ok(())
}

fn main() -> Result<()> {
    init_env_logger()?;

    let args = Args::parse();

    if args.low_mem {
        if args.source == "-" {
            bail!("low-memory mode cannot read from standard input");
        }
        let lender = DekoBufLineLender::from_path(&args.source)?;
        call_with_sorted!(
            args.unsorted,
            rear_coded_list::store_str,
            args.ratio,
            lender,
            args.dest
        )?;
    } else if args.source == "-" {
        let stdin = DekoBufLineLender::new(std::io::BufReader::new(std::io::stdin().lock()))?;
        call_with_sorted!(args.unsorted, compress, stdin, args.dest, args.ratio)?;
    } else {
        let lender = DekoBufLineLender::from_path(&args.source)?;
        call_with_sorted!(args.unsorted, compress, lender, args.dest, args.ratio)?;
    }

    Ok(())
}