combiner 0.2.2

Combiner is a Rust-based command-line tool that processes text files in a given directory, combining their contents into a single output file.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

use combiner::{combine_files, CombinerConfig};
use structopt::StructOpt;

#[derive(Debug, StructOpt)]
#[structopt(name = "combiner", about = "Combines files in a directory")]
struct Opt {
    /// Input directory to process
    #[structopt(short, long, default_value = ".")]
    directory: String,

    /// Output file path/name
    #[structopt(short, long, default_value = "combined_output.txt")]
    output: String,

    /// Tokenizer to use
    #[structopt(
        short,
        long,
        default_value = "p50k_base",
        possible_values = &["o200k_base", "cl100k_base", "p50k_base", "p50k_edit", "r50k_base"]
    )]
    tokenizer: String,
}

fn main() -> anyhow::Result<()> {
    let opt = Opt::from_args();
    let config = CombinerConfig::new(opt.directory, opt.output, opt.tokenizer);
    let stats = combine_files(&config)?;
    stats.print();
    Ok(())
}