use std::fs::{self, File, OpenOptions};
use std::io::{self, BufRead, BufReader, BufWriter, Read, Write};
use std::path::Path;
use std::str;
use clap::Parser;
use indexmap::IndexSet;
use anew::natsort;
#[derive(Parser, Debug)]
#[command(author = "zer0yu", version, about = "A tool for adding new lines to files, skipping duplicates", long_about = None)]
struct Options {
#[arg(short, long, help = "Do not output new lines to stdout")]
quiet_mode: bool,
#[arg(short, long, help = "Sort lines (natsort)")]
sort: bool,
#[arg(short, long, help = "Trim whitespaces")]
trim: bool,
#[arg(
short,
long,
help = "Rewrite existing destination file to remove duplicates"
)]
rewrite: bool,
#[arg(
short,
long,
help = "Do not write to file, only output what would be written"
)]
dry_run: bool,
#[arg(help = "Destination file")]
filepath: Option<String>,
}
fn main() -> io::Result<()> {
let args = Options::parse();
let mut existing_lines = IndexSet::new();
let mut added_lines = Vec::new();
if let Some(filepath) = &args.filepath {
if Path::new(filepath).exists() {
if let Ok(lines) = load_file_content(filepath) {
for line in lines {
let processed_line = if args.trim {
line.trim().to_string()
} else {
line
};
existing_lines.insert(processed_line);
}
}
}
}
let stdin = io::stdin();
let mut handle = stdin.lock();
let mut input = String::new();
handle.read_to_string(&mut input)?;
let lines_iter = input.split('\n');
for line_raw in lines_iter {
let line = if args.trim {
line_raw.trim().to_string()
} else {
line_raw.to_string()
};
if line.is_empty() {
continue;
}
if !existing_lines.contains(&line) {
existing_lines.insert(line.clone());
added_lines.push(line);
}
}
if args.sort {
added_lines.sort_by(|a, b| natsort::compare(a, b, false));
}
if !args.dry_run && args.filepath.is_some() {
let filepath = args.filepath.as_ref().unwrap();
if let Some(parent) = Path::new(filepath).parent() {
fs::create_dir_all(parent)?;
}
if args.rewrite {
let mut all_lines: Vec<_> = existing_lines.into_iter().collect();
if args.sort {
all_lines.sort_by(|a, b| natsort::compare(a, b, false));
}
let file = File::create(filepath)?;
let mut writer = BufWriter::new(file);
for line in all_lines {
writeln!(writer, "{}", line)?;
}
} else {
let need_newline = if Path::new(filepath).exists() {
match fs::read(filepath) {
Ok(content) if !content.is_empty() => {
content.last().map_or(false, |&byte| byte != b'\n')
},
_ => false
}
} else {
false
};
let file = OpenOptions::new()
.write(true)
.append(true)
.create(true)
.open(filepath)?;
let mut writer = BufWriter::new(file);
if need_newline {
writeln!(writer)?;
}
for line in &added_lines {
writeln!(writer, "{}", line)?;
}
}
}
if !args.quiet_mode {
for line in added_lines {
println!("{}", line);
}
}
Ok(())
}
fn load_file_content(filepath: &str) -> io::Result<Vec<String>> {
let file = File::open(filepath)?;
let reader = BufReader::new(file);
let mut lines = Vec::new();
for line in reader.lines() {
lines.push(line?);
}
Ok(lines)
}