use std::fs::{File, OpenOptions};
use std::io::{BufWriter, Write};
use std::path::{Path, PathBuf};
use serde_json::Value;
use url::Url;
use crate::cli::Format;
use crate::exit::{Code, Failure, Run};
pub struct Item {
pub value: Value,
pub text: Option<String>,
pub bytes: Option<Vec<u8>>,
pub url: Option<Url>,
pub extension: &'static str,
}
impl Item {
pub fn structured(value: Value) -> Item {
Item {
value,
text: None,
bytes: None,
url: None,
extension: "txt",
}
}
pub fn with_url(mut self, url: Url) -> Item {
self.url = Some(url);
self
}
pub fn with_text(mut self, text: String, extension: &'static str) -> Item {
self.text = Some(text);
self.extension = extension;
self
}
}
enum Dest {
Stream(Box<dyn Write>),
Dir(PathBuf),
}
#[derive(Debug, Clone, Copy)]
pub struct FileRules {
pub force: bool,
pub mkdir: bool,
pub append: bool,
}
pub struct Emitter {
format: Format,
dest: Dest,
rules: FileRules,
buffered: Vec<Value>,
report: Option<Value>,
sole: bool,
count: usize,
}
impl Emitter {
pub fn open(
format: Format,
output: Option<&str>,
output_dir: Option<&str>,
rules: FileRules,
) -> Run<Emitter> {
let dest = match (output, output_dir) {
(Some("-"), _) => Dest::Stream(Box::new(std::io::stdout())),
(Some(path), _) => {
Dest::Stream(Box::new(BufWriter::new(open_file(Path::new(path), rules)?)))
}
(None, Some(dir)) => {
let dir = PathBuf::from(dir);
if rules.mkdir {
std::fs::create_dir_all(&dir).map_err(|e| {
Failure::output(format!("could not create {}: {e}", dir.display()))
})?;
}
if !dir.is_dir() {
return Err(Failure::output(format!(
"{} is not a directory. Create it, or pass --mkdir.",
dir.display()
)));
}
Dest::Dir(dir)
}
(None, None) => Dest::Stream(Box::new(std::io::stdout())),
};
Ok(Emitter {
format,
dest,
rules,
buffered: Vec::new(),
report: None,
sole: false,
count: 0,
})
}
pub fn to_stream(format: Format, stream: Box<dyn Write>) -> Emitter {
Emitter {
format,
dest: Dest::Stream(stream),
rules: FileRules {
force: false,
mkdir: false,
append: false,
},
buffered: Vec::new(),
report: None,
sole: false,
count: 0,
}
}
pub fn format(&self) -> Format {
self.format
}
pub fn write(&mut self, item: Item) -> Run<()> {
self.count += 1;
match (&mut self.dest, self.format) {
(Dest::Dir(dir), format) => {
let name = file_name(item.url.as_ref(), self.count, format, item.extension);
let path = dir.join(name);
let mut file = open_file(
&path,
FileRules {
append: false,
..self.rules
},
)?;
write_payload(&mut file, &item, format)?;
file.flush().map_err(Failure::from)
}
(Dest::Stream(stream), Format::Json) => {
let _ = stream;
self.buffered.push(item.value);
Ok(())
}
(Dest::Stream(stream), format) => {
write_payload(stream, &item, format)?;
stream.flush().map_err(Failure::from)
}
}
}
pub fn write_sole(&mut self, item: Item) -> Run<()> {
self.sole = true;
self.write(item)
}
pub fn write_report(&mut self, item: Item) -> Run<()> {
if self.format == Format::Json {
self.report = Some(item.value);
return Ok(());
}
self.write(item)
}
pub fn finish(&mut self) -> Run<()> {
if let (Dest::Stream(stream), Format::Json) = (&mut self.dest, self.format) {
let document = if self.sole && self.buffered.len() == 1 {
self.buffered.remove(0)
} else {
let mut document = serde_json::Map::new();
document.insert(
"items".to_string(),
Value::Array(std::mem::take(&mut self.buffered)),
);
if let Some(report) = self.report.take() {
document.insert("report".to_string(), report);
}
Value::Object(document)
};
let encoded = serde_json::to_string(&document)
.map_err(|e| Failure::new(Code::Failed, format!("could not encode: {e}")))?;
stream.write_all(encoded.as_bytes())?;
stream.write_all(b"\n")?;
}
if let Dest::Stream(stream) = &mut self.dest {
stream.flush()?;
}
Ok(())
}
}
fn write_payload(out: &mut dyn Write, item: &Item, format: Format) -> Run<()> {
match format {
Format::Text => match (&item.bytes, &item.text) {
(Some(bytes), _) => out.write_all(bytes).map_err(Failure::from),
(None, Some(text)) => {
out.write_all(text.as_bytes())?;
if !text.ends_with('\n') {
out.write_all(b"\n")?;
}
Ok(())
}
(None, None) => Ok(()),
},
Format::Json | Format::Ndjson => {
let encoded = serde_json::to_string(&item.value)
.map_err(|e| Failure::new(Code::Failed, format!("could not encode: {e}")))?;
out.write_all(encoded.as_bytes())?;
out.write_all(b"\n").map_err(Failure::from)
}
}
}
fn open_file(path: &Path, rules: FileRules) -> Run<File> {
if rules.mkdir {
if let Some(parent) = path.parent() {
if !parent.as_os_str().is_empty() {
std::fs::create_dir_all(parent).map_err(|e| {
Failure::output(format!("could not create {}: {e}", parent.display()))
})?;
}
}
}
if path.exists() && !rules.force && !rules.append {
return Err(Failure::output(format!(
"{} is already there. Pass --force to replace it, or --append to add to it.",
path.display()
)));
}
let mut options = OpenOptions::new();
options.write(true).create(true);
if rules.append {
options.append(true);
} else {
options.truncate(true);
}
options
.open(path)
.map_err(|e| Failure::output(format!("could not open {}: {e}", path.display())))
}
pub fn file_name(url: Option<&Url>, ordinal: usize, format: Format, extension: &str) -> String {
let ext = match format {
Format::Json | Format::Ndjson => "json",
Format::Text => extension,
};
match url {
Some(url) => format!("{}-{}.{ext}", slug(url), short_hash(url.as_str())),
None => format!("item-{ordinal:06}.{ext}"),
}
}
pub fn slug(url: &Url) -> String {
let mut out = String::new();
let host = url.host_str().unwrap_or("address");
push_safe(&mut out, host);
let path = url.path();
if path != "/" {
push_safe(&mut out, path);
}
if let Some(query) = url.query() {
push_safe(&mut out, query);
}
let trimmed: String = out.trim_matches('-').chars().take(80).collect();
let trimmed = trimmed.trim_matches('-').to_string();
if trimmed.is_empty() {
"address".to_string()
} else {
trimmed
}
}
fn push_safe(out: &mut String, text: &str) {
for ch in text.chars() {
if ch.is_ascii_alphanumeric() {
out.push(ch.to_ascii_lowercase());
} else if !out.ends_with('-') {
out.push('-');
}
}
}
pub fn short_hash(text: &str) -> String {
let mut hash: u64 = 0xcbf2_9ce4_8422_2325;
for byte in text.as_bytes() {
hash ^= u64::from(*byte);
hash = hash.wrapping_mul(0x0000_0100_0000_01b3);
}
format!("{:08x}", (hash >> 32) as u32)
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
fn url(text: &str) -> Url {
Url::parse(text).expect("a url")
}
#[test]
fn a_name_is_the_same_on_every_run() {
let one = file_name(Some(&url("https://example.com/a/b")), 1, Format::Text, "md");
let two = file_name(Some(&url("https://example.com/a/b")), 9, Format::Text, "md");
assert_eq!(one, two);
assert_eq!(one, "example-com-a-b-36964733.md");
}
#[test]
fn a_name_carries_no_separator() {
let name = file_name(
Some(&url("https://example.com/../../etc/passwd")),
1,
Format::Json,
"json",
);
assert!(!name.contains('/'), "{name}");
assert!(!name.contains(".."), "{name}");
assert!(name.ends_with(".json"), "{name}");
}
#[test]
fn two_addresses_that_flatten_alike_keep_two_files() {
let one = file_name(
Some(&url("https://example.com/a?b=1")),
1,
Format::Text,
"md",
);
let two = file_name(
Some(&url("https://example.com/a-b-1")),
1,
Format::Text,
"md",
);
assert_ne!(one, two, "the hash is what keeps these apart");
}
#[test]
fn the_extension_follows_the_format() {
let json = file_name(Some(&url("https://example.com/")), 1, Format::Ndjson, "md");
assert!(json.ends_with(".json"), "{json}");
}
#[test]
fn an_item_with_no_address_is_numbered() {
assert_eq!(
file_name(None, 7, Format::Text, "txt"),
"item-000007.txt".to_string()
);
}
}