mod config_cmd;
mod render;
mod schema;
mod skill;
use std::ffi::OsString;
use std::io::{self, IsTerminal, Read, Write};
use std::path::{Path, PathBuf};
use clap::{CommandFactory, FromArgMatches, Parser, Subcommand, ValueEnum};
use crate::config::Config;
use crate::domain::limits::MAX_INPUT_BYTES;
use crate::domain::{self, Document, Format};
const FORMAT_VERSION: u32 = 2;
const EXIT_FOUND: u8 = 0;
const EXIT_EMPTY: u8 = 1;
const EXIT_ERROR: u8 = 2;
const COMMANDS: [&str; 6] = [
"analyze",
"summarize",
"keyphrases",
"clusters",
"config",
"completions",
];
type Fallible<T> = Result<T, Failure>;
#[derive(Debug)]
enum Failure {
Error {
error: domain::Error,
path: Option<PathBuf>,
},
Output(io::Error),
}
impl Failure {
fn refused(reason: &'static str, message: impl Into<String>) -> Failure {
Failure::Error {
error: domain::Error::InvalidInput {
reason,
message: message.into(),
},
path: None,
}
}
}
impl std::fmt::Display for Failure {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Failure::Error {
error,
path: Some(path),
} => write!(f, "{}: {error}", path.display()),
Failure::Error { error, path: None } => write!(f, "{error}"),
Failure::Output(e) => write!(f, "cannot write the output: {e}"),
}
}
}
impl From<domain::Error> for Failure {
fn from(error: domain::Error) -> Failure {
Failure::Error { error, path: None }
}
}
impl From<domain::DocumentError> for Failure {
fn from(e: domain::DocumentError) -> Failure {
Failure::Error {
error: e.error,
path: e.path,
}
}
}
impl From<io::Error> for Failure {
fn from(e: io::Error) -> Failure {
Failure::Output(e)
}
}
pub fn run(
args: impl IntoIterator<Item = OsString>,
out: &mut dyn Write,
err: &mut dyn Write,
) -> u8 {
let args: Vec<OsString> = args.into_iter().collect();
let cli = match parse(args.iter().cloned()) {
Ok(cli) => cli,
Err(early) => {
let sink: &mut dyn Write = if early.to_stderr {
&mut *err
} else {
&mut *out
};
let _ = write!(sink, "{}", early.text);
if early.to_stderr && asks_for_json(&args) {
let error = domain::Error::InvalidInput {
reason: "usage",
message: early.summary,
};
let _ = write_failure(out, command_named(&args), None, &error, Layout::Pretty);
}
return early.code;
}
};
match execute(&cli, out, err) {
Ok(Outcome::Found) => EXIT_FOUND,
Ok(Outcome::Empty) => EXIT_EMPTY,
Ok(Outcome::Failed) => EXIT_ERROR,
Err(Failure::Output(e)) if e.kind() == io::ErrorKind::BrokenPipe => EXIT_FOUND,
Err(failure) => {
let _ = writeln!(err, "matra: {failure}");
if let (true, Failure::Error { error, .. }) = (cli.json, &failure) {
let (command, input) = failure_context(&cli);
let _ = write_failure(out, Some(command), input.as_deref(), error, Layout::Pretty);
}
EXIT_ERROR
}
}
}
#[derive(Parser)]
#[command(
about = "Text in, structured analysis out.",
long_about = "matra parses text into CoNLL-U structure and measures it. \
It reports what is there; interpretation is yours."
)]
struct Cli {
#[arg(long, global = true)]
json: bool,
#[arg(long, global = true, value_name = "DIR")]
model_dir: Option<PathBuf>,
#[arg(long, short, global = true)]
quiet: bool,
#[arg(long, global = true, value_enum, default_value_t = ColorWhen::Auto, value_name = "WHEN")]
color: ColorWhen,
#[arg(long, global = true, value_name = "NAME", default_value = "<stdin>")]
stdin_filename: String,
#[arg(long, global = true)]
skill: bool,
#[arg(
long,
short = 'r',
global = true,
value_name = "NAME",
num_args = 0..=1,
)]
reference: Option<Option<String>>,
#[arg(
long,
global = true,
value_name = "NAME",
num_args = 0..=1,
conflicts_with = "skill",
)]
schema: Option<Option<String>>,
#[command(subcommand)]
command: Option<Command>,
}
#[derive(Subcommand)]
enum Command {
Analyze {
path: PathBuf,
#[arg(long, short)]
sections: bool,
#[arg(long, value_enum, value_name = "FORMAT")]
format: Option<FormatName>,
#[arg(long, value_enum, value_name = "LEVEL", requires = "json")]
depth: Option<Depth>,
},
Summarize {
path: PathBuf,
#[arg(short)]
n: Option<usize>,
#[arg(long, value_enum)]
method: Option<SummaryMethod>,
#[arg(long, value_enum, value_name = "FORMAT")]
format: Option<FormatName>,
},
Keyphrases {
path: PathBuf,
#[arg(short)]
n: Option<usize>,
#[arg(long, value_enum)]
method: Option<KeyphraseMethod>,
#[arg(long, value_enum, value_name = "FORMAT")]
format: Option<FormatName>,
},
Clusters {
path: PathBuf,
#[arg(long, value_name = "SIMILARITY")]
threshold: Option<f64>,
#[arg(long, value_enum, value_name = "FORMAT")]
format: Option<FormatName>,
},
Config {
#[command(subcommand)]
action: ConfigAction,
},
Completions {
#[arg(value_enum)]
shell: CompletionShell,
},
}
#[derive(Subcommand)]
enum ConfigAction {
Show,
Init {
#[arg(long)]
force: bool,
},
}
#[derive(Copy, Clone, PartialEq, Eq, ValueEnum)]
enum SummaryMethod {
Tfidf,
Textrank,
}
#[derive(Copy, Clone, PartialEq, Eq, ValueEnum)]
enum KeyphraseMethod {
Rake,
Yake,
}
#[derive(Copy, Clone, PartialEq, Eq, ValueEnum)]
enum FormatName {
Markdown,
#[value(name = "plain_text")]
PlainText,
Blocks,
}
#[derive(Copy, Clone, PartialEq, Eq, Debug, ValueEnum)]
enum Depth {
Document,
Paragraph,
Sentence,
Token,
}
#[derive(Copy, Clone, PartialEq, Eq, ValueEnum)]
enum ColorWhen {
Auto,
Always,
Never,
}
#[derive(Copy, Clone, PartialEq, Eq, ValueEnum)]
enum CompletionShell {
Bash,
Zsh,
Fish,
}
struct EarlyExit {
text: String,
to_stderr: bool,
code: u8,
summary: String,
}
fn parse(args: impl IntoIterator<Item = OsString>) -> Result<Cli, EarlyExit> {
let mut command = Cli::command()
.name("matra")
.bin_name("matra")
.version(version_line())
.color(clap::ColorChoice::Never);
let matches = command.try_get_matches_from_mut(args).map_err(early_exit)?;
let cli = Cli::from_arg_matches(&matches).map_err(early_exit)?;
if cli.command.is_none() && !cli.skill && cli.schema.is_none() && cli.reference.is_none() {
return Err(EarlyExit {
text: command.render_help().to_string(),
to_stderr: true,
code: EXIT_ERROR,
summary: "no command given; run `matra --help` for the list".to_string(),
});
}
Ok(cli)
}
fn early_exit(e: clap::Error) -> EarlyExit {
let to_stderr = e.use_stderr();
let text = e.render().to_string();
let summary = text
.lines()
.take_while(|line| !line.trim().is_empty())
.map(str::trim)
.collect::<Vec<_>>()
.join(" ")
.trim_start_matches("error: ")
.to_string();
EarlyExit {
text,
to_stderr,
code: if to_stderr { EXIT_ERROR } else { EXIT_FOUND },
summary,
}
}
fn asks_for_json(args: &[OsString]) -> bool {
args.iter()
.skip(1)
.take_while(|arg| *arg != "--")
.any(|arg| arg == "--json")
}
fn command_named(args: &[OsString]) -> Option<&'static str> {
args.iter()
.skip(1)
.take_while(|arg| *arg != "--")
.filter_map(|arg| arg.to_str())
.find_map(|arg| match arg {
"--skill" | "-r" | "--reference" => Some("skill"),
_ if arg == "--schema" || arg.starts_with("--schema=") => Some("schema"),
_ => COMMANDS.iter().copied().find(|name| *name == arg),
})
}
fn version_line() -> &'static str {
static VERSION: std::sync::OnceLock<String> = std::sync::OnceLock::new();
VERSION.get_or_init(|| {
let compiled = [
("udpipe", cfg!(feature = "udpipe")),
("model2vec", cfg!(feature = "model2vec")),
("python", cfg!(feature = "python")),
("cli", cfg!(feature = "cli")),
];
let mut line = String::from(env!("CARGO_PKG_VERSION"));
line.push_str("\nfeatures:");
for (name, on) in compiled {
if on {
line.push(' ');
line.push_str(name);
}
}
line
})
}
enum Outcome {
Found,
Empty,
Failed,
}
fn outcome(empty: bool) -> Outcome {
if empty {
Outcome::Empty
} else {
Outcome::Found
}
}
fn failure_context(cli: &Cli) -> (&'static str, Option<String>) {
if cli.skill || (cli.reference.is_some() && cli.schema.is_none()) {
return ("skill", None);
}
if cli.schema.is_some() {
return ("schema", None);
}
let label = |path: &Path| Some(Target::label(path, &cli.stdin_filename));
match &cli.command {
Some(Command::Analyze { path, .. }) => ("analyze", label(path)),
Some(Command::Summarize { path, .. }) => ("summarize", label(path)),
Some(Command::Keyphrases { path, .. }) => ("keyphrases", label(path)),
Some(Command::Clusters { path, .. }) => ("clusters", label(path)),
Some(Command::Config { .. }) => ("config", None),
Some(Command::Completions { .. }) => ("completions", None),
None => ("matra", None),
}
}
fn execute(cli: &Cli, out: &mut dyn Write, err: &mut dyn Write) -> Fallible<Outcome> {
if cli.skill {
return skill::run(cli, out);
}
if let Some(name) = &cli.schema {
return schema::run(cli, name.as_deref(), out);
}
if cli.reference.is_some() {
return Err(Failure::refused(
"reference_without_skill",
"--reference names a section of the agent skill; pass --skill with it, \
or run `matra --skill -r` to list the references",
));
}
let Some(command) = &cli.command else {
return Err(Failure::refused(
"no_command",
"no command given; run `matra --help` for the list",
));
};
match command {
Command::Config { action } => config_cmd::run(cli, action, out),
Command::Completions { shell } => {
let shell = match shell {
CompletionShell::Bash => clap_complete::Shell::Bash,
CompletionShell::Zsh => clap_complete::Shell::Zsh,
CompletionShell::Fish => clap_complete::Shell::Fish,
};
let mut command = Cli::command().name("matra").bin_name("matra");
let mut script = Vec::new();
clap_complete::generate(shell, &mut command, "matra", &mut script);
out.write_all(&script)?;
Ok(Outcome::Found)
}
Command::Analyze {
path,
sections,
format,
depth,
} => analyze(
cli,
path,
*format,
*sections,
depth.unwrap_or(Depth::Token),
out,
err,
),
Command::Summarize {
path,
n,
method,
format,
} => {
let (label, doc, cfg) = one_document(cli, "summarize", path, *format, err)?;
let n = n.unwrap_or_else(|| cfg.summary_n());
let method = match method {
Some(m) => summary_method_of(*m),
None => summary_method(cfg.summary_method())?,
};
let summary = crate::summarize_with(&doc, n, method)?;
if cli.json {
write_envelope(out, "summarize", Some(&label), &summary)?;
} else if !cli.quiet {
render::sentences(out, &summary)?;
}
if summary.sentences.is_empty() {
say_found_nothing(cli, err, &format!("found nothing to summarize in {label}"));
}
Ok(outcome(summary.sentences.is_empty()))
}
Command::Keyphrases {
path,
n,
method,
format,
} => {
let (label, doc, cfg) = one_document(cli, "keyphrases", path, *format, err)?;
let n = n.unwrap_or_else(|| cfg.keyphrases_n());
let method = match method {
Some(m) => keyphrase_method_of(*m),
None => keyphrase_method(cfg.keyphrases_method())?,
};
let found = crate::keyphrases_with(&doc, n, method)?;
if cli.json {
write_envelope(out, "keyphrases", Some(&label), &found)?;
} else if !cli.quiet {
render::phrases(out, &found)?;
}
if found.phrases.is_empty() {
say_found_nothing(cli, err, &format!("found no keyphrases in {label}"));
}
Ok(outcome(found.phrases.is_empty()))
}
Command::Clusters {
path,
threshold,
format,
} => clusters(cli, path, *threshold, *format, out, err),
}
}
#[cfg(feature = "model2vec")]
fn clusters(
cli: &Cli,
path: &Path,
threshold: Option<f64>,
format: Option<FormatName>,
out: &mut dyn Write,
err: &mut dyn Write,
) -> Fallible<Outcome> {
let (label, doc, cfg) = one_document(cli, "clusters", path, format, err)?;
let threshold = threshold.unwrap_or_else(|| cfg.clusters_threshold());
let embedder = build_embedder(cli, &cfg, err)?;
let found = crate::clusters(&doc, &embedder, threshold)?;
if cli.json {
write_envelope(out, "clusters", Some(&label), &found)?;
} else if !cli.quiet {
render::clusters(out, &doc, &found)?;
}
if found.clusters.is_empty() {
say_found_nothing(
cli,
err,
&format!(
"found no clusters in {label} at --threshold {threshold}; \
a lower threshold joins less similar sentences"
),
);
}
Ok(outcome(found.clusters.is_empty()))
}
fn say_found_nothing(cli: &Cli, err: &mut dyn Write, what: &str) {
if cli.json || cli.quiet {
return;
}
let _ = writeln!(err, "matra: {what}");
}
#[cfg(not(feature = "model2vec"))]
fn clusters(
_cli: &Cli,
_path: &Path,
_threshold: Option<f64>,
_format: Option<FormatName>,
_out: &mut dyn Write,
_err: &mut dyn Write,
) -> Fallible<Outcome> {
Err(Failure::refused(
"embedder_unavailable",
"clusters needs an embedder, and this build was compiled without the `model2vec` \
feature that ships one; install the Python wheel, or build with \
`--features cli,model2vec`",
))
}
fn analyze(
cli: &Cli,
path: &Path,
format: Option<FormatName>,
sections: bool,
depth: Depth,
out: &mut dyn Write,
err: &mut dyn Write,
) -> Fallible<Outcome> {
let style = render::Style::new(colorize(cli.color));
let inputs = match Target::of(path)? {
Target::Dir(inputs) => inputs,
one => {
let (label, doc, _) = analyzed(cli, one, path, format, err)?;
if cli.json {
write_document(out, Some(&label), &doc, depth, Layout::Pretty)?;
} else if !cli.quiet {
render::metrics(out, &label, &doc, style)?;
if sections {
render::sections(out, &doc)?;
}
}
return Ok(outcome(doc.metrics.total_sentences == 0));
}
};
if format.is_some() {
return Err(Failure::refused(
"format_with_directory",
format!(
"--format reads one file or stdin as the format named; {} is a directory, \
and each of its files is read as its own name and bytes say",
path.display()
),
));
}
let cfg = resolve_config(cli)?;
let engine = build_engine(cli, &cfg, err)?;
let mut found = false;
let mut failed = false;
let mut first = true;
for item in engine.analyze_all(inputs) {
match item {
Ok(doc) => {
let label = doc
.provenance
.source
.path
.as_deref()
.map(|p| p.display().to_string());
found |= doc.metrics.total_sentences > 0;
if cli.json {
write_document(out, label.as_deref(), &doc, depth, Layout::Line)?;
} else if !cli.quiet {
if !first {
writeln!(out)?;
}
render::metrics(out, label.as_deref().unwrap_or_default(), &doc, style)?;
if sections {
render::sections(out, &doc)?;
}
}
first = false;
}
Err(e) => {
failed = true;
let _ = writeln!(err, "matra: {e}");
if cli.json {
let label = e.path.as_deref().map(|p| p.display().to_string());
write_failure(
out,
Some("analyze"),
label.as_deref(),
&e.error,
Layout::Line,
)?;
}
}
}
}
Ok(if failed {
Outcome::Failed
} else {
outcome(!found)
})
}
fn one_document(
cli: &Cli,
command: &str,
path: &Path,
format: Option<FormatName>,
err: &mut dyn Write,
) -> Fallible<(String, Document, Config)> {
let target = Target::of(path)?;
if let Target::Dir(_) = target {
return Err(Failure::refused(
"directory_for_one_document",
format!(
"{} is a directory; {command} reads one document, and `matra analyze` reads a \
directory",
path.display()
),
));
}
analyzed(cli, target, path, format, err)
}
fn analyzed(
cli: &Cli,
target: Target,
path: &Path,
format: Option<FormatName>,
err: &mut dyn Write,
) -> Fallible<(String, Document, Config)> {
let format = format.map(format_of);
let (input, file) = match target {
Target::File(file) => {
let input = match &format {
Some(format) => crate::Input::file_as(&file, format.clone()),
None => crate::Input::file(&file),
};
(input, Some(file))
}
Target::Stdin => {
let format = format.unwrap_or_else(|| Format::from_path(&cli.stdin_filename));
(crate::Input::bytes(read_stdin()?, Some(format)), None)
}
Target::Dir(_) => unreachable!("a directory is walked, never analysed as one document"),
};
let cfg = resolve_config(cli)?;
let engine = build_engine(cli, &cfg, err)?;
let doc = engine
.analyze(input)
.map_err(|error| Failure::Error { error, path: file })?;
Ok((Target::label(path, &cli.stdin_filename), doc, cfg))
}
fn build_engine(cli: &Cli, cfg: &Config, err: &mut dyn Write) -> domain::Result<crate::Engine> {
if cli.quiet {
return crate::Engine::from_config(cfg);
}
crate::Engine::from_config_with_notice(cfg, |notice| announce(err, notice))
}
#[cfg(feature = "model2vec")]
fn build_embedder(
cli: &Cli,
cfg: &Config,
err: &mut dyn Write,
) -> domain::Result<crate::embed::model2vec::Model2Vec> {
if cli.quiet {
return crate::embed::model2vec::Model2Vec::from_config(cfg);
}
crate::embed::model2vec::Model2Vec::from_config_with_notice(cfg, |notice| {
announce(err, notice);
})
}
fn announce(err: &mut dyn Write, notice: &domain::ProvisionNotice) {
let _ = err.write_all(notice_text(notice).as_bytes());
let _ = err.flush();
}
pub(crate) fn notice_text(notice: &domain::ProvisionNotice) -> String {
format!(
"matra: downloading {} ({}) into {}\n\
matra: the model is licensed {}, separately from matra: {}\n",
notice.artifact,
human_bytes(notice.bytes),
notice.destination.display(),
notice.license,
notice.license_url
)
}
fn human_bytes(bytes: u64) -> String {
#[allow(clippy::cast_precision_loss)]
let n = bytes as f64;
if bytes >= 1_000_000 {
format!("{:.1} MB", n / 1_000_000.0)
} else if bytes >= 1_000 {
format!("{:.1} kB", n / 1_000.0)
} else {
format!("{bytes} bytes")
}
}
fn resolve_config(cli: &Cli) -> domain::Result<Config> {
let cfg = Config::resolve()?;
Ok(match &cli.model_dir {
Some(dir) => cfg.with_model_dir(dir),
None => cfg,
})
}
fn format_of(name: FormatName) -> Format {
match name {
FormatName::Markdown => Format::Markdown,
FormatName::PlainText => Format::PlainText,
FormatName::Blocks => Format::Blocks,
}
}
fn summary_method_of(method: SummaryMethod) -> crate::SummaryMethod {
match method {
SummaryMethod::Tfidf => crate::SummaryMethod::TfIdf,
SummaryMethod::Textrank => crate::SummaryMethod::TextRank,
}
}
fn keyphrase_method_of(method: KeyphraseMethod) -> crate::KeyphraseMethod {
match method {
KeyphraseMethod::Rake => crate::KeyphraseMethod::Rake,
KeyphraseMethod::Yake => crate::KeyphraseMethod::Yake,
}
}
fn summary_method(name: &str) -> Fallible<crate::SummaryMethod> {
match name {
"tfidf" => Ok(crate::SummaryMethod::TfIdf),
"textrank" => Ok(crate::SummaryMethod::TextRank),
other => Err(Failure::refused(
"method_unavailable",
format!("summary.method is `{other}`, which this build cannot run"),
)),
}
}
fn keyphrase_method(name: &str) -> Fallible<crate::KeyphraseMethod> {
match name {
"rake" => Ok(crate::KeyphraseMethod::Rake),
"yake" => Ok(crate::KeyphraseMethod::Yake),
other => Err(Failure::refused(
"method_unavailable",
format!("keyphrases.method is `{other}`, which this build cannot run"),
)),
}
}
enum Target {
Stdin,
File(PathBuf),
Dir(crate::Inputs),
}
impl Target {
fn of(path: &Path) -> Fallible<Target> {
if path.as_os_str() == "-" {
return Ok(Target::Stdin);
}
match crate::Inputs::dir(path) {
Ok(inputs) => Ok(Target::Dir(inputs)),
Err(domain::Error::InvalidInput {
reason: "not_a_directory",
..
}) => Ok(Target::File(path.to_path_buf())),
Err(error) => Err(Failure::Error { error, path: None }),
}
}
fn label(path: &Path, stdin_filename: &str) -> String {
if path.as_os_str() == "-" {
stdin_filename.to_string()
} else {
path.display().to_string()
}
}
}
fn read_stdin() -> Fallible<Vec<u8>> {
read_capped(&mut io::stdin().lock())
}
fn read_capped(source: &mut dyn Read) -> Fallible<Vec<u8>> {
let mut bytes: Vec<u8> = Vec::new();
source
.take(MAX_INPUT_BYTES as u64 + 1)
.read_to_end(&mut bytes)
.map_err(domain::Error::Io)?;
if bytes.len() > MAX_INPUT_BYTES {
return Err(domain::Error::InputTooLarge {
limit: MAX_INPUT_BYTES,
actual: bytes.len(),
what: "input",
}
.into());
}
Ok(bytes)
}
#[derive(serde::Serialize)]
struct Envelope<'a, T> {
format_version: u32,
command: Option<&'a str>,
input: Option<&'a str>,
result: Option<T>,
error: Option<&'a domain::Error>,
}
#[derive(Copy, Clone)]
enum Layout {
Pretty,
Line,
}
fn write_envelope<T: serde::Serialize>(
out: &mut dyn Write,
command: &str,
input: Option<&str>,
result: T,
) -> Fallible<()> {
let envelope = Envelope {
format_version: FORMAT_VERSION,
command: Some(command),
input,
result: Some(result),
error: None,
};
Ok(emit(out, &envelope, Layout::Pretty)?)
}
fn write_failure(
out: &mut dyn Write,
command: Option<&str>,
input: Option<&str>,
error: &domain::Error,
layout: Layout,
) -> io::Result<()> {
let envelope: Envelope<'_, ()> = Envelope {
format_version: FORMAT_VERSION,
command,
input,
result: None,
error: Some(error),
};
emit(out, &envelope, layout)
}
fn write_document(
out: &mut dyn Write,
input: Option<&str>,
doc: &Document,
depth: Depth,
layout: Layout,
) -> Fallible<()> {
fn envelope<'a, T>(input: Option<&'a str>, result: T) -> Envelope<'a, T> {
Envelope {
format_version: FORMAT_VERSION,
command: Some("analyze"),
input,
result: Some(result),
error: None,
}
}
if depth == Depth::Token {
return Ok(emit(out, &envelope(input, doc), layout)?);
}
let cut = at_depth(doc, depth)?;
Ok(emit(out, &envelope(input, cut), layout)?)
}
fn at_depth(doc: &Document, depth: Depth) -> io::Result<serde_json::Value> {
let mut value = serde_json::to_value(doc).map_err(io::Error::other)?;
let each = |value: &mut serde_json::Value, key: &str| -> Vec<serde_json::Value> {
value
.get_mut(key)
.and_then(serde_json::Value::as_array_mut)
.map(std::mem::take)
.unwrap_or_default()
};
match depth {
Depth::Token => {}
Depth::Document => {
if let Some(object) = value.as_object_mut() {
object.shift_remove("sections");
}
}
Depth::Paragraph | Depth::Sentence => {
let mut sections = each(&mut value, "sections");
for section in &mut sections {
let mut paragraphs = each(section, "paragraphs");
for paragraph in &mut paragraphs {
if depth == Depth::Paragraph {
if let Some(object) = paragraph.as_object_mut() {
object.shift_remove("sentences");
}
continue;
}
let mut sentences = each(paragraph, "sentences");
for sentence in &mut sentences {
if let Some(object) = sentence.as_object_mut() {
object.shift_remove("tokens");
}
}
paragraph["sentences"] = serde_json::Value::Array(sentences);
}
section["paragraphs"] = serde_json::Value::Array(paragraphs);
}
value["sections"] = serde_json::Value::Array(sections);
}
}
Ok(value)
}
fn emit<T: serde::Serialize>(
out: &mut dyn Write,
envelope: &Envelope<'_, T>,
layout: Layout,
) -> io::Result<()> {
let text = match layout {
Layout::Pretty => serde_json::to_string_pretty(envelope),
Layout::Line => serde_json::to_string(envelope),
}
.map_err(io::Error::other)?;
writeln!(out, "{text}")
}
fn colorize(when: ColorWhen) -> bool {
let no_color = std::env::var_os("NO_COLOR").is_some_and(|v| !v.is_empty());
decide_color(when, no_color, io::stdout().is_terminal())
}
fn decide_color(when: ColorWhen, no_color: bool, stdout_is_terminal: bool) -> bool {
match when {
ColorWhen::Always => true,
ColorWhen::Never => false,
ColorWhen::Auto => !no_color && stdout_is_terminal,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn version_line_names_the_compiled_features() {
let line = version_line();
assert!(line.starts_with(env!("CARGO_PKG_VERSION")));
assert!(line.contains("\nfeatures:"));
assert!(line.contains("udpipe"));
assert!(line.contains("cli"));
}
struct ClosesEarly {
left: usize,
}
impl Write for ClosesEarly {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
if self.left == 0 {
return Err(io::Error::from(io::ErrorKind::BrokenPipe));
}
let n = buf.len().min(self.left);
self.left -= n;
Ok(n)
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
#[test]
fn completions_into_a_reader_that_closes_early_exit_0() {
for shell in ["bash", "zsh", "fish"] {
let mut out = ClosesEarly { left: 64 };
let mut err = Vec::new();
let code = run(
["matra", "completions", shell].into_iter().map(Into::into),
&mut out,
&mut err,
);
assert_eq!(code, EXIT_FOUND, "{shell}");
assert!(err.is_empty(), "{shell}: {}", String::from_utf8_lossy(&err));
}
}
#[test]
fn the_command_list_is_the_subcommands_clap_knows() {
let known: Vec<String> = Cli::command()
.get_subcommands()
.map(|c| c.get_name().to_string())
.collect();
assert_eq!(known, COMMANDS);
}
#[test]
fn dash_is_stdin_and_a_directory_is_listed() {
let dir = tempfile::tempdir().expect("temp dir");
assert!(matches!(Target::of(Path::new("-")), Ok(Target::Stdin)));
assert!(matches!(Target::of(dir.path()), Ok(Target::Dir(_))));
let file = dir.path().join("notes.md");
std::fs::write(&file, "text").expect("write");
assert!(matches!(Target::of(&file), Ok(Target::File(_))));
assert_eq!(Target::label(Path::new("-"), "notes.md"), "notes.md");
assert_eq!(Target::label(&file, "notes.md"), file.display().to_string());
}
#[test]
fn a_missing_path_is_the_library_s_not_found() {
let dir = tempfile::tempdir().expect("temp dir");
let missing = dir.path().join("absent.txt");
let Err(Failure::Error { error, .. }) = Target::of(&missing) else {
panic!("a missing path is refused");
};
assert_eq!(error.kind(), "io");
let details = error.details();
assert_eq!(
details.get("io_kind"),
Some(&domain::DetailValue::Text("not_found".to_string()))
);
assert_eq!(
details.get("path"),
Some(&domain::DetailValue::Text(missing.display().to_string()))
);
}
#[test]
fn json_is_asked_for_before_a_bare_double_dash_only() {
let args = |list: &[&str]| -> Vec<OsString> { list.iter().map(OsString::from).collect() };
assert!(asks_for_json(&args(&["matra", "analyze", "--json"])));
assert!(!asks_for_json(&args(&["matra", "analyze", "--", "--json"])));
assert!(!asks_for_json(&args(&["--json"])), "argv[0] is the program");
assert_eq!(
command_named(&args(&["matra", "--json", "summarize", "x"])),
Some("summarize")
);
assert_eq!(
command_named(&args(&["matra", "--schema=x"])),
Some("schema")
);
assert_eq!(command_named(&args(&["matra", "--json", "--bogus"])), None);
}
#[test]
fn oversized_input_is_too_large_not_invalid_utf8() {
let text = "é".repeat(MAX_INPUT_BYTES / 2 + 1);
assert!(text.len() > MAX_INPUT_BYTES);
let Err(Failure::Error { error, .. }) = read_capped(&mut text.as_bytes()) else {
panic!("over the cap");
};
assert!(
matches!(
error,
domain::Error::InputTooLarge {
limit: MAX_INPUT_BYTES,
what: "input",
..
}
),
"{error}"
);
}
#[test]
fn input_at_the_cap_is_read_whole() {
let text = "a".repeat(MAX_INPUT_BYTES);
let read = read_capped(&mut text.as_bytes()).expect("at the cap");
assert_eq!(read.len(), MAX_INPUT_BYTES);
}
#[test]
fn stdin_bytes_are_not_decoded_here() {
let read = read_capped(&mut &b"ok\xff\xfe"[..]).expect("under the cap");
assert_eq!(read, b"ok\xff\xfe");
}
#[test]
fn depth_removes_the_levels_below_it() {
let doc = Document::new(
domain::Provenance::for_test(),
vec![domain::Section::new(
Some("Notes".to_string()),
1,
None,
Vec::new(),
)],
Vec::new(),
);
let document = at_depth(&doc, Depth::Document).expect("value");
assert!(document.get("sections").is_none());
assert!(document.get("metrics").is_some());
assert!(document.get("provenance").is_some());
let paragraph = at_depth(&doc, Depth::Paragraph).expect("value");
assert_eq!(paragraph["sections"][0]["heading"], "Notes");
}
#[test]
fn depth_keeps_the_key_order_of_the_whole_document() {
let mut paragraph = domain::Paragraph::new(
"A sentence.".to_string(),
domain::ParagraphKind::Paragraph,
domain::Span::new(0, 11),
);
paragraph
.sentences
.push(domain::Sentence::new("A sentence.".to_string(), Vec::new()));
let doc = Document::new(
domain::Provenance::for_test(),
vec![domain::Section::new(None, 0, None, vec![paragraph])],
Vec::new(),
);
let whole: serde_json::Value =
serde_json::from_str(&serde_json::to_string(&doc).expect("json")).expect("parse");
let keys = |value: &serde_json::Value| -> Vec<String> {
value
.as_object()
.expect("an object")
.keys()
.cloned()
.collect()
};
let without = |value: &serde_json::Value, gone: &str| -> Vec<String> {
keys(value).into_iter().filter(|k| k != gone).collect()
};
let paragraph_whole = &whole["sections"][0]["paragraphs"][0];
let cut = at_depth(&doc, Depth::Paragraph).expect("value");
assert_eq!(
keys(&cut["sections"][0]["paragraphs"][0]),
without(paragraph_whole, "sentences")
);
let cut = at_depth(&doc, Depth::Sentence).expect("value");
assert_eq!(
keys(&cut["sections"][0]["paragraphs"][0]["sentences"][0]),
without(¶graph_whole["sentences"][0], "tokens")
);
assert_eq!(
keys(&cut["sections"][0]["paragraphs"][0]),
keys(paragraph_whole)
);
let cut = at_depth(&doc, Depth::Document).expect("value");
assert_eq!(keys(&cut), without(&whole, "sections"));
let mut sorted = keys(paragraph_whole);
sorted.sort();
assert_ne!(keys(paragraph_whole), sorted);
}
#[test]
fn every_depth_reads_back_into_a_document() {
let mut paragraph = domain::Paragraph::new(
"A sentence.".to_string(),
domain::ParagraphKind::Paragraph,
domain::Span::new(0, 11),
);
let token = domain::Token::builder(
1,
"A".to_string(),
"a".to_string(),
"DET".to_string(),
0,
"root".to_string(),
)
.build();
paragraph.sentences.push(domain::Sentence::new(
"A sentence.".to_string(),
vec![token],
));
let doc = Document::new(
domain::Provenance::for_test(),
vec![domain::Section::new(None, 0, None, vec![paragraph])],
Vec::new(),
);
for depth in [
Depth::Document,
Depth::Paragraph,
Depth::Sentence,
Depth::Token,
] {
let text = serde_json::to_string(&at_depth(&doc, depth).expect("value")).expect("json");
let back: Document = serde_json::from_str(&text)
.unwrap_or_else(|e| panic!("{depth:?} does not read back: {e}"));
let sentences = back.sentences().count();
let tokens = back.tokens().count();
let expected = match depth {
Depth::Document | Depth::Paragraph => (0, 0),
Depth::Sentence => (1, 0),
Depth::Token => (1, 1),
};
assert_eq!((sentences, tokens), expected, "{depth:?}");
assert_eq!(back.provenance, doc.provenance, "{depth:?}");
}
}
#[test]
fn the_download_notice_names_the_artifact_size_destination_and_license() {
let notice = domain::ProvisionNotice {
artifact: "english-ewt-ud-2.5-191206.udpipe".to_string(),
bytes: 16_309_608,
destination: PathBuf::from("/home/u/.local/share/matra/models"),
license: "CC BY-NC-SA 4.0 (non-commercial)".to_string(),
license_url: "https://creativecommons.org/licenses/by-nc-sa/4.0/".to_string(),
};
assert_eq!(
notice_text(¬ice),
"matra: downloading english-ewt-ud-2.5-191206.udpipe (16.3 MB) \
into /home/u/.local/share/matra/models\n\
matra: the model is licensed CC BY-NC-SA 4.0 (non-commercial), separately from matra: \
https://creativecommons.org/licenses/by-nc-sa/4.0/\n"
);
}
#[test]
fn byte_counts_read_the_way_a_download_size_reads() {
assert_eq!(human_bytes(16_309_608), "16.3 MB");
assert_eq!(human_bytes(1_000_000), "1.0 MB");
assert_eq!(human_bytes(2_048), "2.0 kB");
assert_eq!(human_bytes(999), "999 bytes");
assert_eq!(human_bytes(0), "0 bytes");
}
#[test]
fn color_precedence() {
for tty in [true, false] {
for no_color in [true, false] {
assert!(
decide_color(ColorWhen::Always, no_color, tty),
"--color always is explicit and wins"
);
assert!(
!decide_color(ColorWhen::Never, no_color, tty),
"--color never is explicit and wins"
);
}
}
assert!(
decide_color(ColorWhen::Auto, false, true),
"a bare terminal"
);
assert!(
!decide_color(ColorWhen::Auto, true, true),
"NO_COLOR disables on a terminal"
);
assert!(
!decide_color(ColorWhen::Auto, false, false),
"a pipe is not colored"
);
}
}