mod config_cmd;
mod render;
mod skill;
use std::ffi::OsString;
use std::io::{self, IsTerminal, Read, Write};
use std::path::{Path, PathBuf};
use clap::{CommandFactory, FromArgMatches, Parser, Subcommand, ValueEnum};
use crate::config::Config;
use crate::domain::{self, Document, Format, MAX_INPUT_BYTES, RawDocument, Sentence};
const FORMAT_VERSION: u32 = 1;
const EXIT_FOUND: u8 = 0;
const EXIT_EMPTY: u8 = 1;
const EXIT_ERROR: u8 = 2;
type Fallible<T> = Result<T, Box<dyn std::error::Error>>;
pub fn run(
args: impl IntoIterator<Item = OsString>,
out: &mut dyn Write,
err: &mut dyn Write,
) -> u8 {
let cli = match parse(args) {
Ok(cli) => cli,
Err(early) => {
let sink: &mut dyn Write = if early.to_stderr { err } else { out };
let _ = write!(sink, "{}", early.text);
return early.code;
}
};
match execute(&cli, out, err) {
Ok(Outcome::Found) => EXIT_FOUND,
Ok(Outcome::Empty) => EXIT_EMPTY,
Err(e) => {
if is_broken_pipe(e.as_ref()) {
return EXIT_FOUND;
}
let _ = writeln!(err, "matra: {e}");
EXIT_ERROR
}
}
}
#[derive(Parser)]
#[command(
about = "Text in, structured analysis out.",
long_about = "matra parses text into CoNLL-U structure and measures it. \
It reports what is there; interpretation is yours."
)]
struct Cli {
#[arg(long, global = true)]
json: bool,
#[arg(long, global = true, value_name = "DIR")]
model_dir: Option<PathBuf>,
#[arg(long, short, global = true)]
quiet: bool,
#[arg(long, global = true, value_enum, default_value_t = ColorWhen::Auto, value_name = "WHEN")]
color: ColorWhen,
#[arg(long, global = true, value_name = "NAME", default_value = "<stdin>")]
stdin_filename: String,
#[arg(long, global = true)]
skill: bool,
#[arg(
long,
short = 'r',
global = true,
value_name = "NAME",
num_args = 0..=1,
)]
reference: Option<Option<String>>,
#[command(subcommand)]
command: Option<Command>,
}
#[derive(Subcommand)]
enum Command {
Analyze {
path: PathBuf,
#[arg(long, short)]
sections: bool,
},
Summarize {
path: PathBuf,
#[arg(short)]
n: Option<usize>,
#[arg(long, value_enum)]
method: Option<SummaryMethod>,
},
Keyphrases {
path: PathBuf,
#[arg(short)]
n: Option<usize>,
#[arg(long, value_enum)]
method: Option<KeyphraseMethod>,
},
Config {
#[command(subcommand)]
action: ConfigAction,
},
Completions {
#[arg(value_enum)]
shell: CompletionShell,
},
}
#[derive(Subcommand)]
enum ConfigAction {
Show,
Init {
#[arg(long)]
force: bool,
},
}
#[derive(Copy, Clone, PartialEq, Eq, ValueEnum)]
enum SummaryMethod {
Tfidf,
Textrank,
}
#[derive(Copy, Clone, PartialEq, Eq, ValueEnum)]
enum KeyphraseMethod {
Rake,
Yake,
}
#[derive(Copy, Clone, PartialEq, Eq, ValueEnum)]
enum ColorWhen {
Auto,
Always,
Never,
}
#[derive(Copy, Clone, PartialEq, Eq, ValueEnum)]
enum CompletionShell {
Bash,
Zsh,
Fish,
}
struct EarlyExit {
text: String,
to_stderr: bool,
code: u8,
}
fn parse(args: impl IntoIterator<Item = OsString>) -> Result<Cli, EarlyExit> {
let mut command = Cli::command()
.name("matra")
.bin_name("matra")
.version(version_line())
.color(clap::ColorChoice::Never);
let matches = command.try_get_matches_from_mut(args).map_err(early_exit)?;
let cli = Cli::from_arg_matches(&matches).map_err(early_exit)?;
if cli.command.is_none() && !cli.skill && cli.reference.is_none() {
return Err(EarlyExit {
text: command.render_help().to_string(),
to_stderr: true,
code: EXIT_ERROR,
});
}
Ok(cli)
}
fn early_exit(e: clap::Error) -> EarlyExit {
let to_stderr = e.use_stderr();
EarlyExit {
text: e.render().to_string(),
to_stderr,
code: if to_stderr { EXIT_ERROR } else { EXIT_FOUND },
}
}
fn version_line() -> &'static str {
static VERSION: std::sync::OnceLock<String> = std::sync::OnceLock::new();
VERSION.get_or_init(|| {
let compiled = [
("udpipe", cfg!(feature = "udpipe")),
("model2vec", cfg!(feature = "model2vec")),
("python", cfg!(feature = "python")),
("cli", cfg!(feature = "cli")),
];
let mut line = String::from(env!("CARGO_PKG_VERSION"));
line.push_str("\nfeatures:");
for (name, on) in compiled {
if on {
line.push(' ');
line.push_str(name);
}
}
line
})
}
enum Outcome {
Found,
Empty,
}
fn outcome(empty: bool) -> Outcome {
if empty {
Outcome::Empty
} else {
Outcome::Found
}
}
enum Input {
Path(PathBuf),
Stdin,
}
impl Input {
fn of(path: &Path) -> Input {
if path.as_os_str() == "-" {
Input::Stdin
} else {
Input::Path(path.to_path_buf())
}
}
fn label<'a>(&'a self, stdin_filename: &'a str) -> std::borrow::Cow<'a, str> {
match self {
Input::Path(p) => p.display().to_string().into(),
Input::Stdin => stdin_filename.into(),
}
}
}
fn execute(cli: &Cli, out: &mut dyn Write, err: &mut dyn Write) -> Fallible<Outcome> {
if cli.skill {
return skill::run(cli, out);
}
if cli.reference.is_some() {
return Err(
"--reference names a section of the agent skill; pass --skill with it, \
or run `matra --skill -r` to list the references"
.into(),
);
}
let Some(command) = &cli.command else {
return Err("no command given; run `matra --help` for the list".into());
};
match command {
Command::Config { action } => return config_cmd::run(cli, action, out),
Command::Completions { shell } => {
let shell = match shell {
CompletionShell::Bash => clap_complete::Shell::Bash,
CompletionShell::Zsh => clap_complete::Shell::Zsh,
CompletionShell::Fish => clap_complete::Shell::Fish,
};
let mut command = Cli::command().name("matra").bin_name("matra");
clap_complete::generate(shell, &mut command, "matra", out);
return Ok(Outcome::Found);
}
_ => {}
}
let path = match command {
Command::Analyze { path, .. }
| Command::Summarize { path, .. }
| Command::Keyphrases { path, .. } => path,
Command::Config { .. } | Command::Completions { .. } => unreachable!("dispatched above"),
};
let input = Input::of(path);
if let Input::Path(p) = &input {
check_input(p)?;
}
let cfg = resolve_config(cli)?;
let engine = build_engine(cli, &cfg, err)?;
let doc = document_of(&input, &cli.stdin_filename, &engine)?;
let label = input.label(&cli.stdin_filename);
let style = render::Style::new(colorize(cli.color));
match command {
Command::Analyze { sections, .. } => {
if cli.json {
write_envelope(out, "analyze", Some(&label), &doc)?;
} else if !cli.quiet {
render::metrics(out, &label, &doc, style)?;
if *sections {
render::sections(out, &doc)?;
}
}
Ok(outcome(doc.total_sentences() == 0))
}
Command::Summarize { n, method, .. } => {
let sentences: Vec<Sentence> = doc.sentences().cloned().collect();
let n = n.unwrap_or_else(|| cfg.summarize_n());
let method = match method {
Some(m) => *m,
None => summary_method(cfg.summarize_algorithm())?,
};
let picked = match method {
SummaryMethod::Tfidf => crate::extraction::tfidf_summarize(&sentences, n)?,
SummaryMethod::Textrank => crate::extraction::textrank_summarize(&sentences, n)?,
};
if cli.json {
write_envelope(out, "summarize", Some(&label), &picked)?;
} else if !cli.quiet {
render::sentences(out, &picked)?;
}
Ok(outcome(picked.is_empty()))
}
Command::Keyphrases { n, method, .. } => {
let sentences: Vec<Sentence> = doc.sentences().cloned().collect();
let n = n.unwrap_or_else(|| cfg.keyphrases_n());
let method = match method {
Some(m) => *m,
None => keyphrase_method(cfg.keyphrases_algorithm())?,
};
let phrases = match method {
KeyphraseMethod::Rake => crate::extraction::rake_keyphrases(&sentences, n)?,
KeyphraseMethod::Yake => crate::extraction::yake_keyphrases(&sentences, n)?,
};
if cli.json {
write_envelope(out, "keyphrases", Some(&label), &phrases)?;
} else if !cli.quiet {
render::phrases(out, &phrases)?;
}
Ok(outcome(phrases.is_empty()))
}
Command::Config { .. } | Command::Completions { .. } => unreachable!("dispatched above"),
}
}
fn build_engine(cli: &Cli, cfg: &Config, err: &mut dyn Write) -> domain::Result<crate::Engine> {
if cli.quiet {
return crate::Engine::from_config(cfg);
}
crate::Engine::from_config_with_notice(cfg, |notice| {
let _ = writeln!(
err,
"matra: downloading {} ({}) into {}",
notice.artifact,
human_bytes(notice.bytes),
notice.destination.display()
);
let _ = err.flush();
})
}
fn human_bytes(bytes: u64) -> String {
#[allow(clippy::cast_precision_loss)]
let n = bytes as f64;
if bytes >= 1_000_000 {
format!("{:.1} MB", n / 1_000_000.0)
} else if bytes >= 1_000 {
format!("{:.1} kB", n / 1_000.0)
} else {
format!("{bytes} bytes")
}
}
fn check_input(path: &Path) -> Fallible<()> {
if !path.exists() {
return Err(format!("no such file: {}", path.display()).into());
}
if path.is_dir() {
return Err(format!(
"{} is a directory; pass a file. Directory analysis is on the roadmap.",
path.display()
)
.into());
}
Ok(())
}
fn resolve_config(cli: &Cli) -> domain::Result<Config> {
let cfg = Config::resolve()?;
Ok(match &cli.model_dir {
Some(dir) => cfg.with_model_dir(dir),
None => cfg,
})
}
fn summary_method(name: &str) -> Fallible<SummaryMethod> {
match name {
"tfidf" => Ok(SummaryMethod::Tfidf),
"textrank" => Ok(SummaryMethod::Textrank),
other => {
Err(format!("summarize.algorithm is `{other}`, which this build cannot run").into())
}
}
}
fn keyphrase_method(name: &str) -> Fallible<KeyphraseMethod> {
match name {
"rake" => Ok(KeyphraseMethod::Rake),
"yake" => Ok(KeyphraseMethod::Yake),
other => {
Err(format!("keyphrases.algorithm is `{other}`, which this build cannot run").into())
}
}
}
fn document_of(input: &Input, stdin_filename: &str, engine: &crate::Engine) -> Fallible<Document> {
match input {
Input::Path(path) => {
let mut stream = engine.analyze(crate::Ingest::path(path)?);
match stream.next() {
Some(Ok(entry)) => Ok(entry.analysis),
Some(Err(e)) => Err(Box::new(e)),
None => Err(format!("no documents at {}", path.display()).into()),
}
}
Input::Stdin => {
let text = read_stdin()?;
let raw = RawDocument::new(text, None, Format::from_path(stdin_filename));
Ok(engine.analyze_one(raw)?.analysis)
}
}
}
fn read_stdin() -> Fallible<String> {
read_capped(&mut io::stdin().lock())
}
fn read_capped(source: &mut dyn Read) -> Fallible<String> {
let mut bytes: Vec<u8> = Vec::new();
source
.take(MAX_INPUT_BYTES as u64 + 1)
.read_to_end(&mut bytes)?;
if bytes.len() > MAX_INPUT_BYTES {
return Err(Box::new(domain::Error::InputTooLarge {
limit: MAX_INPUT_BYTES,
actual: bytes.len(),
what: "input",
}));
}
Ok(String::from_utf8(bytes)?)
}
#[derive(serde::Serialize)]
struct Envelope<'a, T> {
format_version: u32,
command: &'a str,
input: Option<&'a str>,
result: T,
}
fn write_envelope<T: serde::Serialize>(
out: &mut dyn Write,
command: &str,
input: Option<&str>,
result: T,
) -> Fallible<()> {
let envelope = Envelope {
format_version: FORMAT_VERSION,
command,
input,
result,
};
writeln!(out, "{}", serde_json::to_string_pretty(&envelope)?)?;
Ok(())
}
fn is_broken_pipe(e: &(dyn std::error::Error + 'static)) -> bool {
let mut source = Some(e);
while let Some(err) = source {
if let Some(io_err) = err.downcast_ref::<io::Error>()
&& io_err.kind() == io::ErrorKind::BrokenPipe
{
return true;
}
source = err.source();
}
false
}
fn colorize(when: ColorWhen) -> bool {
let no_color = std::env::var_os("NO_COLOR").is_some_and(|v| !v.is_empty());
decide_color(when, no_color, io::stdout().is_terminal())
}
fn decide_color(when: ColorWhen, no_color: bool, stdout_is_terminal: bool) -> bool {
match when {
ColorWhen::Always => true,
ColorWhen::Never => false,
ColorWhen::Auto => !no_color && stdout_is_terminal,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn version_line_names_the_compiled_features() {
let line = version_line();
assert!(line.starts_with(env!("CARGO_PKG_VERSION")));
assert!(line.contains("\nfeatures:"));
assert!(line.contains("udpipe"));
assert!(line.contains("cli"));
}
#[test]
fn dash_is_stdin_and_anything_else_is_a_path() {
assert!(matches!(Input::of(Path::new("-")), Input::Stdin));
assert!(matches!(Input::of(Path::new("./-")), Input::Path(_)));
assert!(matches!(Input::of(Path::new("notes.md")), Input::Path(_)));
}
#[test]
fn stdin_filename_selects_the_decomposer() {
assert_eq!(Format::from_path("notes.md"), Format::Markdown);
assert_eq!(Format::from_path("<stdin>"), Format::PlainText);
}
#[test]
fn a_missing_path_and_a_directory_are_both_refused() {
let dir = tempfile::tempdir().expect("temp dir");
let missing = check_input(&dir.path().join("absent.txt")).expect_err("missing is refused");
assert!(missing.to_string().contains("no such file"), "{missing}");
let is_dir = check_input(dir.path()).expect_err("a directory is refused");
let message = is_dir.to_string();
assert!(message.contains("is a directory"), "{message}");
assert!(message.contains("pass a file"), "{message}");
assert!(
message.contains(&dir.path().display().to_string()),
"the message names the path: {message}"
);
let file = dir.path().join("notes.txt");
std::fs::write(&file, "text").expect("write");
check_input(&file).expect("a regular file is accepted");
}
#[test]
fn oversized_input_is_too_large_not_invalid_utf8() {
let text = "é".repeat(MAX_INPUT_BYTES / 2 + 1);
assert!(text.len() > MAX_INPUT_BYTES);
let err = read_capped(&mut text.as_bytes()).expect_err("over the cap");
let too_large = err
.downcast_ref::<domain::Error>()
.expect("a domain error, not an io error");
assert!(
matches!(
too_large,
domain::Error::InputTooLarge {
limit: MAX_INPUT_BYTES,
what: "input",
..
}
),
"{too_large}"
);
}
#[test]
fn input_at_the_cap_is_read_whole() {
let text = "a".repeat(MAX_INPUT_BYTES);
let read = read_capped(&mut text.as_bytes()).expect("at the cap");
assert_eq!(read.len(), MAX_INPUT_BYTES);
}
#[test]
fn invalid_utf8_under_the_cap_still_reports_utf8() {
let err = read_capped(&mut &b"ok\xff\xfe"[..]).expect_err("not utf8");
assert!(err.downcast_ref::<domain::Error>().is_none());
assert!(err.to_string().contains("utf-8"), "{err}");
}
#[test]
fn the_download_notice_names_the_artifact_size_and_destination() {
let notice = domain::ProvisionNotice {
artifact: "english-ewt-ud-2.5-191206.udpipe".to_string(),
bytes: 16_309_608,
destination: PathBuf::from("/home/u/.local/share/matra/models"),
};
let line = format!(
"matra: downloading {} ({}) into {}",
notice.artifact,
human_bytes(notice.bytes),
notice.destination.display()
);
assert_eq!(
line,
"matra: downloading english-ewt-ud-2.5-191206.udpipe (16.3 MB) \
into /home/u/.local/share/matra/models"
);
}
#[test]
fn byte_counts_read_the_way_a_download_size_reads() {
assert_eq!(human_bytes(16_309_608), "16.3 MB");
assert_eq!(human_bytes(1_000_000), "1.0 MB");
assert_eq!(human_bytes(2_048), "2.0 kB");
assert_eq!(human_bytes(999), "999 bytes");
assert_eq!(human_bytes(0), "0 bytes");
}
#[test]
fn color_precedence() {
for tty in [true, false] {
for no_color in [true, false] {
assert!(
decide_color(ColorWhen::Always, no_color, tty),
"--color always is explicit and wins"
);
assert!(
!decide_color(ColorWhen::Never, no_color, tty),
"--color never is explicit and wins"
);
}
}
assert!(
decide_color(ColorWhen::Auto, false, true),
"a bare terminal"
);
assert!(
!decide_color(ColorWhen::Auto, true, true),
"NO_COLOR disables on a terminal"
);
assert!(
!decide_color(ColorWhen::Auto, false, false),
"a pipe is not colored"
);
}
}