use crate::*;
use clap::Args;
use req_file::prelude::*;
use std::path::PathBuf;
#[derive(Args, Debug)]
pub struct TokenizeCommand {
inputs: Option<Vec<PathBuf>>,
#[arg(short, long)]
output: Option<Output>,
#[arg(short, long, default_value_t = OutputFormat::Human)]
format: OutputFormat,
}
impl Command for TokenizeCommand {
fn execute(&self, context: &mut CommandContext) {
let targets = context.target(
self.inputs.as_ref().unwrap_or(&Vec::new()),
&None,
Some("tokens.json"),
WorkspaceTarget::Requirements.into(),
);
let Some(targets) = context.consume_err(targets) else {
return;
};
for target in &targets {
context.consume_err(self.run_tokenize(target));
}
}
}
impl TokenizeCommand {
fn run_tokenize(&self, target: &Target) -> Result<(), CommandError> {
let tokens =
TextTokenizer::tokenize(&mut target.input.reader()?).err_localized(&target.input)?;
match self.format {
OutputFormat::Human => {
for token in &tokens {
println!(
"{}{} => {:?}",
target.input,
token
.source
.span
.map(|v| format!(":{}", v.start))
.unwrap_or_default(),
token.token
);
}
}
OutputFormat::Json => JsonFormatter::serialize_into(
target.output.with_extension("json").writer(true)?,
&tokens,
)
.err_localized(&target.input)?,
}
Ok(())
}
}