use akana_core::*;
use clap::{Parser, Subcommand};
use std::path::PathBuf;
#[derive(Parser)]
#[command(name = "akana")]
#[command(about = "Akana: Modern and blazingly fast Turkish NLP CLI", long_about = None)]
struct Cli {
#[command(subcommand)]
command: Commands,
}
#[derive(Subcommand)]
enum Commands {
Tokenize {
text: Option<String>,
#[arg(short, long)]
file: Option<PathBuf>,
},
Analyze {
word: String,
},
SyntacticAnalyze {
word: String,
},
Generate {
lemma: String,
#[arg(short, long, value_delimiter = ',')]
tags: Vec<String>,
},
Deasciify {
text: Option<String>,
#[arg(short, long)]
file: Option<PathBuf>,
},
Asciify {
text: Option<String>,
#[arg(short, long)]
file: Option<PathBuf>,
},
Normalize {
text: Option<String>,
#[arg(short, long)]
file: Option<PathBuf>,
},
Grammar {
text: Option<String>,
#[arg(short, long)]
file: Option<PathBuf>,
#[arg(long, value_delimiter = ',')]
rules: Option<Vec<String>>,
#[arg(long, value_delimiter = ',')]
enable: Vec<String>,
#[arg(long, value_delimiter = ',')]
disable: Vec<String>,
#[arg(long)]
experimental: bool,
#[arg(short, long)]
correct: bool,
#[arg(short, long)]
json: bool,
#[arg(long)]
list_rules: bool,
},
Spellcheck {
word: String,
},
Parse {
sentence: Option<String>,
#[arg(short, long)]
file: Option<PathBuf>,
},
Readability {
text: Option<String>,
#[arg(short, long)]
file: Option<PathBuf>,
},
Syllabify {
word: String,
},
Number {
num: i64,
},
Ner {
text: Option<String>,
#[arg(short, long)]
file: Option<PathBuf>,
},
Keywords {
text: Option<String>,
#[arg(short, long)]
file: Option<PathBuf>,
#[arg(short, long, default_value_t = 10)]
top_k: usize,
},
Summarize {
text: Option<String>,
#[arg(short, long)]
file: Option<PathBuf>,
#[arg(short, long, default_value_t = 3)]
sentences: usize,
},
Stem {
word: String,
},
AiAudit {
text: Option<String>,
#[arg(short, long)]
file: Option<PathBuf>,
},
HumanizePrompt {
text: Option<String>,
#[arg(short, long)]
file: Option<PathBuf>,
#[arg(short, long, default_value = "blog")]
register: String,
},
Embed {
text: Option<String>,
#[arg(short, long)]
file: Option<PathBuf>,
},
Similarity {
text_a: String,
text_b: String,
},
Chunk {
text: Option<String>,
#[arg(short, long)]
file: Option<PathBuf>,
#[arg(short, long, default_value = "semantic")]
strategy: String,
#[arg(short = 'c', long, default_value_t = 512)]
chunk_size: usize,
#[arg(short, long, default_value_t = 0)]
overlap: usize,
#[arg(short = 'k', long, default_value_t = 20)]
min_chunk_size: usize,
#[arg(short, long)]
threshold: Option<f32>,
#[arg(short = 'm', long, default_value = "percentile")]
mode: String,
#[arg(short, long)]
json: bool,
},
PiiMask {
text: Option<String>,
#[arg(short, long)]
file: Option<PathBuf>,
#[arg(short, long, default_value = "placeholder")]
mode: String,
#[arg(short, long)]
json: bool,
},
PiiRestore {
text: Option<String>,
#[arg(short, long)]
file: Option<PathBuf>,
#[arg(short, long)]
mapping: PathBuf,
},
#[cfg(feature = "pii-tagger")]
PiiTag {
text: Option<String>,
#[arg(short, long)]
file: Option<PathBuf>,
#[arg(long)]
jsonl: Option<PathBuf>,
#[arg(long)]
accurate: bool,
#[arg(long, conflicts_with = "bias")]
strict: bool,
#[arg(long, conflicts_with = "accurate")]
model: Option<PathBuf>,
#[arg(long, requires = "model")]
big: Vec<PathBuf>,
#[arg(long)]
bias: Option<f32>,
#[arg(long, value_delimiter = ',')]
labels: Vec<String>,
#[arg(long)]
threads: Option<usize>,
#[arg(long)]
no_policy: bool,
},
#[cfg(feature = "pii-tagger")]
PiiModels {
#[arg(long)]
accurate: bool,
},
}
#[cfg(feature = "pii-tagger")]
struct TaggerChoice {
accurate: bool,
strict: bool,
model: Option<PathBuf>,
big: Vec<PathBuf>,
bias: Option<f32>,
labels: Vec<String>,
threads: Option<usize>,
no_policy: bool,
}
#[cfg(feature = "pii-tagger")]
fn pii_mode(accurate: bool) -> pii::tagger::Mode {
if accurate {
pii::tagger::Mode::Accurate
} else {
pii::tagger::Mode::Fast
}
}
#[cfg(feature = "pii-tagger")]
fn pii_model_dirs(mode: pii::tagger::Mode) -> Result<Vec<PathBuf>, String> {
let mut report = |path: &str, size: u64| {
eprintln!(
"downloading {path} ({:.1} MB) from {}",
size as f64 / 1e6,
pii::tagger::hub::REPO
);
};
pii::tagger::hub::model_dirs(mode, &mut report).map_err(|e| e.to_string())
}
#[cfg(feature = "pii-tagger")]
fn pii_tag(
text: Option<String>,
file: Option<PathBuf>,
jsonl: Option<PathBuf>,
choice: TaggerChoice,
) -> Result<(), String> {
use std::io::{BufRead, Write};
let mode = pii_mode(choice.accurate || !choice.big.is_empty());
let mut options = if choice.strict {
pii::tagger::TaggerOptions::strict(mode.strict_bias())
} else {
pii::tagger::TaggerOptions {
bias: choice.bias.unwrap_or(0.0),
bias_labels: choice.labels,
..Default::default()
}
};
options.public_policy = !choice.no_policy;
options.threads = choice.threads;
let dirs: Vec<PathBuf> = match choice.model {
Some(model) => std::iter::once(model).chain(choice.big).collect(),
None => pii_model_dirs(mode)?,
};
let tagger = pii::tagger::PiiTagger::load(&dirs, options).map_err(|e| e.to_string())?;
let Some(path) = jsonl else {
let input = resolve_input_text(text, file)?;
let spans = tagger.tag(&input).map_err(|e| e.to_string())?;
let out: Vec<_> = spans
.iter()
.map(|s| {
serde_json::json!({"start": s.start, "end": s.end, "label": s.label,
"text": &input[s.start..s.end]})
})
.collect();
println!(
"{}",
serde_json::to_string_pretty(&out).map_err(|e| e.to_string())?
);
return Ok(());
};
let reader = std::io::BufReader::new(
std::fs::File::open(&path).map_err(|e| format!("{}: {e}", path.display()))?,
);
let mut out = std::io::BufWriter::new(std::io::stdout().lock());
for line in reader.lines() {
let line = line.map_err(|e| e.to_string())?;
if line.trim().is_empty() {
continue;
}
let row: serde_json::Value = serde_json::from_str(&line).map_err(|e| e.to_string())?;
let input = row["text"]
.as_str()
.ok_or("a line without a \"text\" field")?;
let spans = tagger.tag(input).map_err(|e| e.to_string())?;
let spans: Vec<_> = spans
.iter()
.map(|s| serde_json::json!([s.start, s.end, s.label]))
.collect();
writeln!(out, "{}", serde_json::json!({"spans": spans})).map_err(|e| e.to_string())?;
}
Ok(())
}
fn resolve_input_text(text: Option<String>, file: Option<PathBuf>) -> Result<String, String> {
if let Some(f) = file {
std::fs::read_to_string(&f)
.map_err(|e| format!("Failed to read file '{}': {}", f.display(), e))
} else if let Some(t) = text {
Ok(t)
} else {
Err("Please provide input text as argument or use -f/--file <PATH>".to_string())
}
}
fn grammar_rules(
rules: Option<Vec<String>>,
enable: Vec<String>,
disable: Vec<String>,
experimental: bool,
) -> Result<grammar::RuleSet, grammar::UnknownRule> {
let mut set = match rules {
Some(ids) => grammar::RuleSet::from_ids(ids)?,
None => grammar::RuleSet::default(),
};
if experimental {
set = set.with_experimental();
}
for rule in grammar::RuleSet::from_ids(enable)?.iter() {
set.enable(rule);
}
for rule in grammar::RuleSet::from_ids(disable)?.iter() {
set.disable(rule);
}
Ok(set)
}
fn main() {
let cli = Cli::parse();
match cli.command {
Commands::Tokenize { text, file } => {
let input = match resolve_input_text(text, file) {
Ok(t) => t,
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
};
let tokens = tokenization::TurkishTokenizer::tokenize_words(&input);
println!("{}", serde_json::to_string_pretty(&tokens).unwrap());
}
Commands::Analyze { word } => {
let morph = morphology::TurkishMorphology::new();
let parses = morph.analyze(&word);
println!("{}", serde_json::to_string_pretty(&parses).unwrap());
}
Commands::SyntacticAnalyze { word } => {
let morph = syntactic_morphology::TurkishSyntacticMorphology::new();
let parses = morph.analyze(&word);
println!("{}", serde_json::to_string_pretty(&parses).unwrap());
}
Commands::Generate { lemma, tags } => {
let gen = morphology::TurkishGenerator::new();
let tag_refs: Vec<&str> = tags.iter().map(|s| s.as_str()).collect();
if let Some(surface) = gen.generate(&lemma, &tag_refs) {
println!("Generated surface: {surface}");
} else {
println!("Failed to generate surface form.");
}
}
Commands::Deasciify { text, file } => {
let input = match resolve_input_text(text, file) {
Ok(t) => t,
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
};
let res = normalization::TurkishDeasciifier::deasciify(&input);
println!("{res}");
}
Commands::Asciify { text, file } => {
let input = match resolve_input_text(text, file) {
Ok(t) => t,
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
};
let res = normalization::TurkishAsciifier::asciify(&input);
println!("{res}");
}
Commands::Normalize { text, file } => {
let input = match resolve_input_text(text, file) {
Ok(t) => t,
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
};
let res = normalization::TurkishInformalNormalizer::normalize_text(&input);
println!("{res}");
}
Commands::Grammar {
text,
file,
rules,
enable,
disable,
experimental,
correct,
json,
list_rules,
} => {
if list_rules {
for rule in grammar::GrammarRule::ALL {
println!(
"{:<24} {:<12} confidence {:.2} {}",
rule.id(),
if rule.is_default() {
"default"
} else {
"experimental"
},
rule.confidence(),
rule.summary()
);
}
return;
}
let input = match resolve_input_text(text, file) {
Ok(t) => t,
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
};
let set = match grammar_rules(rules, enable, disable, experimental) {
Ok(set) => set,
Err(e) => {
eprintln!("Error: {e} (see --list-rules)");
std::process::exit(1);
}
};
let checker = grammar::TurkishGrammarChecker::with_rules(set);
let result = checker.check(&input);
if json {
println!("{}", serde_json::to_string_pretty(&result).unwrap());
} else if correct {
println!("{}", result.corrected);
} else {
for f in &result.findings {
println!(
"[{}] {:?} -> {:?}: {}",
f.rule, f.original_text, f.replacement, f.message_tr
);
}
println!("{}", result.corrected);
}
}
Commands::Spellcheck { word } => {
let checker = normalization::TurkishSpellChecker::new();
let suggestions = checker.suggest(&word, 2, 5);
println!("{}", serde_json::to_string_pretty(&suggestions).unwrap());
}
Commands::Parse { sentence, file } => {
let input = match resolve_input_text(sentence, file) {
Ok(t) => t,
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
};
let tokens = tokenization::TurkishTokenizer::tokenize_words(&input);
let parser = parser::TurkishDependencyParser::new();
let tree = parser.parse(&tokens);
println!("{}", tree.to_conllu());
}
Commands::Readability { text, file } => {
let input = match resolve_input_text(text, file) {
Ok(t) => t,
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
};
let report = readability::analyze_readability(&input);
println!("{}", serde_json::to_string_pretty(&report).unwrap());
}
Commands::Syllabify { word } => {
let syllables = phonology::TurkishSyllabifier::syllabify(&word);
let hyphenated = phonology::TurkishSyllabifier::hyphenate(&word, "-");
println!(
"Syllables: {:?} (Count: {}, Hyphenated: {})",
syllables,
syllables.len(),
hyphenated
);
}
Commands::Number { num } => {
let words = normalization::TurkishNumberConverter::number_to_words(num);
let ordinal = normalization::TurkishNumberConverter::ordinal_to_words(num);
println!("Cardinal: {words}\nOrdinal: {ordinal}");
}
Commands::Ner { text, file } => {
let input = match resolve_input_text(text, file) {
Ok(t) => t,
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
};
let entities = ner::TurkishNER::extract_entities(&input);
println!("{}", serde_json::to_string_pretty(&entities).unwrap());
}
Commands::Keywords { text, file, top_k } => {
let input = match resolve_input_text(text, file) {
Ok(t) => t,
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
};
let extractor = analysis::TurkishKeywordExtractor::new();
let kw = extractor.extract_keywords(&input, top_k);
println!("{}", serde_json::to_string_pretty(&kw).unwrap());
}
Commands::Summarize {
text,
file,
sentences,
} => {
let input = match resolve_input_text(text, file) {
Ok(t) => t,
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
};
let summarizer = analysis::TurkishSummarizer::new();
let summary = summarizer.summarize(&input, sentences);
for (idx, sent) in summary.iter().enumerate() {
println!("{}. {}", idx + 1, sent);
}
}
Commands::Stem { word } => {
let stemmer = morphology::TurkishStemmer::new();
println!("Stem: {}", stemmer.stem(&word));
}
Commands::AiAudit { text, file } => {
let input = match resolve_input_text(text, file) {
Ok(t) => t,
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
};
let auditor = style::TurkishStyleAuditor::new();
let report = auditor.audit(&input);
println!("{}", serde_json::to_string_pretty(&report).unwrap());
}
Commands::HumanizePrompt {
text,
file,
register,
} => {
let input = match resolve_input_text(text, file) {
Ok(t) => t,
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
};
let prompt = style::TurkishHumanizer::generate_prompt(&input, ®ister);
println!("{prompt}");
}
Commands::Embed { text, file } => {
let input = match resolve_input_text(text, file) {
Ok(t) => t,
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
};
let embeddings = embeddings::TurkishEmbeddings::new();
let vec = embeddings.embed(&input);
println!("{}", serde_json::to_string(&vec).unwrap());
}
Commands::Similarity { text_a, text_b } => {
let embeddings = embeddings::TurkishEmbeddings::new();
let score = embeddings.similarity(&text_a, &text_b);
println!("{score:.4}");
}
Commands::Chunk {
text,
file,
strategy,
chunk_size,
overlap,
min_chunk_size,
threshold,
mode,
json,
} => {
let input = match resolve_input_text(text, file) {
Ok(t) => t,
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
};
let thresh_mode = match mode.to_lowercase().as_str() {
"similarity" | "sim" => {
chunking::ThresholdMode::Similarity(threshold.unwrap_or(0.32))
}
"percentile" | "perc" => {
chunking::ThresholdMode::Percentile(threshold.unwrap_or(0.75))
}
"stdev" | "std" => {
chunking::ThresholdMode::StandardDeviation(threshold.unwrap_or(0.80))
}
"iqr" => chunking::ThresholdMode::Interquartile(threshold.unwrap_or(1.0)),
"auto" => chunking::ThresholdMode::Auto,
_ => chunking::ThresholdMode::Percentile(threshold.unwrap_or(0.75)),
};
let chunks = match strategy.to_lowercase().as_str() {
"sentence" => {
let chunker = chunking::SentenceChunker::new(chunk_size, overlap, 1);
chunker.chunk(&input)
}
"sdpm" => {
let chunker = chunking::SDPMChunker::new(
chunk_size,
thresh_mode,
threshold.unwrap_or(0.32),
)
.with_min_chunk_size(min_chunk_size);
chunker.chunk(&input)
}
_ => {
let chunker = chunking::SemanticChunker::new(chunk_size, thresh_mode)
.with_min_chunk_size(min_chunk_size);
chunker.chunk(&input)
}
};
if json {
println!("{}", serde_json::to_string_pretty(&chunks).unwrap());
} else {
println!("Generated {} chunk(s):\n", chunks.len());
for (i, chunk) in chunks.iter().enumerate() {
println!(
"--- Chunk #{} (tokens: {}, chars: {}-{}) ---",
i + 1,
chunk.token_count,
chunk.start_index,
chunk.end_index
);
println!("{}\n", chunk.text);
}
}
}
Commands::PiiMask {
text,
file,
mode,
json,
} => {
let input = match resolve_input_text(text, file) {
Ok(t) => t,
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
};
let engine = pii::TurkishPiiEngine::new();
let pii_mode = match mode.to_lowercase().as_str() {
"surrogate" => pii::PiiMode::SyntheticSurrogate,
"tag" => pii::PiiMode::Tag,
"anonymize" => pii::PiiMode::Anonymize,
_ => pii::PiiMode::Placeholder,
};
let result = engine.mask(&input, pii_mode);
if json {
println!("{}", serde_json::to_string_pretty(&result).unwrap());
} else {
println!("{}", result.masked_text);
}
}
Commands::PiiRestore {
text,
file,
mapping,
} => {
let input = match resolve_input_text(text, file) {
Ok(t) => t,
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
};
let map_str = match std::fs::read_to_string(&mapping) {
Ok(s) => s,
Err(e) => {
eprintln!("Failed to read mapping file: {e}");
std::process::exit(1);
}
};
let map: std::collections::HashMap<String, String> =
match serde_json::from_str(&map_str) {
Ok(m) => m,
Err(e) => {
eprintln!("Failed to parse mapping JSON: {e}");
std::process::exit(1);
}
};
let engine = pii::TurkishPiiEngine::new();
let restored = engine.restore_with_mapping(&input, &map);
println!("{}", restored);
}
#[cfg(feature = "pii-tagger")]
Commands::PiiTag {
text,
file,
jsonl,
accurate,
strict,
model,
big,
bias,
labels,
threads,
no_policy,
} => {
let choice = TaggerChoice {
accurate,
strict,
model,
big,
bias,
labels,
threads,
no_policy,
};
if let Err(e) = pii_tag(text, file, jsonl, choice) {
eprintln!("Error: {e}");
std::process::exit(1);
}
}
#[cfg(feature = "pii-tagger")]
Commands::PiiModels { accurate } => match pii_model_dirs(pii_mode(accurate)) {
Ok(dirs) => {
for dir in dirs {
println!("{}", dir.display());
}
}
Err(e) => {
eprintln!("Error: {e}");
std::process::exit(1);
}
},
}
}