use std::path::Path;
use anyhow::{Result, bail};
use crate::config::{self, Config, GrammarSource, LanguageRegistry};
use super::assets;
use super::build;
use super::recipe::{GrammarRecipe, builtin_recipes};
pub fn run(args: &[String]) -> Result<()> {
let cfg = Config::load(config::default_path().as_deref())?;
let grammar_dir = cfg.grammar_dir.as_path();
let query_dir = cfg.query_dir.as_path();
let recipes = merged_recipes(&cfg.grammars);
match args.split_first() {
None => {
print_usage();
Ok(())
}
Some((cmd, rest)) => match cmd.as_str() {
"list" | "ls" => list(rest, &recipes, &cfg.languages, grammar_dir, query_dir),
"install" | "add" => install(rest, &recipes, grammar_dir, query_dir),
"install-queries" | "refresh-queries" => install_queries(rest, &recipes, query_dir),
"bundle" => bundle(rest),
"remove" | "rm" | "uninstall" => remove(rest, grammar_dir),
"help" | "-h" | "--help" => {
print_usage();
Ok(())
}
other => {
print_usage();
bail!("unknown grammar subcommand: `{}`", other);
}
},
}
}
pub fn merged_recipes(user: &[GrammarSource]) -> Vec<GrammarRecipe> {
let mut recipes = builtin_recipes();
for g in user {
let recipe =
GrammarRecipe::from_config(&g.name, &g.source, g.rev.as_deref(), g.subpath.as_deref());
match recipes.iter_mut().find(|r| r.name == g.name) {
Some(slot) => *slot = recipe,
None => recipes.push(recipe),
}
}
recipes
}
fn find<'a>(recipes: &'a [GrammarRecipe], name: &str) -> Option<&'a GrammarRecipe> {
recipes.iter().find(|r| r.name == name)
}
fn print_usage() {
eprintln!("usage: vorto grammar <command> [args]");
eprintln!();
eprintln!("commands:");
eprintln!(" list [-v] show install status (grouped; -v for detail)");
eprintln!(" install <name>... | --all build and install one or more grammars");
eprintln!(" install-queries <name>... | --all overwrite installed .scm files from the");
eprintln!(" vendored bundle (no library rebuild)");
eprintln!(" remove <name>... delete installed grammar libraries");
eprintln!(" bundle build all built-ins -> gzip'd libs in");
eprintln!(" assets/grammars-prebuilt/<target>/, and write");
eprintln!(" THIRD_PARTY_LICENSES from the cloned sources");
eprintln!();
eprintln!("examples:");
eprintln!(" vorto grammar install rust python");
eprintln!(" vorto grammar install --all");
eprintln!(" vorto grammar install-queries python");
eprintln!(" vorto grammar list");
eprintln!();
eprintln!("define non-built-in grammars in config under [grammars.<name>]:");
eprintln!(" [grammars.nim]");
eprintln!(" source = \"https://github.com/alaviss/tree-sitter-nim\"");
eprintln!(" rev = \"abc123\" # optional; omit to build the latest default branch");
eprintln!(" subpath = \"sub\" # optional; for monorepos");
}
fn list(
args: &[String],
recipes: &[GrammarRecipe],
languages: &LanguageRegistry,
grammar_dir: &Path,
query_dir: &Path,
) -> Result<()> {
if args.iter().any(|a| a == "-v" || a == "--verbose") {
return list_verbose(recipes, languages, grammar_dir, query_dir);
}
println!("grammar dir: {}", grammar_dir.display());
println!("query dir: {}", query_dir.display());
println!();
let (mut installed, mut lib_only, mut missing) = (Vec::new(), Vec::new(), Vec::new());
for r in recipes {
if build::installed_path(r.name, grammar_dir).is_none() {
missing.push(r.name);
} else if build::is_fully_installed(r.name, grammar_dir, query_dir) {
installed.push(r.name);
} else {
lib_only.push(r.name);
}
}
let total = installed.len() + lib_only.len() + missing.len();
print_group("installed", &installed);
print_group("lib only, queries missing", &lib_only);
print_group("missing", &missing);
println!(
"{} total · {} installed · {} missing",
total,
installed.len() + lib_only.len(),
missing.len()
);
let custom = custom_grammars(recipes, languages, grammar_dir);
if !custom.is_empty() {
println!();
println!("custom (from config) ({}):", custom.len());
for c in &custom {
let glyph = if c.installed { "✓" } else { "✗" };
println!(" {} {}", glyph, c.grammar);
}
if custom.iter().any(|c| !c.installed) {
println!(
" (✗ = no library found and no recipe — add a `[grammars.<name>]` entry to install it, or drop the compiled `.so` in yourself)"
);
}
}
println!("(-v for repo URLs and per-grammar query detail)");
Ok(())
}
struct CustomGrammar {
grammar: String,
installed: bool,
}
fn custom_grammars(
recipes: &[GrammarRecipe],
languages: &LanguageRegistry,
grammar_dir: &Path,
) -> Vec<CustomGrammar> {
use std::collections::HashSet;
let has_recipe: HashSet<&str> = recipes.iter().map(|r| r.name).collect();
let mut seen = HashSet::new();
let mut out = Vec::new();
for lang in languages.iter() {
if has_recipe.contains(lang.grammar.as_str()) || !seen.insert(lang.grammar.clone()) {
continue;
}
let dir = lang.grammar_dir.as_deref().unwrap_or(grammar_dir);
out.push(CustomGrammar {
grammar: lang.grammar.clone(),
installed: build::installed_path(&lang.grammar, dir).is_some(),
});
}
out.sort_by(|a, b| a.grammar.cmp(&b.grammar));
out
}
fn print_group(label: &str, names: &[&str]) {
if names.is_empty() {
return;
}
println!("{} ({}):", label, names.len());
const INDENT: usize = 2;
let term_w = crossterm::terminal::size()
.map(|(c, _)| c as usize)
.unwrap_or(80);
let col_w = names.iter().map(|n| n.len()).max().unwrap_or(1) + 2;
let cols = (term_w.saturating_sub(INDENT) / col_w).max(1);
for row in names.chunks(cols) {
let mut line = " ".repeat(INDENT);
for name in row {
line.push_str(&format!("{:<width$}", name, width = col_w));
}
println!("{}", line.trim_end());
}
println!();
}
fn list_verbose(
recipes: &[GrammarRecipe],
languages: &LanguageRegistry,
grammar_dir: &Path,
query_dir: &Path,
) -> Result<()> {
println!("grammar dir: {}", grammar_dir.display());
println!("query dir: {}", query_dir.display());
println!();
for r in recipes {
let lib_status = match build::installed_path(r.name, grammar_dir) {
Some(_) => "lib ✓",
None => "lib ✗",
};
let installed = build::installed_queries(r.name, query_dir);
let bundled = assets::bundled_query_names(r.name);
let query_status = match (installed.is_empty(), bundled.is_empty()) {
(false, _) => format!("queries: {} (installed)", installed.join(",")),
(true, false) => format!("queries: {} (bundled, not installed)", bundled.join(",")),
(true, true) => "queries: none bundled".to_string(),
};
let subpath = r.subpath.map(|s| format!(" [{}]", s)).unwrap_or_default();
println!(
" {:<12} {}{}\n {} | {}",
r.name, r.repo, subpath, lib_status, query_status
);
}
let custom = custom_grammars(recipes, languages, grammar_dir);
if !custom.is_empty() {
println!();
println!("custom (from config):");
for c in &custom {
let lib_status = if c.installed { "lib ✓" } else { "lib ✗" };
println!(
" {:<12} (no recipe — user-supplied)\n {}",
c.grammar, lib_status
);
}
}
Ok(())
}
fn install(
args: &[String],
recipes: &[GrammarRecipe],
grammar_dir: &Path,
query_dir: &Path,
) -> Result<()> {
let recipes = match args.first().map(String::as_str) {
None => {
bail!("install: need at least one grammar name (or `--all`)");
}
Some("--all") => recipes.to_vec(),
_ => {
let mut out = Vec::new();
for name in args {
match find(recipes, name) {
Some(r) => out.push(r.clone()),
None => bail!(
"unknown grammar `{}`. Run `vorto grammar list` for the catalog, or define it under `[grammars.{}]` in your config.",
name,
name
),
}
}
out
}
};
let mut pending = Vec::new();
for r in &recipes {
if build::is_fully_installed(r.name, grammar_dir, query_dir) {
eprintln!("==> {} already installed, skipping", r.name);
} else {
pending.push(r.clone());
}
}
if pending.is_empty() {
return Ok(());
}
let workers = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(4)
.min(pending.len());
let next = std::sync::atomic::AtomicUsize::new(0);
let stderr_lock = std::sync::Mutex::new(());
let failures = std::sync::Mutex::new(Vec::<&str>::new());
std::thread::scope(|s| {
for _ in 0..workers {
s.spawn(|| {
loop {
let i = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
if i >= pending.len() {
break;
}
let r = &pending[i];
let result = build::install(r, grammar_dir, query_dir);
let mut buf = String::new();
use std::fmt::Write;
writeln!(buf, "==> installing {} ({})", r.name, r.repo).ok();
match result {
Ok(report) => {
writeln!(buf, " lib: {}", report.library.display()).ok();
if report.queries.is_empty() {
writeln!(
buf,
" queries: none bundled — drop your own .scm into {}/{}/",
query_dir.display(),
r.name
)
.ok();
} else {
let names: Vec<String> = report
.queries
.iter()
.filter_map(|p| {
p.file_name().map(|n| n.to_string_lossy().into_owned())
})
.collect();
writeln!(
buf,
" queries: {} ({} files)",
names.join(", "),
report.queries.len()
)
.ok();
}
}
Err(e) => {
writeln!(buf, " failed: {:#}", e).ok();
failures.lock().unwrap().push(r.name);
}
}
let _g = stderr_lock.lock().unwrap();
eprint!("{}", buf);
}
});
}
});
let failures = failures.into_inner().unwrap();
if !failures.is_empty() {
bail!("failed to install: {}", failures.join(", "));
}
Ok(())
}
fn install_queries(args: &[String], recipes: &[GrammarRecipe], query_dir: &Path) -> Result<()> {
let recipes = match args.first().map(String::as_str) {
None => {
bail!("install-queries: need at least one grammar name (or `--all`)");
}
Some("--all") => recipes.to_vec(),
_ => {
let mut out = Vec::new();
for name in args {
match find(recipes, name) {
Some(r) => out.push(r.clone()),
None => bail!(
"unknown grammar `{}`. Run `vorto grammar list` for the catalog.",
name
),
}
}
out
}
};
let mut failures = Vec::new();
for r in &recipes {
eprintln!("==> refreshing queries for {}", r.name);
match build::write_vendored_queries(query_dir, r.name) {
Ok(written) if written.is_empty() => {
eprintln!(" queries: none bundled for `{}`", r.name);
}
Ok(written) => {
let names: Vec<String> = written
.iter()
.filter_map(|p| p.file_name().map(|n| n.to_string_lossy().into_owned()))
.collect();
eprintln!(
" queries: {} ({} files)",
names.join(", "),
written.len()
);
}
Err(e) => {
eprintln!(" failed: {:#}", e);
failures.push(r.name);
}
}
}
if !failures.is_empty() {
bail!("failed to refresh: {}", failures.join(", "));
}
Ok(())
}
type GrammarLicenses = (&'static str, &'static str, Vec<(String, String)>);
fn bundle(_args: &[String]) -> Result<()> {
fn gzip_to(lib: &Path, out_dir: &Path, name: &str, ext: &str) -> Result<u64> {
use std::io::Write;
let raw = std::fs::read(lib)?;
let dst = out_dir.join(format!("{name}.{ext}.gz"));
let f = std::fs::File::create(&dst)?;
let mut enc = flate2::write::GzEncoder::new(f, flate2::Compression::best());
enc.write_all(&raw)?;
enc.finish()?;
Ok(std::fs::metadata(&dst)?.len())
}
let target = env!("BUILD_TARGET");
let ext = build::dylib_ext();
let out_dir = Path::new(env!("CARGO_MANIFEST_DIR"))
.join("assets")
.join("grammars-prebuilt")
.join(target);
std::fs::create_dir_all(&out_dir)?;
eprintln!("bundling for target {target} -> {}", out_dir.display());
let tmp = std::env::temp_dir().join(format!("vorto-bundle-{}", std::process::id()));
let build_grammar_dir = tmp.join("grammars");
let build_query_dir = tmp.join("queries");
let recipes = builtin_recipes();
let next = std::sync::atomic::AtomicUsize::new(0);
let io_lock = std::sync::Mutex::new(());
let failures = std::sync::Mutex::new(Vec::<&str>::new());
let collected = std::sync::Mutex::new(Vec::<GrammarLicenses>::new());
let workers = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(4)
.min(recipes.len());
std::thread::scope(|s| {
for _ in 0..workers {
s.spawn(|| {
loop {
let i = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
if i >= recipes.len() {
break;
}
let r = &recipes[i];
let res = build::install(r, &build_grammar_dir, &build_query_dir).and_then(
|report| {
let bytes = gzip_to(&report.library, &out_dir, r.name, ext)?;
collected
.lock()
.unwrap()
.push((r.name, r.repo, report.licenses));
Ok(bytes)
},
);
let _g = io_lock.lock().unwrap();
match res {
Ok(bytes) => eprintln!("==> {} ({} bytes gz)", r.name, bytes),
Err(e) => {
eprintln!("==> {} FAILED: {:#}", r.name, e);
failures.lock().unwrap().push(r.name);
}
}
}
});
}
});
let _ = std::fs::remove_dir_all(&tmp);
let failures = failures.into_inner().unwrap();
if !failures.is_empty() {
bail!("failed to bundle: {}", failures.join(", "));
}
eprintln!("done: {} grammars -> {}", recipes.len(), out_dir.display());
let mut collected = collected.into_inner().unwrap();
collected.sort_by(|a, b| a.0.cmp(b.0));
write_third_party_licenses(&collected)?;
Ok(())
}
fn write_third_party_licenses(grammars: &[GrammarLicenses]) -> Result<()> {
use std::fmt::Write as _;
let manifest = Path::new(env!("CARGO_MANIFEST_DIR"));
let mut doc = String::new();
doc.push_str(
"THIRD-PARTY LICENSES\n\
====================\n\n\
vorto itself is licensed under MIT OR Apache-2.0 (see LICENSE-MIT,\n\
LICENSE-APACHE). The components below are redistributed in release\n\
binaries built with the `bundled-grammars` feature: pre-built\n\
tree-sitter grammar libraries embedded in the binary, and the\n\
tree-sitter query files (`.scm`) bundled via include_dir!.\n\n",
);
doc.push_str(
"================================================================\n\
Tree-sitter grammars (embedded native libraries)\n\
================================================================\n\n",
);
let mut seen_repos: Vec<&str> = Vec::new();
for (name, repo, files) in grammars {
if seen_repos.contains(repo) {
continue;
}
seen_repos.push(repo);
let names: Vec<&str> = grammars
.iter()
.filter(|(_, r, _)| r == repo)
.map(|(n, _, _)| *n)
.collect();
let _ = writeln!(
doc,
"----------------------------------------------------------------"
);
let _ = writeln!(doc, "grammar(s): {}", names.join(", "));
let _ = writeln!(doc, "source: {repo}");
let _ = writeln!(
doc,
"----------------------------------------------------------------"
);
if files.is_empty() {
let _ = writeln!(
doc,
"(!) no LICENSE file found in the upstream source — verify manually for `{name}`."
);
}
for (fname, text) in files {
let _ = writeln!(doc, "[{fname}]");
doc.push_str(text);
if !text.ends_with('\n') {
doc.push('\n');
}
}
doc.push('\n');
}
let derived = nvim_derived_langs();
doc.push_str(
"================================================================\n\
Tree-sitter query files (.scm)\n\
================================================================\n\n",
);
let _ = writeln!(
doc,
"Many bundled `.scm` query files are derived from nvim-treesitter\n\
(https://github.com/nvim-treesitter/nvim-treesitter), licensed\n\
Apache-2.0. Each derived file carries a `;; Based on nvim-treesitter`\n\
header naming its exact source commit. Languages affected:\n {}\n",
derived.join(", ")
);
doc.push('\n');
let nvim_license = manifest.join("assets/licenses/Apache-2.0.txt");
match std::fs::read_to_string(&nvim_license) {
Ok(text) => {
doc.push_str("----------------------------------------------------------------\n");
doc.push_str("nvim-treesitter — Apache-2.0\n");
doc.push_str("----------------------------------------------------------------\n");
doc.push_str(&text);
if !text.ends_with('\n') {
doc.push('\n');
}
}
Err(_) => {
bail!(
"missing vendored license {} — commit nvim-treesitter's Apache-2.0 LICENSE there",
nvim_license.display()
);
}
}
let out = manifest.join("THIRD_PARTY_LICENSES");
std::fs::write(&out, doc)?;
eprintln!("wrote {}", out.display());
Ok(())
}
fn nvim_derived_langs() -> Vec<String> {
let mut langs: Vec<String> = assets::QUERIES
.dirs()
.filter_map(|dir| {
let derived = dir
.files()
.any(|f| String::from_utf8_lossy(f.contents()).contains("nvim-treesitter"));
if derived {
dir.path()
.file_name()
.and_then(|s| s.to_str())
.map(|s| s.to_string())
} else {
None
}
})
.collect();
langs.sort();
langs
}
fn remove(args: &[String], grammar_dir: &Path) -> Result<()> {
if args.is_empty() {
bail!("remove: need at least one grammar name");
}
for name in args {
let removed = build::remove(name, grammar_dir)?;
if removed {
eprintln!("removed: {}", name);
} else {
eprintln!("not installed: {}", name);
}
}
Ok(())
}