use anyhow::{Context, Result, bail};
use quarb::{AllowShell, NodeId, QueryResult, WithNow};
#[cfg(feature = "native")]
use std::path::Path;
use std::rc::Rc;
#[derive(Clone, Default)]
pub struct Options {
pub hidden: bool,
pub respect_ignore: bool,
pub graft: bool,
pub no_graft: bool,
pub refs: Rc<Vec<(String, String)>>,
pub allow_shell: bool,
pub bonds: Option<Rc<syndesmos::Syndesmos>>,
}
#[cfg(feature = "native")]
fn declared_encoding(name: &str, bytes: &[u8]) -> Option<String> {
let ext = name.rsplit('.').next().unwrap_or("").to_ascii_lowercase();
match ext.as_str() {
"xml" | "svg" | "xhtml" | "fb2" | "tei" | "rss" | "atom" | "opml" | "xsd" | "xsl" => {
quarb_xml::declared_encoding(bytes)
}
"html" | "htm" => quarb_html::declared_encoding(bytes),
"rtf" => quarb_text_rtf::declared_encoding(bytes),
_ => quarb_xml::declared_encoding(bytes).or_else(|| quarb_html::declared_encoding(bytes)),
}
}
#[cfg(feature = "native")]
fn read_source(path: &Path, explicit: Option<&str>) -> Result<String> {
let bytes = std::fs::read(path).with_context(|| format!("reading {}", path.display()))?;
let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("");
quarb::source::SourceDecoding::default()
.with_declared(declared_encoding)
.with_option(explicit)
.and_then(|d| d.decode(name, &bytes))
.map(|d| d.text)
.map_err(|e| anyhow::anyhow!("reading {}: {e}", path.display()))
}
#[cfg(feature = "native")]
fn take_encoding_option(path: &Path) -> Result<(std::path::PathBuf, Option<String>)> {
let (p, explicit, _) = take_source_options(path)?;
Ok((p, explicit))
}
#[cfg(feature = "native")]
fn take_references_option(path: &Path) -> Result<(std::path::PathBuf, Option<String>)> {
let Some(s) = path.to_str() else {
return Ok((path.to_path_buf(), None));
};
if path.exists() || !s.contains("references=") {
return Ok((path.to_path_buf(), None));
}
let Some((p, query)) = s.rsplit_once('?') else {
return Ok((path.to_path_buf(), None));
};
let mut standard = None;
let mut kept = Vec::new();
for pair in query.split('&').filter(|p| !p.is_empty()) {
match pair.split_once('=') {
Some(("references", v)) => {
anyhow::ensure!(
quarb::bibref::STANDARDS.contains(&v),
"{s}: references={v} — references= takes {}",
quarb::bibref::STANDARDS.join(", ")
);
standard = Some(v.to_string());
}
_ => kept.push(pair),
}
}
let rebuilt = if kept.is_empty() {
p.to_string()
} else {
format!("{p}?{}", kept.join("&"))
};
Ok((std::path::PathBuf::from(rebuilt), standard))
}
#[cfg(feature = "native")]
fn take_source_options(
path: &Path,
) -> Result<(std::path::PathBuf, Option<String>, quarb_csv::CsvOptions)> {
let mut csv = quarb_csv::CsvOptions::default();
let Some(s) = path.to_str() else {
return Ok((path.to_path_buf(), None, csv));
};
let prefixed = s.split_once(':').is_some_and(|(pre, _)| {
!pre.is_empty()
&& pre
.chars()
.all(|c| c.is_ascii_alphanumeric() || "+.-".contains(c))
});
if path.exists() || !s.contains('?') || prefixed {
return Ok((path.to_path_buf(), None, csv));
}
let (p, query) = s.rsplit_once('?').unwrap();
let mut explicit = None;
for pair in query.split('&').filter(|p| !p.is_empty()) {
match pair.split_once('=') {
Some(("encoding", label)) => {
quarb::source::encoding_for_label(label)
.ok_or_else(|| anyhow::anyhow!("{s}: unknown encoding label {label:?}"))?;
explicit = Some(label.to_string());
}
Some(("delimiter", label)) => {
csv = csv
.with_delimiter_label(label)
.map_err(|e| anyhow::anyhow!("{s}: {e}"))?;
}
Some(("decimal", label)) => {
csv = csv
.with_decimal_label(label)
.map_err(|e| anyhow::anyhow!("{s}: {e}"))?;
}
_ => bail!(
"unknown option {pair:?} on {p} — a file or folder takes encoding=, \
delimiter=, decimal="
),
}
}
Ok((std::path::PathBuf::from(p), explicit, csv))
}
pub enum Doc {
Json(quarb_json::JsonAdapter),
Csv(quarb_csv::CsvAdapter),
Xml(quarb_xml::XmlAdapter),
Html(quarb_html::HtmlAdapter),
Text(quarb_text::TextModel),
#[cfg(feature = "sqlite")]
Sqlite(quarb_sqlite::SqliteAdapter),
#[cfg(feature = "native")]
Fs(quarb_fs::FsAdapter),
#[cfg(feature = "native")]
FsDeep(quarb_compose::ComposeAdapter<quarb_fs::FsAdapter>),
#[cfg(feature = "native")]
Git(quarb_git::GitAdapter),
#[cfg(feature = "native")]
Archive(quarb_compose::ComposeAdapter<quarb_archive::ArchiveAdapter>),
#[cfg(feature = "native")]
ArchiveRaw(quarb_archive::ArchiveAdapter),
#[cfg(feature = "native")]
Xlsx(quarb_xlsx::XlsxAdapter),
#[cfg(feature = "native")]
Syntax(quarb_tree_sitter::TreeSitterAdapter),
#[cfg(feature = "native")]
Code(quarb_code::CodeModel),
Mount(quarb_mount::MountAdapter),
Boxed(Dyn, Box<dyn Fn(NodeId) -> String>),
}
pub struct Dyn(pub Box<dyn quarb::AstAdapter>);
impl quarb::AstAdapter for Dyn {
quarb::forward_ast_adapter!(0);
}
impl Doc {
fn boxed_kaiv(a: quarb_kaiv::KaivAdapter) -> Doc {
let a = std::rc::Rc::new(a);
let r = a.clone();
Doc::Boxed(
Dyn(Box::new(quarb_mount::Shared(a))),
Box::new(move |n| r.locator(n)),
)
}
pub fn parse(input: &str, format: &str) -> Result<Doc> {
match format {
"kaiv" => {
let a = quarb_kaiv::KaivAdapter::parse_kaiv(input)
.map_err(|e| anyhow::anyhow!("parsing kaiv: {e}"))?;
return Ok(Self::boxed_kaiv(a));
}
"daiv" => {
let a = quarb_kaiv::KaivAdapter::parse_daiv(input)
.map_err(|e| anyhow::anyhow!("parsing daiv: {e}"))?;
return Ok(Self::boxed_kaiv(a));
}
"json" => quarb_json::JsonAdapter::parse(input)
.map(Doc::Json)
.context("parsing JSON"),
"json2" => quarb_json::JsonAdapter::parse_as(input, quarb_json::Reading::Property)
.map(Doc::Json)
.context("parsing JSON"),
"jsonl" | "ndjson" => quarb_json::JsonAdapter::parse_lines(input)
.map(Doc::Json)
.context("parsing JSONL"),
"yaml" | "yml" => quarb_yaml::parse(input)
.map(Doc::Json)
.context("parsing YAML"),
"toml" => quarb_toml::parse(input)
.map(Doc::Json)
.context("parsing TOML"),
"csv" => quarb_csv::CsvAdapter::parse_declared(input, b',', Default::default())
.map(Doc::Csv)
.context("parsing CSV"),
"tsv" => quarb_csv::CsvAdapter::parse_declared(input, b'\t', Default::default())
.map(Doc::Csv)
.context("parsing TSV"),
"xml" => quarb_xml::XmlAdapter::parse(input)
.map(Doc::Xml)
.context("parsing XML"),
"html" => Ok(Doc::Html(quarb_html::HtmlAdapter::parse(input))),
"markdown" | "md" => Ok(Doc::Html(quarb_markdown::parse(input))),
"tex" | "latex" => Ok(Doc::Text(quarb_text_latex::parse(input))),
"text-html" => Ok(Doc::Text(quarb_text_html::parse(input))),
"text-markdown" | "text-md" => Ok(Doc::Text(quarb_text_markdown::parse(input))),
"text" => Ok(Doc::Text(quarb_text::TextModel::parse_plain(input))),
"text-rtf" => quarb_text_rtf::parse(input.as_bytes())
.map(Doc::Text)
.map_err(|e| anyhow::anyhow!("reading RTF: {e}")),
"conllu" => quarb_text::TextModel::parse_conllu_text(input)
.map(Doc::Text)
.map_err(|e| anyhow::anyhow!("reading CoNLL-U: {e}")),
"corpus-conllu" => quarb_text::TextModel::parse_conllu_corpus(input)
.map(Doc::Text)
.map_err(|e| anyhow::anyhow!("reading CoNLL-U: {e}")),
"corpus-html" | "corpus-markdown" | "corpus" => {
let mut model = match format {
"corpus-html" => quarb_text_html::parse(input),
"corpus-markdown" => quarb_text_markdown::parse(input),
_ => quarb_text::TextModel::parse_plain(input),
};
model.tokenize();
Ok(Doc::Text(model))
}
#[cfg(feature = "native")]
"text-bibtex" | "bibtex" => quarb_text_koine::parse_bibtex(input)
.map(Doc::Text)
.map_err(|e| anyhow::anyhow!("{e}")),
#[cfg(feature = "native")]
"text-fb2" | "fb2" => quarb_text_koine::parse_xml_as(input, "fb2")
.map(Doc::Text)
.map_err(|e| anyhow::anyhow!("{e}")),
#[cfg(feature = "native")]
"code-rust" => quarb_code::CodeModel::parse(input, "rs")
.map(Doc::Code)
.context("parsing Rust at the code level"),
#[cfg(feature = "native")]
"code-python" => quarb_code::CodeModel::parse(input, "py")
.map(Doc::Code)
.context("parsing Python at the code level"),
#[cfg(feature = "native")]
"code-javascript" => quarb_code::CodeModel::parse(input, "js")
.map(Doc::Code)
.context("parsing JavaScript at the code level"),
#[cfg(feature = "native")]
"code-c" => quarb_code::CodeModel::parse(input, "c")
.map(Doc::Code)
.context("parsing C at the code level"),
other => bail!("unknown format: {other}"),
}
}
pub(crate) fn base_dyn(&self) -> &dyn quarb::AstAdapter {
match self {
Doc::Json(a) => a,
Doc::Csv(a) => a,
Doc::Xml(a) => a,
Doc::Html(a) => a,
Doc::Text(a) => a,
#[cfg(feature = "sqlite")]
Doc::Sqlite(a) => a,
#[cfg(feature = "native")]
Doc::Fs(a) => a,
#[cfg(feature = "native")]
Doc::FsDeep(a) => a,
#[cfg(feature = "native")]
Doc::Git(a) => a,
#[cfg(feature = "native")]
Doc::Archive(a) => a,
#[cfg(feature = "native")]
Doc::ArchiveRaw(a) => a,
#[cfg(feature = "native")]
Doc::Xlsx(a) => a,
#[cfg(feature = "native")]
Doc::Syntax(a) => a,
#[cfg(feature = "native")]
Doc::Code(a) => a,
Doc::Mount(a) => a,
Doc::Boxed(a, _) => &*a.0,
}
}
pub fn export(
&self,
query: &str,
now: (i64, u32),
allow_shell: bool,
kind: &str,
) -> Result<String> {
let render = quarb_text::Render::from_name(kind)
.ok_or_else(|| anyhow::anyhow!("unknown export format: {kind} (md, html, txt)"))?;
if query.trim().is_empty() {
let base = self.base_dyn();
return Ok(quarb_text::render_nodes(base, &[base.root()], render));
}
match self
.run(query, now, allow_shell)
.map_err(|e| anyhow::anyhow!("{e}"))?
{
QueryResult::Nodes(nodes) => {
Ok(quarb_text::render_nodes(self.base_dyn(), &nodes, render))
}
QueryResult::Values(values) => Ok(quarb_text::render::render_values(&values, render)),
}
}
pub fn run_modeled(
&self,
query: &str,
now: (i64, u32),
allow_shell: bool,
model: &quarb_model::Model,
) -> quarb::Result<QueryResult> {
let (secs, nanos) = now;
let base = quarb_model::Borrowed(self.base_dyn());
let nowed = WithNow {
inner: &base,
secs,
nanos,
};
let enriched = quarb_model::ModelAdapter::new(nowed, model.clone());
enriched
.check()
.map_err(|e| quarb::QuarbError::Parse(format!("reading the model: {e}")))?;
if allow_shell {
quarb::run(query, &AllowShell { inner: &enriched })
} else {
quarb::run(query, &enriched)
}
}
pub fn render_modeled(&self, node: NodeId, model: &quarb_model::Model) -> String {
let enriched =
quarb_model::ModelAdapter::new(quarb_model::Borrowed(self.base_dyn()), model.clone());
enriched.locator(node, |bn| self.render(bn))
}
pub fn run(
&self,
query: &str,
now: (i64, u32),
allow_shell: bool,
) -> quarb::Result<QueryResult> {
let (secs, nanos) = now;
macro_rules! go {
($a:expr) => {{
let nowed = WithNow {
inner: $a,
secs,
nanos,
};
if allow_shell {
quarb::run(query, &AllowShell { inner: &nowed })
} else {
quarb::run(query, &nowed)
}
}};
}
match self {
Doc::Json(a) => go!(a),
Doc::Csv(a) => go!(a),
Doc::Xml(a) => go!(a),
Doc::Html(a) => go!(a),
Doc::Text(a) => go!(a),
#[cfg(feature = "sqlite")]
Doc::Sqlite(a) => go!(a),
#[cfg(feature = "native")]
Doc::Fs(a) => go!(a),
#[cfg(feature = "native")]
Doc::FsDeep(a) => go!(a),
#[cfg(feature = "native")]
Doc::Git(a) => go!(a),
#[cfg(feature = "native")]
Doc::Archive(a) => go!(a),
#[cfg(feature = "native")]
Doc::ArchiveRaw(a) => go!(a),
#[cfg(feature = "native")]
Doc::Xlsx(a) => go!(a),
#[cfg(feature = "native")]
Doc::Syntax(a) => go!(a),
#[cfg(feature = "native")]
Doc::Code(a) => go!(a),
Doc::Mount(a) => go!(a),
Doc::Boxed(a, _) => go!(a),
}
}
pub fn family(&self) -> &'static str {
match self {
Doc::Json(_) => "json",
Doc::Csv(_) => "csv",
Doc::Xml(_) => "xml",
Doc::Html(_) => "html",
Doc::Text(_) => "text",
#[cfg(feature = "sqlite")]
Doc::Sqlite(_) => "sqlite",
#[cfg(feature = "native")]
Doc::Fs(_) | Doc::FsDeep(_) => "fs",
#[cfg(feature = "native")]
Doc::Git(_) => "git",
#[cfg(feature = "native")]
Doc::Archive(_) | Doc::ArchiveRaw(_) => "archive",
#[cfg(feature = "native")]
Doc::Xlsx(_) => "xlsx",
#[cfg(feature = "native")]
Doc::Syntax(_) => "tree-sitter",
#[cfg(feature = "native")]
Doc::Code(_) => "code",
Doc::Mount(_) => "mount",
Doc::Boxed(..) => "boxed",
}
}
pub fn trace(
&self,
committed: &str,
staged: Option<&str>,
opts: &quarb::trace::Options,
) -> quarb::Result<quarb_trace::Payload> {
let base = quarb_model::Borrowed(self.base_dyn());
quarb::trace::trace(committed, staged, &base, &|n| self.render(n), opts)
}
pub fn render(&self, node: NodeId) -> String {
match self {
Doc::Json(a) => a.pointer(node),
Doc::Csv(a) => a.locator(node),
Doc::Xml(a) => a.locator(node),
Doc::Html(a) => a.locator(node),
Doc::Text(a) => a.locator(node),
#[cfg(feature = "sqlite")]
Doc::Sqlite(a) => a.locator(node),
#[cfg(feature = "native")]
Doc::Fs(a) => a.path(node).display().to_string(),
#[cfg(feature = "native")]
Doc::FsDeep(a) => a.locator(node, |o| a.outer().path(o).display().to_string()),
#[cfg(feature = "native")]
Doc::Git(a) => a.locator(node),
#[cfg(feature = "native")]
Doc::Archive(a) => a.locator(node, |o| a.outer().locator(o)),
#[cfg(feature = "native")]
Doc::ArchiveRaw(a) => a.locator(node),
#[cfg(feature = "native")]
Doc::Xlsx(a) => a.locator(node),
#[cfg(feature = "native")]
Doc::Syntax(a) => a.locator(node),
#[cfg(feature = "native")]
Doc::Code(a) => a.locator(node),
Doc::Mount(a) => generic_locator(a, node),
Doc::Boxed(_, render) => render(node),
}
}
#[cfg(feature = "sqlite")]
pub fn sqlite_bytes(bytes: &[u8]) -> Result<Doc> {
Ok(Doc::Sqlite(
quarb_sqlite::SqliteAdapter::from_bytes(bytes)
.map_err(|e| anyhow::anyhow!("{e}"))
.context("opening SQLite bytes")?,
))
}
#[cfg(not(feature = "sqlite"))]
pub fn sqlite_bytes(_bytes: &[u8]) -> Result<Doc> {
anyhow::bail!("this quarb-session was built without the `sqlite` feature")
}
pub fn mount_docs(parts: Vec<(String, Doc)>) -> Result<Doc> {
let mut mounts: Vec<quarb_mount::Mount> = Vec::new();
for (name, doc) in parts {
if mounts.iter().any(|m| m.name == name) {
bail!("two sources mount as '{name}'; give each a distinct name");
}
mounts.push(quarb_mount::Mount {
name,
target: None,
adapter: doc.into_boxed()?,
});
}
Ok(Doc::Mount(
quarb_mount::MountAdapter::try_new(mounts)
.map_err(|e| anyhow::anyhow!("mount: {e}"))?,
))
}
pub fn attach_url(&mut self, url: &str) {
match self {
Doc::Html(a) => a.set_document_url(url),
Doc::Text(a) => a.set_document_url(url),
_ => {}
}
}
pub fn url_roots(
&self,
urls: &[(String, String)],
) -> std::collections::HashMap<String, quarb::NodeId> {
use quarb::AstAdapter;
let a = self.base_dyn();
let root = a.root();
match self {
Doc::Mount(_) => urls
.iter()
.filter_map(|(name, url)| {
a.children_named(root, name)
.first()
.map(|&n| (url.clone(), n))
})
.collect(),
_ => urls
.iter()
.take(1)
.map(|(_, url)| (url.clone(), root))
.collect(),
}
}
pub fn mount_texts(parts: &[(String, String, String)]) -> Result<Doc> {
let mut docs: Vec<(String, Doc)> = Vec::new();
for (name, format, text) in parts {
let doc = Doc::parse(text, format)
.with_context(|| format!("parsing '{name}' as {format}"))?;
docs.push((name.clone(), doc));
}
Doc::mount_docs(docs)
}
fn into_boxed(self) -> Result<Box<dyn quarb::AstAdapter>> {
use quarb_mount::Shared;
Ok(match self {
Doc::Json(a) => Box::new(Shared(Rc::new(a))),
Doc::Csv(a) => Box::new(Shared(Rc::new(a))),
Doc::Xml(a) => Box::new(Shared(Rc::new(a))),
Doc::Html(a) => Box::new(Shared(Rc::new(a))),
Doc::Text(a) => Box::new(Shared(Rc::new(a))),
#[cfg(feature = "sqlite")]
Doc::Sqlite(a) => Box::new(Shared(Rc::new(a))),
#[cfg(feature = "native")]
Doc::Fs(a) => Box::new(Shared(Rc::new(a))),
#[cfg(feature = "native")]
Doc::FsDeep(a) => Box::new(Shared(Rc::new(a))),
#[cfg(feature = "native")]
Doc::Git(a) => Box::new(Shared(Rc::new(a))),
#[cfg(feature = "native")]
Doc::Archive(a) => Box::new(Shared(Rc::new(a))),
#[cfg(feature = "native")]
Doc::ArchiveRaw(a) => Box::new(Shared(Rc::new(a))),
#[cfg(feature = "native")]
Doc::Xlsx(a) => Box::new(Shared(Rc::new(a))),
#[cfg(feature = "native")]
Doc::Syntax(a) => Box::new(Shared(Rc::new(a))),
#[cfg(feature = "native")]
Doc::Code(a) => Box::new(Shared(Rc::new(a))),
Doc::Mount(_) => bail!("cannot nest a mount inside a mount"),
Doc::Boxed(a, _) => a.0,
})
}
}
#[cfg(feature = "native")]
impl Doc {
pub fn open(path: &Path, opts: &Options) -> Result<Doc> {
let (stripped, references) = take_references_option(path)?;
let doc = Self::open_inner(stripped.as_path(), opts)?;
match (doc, references) {
(Doc::Text(mut model), Some(standard)) => {
model
.lower_references(&standard)
.map_err(|e| anyhow::anyhow!("{}: {e}", path.display()))?;
Ok(Doc::Text(model))
}
(doc, Some(_)) => {
let _ = doc;
bail!(
"{}: references= applies to a text-level reading (text:, koine:)",
path.display()
)
}
(doc, None) => Ok(doc),
}
}
fn open_inner(path: &Path, opts: &Options) -> Result<Doc> {
let (path, explicit, csv) = take_source_options(path)?;
let path = path.as_path();
let explicit = explicit.as_deref();
if path.is_dir() {
let fsopts = quarb_fs::FsOptions {
hidden: opts.hidden,
respect_ignore: opts.respect_ignore,
};
let fs = quarb_fs::FsAdapter::with_options(path, fsopts)
.with_context(|| format!("opening directory {}", path.display()))?;
return Ok(if opts.graft {
Doc::FsDeep(
quarb_compose::ComposeAdapter::with_source_paths(fs, |fs, n| Some(fs.path(n)))
.with_decoding(
quarb_compose::source_decoding(explicit)
.map_err(|e| anyhow::anyhow!("{e}"))?,
)
.with_csv_options(csv),
)
} else {
Doc::Fs(fs)
});
}
let s = path.to_string_lossy();
if let Some(repo) = s.strip_prefix("git:") {
let a =
quarb_git::GitAdapter::open(Path::new(repo)).context("opening git repository")?;
return Ok(Doc::Git(a));
}
if let Some(rest) = s.strip_prefix("lines:")
&& !rest.is_empty()
{
let (target, opt) = take_encoding_option(Path::new(rest))?;
let text = read_source(&target, opt.as_deref().or(explicit))?;
let a = std::rc::Rc::new(quarb_lines::LinesAdapter::parse(&text));
let r = a.clone();
return Ok(Doc::Boxed(
Dyn(Box::new(quarb_mount::Shared(a))),
Box::new(move |n| r.locator(n)),
));
}
if let Some(rest) = s.strip_prefix("web:")
&& !rest.is_empty()
{
let (path_part, base) = match rest.split_once('?') {
Some((p, q)) => {
let mut base = String::new();
for pair in q.split('&') {
match pair.split_once('=') {
Some(("base", v)) => base = v.to_string(),
_ => {
return Err(anyhow::anyhow!(
"unknown web option {pair:?} — supported: base="
));
}
}
}
(p, base)
}
None => (rest, String::new()),
};
let decoding =
quarb_compose::source_decoding(explicit).map_err(|e| anyhow::anyhow!("{e}"))?;
let target = Path::new(path_part);
let site = if target.is_dir() {
quarb_web::fs::open_dir_with_decoding(target, &base, decoding)
.with_context(|| format!("reading {} as a site", target.display()))?
} else if target.is_file() && !path_part.starts_with("postgres") {
quarb_web::archive::open_path_with_decoding(target, &base, decoding)
.with_context(|| format!("reading {} as a site", target.display()))?
} else {
return Err(anyhow::anyhow!(
"web: in a session takes a directory of pages or an archive of them; a site store ({path_part}) is read on the command line (qua … web:{path_part})"
));
};
let a = std::rc::Rc::new(site);
let r = a.clone();
return Ok(Doc::Boxed(
Dyn(Box::new(quarb_mount::Shared(a))),
Box::new(move |n| r.locator(n)),
));
}
if let Some(rest) = s.strip_prefix("koine:")
&& !rest.is_empty()
{
let (path_part, format) = match rest.split_once('?') {
Some((p, q)) => {
let mut fmt = None;
for pair in q.split('&') {
match pair.split_once('=') {
Some(("format", v)) => fmt = Some(v.to_string()),
_ => {
return Err(anyhow::anyhow!(
"unknown koine option {pair:?} — supported: format="
));
}
}
}
(p, fmt)
}
None => (rest, None),
};
let target = Path::new(path_part);
let ext = target
.extension()
.and_then(|e| e.to_str())
.map(|e| e.to_ascii_lowercase());
let read = || -> Result<String> {
let text = read_source(target, explicit)?;
Ok(text
.strip_prefix('\u{feff}')
.map(str::to_owned)
.unwrap_or(text))
};
if let Some(fmt) = format {
let imported = match fmt.as_str() {
"atd" | "atk" => {
let model = quarb_text_koine::parse_file(target).with_context(|| {
format!("reading {} as an atrep document", target.display())
})?;
return Ok(Doc::Text(model));
}
"md" | "markdown" => quarb_text_koine::parse_markdown(&read()?),
"html" => quarb_text_koine::parse_html(&read()?),
"rst" => quarb_text_koine::parse_rst(&read()?),
"org" => quarb_text_koine::parse_org(&read()?),
"dj" | "djot" => quarb_text_koine::parse_djot(&read()?),
"tei" | "docbook" | "jats" | "usx" | "osis" | "fb2" | "rnc" | "opencorpora"
| "proiel" => quarb_text_koine::parse_xml_as(&read()?, &fmt),
"docx" => quarb_text_koine::parse_docx(&std::fs::read(target)?),
"dsl" => quarb_text_koine::parse_dsl_file(target),
other => {
return Err(anyhow::anyhow!(
"unknown koine format {other:?} — known: md, html, rst, org, djot, tei, docbook, jats, usx, osis, fb2, docx, dsl, atd"
));
}
};
return imported.map(Doc::Text).with_context(|| {
format!("importing {} through atrep as {fmt}", target.display())
});
}
if quarb_text_koine::is_dsl_path(target) {
let model = quarb_text_koine::parse_dsl_file(target).with_context(|| {
format!("importing {} as a Lingvo DSL dictionary", target.display())
})?;
return Ok(Doc::Text(model));
}
let imported = match ext.as_deref() {
Some("atd" | "atk") => {
let model = quarb_text_koine::parse_file(target).with_context(|| {
format!("reading {} as an atrep document", target.display())
})?;
return Ok(Doc::Text(model));
}
Some("docx") => quarb_text_koine::parse_docx(&std::fs::read(target)?),
Some("usfm" | "sfm") => quarb_text_koine::parse_xml_as(&read()?, "usfm"),
Some("fb2") => quarb_text_koine::parse_xml_as(&read()?, "fb2"),
Some("bib") => quarb_text_koine::parse_bibtex(&read()?),
Some("md" | "markdown") => quarb_text_koine::parse_markdown(&read()?),
Some("html" | "htm") => quarb_text_koine::parse_html(&read()?),
Some("rst") => quarb_text_koine::parse_rst(&read()?),
Some("org") => quarb_text_koine::parse_org(&read()?),
Some("dj" | "djot") => quarb_text_koine::parse_djot(&read()?),
Some("xml") => {
let text = read()?;
match quarb_text_koine::detect_xml_kind(&text) {
Some(kind) => quarb_text_koine::parse_xml_as(&text, kind),
None => {
return Err(anyhow::anyhow!(
"{} declares no XML identity this route knows — force one with koine:{}?format=tei|docbook|jats|usx|osis|rnc|opencorpora|proiel",
target.display(),
target.display()
));
}
}
}
Some("pdf") => {
return Err(anyhow::anyhow!(
"atrep cannot import a PDF — the print reading is text:{}",
target.display()
));
}
_ => {
return Err(anyhow::anyhow!(
"no atrep import for this format yet — the native reading is text:{}",
target.display()
));
}
};
return imported
.map(Doc::Text)
.with_context(|| format!("importing {} through atrep", target.display()));
}
if let Some(rest) = s.strip_prefix("lit:")
&& !rest.is_empty()
{
let (rest, conllu, annotate, desm, format, cast, places, scheme, modernize, notes) =
match rest.split_once('?') {
Some((p, q)) => {
let (
mut conllu,
mut annotate,
mut desm,
mut format,
mut cast,
mut places,
mut scheme,
mut modernize,
mut notes,
) = (None, None, Vec::new(), None, None, None, None, None, None);
for pair in q.split('&') {
match pair.split_once('=') {
Some(("conllu", v)) => conllu = Some(v.to_string()),
Some(("annotate", v)) => annotate = Some(v.to_string()),
Some(("desm", v)) => desm.push(v.to_string()),
Some(("cast", v)) => cast = Some(v.to_string()),
Some(("places", v)) => places = Some(v.to_string()),
Some(("scheme", v)) => scheme = Some(v.to_string()),
Some(("modernize", v)) => modernize = Some(v.to_string()),
Some(("notes", v)) => notes = Some(v.to_string()),
Some(("format", v)) => format = Some(v.to_string()),
Some(("encoding", _)) => {}
_ => bail!(
"unknown lit option {pair:?} — supported: conllu=, annotate=, desm=, cast=, places=, format=, scheme=, modernize=, notes=, encoding="
),
}
}
(
p, conllu, annotate, desm, format, cast, places, scheme, modernize,
notes,
)
}
None => (
rest,
None,
None,
Vec::new(),
None,
None,
None,
None,
None,
None,
),
};
let target = Path::new(rest);
let ext = target
.extension()
.and_then(|e| e.to_str())
.map(|e| e.to_ascii_lowercase());
let read = || -> Result<String> {
let text = read_source(target, explicit)?;
Ok(text
.strip_prefix('\u{feff}')
.map(str::to_owned)
.unwrap_or(text))
};
let kind = match format {
Some(k) => k,
None => match ext.as_deref() {
Some("atd" | "atk") => "atd".to_string(),
Some("usx") => "usx".to_string(),
Some("usfm" | "sfm") => "usfm".to_string(),
Some("fb2") => "fb2".to_string(),
_ => match quarb_text_koine::detect_xml_kind(&read()?) {
Some(k) => k.to_string(),
None => bail!(
"lit: reads TEI, USX, OSIS, USFM, FictionBook, RNC, OpenCorpora, PROIEL and litogramma — {rest} declares none of them (koine:{rest} reads the other XML vocabularies, corpus:{rest} plain prose)"
),
},
},
};
let mut model = match kind.as_str() {
"atd" | "atk" => quarb_text_koine::parse_lit_file_with(target, scheme.as_deref())
.with_context(|| {
format!("reading {} as an atrep document", target.display())
})?,
"tei" | "usx" | "osis" | "fb2" | "usfm" | "rnc" | "opencorpora" | "proiel" => {
quarb_text_koine::parse_lit_xml_as_with(&read()?, &kind, scheme.as_deref())
.with_context(|| {
format!("reading {} as {}", target.display(), kind.to_uppercase())
})?
}
other => bail!(
"lit: reads TEI, USX, OSIS, USFM, FictionBook, RNC, OpenCorpora, PROIEL and litogramma — not {other} (koine:{rest} reads the other XML vocabularies)"
),
};
model.set_document_path(rest);
return finish_corpus(
model, conllu, annotate, &desm, cast, places, modernize, notes, opts, "lit",
);
}
if let Some(rest) = s.strip_prefix("corpus:")
&& !rest.is_empty()
{
let (rest, conllu, annotate, desm, format, cast, places, modernize, notes) = match rest
.split_once('?')
{
Some((p, q)) => {
let (
mut conllu,
mut annotate,
mut desm,
mut format,
mut cast,
mut places,
mut modernize,
mut notes,
) = (None, None, Vec::new(), None, None, None, None, None);
for pair in q.split('&') {
match pair.split_once('=') {
Some(("conllu", v)) => conllu = Some(v.to_string()),
Some(("annotate", v)) => annotate = Some(v.to_string()),
Some(("desm", v)) => desm.push(v.to_string()),
Some(("cast", v)) => cast = Some(v.to_string()),
Some(("places", v)) => places = Some(v.to_string()),
Some(("modernize", v)) => modernize = Some(v.to_string()),
Some(("notes", v)) => notes = Some(v.to_string()),
Some(("format", "conllu")) => format = Some("conllu"),
Some(("format", v)) => bail!(
"corpus: format={v} is not a treebank format — format= takes conllu"
),
_ => bail!(
"unknown corpus option {pair:?} — supported: conllu=, annotate=, desm=, cast=, places=, modernize=, notes=, format="
),
}
}
(
p, conllu, annotate, desm, format, cast, places, modernize, notes,
)
}
None => (rest, None, None, Vec::new(), None, None, None, None, None),
};
let target = Path::new(rest);
let ext = target
.extension()
.and_then(|e| e.to_str())
.map(|e| e.to_ascii_lowercase());
if ext.as_deref() == Some("pdf") {
bail!(
"corpus: needs a prose reading, and a PDF's is line geometry — text:{rest} reads its lines"
);
}
let own_annotation = || {
if conllu.is_some() || annotate.is_some() {
bail!(
"corpus: {rest} is its own annotation — conllu= and annotate= apply to prose"
);
}
Ok(())
};
if target.is_dir() {
own_annotation()?;
let files = quarb_text::read_conllu_dir(target).map_err(|e| {
anyhow::anyhow!("corpus: over a directory reads its treebank — {e}")
})?;
let refs: Vec<(&str, &str)> = files
.iter()
.map(|(n, t)| (n.as_str(), t.as_str()))
.collect();
let mut model = quarb_text::TextModel::parse_conllu_corpus_set(&refs)
.map_err(|e| anyhow::anyhow!("reading {rest} as CoNLL-U: {e}"))?;
model.set_document_path(rest);
return treebank_options(model, &cast, &places, &modernize);
}
let treebank = matches!(ext.as_deref(), Some("conllu" | "conllup"))
|| format == Some("conllu")
|| (ext.is_none()
&& read_source(target, explicit)
.is_ok_and(|t| quarb_text::looks_like_conllu(&t)));
if treebank {
own_annotation()?;
let text = read_source(target, explicit)?;
let text = text
.strip_prefix('\u{feff}')
.map(str::to_owned)
.unwrap_or(text);
let Doc::Text(mut model) = Doc::parse(&text, "corpus-conllu")? else {
bail!("corpus: needs a prose reading of {rest}")
};
model.set_document_path(rest);
return treebank_options(model, &cast, &places, &modernize);
}
let route = if matches!(ext.as_deref(), Some("xml" | "tei")) {
format!("koine:{rest}")
} else {
format!("text:{rest}")
};
let Doc::Text(model) = Doc::open(Path::new(&route), opts)? else {
bail!("corpus: needs a prose reading of {rest}")
};
return finish_corpus(
model, conllu, annotate, &desm, cast, places, modernize, notes, opts, "corpus",
);
}
if let Some(rest) = s.strip_prefix("json2:")
&& !rest.is_empty()
{
use quarb_json::Reading;
let target = Path::new(rest);
let text = read_source(target, explicit)?;
let ext = target
.extension()
.and_then(|e| e.to_str())
.map(|e| e.to_ascii_lowercase())
.unwrap_or_default();
let a = match ext.as_str() {
"yaml" | "yml" => {
quarb_yaml::parse_as(&text, Reading::Property).context("parsing YAML")?
}
"toml" => quarb_toml::parse_as(&text, Reading::Property).context("parsing TOML")?,
"jsonl" | "ndjson" => {
quarb_json::JsonAdapter::parse_lines_as(&text, Reading::Property)
.context("parsing JSONL")?
}
_ => quarb_json::JsonAdapter::parse_as(&text, Reading::Property)
.context("parsing JSON")?,
};
return Ok(Doc::Json(a));
}
if let Some(rest) = s.strip_prefix("text:")
&& !rest.is_empty()
{
let target = Path::new(rest);
if target.is_dir() {
let files = quarb_text::read_conllu_dir(target).map_err(|e| {
anyhow::anyhow!("text: over a directory reads its treebank — {e}")
})?;
let refs: Vec<(&str, &str)> = files
.iter()
.map(|(n, t)| (n.as_str(), t.as_str()))
.collect();
let mut model = quarb_text::TextModel::parse_conllu_text_set(&refs)
.map_err(|e| anyhow::anyhow!("reading {rest} as CoNLL-U: {e}"))?;
model.set_document_path(rest);
return Ok(Doc::Text(model));
}
let ext = target
.extension()
.and_then(|e| e.to_str())
.map(|e| e.to_ascii_lowercase());
if let Some("usfm" | "sfm") = ext.as_deref() {
let text = read_source(target, explicit)?;
let text = text
.strip_prefix('\u{feff}')
.map(str::to_owned)
.unwrap_or(text);
let model = quarb_text_koine::parse_xml_as(&text, "usfm")
.with_context(|| format!("reading {} as USFM", target.display()))?;
return Ok(Doc::Text(model));
}
if let Some("atd" | "atk") = ext.as_deref() {
let model = quarb_text_koine::parse_file(target).with_context(|| {
format!("reading {} as an atrep document", target.display())
})?;
return Ok(Doc::Text(model));
}
if ext.as_deref() == Some("pdf") {
let bytes = std::fs::read(target)
.with_context(|| format!("reading {}", target.display()))?;
let model = quarb_text_pdf::parse(&bytes)
.with_context(|| format!("reading {} as PDF", target.display()))?;
let a = std::rc::Rc::new(model);
let r = a.clone();
return Ok(Doc::Boxed(
Dyn(Box::new(quarb_mount::Shared(a))),
Box::new(move |n| r.locator(n)),
));
}
if matches!(ext.as_deref(), Some("djvu" | "djv")) {
let model = quarb_text_djvu::parse_file(target)
.with_context(|| format!("reading {} as DjVu", target.display()))?;
let a = std::rc::Rc::new(model);
let r = a.clone();
return Ok(Doc::Boxed(
Dyn(Box::new(quarb_mount::Shared(a))),
Box::new(move |n| r.locator(n)),
));
}
if quarb_text_koine::is_dsl_path(target) {
let model = quarb_text_koine::parse_dsl_file(target).with_context(|| {
format!("reading {} as a Lingvo DSL dictionary", target.display())
})?;
return Ok(Doc::Text(model));
}
let word97 = ext.as_deref() == Some("doc")
&& std::fs::read(target).is_ok_and(|b| quarb_text_doc::is_ole2(&b));
if let Some(ext @ ("docx" | "epub" | "odt" | "fodt" | "sxw" | "doc")) =
ext.as_deref().filter(|e| *e != "doc" || word97)
{
let bytes = std::fs::read(target)
.with_context(|| format!("reading {}", target.display()))?;
let decoding =
quarb_compose::source_decoding(explicit).map_err(|e| anyhow::anyhow!("{e}"))?;
let model = match ext {
"doc" => quarb_text_doc::parse(&bytes)
.with_context(|| format!("reading {} as Word 97", target.display()))?,
"docx" => {
let mut m = quarb_text_docx::parse_with_decoding(&bytes, &decoding)
.with_context(|| format!("reading {} as Word", target.display()))?;
quarb_text_koine::attach_bib_aliases(&mut m);
m
}
"odt" | "fodt" | "sxw" => quarb_text_odt::parse_with_decoding(
&bytes, &decoding,
)
.with_context(|| format!("reading {} as OpenDocument", target.display()))?,
_ => quarb_text_epub::parse_with_decoding(&bytes, &decoding)
.with_context(|| format!("reading {} as EPUB", target.display()))?,
};
return Ok(Doc::Text(model));
}
let text = read_source(target, explicit)?;
let text = text
.strip_prefix('\u{feff}')
.map(str::to_owned)
.unwrap_or(text);
let format = match target
.extension()
.and_then(|e| e.to_str())
.map(|e| e.to_ascii_lowercase())
.as_deref()
{
Some("html" | "htm") => "text-html",
Some("md" | "markdown") => "text-markdown",
Some("tex" | "latex") => "tex",
Some("txt") => "text",
Some("rtf") => "text-rtf",
Some("conllu" | "conllup") => "conllu",
#[cfg(feature = "native")]
Some("bib") => "text-bibtex",
#[cfg(feature = "native")]
Some("fb2") => "text-fb2",
#[cfg(feature = "native")]
_ if text.trim_start().starts_with('<')
&& quarb_text_koine::detect_xml_kind(&text) == Some("fb2") =>
{
"text-fb2"
}
_ if text.trim_start().starts_with('<') => "text-html",
None if quarb_text::looks_like_conllu(&text) => "conllu",
_ => "text",
};
return Doc::parse(&text, format);
}
if let Some(rest) = s.strip_prefix("code:")
&& !rest.is_empty()
{
if opts.no_graft {
bail!(
"no_graft refuses the code: prefix: the prefix's whole \
meaning is the grafted code-level view"
);
}
let target = Path::new(rest);
if target.is_dir() {
let fsopts = quarb_fs::FsOptions {
hidden: opts.hidden,
respect_ignore: opts.respect_ignore,
};
let fs = quarb_fs::FsAdapter::with_options(target, fsopts)
.with_context(|| format!("opening directory {}", target.display()))?;
return Ok(Doc::FsDeep(
quarb_compose::ComposeAdapter::with_source_paths(fs, |fs, n| Some(fs.path(n)))
.with_source_graft(quarb_compose::SourceGraft::Code),
));
}
let a = quarb_code::CodeModel::open(target)
.with_context(|| format!("parsing {} at the code level", target.display()))?;
return Ok(Doc::Code(a));
}
let ext = path
.extension()
.and_then(|e| e.to_str())
.map(|e| e.to_ascii_lowercase());
if let Some(e) = &ext
&& quarb_tree_sitter::supported(e)
{
let a =
quarb_tree_sitter::TreeSitterAdapter::open(path).context("parsing source file")?;
return Ok(Doc::Syntax(a));
}
if matches!(ext.as_deref(), Some("xlsx" | "xls" | "ods")) {
let a = quarb_xlsx::XlsxAdapter::open(path).context("opening workbook")?;
return Ok(Doc::Xlsx(a));
}
if is_sqlite(path) {
#[cfg(not(feature = "sqlite"))]
anyhow::bail!(
"{}: this quarb-session was built without the `sqlite` feature",
path.display()
);
#[cfg(feature = "sqlite")]
{
let a = quarb_sqlite::SqliteAdapter::open_with_refs(path, &opts.refs)
.context("opening SQLite database")?;
return Ok(Doc::Sqlite(a));
}
}
if is_archive(path) {
let a = quarb_archive::ArchiveAdapter::open(path).context("opening archive")?;
return Ok(if opts.no_graft {
Doc::ArchiveRaw(a)
} else {
Doc::Archive(quarb_compose::ComposeAdapter::new(a))
});
}
let text = read_source(path, explicit)?;
let text = text
.strip_prefix('\u{feff}')
.map(str::to_owned)
.unwrap_or(text);
match ext.as_deref() {
Some("kaiv") => quarb_kaiv::KaivAdapter::parse_kaiv_at(&text, path.parent())
.map(Doc::boxed_kaiv)
.map_err(|e| anyhow::anyhow!("parsing kaiv: {e}")),
Some("daiv") => quarb_kaiv::KaivAdapter::parse_daiv_at(&text, path.parent())
.map(Doc::boxed_kaiv)
.map_err(|e| anyhow::anyhow!("parsing daiv: {e}")),
Some("raiv") => quarb_kaiv::KaivAdapter::parse_raiv_at(&text, path.parent())
.map(Doc::boxed_kaiv)
.map_err(|e| anyhow::anyhow!("parsing raiv: {e}")),
Some("quarb") => {
let a = std::rc::Rc::new(
quarb::reflect::QueryArbor::parse(&text).context("parsing Quarb query")?,
);
let r = a.clone();
Ok(Doc::Boxed(
Dyn(Box::new(quarb_mount::Shared(a))),
Box::new(move |n| r.locator(n)),
))
}
Some("csv") => quarb_csv::CsvAdapter::parse_declared(&text, b',', csv)
.map(Doc::Csv)
.context("parsing CSV"),
Some("tsv") => quarb_csv::CsvAdapter::parse_declared(&text, b'\t', csv)
.map(Doc::Csv)
.context("parsing TSV"),
Some("yaml" | "yml") => Doc::parse(&text, "yaml"),
Some("toml") => Doc::parse(&text, "toml"),
Some("md" | "markdown") => Doc::parse(&text, "markdown"),
Some("txt") => Doc::parse(&text, "text"),
Some("conllu" | "conllup") => Doc::parse(&text, "conllu"),
Some("jsonl" | "ndjson") => Doc::parse(&text, "jsonl"),
_ => {
if is_xml(path, &text) {
Doc::parse(&text, "xml")
} else if is_html(path, &text) {
Doc::parse(&text, "html")
} else {
Doc::parse(&text, "json")
}
}
}
}
pub fn mount(paths: &[std::path::PathBuf], opts: &Options) -> Result<Doc> {
let specs: Vec<crate::MountSpec> = paths
.iter()
.map(|p| crate::MountSpec {
name: None,
path: p.clone(),
})
.collect();
Doc::mount_specs(&specs, opts)
}
pub fn mount_specs(specs: &[crate::MountSpec], opts: &Options) -> Result<Doc> {
let mut mounts: Vec<quarb_mount::Mount> = Vec::new();
for (i, spec) in specs.iter().enumerate() {
let name = spec.name.clone().unwrap_or_else(|| {
spec.path
.file_stem()
.map(|s| s.to_string_lossy().into_owned())
.unwrap_or_else(|| format!("doc{i}"))
});
if mounts.iter().any(|m| m.name == name) {
bail!(
"input '{}' mounts as '{name}', colliding with an earlier input of the \
same name; give each a distinct basename (or a NAME=TARGET alias)",
spec.path.display()
);
}
let adapter = Doc::open(&spec.path, opts)?.into_boxed()?;
mounts.push(quarb_mount::Mount {
name,
target: Some(spec.path.display().to_string()),
adapter,
});
}
Ok(Doc::Mount(
quarb_mount::MountAdapter::try_new(mounts)
.map_err(|e| anyhow::anyhow!("mount: {e}"))?,
))
}
}
fn generic_locator<A: quarb::AstAdapter>(a: &A, node: NodeId) -> String {
let mut parts = Vec::new();
let mut cur = Some(node);
while let Some(n) = cur {
if let Some(nm) = a.name(n) {
parts.push(nm);
}
cur = a.parent(n);
}
parts.reverse();
format!("/{}", parts.join("/"))
}
#[cfg(feature = "native")]
fn is_sqlite(path: &Path) -> bool {
if path
.extension()
.and_then(|e| e.to_str())
.is_some_and(|e| matches!(e.to_ascii_lowercase().as_str(), "db" | "sqlite" | "sqlite3"))
{
return true;
}
use std::io::Read as _;
let mut buf = [0u8; 16];
std::fs::File::open(path)
.and_then(|mut f| f.read_exact(&mut buf))
.is_ok()
&& &buf == b"SQLite format 3\0"
}
#[cfg(feature = "native")]
fn is_archive(path: &Path) -> bool {
if path.extension().and_then(|e| e.to_str()).is_some_and(|e| {
matches!(
e.to_ascii_lowercase().as_str(),
"zip"
| "tar"
| "gz"
| "tgz"
| "jar"
| "war"
| "docx"
| "pptx"
| "odt"
| "odp"
| "7z"
| "xz"
| "txz"
| "bz2"
| "tbz2"
| "zst"
| "tzst"
)
}) {
return true;
}
use std::io::Read as _;
let mut buf = [0u8; 2];
std::fs::File::open(path)
.and_then(|mut f| f.read_exact(&mut buf))
.is_ok()
&& (&buf == b"PK" || buf == [0x1f, 0x8b])
}
#[cfg(feature = "native")]
fn is_xml(path: &Path, text: &str) -> bool {
path.extension()
.and_then(|e| e.to_str())
.is_some_and(|e| matches!(e.to_ascii_lowercase().as_str(), "xml" | "svg" | "xhtml"))
|| text.trim_start().starts_with("<?xml")
}
#[cfg(feature = "native")]
fn is_html(path: &Path, text: &str) -> bool {
path.extension()
.and_then(|e| e.to_str())
.is_some_and(|e| matches!(e.to_ascii_lowercase().as_str(), "html" | "htm"))
|| text.trim_start().starts_with('<')
}
#[cfg(feature = "native")]
fn treebank_options(
mut model: quarb_text::TextModel,
cast: &Option<String>,
places: &Option<String>,
modernize: &Option<String>,
) -> Result<Doc> {
if let Some(t) = modernize {
model
.set_orthography(t)
.map_err(|e| anyhow::anyhow!("corpus: {e}"))?;
}
if let Some(f) = cast {
let rows = quarb_text::CastRow::read_csv(f)
.map_err(|e| anyhow::anyhow!("corpus: reading the cast table {f}: {e}"))?;
model.apply_cast(&rows);
}
if let Some(f) = places {
let rows = quarb_text::CastRow::read_csv(f)
.map_err(|e| anyhow::anyhow!("corpus: reading the places table {f}: {e}"))?;
model.apply_places(&rows);
}
Ok(Doc::Text(model))
}
#[cfg(feature = "native")]
fn finish_corpus(
mut model: quarb_text::TextModel,
conllu: Option<String>,
annotate: Option<String>,
desm: &[String],
cast: Option<String>,
places: Option<String>,
modernize: Option<String>,
notes: Option<String>,
opts: &Options,
level: &str,
) -> Result<Doc> {
match notes.as_deref() {
None => {}
Some("omit") => model.omit_notes(true),
Some(v) => {
bail!("{level}: notes={v} — notes= takes omit (a note is part of the text by default)")
}
}
if let Some(table) = &modernize {
model
.set_orthography(table)
.map_err(|e| anyhow::anyhow!("{level}: {e}"))?;
}
let conllu_text = match (conllu, annotate) {
(Some(_), Some(_)) => bail!("{level}: takes conllu= or annotate=, not both"),
(Some(file), None) => Some(
read_source(Path::new(&file), None)
.with_context(|| format!("reading {file} as CoNLL-U"))?,
),
(None, Some(cmd)) => {
if !opts.allow_shell {
bail!("{level}: annotate= runs a command; the shell gate is closed");
}
use std::io::Write;
let mut child = std::process::Command::new("sh")
.arg("-c")
.arg(&cmd)
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.spawn()
.with_context(|| format!("running annotator {cmd:?}"))?;
if let Some(mut stdin) = child.stdin.take() {
stdin
.write_all(model.corpus_text().as_bytes())
.context("feeding the annotator")?;
}
let out = child
.wait_with_output()
.context("waiting for the annotator")?;
if !out.status.success() {
bail!("annotator {cmd:?} failed: {}", out.status);
}
Some(String::from_utf8(out.stdout).context("annotator output is not UTF-8")?)
}
(None, None) => None,
};
match conllu_text {
Some(text) => model
.annotate_conllu(&text)
.map_err(|e| anyhow::anyhow!("{level}: {e}"))?,
None => {
let mut bonds = opts.bonds.as_ref().map(|b| (**b).clone());
for f in desm {
let more = syndesmos::Syndesmos::load(f)
.map_err(|e| anyhow::anyhow!("reading {f}: {e}"))?;
bonds
.get_or_insert_with(syndesmos::Syndesmos::empty)
.extend(more);
}
model.tokenize_with(bonds.as_ref())
}
}
if let Some(file) = cast {
let rows = quarb_text::CastRow::read_csv(&file)
.map_err(|e| anyhow::anyhow!("{level}: reading the cast table {file}: {e}"))?;
model.apply_cast(&rows);
}
if let Some(file) = places {
let rows = quarb_text::CastRow::read_csv(&file)
.map_err(|e| anyhow::anyhow!("{level}: reading the places table {file}: {e}"))?;
model.apply_places(&rows);
}
Ok(Doc::Text(model))
}