use anyhow::{Context, Result, bail};
use quarb::{AllowShell, NodeId, QueryResult, WithNow};
#[cfg(feature = "native")]
use std::path::Path;
use std::rc::Rc;
#[derive(Clone, Default)]
pub struct Options {
pub hidden: bool,
pub respect_ignore: bool,
pub graft: bool,
pub no_graft: bool,
pub refs: Rc<Vec<(String, String)>>,
pub allow_shell: bool,
pub bonds: Option<Rc<syndesmos::Syndesmos>>,
}
pub enum Doc {
Json(quarb_json::JsonAdapter),
Csv(quarb_csv::CsvAdapter),
Xml(quarb_xml::XmlAdapter),
Html(quarb_html::HtmlAdapter),
Text(quarb_text::TextModel),
#[cfg(feature = "sqlite")]
Sqlite(quarb_sqlite::SqliteAdapter),
#[cfg(feature = "native")]
Fs(quarb_fs::FsAdapter),
#[cfg(feature = "native")]
FsDeep(quarb_compose::ComposeAdapter<quarb_fs::FsAdapter>),
#[cfg(feature = "native")]
Git(quarb_git::GitAdapter),
#[cfg(feature = "native")]
Archive(quarb_compose::ComposeAdapter<quarb_archive::ArchiveAdapter>),
#[cfg(feature = "native")]
ArchiveRaw(quarb_archive::ArchiveAdapter),
#[cfg(feature = "native")]
Xlsx(quarb_xlsx::XlsxAdapter),
#[cfg(feature = "native")]
Syntax(quarb_tree_sitter::TreeSitterAdapter),
#[cfg(feature = "native")]
Code(quarb_code::CodeModel),
Mount(quarb_mount::MountAdapter),
Boxed(Dyn, Box<dyn Fn(NodeId) -> String>),
}
pub struct Dyn(pub Box<dyn quarb::AstAdapter>);
impl quarb::AstAdapter for Dyn {
fn root(&self) -> NodeId {
self.0.root()
}
fn children(&self, node: NodeId) -> Vec<NodeId> {
self.0.children(node)
}
fn name(&self, node: NodeId) -> Option<String> {
self.0.name(node)
}
fn parent(&self, node: NodeId) -> Option<NodeId> {
self.0.parent(node)
}
fn traits(&self, node: NodeId) -> Vec<String> {
self.0.traits(node)
}
fn property(&self, node: NodeId, name: &str) -> Option<quarb::Value> {
self.0.property(node, name)
}
fn children_named(&self, node: NodeId, name: &str) -> Vec<NodeId> {
self.0.children_named(node, name)
}
fn child_host(&self, node: NodeId) -> NodeId {
self.0.child_host(node)
}
fn non_tree_nodes(&self) -> Vec<NodeId> {
self.0.non_tree_nodes()
}
fn properties(&self, node: NodeId) -> Vec<(String, quarb::Value)> {
self.0.properties(node)
}
fn default_value(&self, node: NodeId) -> Option<quarb::Value> {
self.0.default_value(node)
}
fn metadata(&self, node: NodeId, key: &str) -> Option<quarb::Value> {
self.0.metadata(node, key)
}
fn aliased_metadata(&self, node: NodeId) -> &'static [&'static str] {
self.0.aliased_metadata(node)
}
fn links(&self, node: NodeId) -> Vec<(String, NodeId)> {
self.0.links(node)
}
fn backlinks(&self, node: NodeId) -> Vec<(String, NodeId)> {
self.0.backlinks(node)
}
fn resolve(&self, node: NodeId, property: &str, hint: Option<&str>) -> Option<NodeId> {
self.0.resolve(node, property, hint)
}
fn external_ref(&self, node: NodeId, property: &str, hint: Option<&str>) -> Option<String> {
self.0.external_ref(node, property, hint)
}
fn resolve_fragment(&self, node: NodeId, fragment: &str) -> Option<NodeId> {
self.0.resolve_fragment(node, fragment)
}
fn ref_property(&self, node: NodeId) -> Option<String> {
self.0.ref_property(node)
}
fn ref_label(&self, node: NodeId, property: &str) -> Option<String> {
self.0.ref_label(node, property)
}
fn link_property(
&self,
source: NodeId,
label: &str,
target: NodeId,
name: &str,
) -> Option<quarb::Value> {
self.0.link_property(source, label, target, name)
}
fn quantifier_bound(&self) -> usize {
self.0.quantifier_bound()
}
fn invocation_instant(&self) -> Option<(i64, u32)> {
self.0.invocation_instant()
}
fn unit_scale(&self, expr: &str) -> Option<(f64, String)> {
self.0.unit_scale(expr)
}
fn answers_to(&self, node: NodeId, name: &str) -> bool {
self.0.answers_to(node, name)
}
fn has_trait(&self, node: NodeId, name: &str) -> bool {
self.0.has_trait(node, name)
}
fn has_node(&self, node: NodeId) -> bool {
self.0.has_node(node)
}
fn document_by_ref(&self, id: &str) -> Option<NodeId> {
self.0.document_by_ref(id)
}
fn reverse_resolve(
&self,
node: NodeId,
property: Option<&str>,
hint: Option<&str>,
) -> Option<Vec<NodeId>> {
self.0.reverse_resolve(node, property, hint)
}
fn descendants_named(&self, node: NodeId, name: &str) -> Option<Vec<(NodeId, usize)>> {
self.0.descendants_named(node, name)
}
fn prefetch_links(&self, nodes: &[NodeId], dir: quarb::LinkDir) {
self.0.prefetch_links(nodes, dir)
}
fn provenance(&self, node: NodeId) -> quarb::Provenance {
self.0.provenance(node)
}
fn provenance_list(&self, node: NodeId) -> quarb::ProvenanceList {
self.0.provenance_list(node)
}
fn allow_shell(&self) -> bool {
self.0.allow_shell()
}
}
impl Doc {
fn boxed_kaiv(a: quarb_kaiv::KaivAdapter) -> Doc {
let a = std::rc::Rc::new(a);
let r = a.clone();
Doc::Boxed(
Dyn(Box::new(quarb_mount::Shared(a))),
Box::new(move |n| r.locator(n)),
)
}
pub fn parse(input: &str, format: &str) -> Result<Doc> {
match format {
"kaiv" => {
let a = quarb_kaiv::KaivAdapter::parse_kaiv(input)
.map_err(|e| anyhow::anyhow!("parsing kaiv: {e}"))?;
return Ok(Self::boxed_kaiv(a));
}
"daiv" => {
let a = quarb_kaiv::KaivAdapter::parse_daiv(input)
.map_err(|e| anyhow::anyhow!("parsing daiv: {e}"))?;
return Ok(Self::boxed_kaiv(a));
}
"json" => quarb_json::JsonAdapter::parse(input)
.map(Doc::Json)
.context("parsing JSON"),
"json2" => quarb_json::JsonAdapter::parse_as(input, quarb_json::Reading::Property)
.map(Doc::Json)
.context("parsing JSON"),
"jsonl" | "ndjson" => quarb_json::JsonAdapter::parse_lines(input)
.map(Doc::Json)
.context("parsing JSONL"),
"yaml" | "yml" => quarb_yaml::parse(input)
.map(Doc::Json)
.context("parsing YAML"),
"toml" => quarb_toml::parse(input)
.map(Doc::Json)
.context("parsing TOML"),
"csv" => quarb_csv::CsvAdapter::parse_with_delimiter(input, b',')
.map(Doc::Csv)
.context("parsing CSV"),
"tsv" => quarb_csv::CsvAdapter::parse_with_delimiter(input, b'\t')
.map(Doc::Csv)
.context("parsing TSV"),
"xml" => quarb_xml::XmlAdapter::parse(input)
.map(Doc::Xml)
.context("parsing XML"),
"html" => Ok(Doc::Html(quarb_html::HtmlAdapter::parse(input))),
"markdown" | "md" => Ok(Doc::Html(quarb_markdown::parse(input))),
"tex" | "latex" => Ok(Doc::Text(quarb_text_latex::parse(input))),
"text-html" => Ok(Doc::Text(quarb_text_html::parse(input))),
"text-markdown" | "text-md" => Ok(Doc::Text(quarb_text_markdown::parse(input))),
"text" => Ok(Doc::Text(quarb_text::TextModel::parse_plain(input))),
"conllu" => quarb_text::TextModel::parse_conllu_text(input)
.map(Doc::Text)
.map_err(|e| anyhow::anyhow!("reading CoNLL-U: {e}")),
"corpus-conllu" => quarb_text::TextModel::parse_conllu_corpus(input)
.map(Doc::Text)
.map_err(|e| anyhow::anyhow!("reading CoNLL-U: {e}")),
"corpus-html" | "corpus-markdown" | "corpus" => {
let mut model = match format {
"corpus-html" => quarb_text_html::parse(input),
"corpus-markdown" => quarb_text_markdown::parse(input),
_ => quarb_text::TextModel::parse_plain(input),
};
model.tokenize();
Ok(Doc::Text(model))
}
#[cfg(feature = "koine")]
"text-bibtex" | "bibtex" => quarb_text_koine::parse_bibtex(input)
.map(Doc::Text)
.map_err(|e| anyhow::anyhow!("{e}")),
#[cfg(feature = "native")]
"code-rust" => quarb_code::CodeModel::parse(input, "rs")
.map(Doc::Code)
.context("parsing Rust at the code level"),
#[cfg(feature = "native")]
"code-python" => quarb_code::CodeModel::parse(input, "py")
.map(Doc::Code)
.context("parsing Python at the code level"),
#[cfg(feature = "native")]
"code-javascript" => quarb_code::CodeModel::parse(input, "js")
.map(Doc::Code)
.context("parsing JavaScript at the code level"),
#[cfg(feature = "native")]
"code-c" => quarb_code::CodeModel::parse(input, "c")
.map(Doc::Code)
.context("parsing C at the code level"),
other => bail!("unknown format: {other}"),
}
}
pub(crate) fn base_dyn(&self) -> &dyn quarb::AstAdapter {
match self {
Doc::Json(a) => a,
Doc::Csv(a) => a,
Doc::Xml(a) => a,
Doc::Html(a) => a,
Doc::Text(a) => a,
#[cfg(feature = "sqlite")]
Doc::Sqlite(a) => a,
#[cfg(feature = "native")]
Doc::Fs(a) => a,
#[cfg(feature = "native")]
Doc::FsDeep(a) => a,
#[cfg(feature = "native")]
Doc::Git(a) => a,
#[cfg(feature = "native")]
Doc::Archive(a) => a,
#[cfg(feature = "native")]
Doc::ArchiveRaw(a) => a,
#[cfg(feature = "native")]
Doc::Xlsx(a) => a,
#[cfg(feature = "native")]
Doc::Syntax(a) => a,
#[cfg(feature = "native")]
Doc::Code(a) => a,
Doc::Mount(a) => a,
Doc::Boxed(a, _) => &*a.0,
}
}
pub fn export(
&self,
query: &str,
now: (i64, u32),
allow_shell: bool,
kind: &str,
) -> Result<String> {
let render = quarb_text::Render::from_name(kind)
.ok_or_else(|| anyhow::anyhow!("unknown export format: {kind} (md, html, txt)"))?;
if query.trim().is_empty() {
let base = self.base_dyn();
return Ok(quarb_text::render_nodes(base, &[base.root()], render));
}
match self
.run(query, now, allow_shell)
.map_err(|e| anyhow::anyhow!("{e}"))?
{
QueryResult::Nodes(nodes) => {
Ok(quarb_text::render_nodes(self.base_dyn(), &nodes, render))
}
QueryResult::Values(values) => Ok(quarb_text::render::render_values(&values, render)),
}
}
pub fn run_modeled(
&self,
query: &str,
now: (i64, u32),
allow_shell: bool,
model: &quarb_model::Model,
) -> quarb::Result<QueryResult> {
let (secs, nanos) = now;
let base = quarb_model::Borrowed(self.base_dyn());
let nowed = WithNow {
inner: &base,
secs,
nanos,
};
let enriched = quarb_model::ModelAdapter::new(nowed, model.clone());
enriched
.check_declared_traits()
.map_err(|e| quarb::QuarbError::Parse(format!("reading the model: {e}")))?;
if allow_shell {
quarb::run(query, &AllowShell { inner: &enriched })
} else {
quarb::run(query, &enriched)
}
}
pub fn render_modeled(&self, node: NodeId, model: &quarb_model::Model) -> String {
let enriched =
quarb_model::ModelAdapter::new(quarb_model::Borrowed(self.base_dyn()), model.clone());
enriched.locator(node, |bn| self.render(bn))
}
pub fn run(
&self,
query: &str,
now: (i64, u32),
allow_shell: bool,
) -> quarb::Result<QueryResult> {
let (secs, nanos) = now;
macro_rules! go {
($a:expr) => {{
let nowed = WithNow {
inner: $a,
secs,
nanos,
};
if allow_shell {
quarb::run(query, &AllowShell { inner: &nowed })
} else {
quarb::run(query, &nowed)
}
}};
}
match self {
Doc::Json(a) => go!(a),
Doc::Csv(a) => go!(a),
Doc::Xml(a) => go!(a),
Doc::Html(a) => go!(a),
Doc::Text(a) => go!(a),
#[cfg(feature = "sqlite")]
Doc::Sqlite(a) => go!(a),
#[cfg(feature = "native")]
Doc::Fs(a) => go!(a),
#[cfg(feature = "native")]
Doc::FsDeep(a) => go!(a),
#[cfg(feature = "native")]
Doc::Git(a) => go!(a),
#[cfg(feature = "native")]
Doc::Archive(a) => go!(a),
#[cfg(feature = "native")]
Doc::ArchiveRaw(a) => go!(a),
#[cfg(feature = "native")]
Doc::Xlsx(a) => go!(a),
#[cfg(feature = "native")]
Doc::Syntax(a) => go!(a),
#[cfg(feature = "native")]
Doc::Code(a) => go!(a),
Doc::Mount(a) => go!(a),
Doc::Boxed(a, _) => go!(a),
}
}
pub fn family(&self) -> &'static str {
match self {
Doc::Json(_) => "json",
Doc::Csv(_) => "csv",
Doc::Xml(_) => "xml",
Doc::Html(_) => "html",
Doc::Text(_) => "text",
#[cfg(feature = "sqlite")]
Doc::Sqlite(_) => "sqlite",
#[cfg(feature = "native")]
Doc::Fs(_) | Doc::FsDeep(_) => "fs",
#[cfg(feature = "native")]
Doc::Git(_) => "git",
#[cfg(feature = "native")]
Doc::Archive(_) | Doc::ArchiveRaw(_) => "archive",
#[cfg(feature = "native")]
Doc::Xlsx(_) => "xlsx",
#[cfg(feature = "native")]
Doc::Syntax(_) => "tree-sitter",
#[cfg(feature = "native")]
Doc::Code(_) => "code",
Doc::Mount(_) => "mount",
Doc::Boxed(..) => "boxed",
}
}
pub fn trace(
&self,
committed: &str,
staged: Option<&str>,
opts: &quarb::trace::Options,
) -> quarb::Result<quarb_trace::Payload> {
let base = quarb_model::Borrowed(self.base_dyn());
quarb::trace::trace(committed, staged, &base, &|n| self.render(n), opts)
}
pub fn render(&self, node: NodeId) -> String {
match self {
Doc::Json(a) => a.pointer(node),
Doc::Csv(a) => a.locator(node),
Doc::Xml(a) => a.locator(node),
Doc::Html(a) => a.locator(node),
Doc::Text(a) => a.locator(node),
#[cfg(feature = "sqlite")]
Doc::Sqlite(a) => a.locator(node),
#[cfg(feature = "native")]
Doc::Fs(a) => a.path(node).display().to_string(),
#[cfg(feature = "native")]
Doc::FsDeep(a) => a.locator(node, |o| a.outer().path(o).display().to_string()),
#[cfg(feature = "native")]
Doc::Git(a) => a.locator(node),
#[cfg(feature = "native")]
Doc::Archive(a) => a.locator(node, |o| a.outer().locator(o)),
#[cfg(feature = "native")]
Doc::ArchiveRaw(a) => a.locator(node),
#[cfg(feature = "native")]
Doc::Xlsx(a) => a.locator(node),
#[cfg(feature = "native")]
Doc::Syntax(a) => a.locator(node),
#[cfg(feature = "native")]
Doc::Code(a) => a.locator(node),
Doc::Mount(a) => generic_locator(a, node),
Doc::Boxed(_, render) => render(node),
}
}
#[cfg(feature = "sqlite")]
pub fn sqlite_bytes(bytes: &[u8]) -> Result<Doc> {
Ok(Doc::Sqlite(
quarb_sqlite::SqliteAdapter::from_bytes(bytes)
.map_err(|e| anyhow::anyhow!("{e}"))
.context("opening SQLite bytes")?,
))
}
#[cfg(not(feature = "sqlite"))]
pub fn sqlite_bytes(_bytes: &[u8]) -> Result<Doc> {
anyhow::bail!("this quarb-session was built without the `sqlite` feature")
}
pub fn mount_docs(parts: Vec<(String, Doc)>) -> Result<Doc> {
let mut mounts: Vec<quarb_mount::Mount> = Vec::new();
for (name, doc) in parts {
if mounts.iter().any(|m| m.name == name) {
bail!("two sources mount as '{name}'; give each a distinct name");
}
mounts.push(quarb_mount::Mount {
name,
target: None,
adapter: doc.into_boxed()?,
});
}
Ok(Doc::Mount(quarb_mount::MountAdapter::new(mounts)))
}
pub fn attach_url(&mut self, url: &str) {
match self {
Doc::Html(a) => a.set_document_url(url),
Doc::Text(a) => a.set_document_url(url),
_ => {}
}
}
pub fn url_roots(
&self,
urls: &[(String, String)],
) -> std::collections::HashMap<String, quarb::NodeId> {
use quarb::AstAdapter;
let a = self.base_dyn();
let root = a.root();
match self {
Doc::Mount(_) => urls
.iter()
.filter_map(|(name, url)| {
a.children_named(root, name)
.first()
.map(|&n| (url.clone(), n))
})
.collect(),
_ => urls
.iter()
.take(1)
.map(|(_, url)| (url.clone(), root))
.collect(),
}
}
pub fn mount_texts(parts: &[(String, String, String)]) -> Result<Doc> {
let mut docs: Vec<(String, Doc)> = Vec::new();
for (name, format, text) in parts {
let doc = Doc::parse(text, format)
.with_context(|| format!("parsing '{name}' as {format}"))?;
docs.push((name.clone(), doc));
}
Doc::mount_docs(docs)
}
fn into_boxed(self) -> Result<Box<dyn quarb::AstAdapter>> {
use quarb_mount::Shared;
Ok(match self {
Doc::Json(a) => Box::new(Shared(Rc::new(a))),
Doc::Csv(a) => Box::new(Shared(Rc::new(a))),
Doc::Xml(a) => Box::new(Shared(Rc::new(a))),
Doc::Html(a) => Box::new(Shared(Rc::new(a))),
Doc::Text(a) => Box::new(Shared(Rc::new(a))),
#[cfg(feature = "sqlite")]
Doc::Sqlite(a) => Box::new(Shared(Rc::new(a))),
#[cfg(feature = "native")]
Doc::Fs(a) => Box::new(Shared(Rc::new(a))),
#[cfg(feature = "native")]
Doc::FsDeep(a) => Box::new(Shared(Rc::new(a))),
#[cfg(feature = "native")]
Doc::Git(a) => Box::new(Shared(Rc::new(a))),
#[cfg(feature = "native")]
Doc::Archive(a) => Box::new(Shared(Rc::new(a))),
#[cfg(feature = "native")]
Doc::ArchiveRaw(a) => Box::new(Shared(Rc::new(a))),
#[cfg(feature = "native")]
Doc::Xlsx(a) => Box::new(Shared(Rc::new(a))),
#[cfg(feature = "native")]
Doc::Syntax(a) => Box::new(Shared(Rc::new(a))),
#[cfg(feature = "native")]
Doc::Code(a) => Box::new(Shared(Rc::new(a))),
Doc::Mount(_) => bail!("cannot nest a mount inside a mount"),
Doc::Boxed(a, _) => a.0,
})
}
}
#[cfg(feature = "native")]
impl Doc {
pub fn open(path: &Path, opts: &Options) -> Result<Doc> {
if path.is_dir() {
let fsopts = quarb_fs::FsOptions {
hidden: opts.hidden,
respect_ignore: opts.respect_ignore,
};
let fs = quarb_fs::FsAdapter::with_options(path, fsopts)
.with_context(|| format!("opening directory {}", path.display()))?;
return Ok(if opts.graft {
Doc::FsDeep(quarb_compose::ComposeAdapter::with_source_paths(
fs,
|fs, n| Some(fs.path(n)),
))
} else {
Doc::Fs(fs)
});
}
let s = path.to_string_lossy();
if let Some(repo) = s.strip_prefix("git:") {
let a =
quarb_git::GitAdapter::open(Path::new(repo)).context("opening git repository")?;
return Ok(Doc::Git(a));
}
if let Some(rest) = s.strip_prefix("koine:")
&& !rest.is_empty()
{
let (path_part, format) = match rest.split_once('?') {
Some((p, q)) => {
let mut fmt = None;
for pair in q.split('&') {
match pair.split_once('=') {
Some(("format", v)) => fmt = Some(v.to_string()),
_ => {
return Err(anyhow::anyhow!(
"unknown koine option {pair:?} — supported: format="
));
}
}
}
(p, fmt)
}
None => (rest, None),
};
let target = Path::new(path_part);
let ext = target
.extension()
.and_then(|e| e.to_str())
.map(|e| e.to_ascii_lowercase());
let read = || -> Result<String> {
let text = std::fs::read_to_string(target)
.with_context(|| format!("reading {}", target.display()))?;
Ok(text
.strip_prefix('\u{feff}')
.map(str::to_owned)
.unwrap_or(text))
};
if let Some(fmt) = format {
let imported = match fmt.as_str() {
"atd" | "atk" => {
let model = quarb_text_koine::parse_file(target).with_context(|| {
format!("reading {} as an atrep document", target.display())
})?;
return Ok(Doc::Text(model));
}
"md" | "markdown" => quarb_text_koine::parse_markdown(&read()?),
"html" => quarb_text_koine::parse_html(&read()?),
"rst" => quarb_text_koine::parse_rst(&read()?),
"org" => quarb_text_koine::parse_org(&read()?),
"dj" | "djot" => quarb_text_koine::parse_djot(&read()?),
"tei" | "docbook" | "jats" | "usx" | "osis" | "rnc" | "opencorpora"
| "proiel" => quarb_text_koine::parse_xml_as(&read()?, &fmt),
other => {
return Err(anyhow::anyhow!(
"unknown koine format {other:?} — known: md, html, rst, org, djot, tei, docbook, jats, usx, osis, atd"
));
}
};
return imported.map(Doc::Text).with_context(|| {
format!("importing {} through atrep as {fmt}", target.display())
});
}
let imported = match ext.as_deref() {
Some("atd" | "atk") => {
let model = quarb_text_koine::parse_file(target).with_context(|| {
format!("reading {} as an atrep document", target.display())
})?;
return Ok(Doc::Text(model));
}
Some("usfm" | "sfm") => quarb_text_koine::parse_xml_as(&read()?, "usfm"),
Some("bib") => quarb_text_koine::parse_bibtex(&read()?),
Some("md" | "markdown") => quarb_text_koine::parse_markdown(&read()?),
Some("html" | "htm") => quarb_text_koine::parse_html(&read()?),
Some("rst") => quarb_text_koine::parse_rst(&read()?),
Some("org") => quarb_text_koine::parse_org(&read()?),
Some("dj" | "djot") => quarb_text_koine::parse_djot(&read()?),
Some("xml") => {
let text = read()?;
match quarb_text_koine::detect_xml_kind(&text) {
Some(kind) => quarb_text_koine::parse_xml_as(&text, kind),
None => {
return Err(anyhow::anyhow!(
"{} declares no XML identity this route knows — force one with koine:{}?format=tei|docbook|jats|usx|osis|rnc|opencorpora|proiel",
target.display(),
target.display()
));
}
}
}
Some("pdf") => {
return Err(anyhow::anyhow!(
"atrep cannot import a PDF — the print reading is text:{}",
target.display()
));
}
_ => {
return Err(anyhow::anyhow!(
"no atrep import for this format yet — the native reading is text:{}",
target.display()
));
}
};
return imported
.map(Doc::Text)
.with_context(|| format!("importing {} through atrep", target.display()));
}
if let Some(rest) = s.strip_prefix("lit:")
&& !rest.is_empty()
{
let (rest, conllu, annotate, desm, format, cast, places, modernize) = match rest
.split_once('?')
{
Some((p, q)) => {
let (
mut conllu,
mut annotate,
mut desm,
mut format,
mut cast,
mut places,
mut modernize,
) = (None, None, Vec::new(), None, None, None, None);
for pair in q.split('&') {
match pair.split_once('=') {
Some(("conllu", v)) => conllu = Some(v.to_string()),
Some(("annotate", v)) => annotate = Some(v.to_string()),
Some(("desm", v)) => desm.push(v.to_string()),
Some(("cast", v)) => cast = Some(v.to_string()),
Some(("places", v)) => places = Some(v.to_string()),
Some(("modernize", v)) => modernize = Some(v.to_string()),
Some(("format", v)) => format = Some(v.to_string()),
_ => bail!(
"unknown lit option {pair:?} — supported: conllu=, annotate=, desm=, cast=, places=, modernize=, format="
),
}
}
(p, conllu, annotate, desm, format, cast, places, modernize)
}
None => (rest, None, None, Vec::new(), None, None, None, None),
};
let target = Path::new(rest);
let ext = target
.extension()
.and_then(|e| e.to_str())
.map(|e| e.to_ascii_lowercase());
let read = || -> Result<String> {
let text = std::fs::read_to_string(target)
.with_context(|| format!("reading {}", target.display()))?;
Ok(text
.strip_prefix('\u{feff}')
.map(str::to_owned)
.unwrap_or(text))
};
let kind = match format {
Some(k) => k,
None => match ext.as_deref() {
Some("atd" | "atk") => "atd".to_string(),
Some("usx") => "usx".to_string(),
_ => match quarb_text_koine::detect_xml_kind(&read()?) {
Some(k) => k.to_string(),
None => bail!(
"lit: reads TEI, USX, OSIS, RNC, OpenCorpora, PROIEL and litogramma — {rest} declares none of them (koine:{rest} reads the other XML vocabularies, corpus:{rest} plain prose)"
),
},
},
};
let mut model = match kind.as_str() {
"atd" | "atk" => quarb_text_koine::parse_lit_file(target).with_context(|| {
format!("reading {} as an atrep document", target.display())
})?,
"tei" | "usx" | "osis" | "rnc" | "opencorpora" | "proiel" => {
quarb_text_koine::parse_lit_xml_as(&read()?, &kind).with_context(|| {
format!("reading {} as {}", target.display(), kind.to_uppercase())
})?
}
other => bail!(
"lit: reads TEI, USX, OSIS, RNC, OpenCorpora, PROIEL and litogramma — not {other} (koine:{rest} reads the other XML vocabularies)"
),
};
model.set_document_path(rest);
return finish_corpus(
model, conllu, annotate, &desm, cast, places, modernize, opts, "lit",
);
}
if let Some(rest) = s.strip_prefix("corpus:")
&& !rest.is_empty()
{
let (rest, conllu, annotate, desm, format, cast, places, modernize) = match rest
.split_once('?')
{
Some((p, q)) => {
let (
mut conllu,
mut annotate,
mut desm,
mut format,
mut cast,
mut places,
mut modernize,
) = (None, None, Vec::new(), None, None, None, None);
for pair in q.split('&') {
match pair.split_once('=') {
Some(("conllu", v)) => conllu = Some(v.to_string()),
Some(("annotate", v)) => annotate = Some(v.to_string()),
Some(("desm", v)) => desm.push(v.to_string()),
Some(("cast", v)) => cast = Some(v.to_string()),
Some(("places", v)) => places = Some(v.to_string()),
Some(("modernize", v)) => modernize = Some(v.to_string()),
Some(("format", "conllu")) => format = Some("conllu"),
Some(("format", v)) => bail!(
"corpus: format={v} is not a treebank format — format= takes conllu"
),
_ => bail!(
"unknown corpus option {pair:?} — supported: conllu=, annotate=, desm=, cast=, places=, modernize=, format="
),
}
}
(p, conllu, annotate, desm, format, cast, places, modernize)
}
None => (rest, None, None, Vec::new(), None, None, None, None),
};
let target = Path::new(rest);
let ext = target
.extension()
.and_then(|e| e.to_str())
.map(|e| e.to_ascii_lowercase());
if ext.as_deref() == Some("pdf") {
bail!(
"corpus: needs a prose reading, and a PDF's is line geometry — text:{rest} reads its lines"
);
}
let own_annotation = || {
if conllu.is_some() || annotate.is_some() {
bail!(
"corpus: {rest} is its own annotation — conllu= and annotate= apply to prose"
);
}
Ok(())
};
if target.is_dir() {
own_annotation()?;
let files = quarb_text::read_conllu_dir(target).map_err(|e| {
anyhow::anyhow!("corpus: over a directory reads its treebank — {e}")
})?;
let refs: Vec<(&str, &str)> = files
.iter()
.map(|(n, t)| (n.as_str(), t.as_str()))
.collect();
let mut model = quarb_text::TextModel::parse_conllu_corpus_set(&refs)
.map_err(|e| anyhow::anyhow!("reading {rest} as CoNLL-U: {e}"))?;
model.set_document_path(rest);
return treebank_options(model, &cast, &places, &modernize);
}
let treebank = matches!(ext.as_deref(), Some("conllu" | "conllup"))
|| format == Some("conllu")
|| (ext.is_none()
&& std::fs::read_to_string(target)
.is_ok_and(|t| quarb_text::looks_like_conllu(&t)));
if treebank {
own_annotation()?;
let text = std::fs::read_to_string(target)
.with_context(|| format!("reading {}", target.display()))?;
let text = text
.strip_prefix('\u{feff}')
.map(str::to_owned)
.unwrap_or(text);
let Doc::Text(mut model) = Doc::parse(&text, "corpus-conllu")? else {
bail!("corpus: needs a prose reading of {rest}")
};
model.set_document_path(rest);
return treebank_options(model, &cast, &places, &modernize);
}
let route = if matches!(ext.as_deref(), Some("xml" | "tei")) {
format!("koine:{rest}")
} else {
format!("text:{rest}")
};
let Doc::Text(model) = Doc::open(Path::new(&route), opts)? else {
bail!("corpus: needs a prose reading of {rest}")
};
return finish_corpus(
model, conllu, annotate, &desm, cast, places, modernize, opts, "corpus",
);
}
if let Some(rest) = s.strip_prefix("json2:")
&& !rest.is_empty()
{
use quarb_json::Reading;
let target = Path::new(rest);
let text = std::fs::read_to_string(target)
.with_context(|| format!("reading {}", target.display()))?;
let ext = target
.extension()
.and_then(|e| e.to_str())
.map(|e| e.to_ascii_lowercase())
.unwrap_or_default();
let a = match ext.as_str() {
"yaml" | "yml" => {
quarb_yaml::parse_as(&text, Reading::Property).context("parsing YAML")?
}
"toml" => quarb_toml::parse_as(&text, Reading::Property).context("parsing TOML")?,
"jsonl" | "ndjson" => {
quarb_json::JsonAdapter::parse_lines_as(&text, Reading::Property)
.context("parsing JSONL")?
}
_ => quarb_json::JsonAdapter::parse_as(&text, Reading::Property)
.context("parsing JSON")?,
};
return Ok(Doc::Json(a));
}
if let Some(rest) = s.strip_prefix("text:")
&& !rest.is_empty()
{
let target = Path::new(rest);
if target.is_dir() {
let files = quarb_text::read_conllu_dir(target).map_err(|e| {
anyhow::anyhow!("text: over a directory reads its treebank — {e}")
})?;
let refs: Vec<(&str, &str)> = files
.iter()
.map(|(n, t)| (n.as_str(), t.as_str()))
.collect();
let mut model = quarb_text::TextModel::parse_conllu_text_set(&refs)
.map_err(|e| anyhow::anyhow!("reading {rest} as CoNLL-U: {e}"))?;
model.set_document_path(rest);
return Ok(Doc::Text(model));
}
let ext = target
.extension()
.and_then(|e| e.to_str())
.map(|e| e.to_ascii_lowercase());
if let Some("usfm" | "sfm") = ext.as_deref() {
let text = std::fs::read_to_string(target)
.with_context(|| format!("reading {}", target.display()))?;
let text = text
.strip_prefix('\u{feff}')
.map(str::to_owned)
.unwrap_or(text);
let model = quarb_text_koine::parse_xml_as(&text, "usfm")
.with_context(|| format!("reading {} as USFM", target.display()))?;
return Ok(Doc::Text(model));
}
if let Some("atd" | "atk") = ext.as_deref() {
let model = quarb_text_koine::parse_file(target).with_context(|| {
format!("reading {} as an atrep document", target.display())
})?;
return Ok(Doc::Text(model));
}
if ext.as_deref() == Some("pdf") {
let bytes = std::fs::read(target)
.with_context(|| format!("reading {}", target.display()))?;
let model = quarb_text_pdf::parse(&bytes)
.with_context(|| format!("reading {} as PDF", target.display()))?;
let a = std::rc::Rc::new(model);
let r = a.clone();
return Ok(Doc::Boxed(
Dyn(Box::new(quarb_mount::Shared(a))),
Box::new(move |n| r.locator(n)),
));
}
if let Some(ext @ ("docx" | "epub")) = ext.as_deref() {
let bytes = std::fs::read(target)
.with_context(|| format!("reading {}", target.display()))?;
let model = if ext == "docx" {
let mut m = quarb_text_docx::parse(&bytes)
.with_context(|| format!("reading {} as Word", target.display()))?;
quarb_text_koine::attach_bib_aliases(&mut m);
m
} else {
quarb_text_epub::parse(&bytes)
.with_context(|| format!("reading {} as EPUB", target.display()))?
};
return Ok(Doc::Text(model));
}
let text = std::fs::read_to_string(target)
.with_context(|| format!("reading {}", target.display()))?;
let text = text
.strip_prefix('\u{feff}')
.map(str::to_owned)
.unwrap_or(text);
let format = match target
.extension()
.and_then(|e| e.to_str())
.map(|e| e.to_ascii_lowercase())
.as_deref()
{
Some("html" | "htm") => "text-html",
Some("md" | "markdown") => "text-markdown",
Some("tex" | "latex") => "tex",
Some("txt") => "text",
Some("conllu" | "conllup") => "conllu",
#[cfg(feature = "koine")]
Some("bib") => "text-bibtex",
_ if text.trim_start().starts_with('<') => "text-html",
None if quarb_text::looks_like_conllu(&text) => "conllu",
_ => "text",
};
return Doc::parse(&text, format);
}
if let Some(rest) = s.strip_prefix("code:")
&& !rest.is_empty()
{
if opts.no_graft {
bail!(
"no_graft refuses the code: prefix: the prefix's whole \
meaning is the grafted code-level view"
);
}
let target = Path::new(rest);
if target.is_dir() {
let fsopts = quarb_fs::FsOptions {
hidden: opts.hidden,
respect_ignore: opts.respect_ignore,
};
let fs = quarb_fs::FsAdapter::with_options(target, fsopts)
.with_context(|| format!("opening directory {}", target.display()))?;
return Ok(Doc::FsDeep(
quarb_compose::ComposeAdapter::with_source_paths(fs, |fs, n| Some(fs.path(n)))
.with_source_graft(quarb_compose::SourceGraft::Code),
));
}
let a = quarb_code::CodeModel::open(target)
.with_context(|| format!("parsing {} at the code level", target.display()))?;
return Ok(Doc::Code(a));
}
let ext = path
.extension()
.and_then(|e| e.to_str())
.map(|e| e.to_ascii_lowercase());
if let Some(e) = &ext
&& quarb_tree_sitter::supported(e)
{
let a =
quarb_tree_sitter::TreeSitterAdapter::open(path).context("parsing source file")?;
return Ok(Doc::Syntax(a));
}
if matches!(ext.as_deref(), Some("xlsx" | "xls" | "ods")) {
let a = quarb_xlsx::XlsxAdapter::open(path).context("opening workbook")?;
return Ok(Doc::Xlsx(a));
}
if is_sqlite(path) {
#[cfg(not(feature = "sqlite"))]
anyhow::bail!(
"{}: this quarb-session was built without the `sqlite` feature",
path.display()
);
#[cfg(feature = "sqlite")]
{
let a = quarb_sqlite::SqliteAdapter::open_with_refs(path, &opts.refs)
.context("opening SQLite database")?;
return Ok(Doc::Sqlite(a));
}
}
if is_archive(path) {
let a = quarb_archive::ArchiveAdapter::open(path).context("opening archive")?;
return Ok(if opts.no_graft {
Doc::ArchiveRaw(a)
} else {
Doc::Archive(quarb_compose::ComposeAdapter::new(a))
});
}
let text =
std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?;
let text = text
.strip_prefix('\u{feff}')
.map(str::to_owned)
.unwrap_or(text);
match ext.as_deref() {
Some("csv") => Doc::parse(&text, "csv"),
Some("tsv") => Doc::parse(&text, "tsv"),
Some("yaml" | "yml") => Doc::parse(&text, "yaml"),
Some("toml") => Doc::parse(&text, "toml"),
Some("md" | "markdown") => Doc::parse(&text, "markdown"),
Some("txt") => Doc::parse(&text, "text"),
Some("conllu" | "conllup") => Doc::parse(&text, "conllu"),
Some("jsonl" | "ndjson") => Doc::parse(&text, "jsonl"),
_ => {
if is_xml(path, &text) {
Doc::parse(&text, "xml")
} else if is_html(path, &text) {
Doc::parse(&text, "html")
} else {
Doc::parse(&text, "json")
}
}
}
}
pub fn mount(paths: &[std::path::PathBuf], opts: &Options) -> Result<Doc> {
let specs: Vec<crate::MountSpec> = paths
.iter()
.map(|p| crate::MountSpec {
name: None,
path: p.clone(),
})
.collect();
Doc::mount_specs(&specs, opts)
}
pub fn mount_specs(specs: &[crate::MountSpec], opts: &Options) -> Result<Doc> {
let mut mounts: Vec<quarb_mount::Mount> = Vec::new();
for (i, spec) in specs.iter().enumerate() {
let name = spec.name.clone().unwrap_or_else(|| {
spec.path
.file_stem()
.map(|s| s.to_string_lossy().into_owned())
.unwrap_or_else(|| format!("doc{i}"))
});
if mounts.iter().any(|m| m.name == name) {
bail!(
"input '{}' mounts as '{name}', colliding with an earlier input of the \
same name; give each a distinct basename (or a NAME=TARGET alias)",
spec.path.display()
);
}
let adapter = Doc::open(&spec.path, opts)?.into_boxed()?;
mounts.push(quarb_mount::Mount {
name,
target: Some(spec.path.display().to_string()),
adapter,
});
}
Ok(Doc::Mount(quarb_mount::MountAdapter::new(mounts)))
}
}
fn generic_locator<A: quarb::AstAdapter>(a: &A, node: NodeId) -> String {
let mut parts = Vec::new();
let mut cur = Some(node);
while let Some(n) = cur {
if let Some(nm) = a.name(n) {
parts.push(nm);
}
cur = a.parent(n);
}
parts.reverse();
format!("/{}", parts.join("/"))
}
#[cfg(feature = "native")]
fn is_sqlite(path: &Path) -> bool {
if path
.extension()
.and_then(|e| e.to_str())
.is_some_and(|e| matches!(e.to_ascii_lowercase().as_str(), "db" | "sqlite" | "sqlite3"))
{
return true;
}
use std::io::Read as _;
let mut buf = [0u8; 16];
std::fs::File::open(path)
.and_then(|mut f| f.read_exact(&mut buf))
.is_ok()
&& &buf == b"SQLite format 3\0"
}
#[cfg(feature = "native")]
fn is_archive(path: &Path) -> bool {
if path.extension().and_then(|e| e.to_str()).is_some_and(|e| {
matches!(
e.to_ascii_lowercase().as_str(),
"zip" | "tar" | "gz" | "tgz" | "jar" | "war" | "docx" | "pptx" | "odt" | "odp"
)
}) {
return true;
}
use std::io::Read as _;
let mut buf = [0u8; 2];
std::fs::File::open(path)
.and_then(|mut f| f.read_exact(&mut buf))
.is_ok()
&& (&buf == b"PK" || buf == [0x1f, 0x8b])
}
#[cfg(feature = "native")]
fn is_xml(path: &Path, text: &str) -> bool {
path.extension()
.and_then(|e| e.to_str())
.is_some_and(|e| matches!(e.to_ascii_lowercase().as_str(), "xml" | "svg" | "xhtml"))
|| text.trim_start().starts_with("<?xml")
}
#[cfg(feature = "native")]
fn is_html(path: &Path, text: &str) -> bool {
path.extension()
.and_then(|e| e.to_str())
.is_some_and(|e| matches!(e.to_ascii_lowercase().as_str(), "html" | "htm"))
|| text.trim_start().starts_with('<')
}
fn treebank_options(
mut model: quarb_text::TextModel,
cast: &Option<String>,
places: &Option<String>,
modernize: &Option<String>,
) -> Result<Doc> {
if let Some(t) = modernize {
model
.set_orthography(t)
.map_err(|e| anyhow::anyhow!("corpus: {e}"))?;
}
if let Some(f) = cast {
let rows = quarb_text::CastRow::read_csv(f)
.map_err(|e| anyhow::anyhow!("corpus: reading the cast table {f}: {e}"))?;
model.apply_cast(&rows);
}
if let Some(f) = places {
let rows = quarb_text::CastRow::read_csv(f)
.map_err(|e| anyhow::anyhow!("corpus: reading the places table {f}: {e}"))?;
model.apply_places(&rows);
}
Ok(Doc::Text(model))
}
fn finish_corpus(
mut model: quarb_text::TextModel,
conllu: Option<String>,
annotate: Option<String>,
desm: &[String],
cast: Option<String>,
places: Option<String>,
modernize: Option<String>,
opts: &Options,
level: &str,
) -> Result<Doc> {
if let Some(table) = &modernize {
model
.set_orthography(table)
.map_err(|e| anyhow::anyhow!("{level}: {e}"))?;
}
let conllu_text = match (conllu, annotate) {
(Some(_), Some(_)) => bail!("{level}: takes conllu= or annotate=, not both"),
(Some(file), None) => Some(
std::fs::read_to_string(&file).with_context(|| format!("reading {file} as CoNLL-U"))?,
),
(None, Some(cmd)) => {
if !opts.allow_shell {
bail!("{level}: annotate= runs a command; the shell gate is closed");
}
use std::io::Write;
let mut child = std::process::Command::new("sh")
.arg("-c")
.arg(&cmd)
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.spawn()
.with_context(|| format!("running annotator {cmd:?}"))?;
if let Some(mut stdin) = child.stdin.take() {
stdin
.write_all(model.corpus_text().as_bytes())
.context("feeding the annotator")?;
}
let out = child
.wait_with_output()
.context("waiting for the annotator")?;
if !out.status.success() {
bail!("annotator {cmd:?} failed: {}", out.status);
}
Some(String::from_utf8(out.stdout).context("annotator output is not UTF-8")?)
}
(None, None) => None,
};
match conllu_text {
Some(text) => model
.annotate_conllu(&text)
.map_err(|e| anyhow::anyhow!("{level}: {e}"))?,
None => {
let mut bonds = opts.bonds.as_ref().map(|b| (**b).clone());
for f in desm {
let more = syndesmos::Syndesmos::load(f)
.map_err(|e| anyhow::anyhow!("reading {f}: {e}"))?;
bonds
.get_or_insert_with(syndesmos::Syndesmos::empty)
.extend(more);
}
model.tokenize_with(bonds.as_ref())
}
}
if let Some(file) = cast {
let rows = quarb_text::CastRow::read_csv(&file)
.map_err(|e| anyhow::anyhow!("{level}: reading the cast table {file}: {e}"))?;
model.apply_cast(&rows);
}
if let Some(file) = places {
let rows = quarb_text::CastRow::read_csv(&file)
.map_err(|e| anyhow::anyhow!("{level}: reading the places table {file}: {e}"))?;
model.apply_places(&rows);
}
Ok(Doc::Text(model))
}