quarb 0.34.0

//! The Quarb query engine.
//!
//! Quarb is a query language for *arbors* — tree-spanned graphs. This
//! crate is the engine: it lexes and parses a query, then evaluates
//! it against an [`AstAdapter`] that maps some data source onto the
//! arbor model.
//!
//! It currently implements tree navigation in full (child, descendant,
//! parent, ancestor, and sibling hops; proximal/distal reach; root and
//! leaf anchors; literal, glob, and `~(...)` regex name matching;
//! `<trait>` filters), scalar projection (`::` properties, `:::` core
//! metadata, `;;;` adapter metadata), `[...]` predicates (comparisons,
//! `and`/`or`/`not`/`!`, structural conditions, index selection),
//! `||` union, `->` / `<-` crosslink navigation, path patterns
//! (`(...)` groups with subpath alternation and `+`/`*`/`{m,n}`
//! quantifiers — simple-path expansion under the adapter's
//! [quantifier bound](AstAdapter::quantifier_bound)), and the register
//! system: per-capsa `| f` transforms, `@| f` whole-context
//! aggregation, `| .` push and `| $.` recall of breadcrumbs, and
//! `| .(expr)` subcontexts for grouped aggregation. It also does
//! correlation: `E1 <=> E2[…$*1…]` joins two contexts, a predicate
//! referencing a prior expression's context via `$*N`. It also
//! resolves cross-references: `::prop-->` maps a reference to its
//! target node (a JSON adapter follows a `$ref` JSON Pointer), the
//! reverse resolution `<--` (who refers here) and the value match
//! `=>`. See `doc/impl.tex`.
//!
//! ```
//! # use quarb::{AstAdapter, NodeId, QueryResult, run};
//! # struct Empty;
//! # impl AstAdapter for Empty {
//! #     fn root(&self) -> NodeId { NodeId(0) }
//! #     fn children(&self, _: NodeId) -> Vec<NodeId> { vec![] }
//! #     fn name(&self, _: NodeId) -> Option<String> { None }
//! # }
//! let hits = run("//*.rs", &Empty).unwrap();
//! assert!(matches!(hits, QueryResult::Nodes(ns) if ns.is_empty()));
//! ```

pub mod adapter;
mod ast;
pub mod bibref;
pub mod encoding;
pub mod error;
mod exec;
#[cfg(feature = "kaiv")]
pub mod kaiv_out;
mod lexer;
pub mod locale;
pub mod number;
mod parser;
pub mod reflect;
pub mod source;
pub mod trace;
pub use encoding::{base64, base64_decode, sha256, sha256_hex};
pub mod complete;
pub mod highlight;
pub mod quantity;
mod stdlib;
pub use stdlib::{tabulate, tabulate_in, tabulate_with};
pub mod plan;
pub mod registry;
pub use quarb_time::datenames;
/// The temporal fragment (the `quarb-time` crate, at its old paths).
pub use quarb_time::{period, temporal, timex};
pub mod target;
pub mod translit;
mod unparse;
mod value;

pub use adapter::{
    AstAdapter, Confidence, Conjunct, Estimate, ExternalRef, InstantFrom, Latency, LinkDir, NodeId,
    PlanAxis, Provenance, ProvenanceList, Selection, StepPlan,
};
pub mod conformance;
pub use error::{Kind as ErrorKind, QuarbError, Result};
pub mod koine;

/// The invocation instant, pinned once by the tool layer (`qua
/// --now`, or the clock read once at startup) and read by
/// adapters that would otherwise consult the wall clock when
/// resolving a relative window (`since=30m`). A pinned run
/// replays: evaluation never reads a clock, and neither does a
/// mount. `None` before the tool sets it — a library embedder
/// that never pins gets the old wall-clock behavior.
mod pinned {
    use std::cell::Cell;
    thread_local! {
        pub(super) static NOW: Cell<Option<(i64, u32)>> = const { Cell::new(None) };
    }
}

/// Pin the invocation instant for this thread (seconds and nanos
/// since the epoch). Called once, by the tool, before mounting.
pub fn set_invocation_instant(secs: i64, nanos: u32) {
    pinned::NOW.with(|c| c.set(Some((secs, nanos))));
}

/// How a text is cut into sentences: the session's sentence bonds
/// (a `.desm` set — "Dr." binds to what follows), behind which the
/// engine does not care what segments. `sentences`, `sc` and the
/// corpus reading segment with it, so a pipeline and a `corpus:`
/// mount agree on "Dr. Smith". Without one, UAX #29 alone.
pub trait SentenceBonds {
    /// The sentences of `text`, in order, as slices of it.
    fn unicode_sentences<'a>(&self, text: &'a str) -> Vec<&'a str>;
    /// The concrete set, for a tool that extends it (`?desm=` on a
    /// mount adds to the session's).
    fn as_any(&self) -> &dyn std::any::Any;
}

#[cfg(feature = "desm")]
impl SentenceBonds for syndesmos::Syndesmos {
    fn unicode_sentences<'a>(&self, text: &'a str) -> Vec<&'a str> {
        syndesmos::Syndesmos::unicode_sentences(self, text)
    }
    fn as_any(&self) -> &dyn std::any::Any {
        self
    }
}

thread_local! {
    /// The session's sentence bonds, set by the tool at open time.
    static SENTENCE_BONDS: std::cell::RefCell<Option<std::rc::Rc<dyn SentenceBonds>>> =
        const { std::cell::RefCell::new(None) };
}

/// Set the session's sentence bonds (`qua --desm`, quai's).
pub fn set_sentence_bonds(bonds: Option<std::rc::Rc<dyn SentenceBonds>>) {
    SENTENCE_BONDS.with(|b| *b.borrow_mut() = bonds);
}

/// The session's sentence bonds, if any were loaded.
pub fn sentence_bonds() -> Option<std::rc::Rc<dyn SentenceBonds>> {
    SENTENCE_BONDS.with(|b| b.borrow().clone())
}

/// The pinned invocation instant, if the tool layer set one.
pub fn invocation_instant() -> Option<(i64, u32)> {
    pinned::NOW.with(|c| c.get())
}

/// The pinned instant in whole seconds, falling back to the wall
/// clock — the one call an adapter needs when resolving a
/// relative window. Reading it in `open` (never during
/// evaluation) keeps a mount as reproducible as a query.
pub fn now_secs() -> i64 {
    match invocation_instant() {
        Some((secs, _)) => secs,
        None => std::time::SystemTime::now()
            .duration_since(std::time::UNIX_EPOCH)
            .map(|d| d.as_secs() as i64)
            .unwrap_or(0),
    }
}
pub use exec::{
    ORIGIN_CAP, Origin, Origins, Prov, QueryResult, RunOptions as Run, Traced, canonical_identity,
    canonical_identity_at_open, fetched, fetched_opt, is_url, name_path, record_refusal,
    resolved_provenance,
};
pub use parser::{Defs, parse_scope_path};
pub use value::Value;

/// The universal search over a step body: `*//BODY`, with the
/// wildcard supplied when the body opens with a trait expression or
/// a predicate (`trait <long-text>: [:: | cc > 300];` means every
/// node that passes).
pub fn step_body_query(body: &str) -> String {
    let body = body.trim();
    if body.starts_with(['<', '[']) {
        format!("*//*{body}")
    } else {
        format!("*//{body}")
    }
}

/// One path step's body, parsed — a name, a pattern, an
/// alternation, a trait expression, predicates, in any combination,
/// without the navigation operator — as a model's `trait <name>:
/// BODY;` declaration takes it. Test a node against it with
/// [`node_matches_step_body`]: decided at the node, no walk.
pub struct StepBody(ast::PathElem);

/// Parse `body` as one step's body; refuses anything beyond that
/// single step, and positional predicates (`[1]`, `[2..3]`), which
/// have no meaning at a single node.
pub fn parse_step_body(body: &str) -> Result<StepBody> {
    let q = parser::parse(&lexer::lex(&step_body_query(body))?)?;
    let refuse = |what: &str| {
        Err(QuarbError::parse(format!(
            "a trait declaration takes one step's body (a name, a pattern, \
             a trait expression, predicates), not {what}: '{body}'"
        )))
    };
    if !q.correlations.is_empty() || !q.pipeline.is_empty() {
        return refuse("a pipeline");
    }
    if q.branches.len() != 1 || q.branches[0].projection.is_some() {
        return refuse("a projection or a union");
    }
    if q.branches[0].steps.len() != 1 {
        return refuse("a path");
    }
    let elem = q
        .branches
        .into_iter()
        .next()
        .unwrap()
        .steps
        .into_iter()
        .next()
        .unwrap();
    let preds = match &elem {
        ast::PathElem::Step(s) => &s.predicates,
        ast::PathElem::Group(g) => &g.predicates,
        _ => return refuse("that"),
    };
    if preds
        .iter()
        .any(|p| matches!(p, ast::Predicate::Index(_) | ast::Predicate::Range(_, _)))
    {
        return refuse("a positional predicate");
    }
    Ok(StepBody(elem))
}

/// Check that `body` parses as one step's body (see
/// [`parse_step_body`]).
pub fn validate_step_body(body: &str) -> Result<()> {
    parse_step_body(body).map(|_| ())
}

/// Whether `node` would match the step body as a step of a path:
/// decided at the node, the predicates evaluated there, no hop.
pub fn node_matches_step_body(adapter: &impl AstAdapter, node: NodeId, body: &StepBody) -> bool {
    exec::node_matches_elem(adapter, node, &body.0)
}

/// Convert a refusal recorded during evaluation into an error. The
/// eval layer below the entry points has no error channel; a walk
/// that would return a silently incomplete answer (e.g. a
/// quantifier cut off at the bound) records a refusal instead, and
/// the entry points surface it here.
fn checked<T>(result: T) -> Result<T> {
    match exec::take_refusal() {
        Some(msg) => Err(QuarbError::refused(msg)),
        None => Ok(result),
    }
}

impl Run {
    /// The options an embedder gets without choosing any: `now()`
    /// from the instant the tool pinned ([`set_invocation_instant`],
    /// if it did), the sentence bonds it set ([`set_sentence_bonds`]),
    /// the shell gate closed, every other default.
    pub fn ambient() -> Run {
        Run {
            now: invocation_instant(),
            bonds: sentence_bonds(),
            ..Run::default()
        }
    }
}

/// The options a [`run`]-family entry runs under: the enclosing
/// run's when one is in progress (an adapter answering through the
/// engine mid-evaluation keeps its caller's instant, bound, gate
/// and targets), the ambient ones at the top.
fn inherited() -> Run {
    if exec::in_run() {
        exec::run_option(|o| o.clone())
    } else {
        Run::ambient()
    }
}

/// A query's closing serialization for people — `@| csv` or
/// `@| table` as the last stage — taken off for the tool to render.
#[derive(Debug, Clone)]
pub struct ClosingTable {
    pub csv: bool,
    /// The named columns, in order; empty for every field.
    pub columns: Vec<String>,
    /// The delimiter `csv(";")` names, when it names one.
    pub delimiter: Option<u8>,
}

/// What a run leaves for its caller.
#[derive(Debug, Clone)]
pub struct Outcome {
    /// A node set, or scalar values if the query ends in a
    /// projection.
    pub result: QueryResult,
    /// The closing `@| csv` / `@| table` taken off for rendering
    /// ([`Query::for_rendering`]), with the rows standing in
    /// `result` as they were before it.
    pub closing: Option<ClosingTable>,
    /// The unresolved external references: the identifiers of
    /// documents the query referenced but could not reach (not
    /// mounted, no tree declared for them).
    pub refs: Vec<String>,
    /// The static plan the run was made under, when the run asked
    /// for it (`Run::plan`) or held it to a budget.
    pub plan: Option<plan::Plan>,
}

/// A parsed query, ready to run against an adapter under a
/// [`Run`]: parse once, run many.
#[derive(Debug, Clone)]
pub struct Query {
    ast: ast::Query,
    closing: Option<ClosingTable>,
}

impl Query {
    /// Lex and parse `text` with the dataset at hand (data-aware
    /// macros read it at expansion time), under the ambient options.
    pub fn parse(text: &str, adapter: &impl AstAdapter) -> Result<Query> {
        Query::parse_with(text, &Defs::default(), adapter, &inherited())
    }

    /// [`Query::parse`] with a pre-seeded fragment table and the
    /// options a macro body expands under (its shell gate, its
    /// instant).
    pub fn parse_with(
        text: &str,
        defs: &Defs,
        adapter: &impl AstAdapter,
        run: &Run,
    ) -> Result<Query> {
        let _scope = exec::begin_run(Some(run.clone()));
        let (tokens, spans) = lexer::lex_spanned(text)?;
        let ast =
            parser::parse_with_data_spanned(&tokens, Some(&spans), defs.clone(), Some(adapter))?;
        Ok(Query { ast, closing: None })
    }

    /// For a tool that renders under an output locale: when the
    /// query closes with `@| csv` or `@| table`, the run returns the
    /// rows as they stand before that stage, with its shape in
    /// [`Outcome::closing`], and the tool tabulates them itself
    /// (`tabulate_in` with its locale's cells and delimiter). The
    /// stage anywhere else, and every other stage, yields the same
    /// values as under a plain run — a value never depends on a
    /// locale.
    pub fn for_rendering(mut self) -> Query {
        if let Some(ast::Stage::Agg(call)) = self.ast.pipeline.last_mut()
            && matches!(call.name.as_str(), "csv" | "table")
        {
            let (delimiter, columns) = stdlib::split_csv_delimiter(call);
            self.closing = Some(ClosingTable {
                csv: call.name == "csv",
                columns: columns
                    .iter()
                    .filter_map(|a| match a {
                        ast::Arg::Lit(Value::Str(s)) => Some(s.clone()),
                        _ => None,
                    })
                    .collect(),
                delimiter,
            });
            // The rows, with a node standing for the record of the
            // columns asked of it, exactly as the stage would read
            // them.
            let columns: Vec<ast::Arg> = columns.to_vec();
            call.name = stdlib::ROWS_FOR_RENDERING.to_string();
            call.args = columns;
        }
        self
    }

    /// Evaluate against `adapter` under `run`.
    ///
    /// Errors: a refusal (a feature the engine does not implement, a
    /// shell stage with the gate closed, an argument of the wrong
    /// kind); a malformed query is refused at [`Query::parse`].
    pub fn run(&self, adapter: &impl AstAdapter, run: &Run) -> Result<Outcome> {
        exec::gate_shell(&self.ast, run.allow_shell)?;
        let plan = self.planned(adapter, run)?;
        let result = checked(exec::eval_with(&self.ast, adapter, Some(run.clone())))?;
        Ok(Outcome {
            result,
            closing: self.closing.clone(),
            refs: exec::take_refs(),
            plan,
        })
    }

    /// The static plan when the run asks for it, held to the run's
    /// budget: a query the estimate puts beyond the budget is
    /// refused here, before anything runs.
    fn planned(&self, adapter: &impl AstAdapter, run: &Run) -> Result<Option<plan::Plan>> {
        if !run.plan && run.budget.is_none() {
            return Ok(None);
        }
        let plan = plan::of(&self.ast, adapter);
        if let Some(budget) = &run.budget
            && let Some(why) = budget.exceeded_by(&plan)
        {
            return Err(QuarbError::refused(why));
        }
        Ok(Some(plan))
    }

    /// Like [`Query::run`], returning the final capsae as `(node,
    /// topic)` pairs, so each result value still knows which node
    /// produced it — the provenance-bearing form used by typed
    /// emission (`qua --daiv`). A `None` topic is a node result.
    pub fn run_traced(
        &self,
        adapter: &impl AstAdapter,
        run: &Run,
    ) -> Result<Vec<(NodeId, Option<Value>)>> {
        Ok(self
            .run_traced_prov(adapter, run)?
            .into_iter()
            .map(|t| (t.node, t.topic))
            .collect())
    }

    /// Like [`Query::run_traced`], with each value's provenance: the
    /// origins of its topic, per field for a record
    /// ([`exec::Traced`]).
    pub fn run_traced_prov(&self, adapter: &impl AstAdapter, run: &Run) -> Result<Vec<Traced>> {
        exec::gate_shell(&self.ast, run.allow_shell)?;
        checked(exec::eval_traced_prov_with(
            &self.ast,
            adapter,
            Some(run.clone()),
        ))
    }

    /// The static plan: what the query will touch, step by step,
    /// with what `adapter` says it would apply from its catalog —
    /// no node is read ([`plan::of`]).
    pub fn plan(&self, adapter: &dyn AstAdapter) -> plan::Plan {
        plan::of(&self.ast, adapter)
    }

    /// The query rendered back to canonical text (defs expanded).
    pub fn canonical(&self) -> String {
        unparse::unparse(&self.ast)
    }

    /// The syntax tree.
    pub fn ast(&self) -> &ast::Query {
        &self.ast
    }
}

/// Lex, parse, and evaluate `query` against `adapter` under the
/// ambient options ([`Run::ambient`]; inside a run, that run's).
///
/// Returns the [`QueryResult`] — a node set, or scalar values if the
/// query ends in a projection — or an error if the query is malformed
/// or uses a feature the engine does not implement yet. A tool
/// choosing its options runs a [`Query`] under a [`Run`].
pub fn run(query: &str, adapter: &impl AstAdapter) -> Result<QueryResult> {
    run_with_defs(query, &Defs::default(), adapter)
}

/// Like [`run`], for a tool that renders under an output locale
/// ([`Query::for_rendering`]).
pub fn run_for_rendering(
    query: &str,
    adapter: &impl AstAdapter,
) -> Result<(QueryResult, Option<ClosingTable>)> {
    let r = inherited();
    let out = Query::parse_with(query, &Defs::default(), adapter, &r)?
        .for_rendering()
        .run(adapter, &r)?;
    Ok((out.result, out.closing))
}

/// Like [`run`], returning `(node, topic)` pairs
/// ([`Query::run_traced`]).
pub fn run_traced(query: &str, adapter: &impl AstAdapter) -> Result<Vec<(NodeId, Option<Value>)>> {
    run_traced_with_defs(query, &Defs::default(), adapter)
}

/// Like [`run_traced`], with each value's provenance
/// ([`Query::run_traced_prov`]).
pub fn run_traced_prov(query: &str, adapter: &impl AstAdapter) -> Result<Vec<exec::Traced>> {
    run_traced_prov_with_defs(query, &Defs::default(), adapter)
}

/// Parse a definitions file — `def &name(params): body;` statements
/// only — into a fragment table for [`run_with_defs`]. Lines whose
/// first non-blank character is `#` are comments: a defs file is a
/// library, and a library wants a header. (Query text proper has no
/// comment syntax — queries are one-liners; the annotation lives in
/// the defs file.)
pub fn parse_defs(text: &str) -> Result<Defs> {
    let tokens = lexer::lex(&strip_defs_comments(text))?;
    parser::parse_defs(&tokens)
}

/// Blank out the `#` comment lines of a defs file, preserving line
/// structure. Session layers that *prepend defs text to query text*
/// (quai, the notebook kernels) must store the stripped form — the
/// query lexer itself has no comment syntax.
pub fn strip_defs_comments(text: &str) -> String {
    text.lines()
        .map(|l| {
            if l.trim_start().starts_with('#') {
                ""
            } else {
                l
            }
        })
        .collect::<Vec<_>>()
        .join("\n")
}

/// Like [`run`], with a pre-seeded fragment table (a `--defs` file);
/// inline `def` statements extend it.
pub fn run_with_defs(query: &str, defs: &Defs, adapter: &impl AstAdapter) -> Result<QueryResult> {
    let r = inherited();
    Ok(Query::parse_with(query, defs, adapter, &r)?
        .run(adapter, &r)?
        .result)
}

/// Like [`run_traced`], with a pre-seeded fragment table.
pub fn run_traced_with_defs(
    query: &str,
    defs: &Defs,
    adapter: &impl AstAdapter,
) -> Result<Vec<(NodeId, Option<Value>)>> {
    let r = inherited();
    Query::parse_with(query, defs, adapter, &r)?.run_traced(adapter, &r)
}

/// Like [`run_traced_prov`], with a pre-seeded fragment table.
pub fn run_traced_prov_with_defs(
    query: &str,
    defs: &Defs,
    adapter: &impl AstAdapter,
) -> Result<Vec<exec::Traced>> {
    let r = inherited();
    Query::parse_with(query, defs, adapter, &r)?.run_traced_prov(adapter, &r)
}

/// True if `query` (inline defs allowed) opens with an expression
/// head or a lone root anchor and never navigates a document: every
/// branch is a step-less, projection-less root anchor and there are
/// no correlations. Such a query can run against a bare root with
/// no mounted target — the calculator invocation (spec: The
/// Expression Head). Unparseable text answers `false`; the caller's
/// normal run path owns the error message.
pub fn is_calculator(query: &str) -> bool {
    let Ok(tokens) = lexer::lex(query) else {
        return false;
    };
    let Ok(ast) = parser::parse_with_defs(&tokens, Defs::default()) else {
        return false;
    };
    ast.correlations.is_empty()
        && !ast.branches.is_empty()
        && ast.branches.iter().all(|b| {
            matches!(b.anchor, ast::Anchor::Root) && b.steps.is_empty() && b.projection.is_none()
        })
}

/// Parse `query` (expanding defs) and render it back to canonical
/// query text — the expansion lens (`qua --expand`). Pure fragments
/// only: a data-aware macro (`&name!`) needs [`expand_with`].
pub fn expand(query: &str, defs: &Defs) -> Result<String> {
    let tokens = lexer::lex(query)?;
    let ast = parser::parse_with_defs(&tokens, defs.clone())?;
    Ok(unparse::unparse(&ast))
}

/// The `macroexpand-1` lens (`qua --expand-1`): parse fully, but
/// return each directly-invoked macro's generated text before its
/// re-expansion — one entry per direct invocation, in source
/// order. Run the printed text again to take the next step;
/// [`expand`] is the fixed point.
pub fn expand_first(query: &str, defs: &Defs) -> Result<Vec<String>> {
    let tokens = lexer::lex(query)?;
    parser::parse_first_steps(&tokens, defs.clone(), None)
}

/// Like [`expand_first`], with the dataset at hand, so data-aware
/// macros (`&name!`) can read it at expansion time, under `run`
/// (their shell gate, their instant).
pub fn expand_first_with(
    query: &str,
    defs: &Defs,
    adapter: &impl AstAdapter,
    run: &Run,
) -> Result<Vec<String>> {
    let _scope = exec::begin_run(Some(run.clone()));
    let tokens = lexer::lex(query)?;
    parser::parse_first_steps(&tokens, defs.clone(), Some(adapter))
}

/// Like [`expand`], with the dataset at hand, so data-aware macros
/// (`&name!`) can read it at expansion time, under `run`.
pub fn expand_with(
    query: &str,
    defs: &Defs,
    adapter: &impl AstAdapter,
    run: &Run,
) -> Result<String> {
    Ok(Query::parse_with(query, defs, adapter, run)?.canonical())
}