kaish-kernel 0.16.0

//! jq — Native JSON query tool using jaq.
//!
//! This implementation uses the jaq crate for native jq execution
//! without spawning an external process. Benefits:
//! - Parse-time filter validation (fail fast)
//! - No subprocess overhead
//! - Consistent behavior across platforms
//! - Type-safe Rust API
//!
//! # Examples
//!
//! ```kaish
//! echo '{"name": "Alice"}' | jq ".name"
//! echo '{"name": "Alice"}' | jq ".name" -r
//! jq ".items[]" path=/data/items.json
//! jq ".[] | select(.active)" -c
//! ```

use std::path::Path;

use async_trait::async_trait;
use clap::{CommandFactory, Parser};
use jaq_core::{data, load, unwrap_valr, Compiler, Ctx, Vars};
use jaq_json::write::Pp;
use jaq_json::{Num, Val};
use jaq_std::ValT as _;

use crate::ast::Value;
use crate::interpreter::{ExecResult, OutputData};
use crate::tools::builtin::get_path_string;
use crate::tools::{schema_from_clap, validate_against_schema, ExecContext, ToolCtx, GlobalFlags, ParamSchema, Tool, ToolArgs, ToolSchema};
use crate::validator::{IssueCode, ValidationIssue};

/// Native jq tool using jaq (pure Rust jq implementation).
pub struct JqNative;

/// clap-derived argv layer for jq.
///
/// jq's filter syntax stays hand-rolled — only argv-level flags are declared
/// here. `--arg` / `--argjson` are accept-and-ignore at the clap level; the
/// body reads them from `args.named` (kernel pre-parses them as
/// `consumes=2` `Json(Array(Array([NAME, VALUE])))` pairs).
#[derive(Parser, Debug)]
#[command(name = "jq", about = "Native JSON query processor")]
struct JqArgs {
    /// Raw output mode (-r): output strings without quotes.
    #[arg(short = 'r', long = "raw")]
    raw: bool,

    /// Compact output mode (-c): no pretty-printing.
    #[arg(short = 'c', long = "compact")]
    compact: bool,

    /// Use null as input instead of reading stdin (-n).
    #[arg(short = 'n', long = "null-input", visible_alias = "null_input")]
    null_input: bool,

    /// Slurp mode (-s): read the input as a document stream and wrap it in
    /// one array — always, even for a single document, matching real jq. A
    /// value arriving on `.data` is one document, so `-s` wraps it in a
    /// one-element array.
    #[arg(short = 's', long = "slurp")]
    slurp: bool,

    /// Read from VFS file instead of stdin.
    #[arg(long = "path")]
    path: Option<String>,

    /// Bind a kaish variable as a jq string: `--arg NAME VALUE` sets `$NAME`.
    /// Repeatable: `--arg a 1 --arg b 2`.
    // Accept-and-ignore at this layer: the body reads the pairs from
    // `args.named`, which the kernel pre-parses as `consumes=2`.
    #[arg(id = "arg", long = "arg", action = clap::ArgAction::Append, value_name = "NAME VALUE")]
    _arg: Vec<String>,

    /// Bind a kaish variable as a jq JSON value: `--argjson NAME JSON` sets
    /// `$NAME`. Repeatable: `--argjson a '[1]' --argjson b '{}'`.
    // Same accept-and-ignore split as `--arg` above.
    #[arg(id = "argjson", long = "argjson", action = clap::ArgAction::Append, value_name = "NAME JSON")]
    _argjson: Vec<String>,

    #[command(flatten)]
    global: GlobalFlags,

    /// File to read instead of stdin, given after the filter:
    /// `jq '.name' data.json`.
    #[arg(hide = true)]
    rest: Vec<String>,
}

/// jaq 3.x data kind: filters that process `Val` and carry only the LUT as
/// data (no lazy inputs). `JustLut` requires `V: 'static`, which `Val` is.
type DataKind = data::JustLut<Val>;
type Filter = jaq_core::Filter<DataKind>;

/// True when `key` (`arg`/`argjson`) carries a legal `consumes=2` binding: an
/// array whose elements are themselves arrays (NAME/VALUE pairs from the space
/// form `--arg NAME VAL`). The illegal `--arg=NAME` equals form binds scalar
/// elements instead, so it returns false and the filter compile check still runs.
fn has_arg_pair_binding(args: &ToolArgs, key: &str) -> bool {
    matches!(
        args.named.get(key),
        Some(Value::Json(serde_json::Value::Array(items)))
            if items.iter().any(|it| matches!(it, serde_json::Value::Array(_)))
    )
}

/// Parse and compile a jq filter expression.
///
/// `global_vars` are the bindings introduced by `--arg` / `--argjson`, in
/// declaration order. Each name must start with `$` (per `jaq_core`'s
/// `with_global_vars` contract). Values are supplied at execution time via
/// `Ctx::new` in the same order.
fn compile_filter(filter_str: &str, global_vars: &[String]) -> Result<Filter, String> {
    // Create arena for parsing
    let arena = load::Arena::default();

    // Load standard library definitions (jaq 3.x splits core defs out).
    let defs = jaq_core::defs().chain(jaq_std::defs()).chain(jaq_json::defs());
    let loader = load::Loader::new(defs);

    // Parse the filter
    let modules = loader
        .load(&arena, load::File { path: (), code: filter_str })
        .map_err(|errs| {
            let msgs: Vec<String> = errs
                .into_iter()
                .flat_map(|(_, e)| -> Vec<String> {
                    match e {
                        load::Error::Io(io_errs) => io_errs.into_iter().map(|(_, msg)| msg).collect(),
                        load::Error::Lex(lex_errs) => lex_errs.into_iter().map(|(expected, _)| format!("expected {}", expected.as_str())).collect(),
                        load::Error::Parse(parse_errs) => parse_errs.into_iter().map(|(expected, _)| format!("expected {}", expected.as_str())).collect(),
                    }
                })
                .collect();
            format!("jq parse error: {}", msgs.join(", "))
        })?;

    // Compile with standard library functions and any `--arg` / `--argjson` bindings.
    let funs = jaq_core::funs::<DataKind>()
        .chain(jaq_std::funs())
        .chain(jaq_json::funs());
    let compiler = Compiler::default()
        .with_funs(funs)
        .with_global_vars(global_vars.iter().map(String::as_str));
    let filter = compiler.compile(modules).map_err(|errs| {
        let msgs: Vec<String> = errs
            .into_iter()
            .flat_map(|(_, errors)| {
                errors.into_iter().map(|(_, undefined)| format!("undefined {}", undefined.as_str()))
            })
            .collect();
        format!("jq compile error: {}", msgs.join(", "))
    })?;

    Ok(filter)
}

/// Outcome of running a compiled jq filter: rendered text for pipes/stdout
/// plus the raw per-output JSON values so callers can populate `.data`.
///
/// `text` uses `-r` or pretty JSON per the caller's preference and is the
/// canonical representation for downstream pipe stages. `values` preserves
/// the filter's output stream as a list of JSON values regardless of how
/// they were rendered — this is what enables `for i in $(jq -r '.[]' …)`
/// to iterate each element instead of binding the whole newline-joined
/// stdout to a single loop variable.
struct JqRun {
    text: String,
    values: Vec<serde_json::Value>,
}

/// Execute a compiled jq filter on pre-parsed JSON.
///
/// `var_values` holds JSON values for any `--arg` / `--argjson` bindings
/// declared at compile time, in the **same order** as `global_vars` passed
/// to `compile_filter`. They're converted to `Val` inside this function
/// because `Val` contains `Rc` pointers and is therefore `!Send` — keeping
/// the conversion here lets the async caller stay `Send` across `.await`.
fn execute_filter_json(
    filter: &Filter,
    json: serde_json::Value,
    raw_output: bool,
    compact: bool,
    var_values: Vec<serde_json::Value>,
) -> Result<JqRun, String> {
    // Convert serde_json::Value to jaq_json::Val (jaq 3.x ships a serde
    // `Deserialize for Val`, which preserves object insertion order into the
    // IndexMap-backed `Val::Obj`).
    let input_val = json_to_val(json);
    let vars: Vec<Val> = var_values.into_iter().map(json_to_val).collect();

    // jaq 3.x execution: the context carries the compiled filter's LUT plus the
    // `--arg`/`--argjson` bindings (in declaration order); `filter.id` is the
    // entry term, run against the input value. `unwrap_valr` flattens the
    // exception layer into `Result<Val, Error>`.
    let ctx = Ctx::<DataKind>::new(&filter.lut, Vars::new(vars));

    let mut text = String::new();
    let mut values = Vec::new();
    for result in filter.id.run((ctx, input_val)).map(unwrap_valr) {
        match result {
            Ok(val) => {
                // jaq still evaluates `n / 0` to a non-finite float
                // (`inf`/`-inf`/`NaN`) rather than erroring like real jq, and
                // JSON has no infinity. Fail loudly instead of rendering it.
                if has_nonfinite_float(&val) {
                    return Err(
                        "jq runtime error: numeric result is not finite (division by zero?)"
                            .to_string(),
                    );
                }
                let formatted = render_value(&val, raw_output, compact);
                if !text.is_empty() {
                    text.push('\n');
                }
                text.push_str(&formatted);
                values.push(val_to_json(&val));
            }
            Err(e) => {
                let msg = e.to_string();
                // jaq's own array-index error renders as `cannot index [...]
                // with ...` (the array prints as its JSON text). That's the
                // exact signature of writing real-jq's per-row streaming
                // form (`.foo`) against kaish's always-slurped array input
                // (GH #80's `.[]` watch item from scatter/gather) — hint the
                // fix rather than leaving the agent to guess.
                let hint = if msg.starts_with("cannot index [") {
                    " — kaish jq is always-slurped — try '.[] | …'"
                } else {
                    ""
                };
                return Err(format!("jq runtime error: {msg}{hint}"));
            }
        }
    }

    // Terminate the output with a newline (builtin-sweep P4.1) so jq matches
    // the consensus and real jq, which newline-terminate each value. Empty
    // output (no values) stays empty.
    if !text.is_empty() {
        text.push('\n');
    }

    Ok(JqRun { text, values })
}

/// Convert serde_json::Value to jaq_json::Val.
///
/// Objects keep insertion order: with the `preserve_order` feature a
/// `serde_json::Map` is an IndexMap, and `Val::Obj` is too, so `keys_unsorted`
/// and plain object output stay in source order (jq parity).
fn json_to_val(json: serde_json::Value) -> Val {
    use std::rc::Rc;
    match json {
        serde_json::Value::Null => Val::Null,
        serde_json::Value::Bool(b) => Val::Bool(b),
        // Without serde_json's `arbitrary_precision`, every number is exactly
        // one of i64/u64/f64; `Num::from_integral` keeps integers exact (via
        // BigInt when they exceed isize).
        serde_json::Value::Number(n) => {
            if let Some(i) = n.as_i64() {
                Val::Num(Num::from_integral(i))
            } else if let Some(u) = n.as_u64() {
                Val::Num(Num::from_integral(u))
            } else if let Some(f) = n.as_f64() {
                Val::Num(Num::Float(f))
            } else {
                // Unreachable without serde_json's `arbitrary_precision`, which
                // we don't enable. Panic rather than silently coerce a number we
                // can't represent to null (crash over corruption); if a future
                // feature pulls in arbitrary_precision this fails loudly.
                panic!("jq: serde_json number is neither i64/u64/f64: {n}")
            }
        }
        serde_json::Value::String(s) => Val::from(s),
        serde_json::Value::Array(arr) => {
            Val::Arr(Rc::new(arr.into_iter().map(json_to_val).collect()))
        }
        serde_json::Value::Object(obj) => Val::obj(
            obj.into_iter()
                .map(|(k, v)| (Val::from(k), json_to_val(v)))
                .collect(),
        ),
    }
}

/// Pretty-printer for `jq` default (multi-line) output: two-space indent and a
/// space after `:`, objects in insertion order (jaq's writer, not serde_json).
fn pretty_pp() -> Pp {
    Pp {
        indent: Some("  ".to_string()),
        sep_space: true,
        ..Pp::default()
    }
}

/// jq prints an integral number without a decimal point (`6/2` → `3`, the
/// literal `1e10` → `10000000000`), but jaq keeps the float (`3.0`, `1e10`).
/// Return an integer `Num` for an integral value within f64's exact-integer
/// range (2^53), else `None` to leave it as jaq rendered it (a fractional value
/// like `2.5`, or a huge float like `1e100` that jq also keeps in float form).
fn canonical_integral(num: &Num) -> Option<Num> {
    let f = match num {
        Num::Float(f) => *f,
        Num::Dec(s) => s.parse::<f64>().ok()?,
        // Int/BigInt are already integral; jaq prints them correctly.
        Num::Int(_) | Num::BigInt(_) => return None,
    };
    // 2^53 is the largest magnitude where every integer is uniquely f64-exact;
    // the bound is inclusive because 2^53 itself is exactly representable (jq
    // prints `pow(2;53)` as the integer `9007199254740992`). Above it, an
    // integral-looking float isn't a distinct integer and jq's own
    // literal-vs-computed preservation diverges, so we leave it to jaq's writer.
    const EXACT_INT_LIMIT: f64 = 9_007_199_254_740_992.0;
    // `f as i64` is exact here; `Num::from_integral` then picks `Int` (or
    // `BigInt` on a 32-bit `isize` target like wasm32), so this stays correct
    // off 64-bit — a plain `as isize` would saturate.
    (f.is_finite() && f.fract() == 0.0 && f.abs() <= EXACT_INT_LIMIT)
        .then(|| Num::from_integral(f as i64))
}

/// Recursively canonicalize integral floats to integers so jaq's writer prints
/// them the jq way (see [`canonical_integral`]). Other values pass through.
fn canonicalize_numbers(val: &Val) -> Val {
    use std::rc::Rc;
    match val {
        Val::Num(n) => Val::Num(canonical_integral(n).unwrap_or_else(|| n.clone())),
        Val::Arr(arr) => Val::Arr(Rc::new(arr.iter().map(canonicalize_numbers).collect())),
        Val::Obj(obj) => Val::obj(
            obj.iter()
                // Canonicalize keys too: jaq (unlike jq) tolerates a numeric
                // object key, and a number should render the same in key
                // position as in value position.
                .map(|(k, v)| (canonicalize_numbers(k), canonicalize_numbers(v)))
                .collect(),
        ),
        other => other.clone(),
    }
}

/// Render a jaq value to text the way jq does, using jaq's native writer
/// (insertion-ordered objects, exact big integers) after canonicalizing
/// integral floats (`6/2` → `3`). `-c` is compact (`Pp::default`); the default
/// is pretty. `-r` prints a top-level string's bytes verbatim (no quotes) and
/// otherwise falls back to compact JSON.
fn render_value(val: &Val, raw_output: bool, compact: bool) -> String {
    if raw_output {
        if let Some(bytes) = val.as_bytes() {
            return String::from_utf8_lossy(bytes).into_owned();
        }
    }
    let val = canonicalize_numbers(val);
    let pp = if compact || raw_output {
        Pp::default()
    } else {
        pretty_pp()
    };
    let mut buf: Vec<u8> = Vec::new();
    // Writing to a Vec never fails.
    let _ = jaq_json::write::write(&mut buf, &pp, 0, &val);
    String::from_utf8_lossy(&buf).into_owned()
}

/// Convert ast::Value to serde_json::Value for jq processing.
///
/// Value::Json is returned directly as it's already a serde_json::Value.
/// Value::String converts to a JSON string verbatim — it is never re-parsed
/// as a JSON document. `.data` structure is opt-in (set by an upstream
/// producer like `fromjson`), never sniffed from string content: a
/// `Value::String("1")` is the kaish string `1`, and must stay the JSON
/// string `"1"` here, not become the JSON number `1` (see
/// `docs/arch_no_json_sniffing` / CLAUDE.md's ".data is opt-in, never
/// inferred from stdout text" invariant).
fn ast_value_to_json(value: &Value) -> serde_json::Value {
    match value {
        Value::Null => serde_json::Value::Null,
        Value::Bool(b) => serde_json::Value::Bool(*b),
        Value::Int(i) => serde_json::Value::Number((*i).into()),
        Value::Float(f) => {
            serde_json::Number::from_f64(*f)
                .map(serde_json::Value::Number)
                .unwrap_or(serde_json::Value::Null)
        }
        Value::String(s) => serde_json::Value::String(s.clone()),
        Value::Json(json) => json.clone(),
        // Binary jq input surfaces as the self-describing base64 envelope.
        Value::Bytes(b) => kaish_types::bytes_to_envelope(b),
    }
}

/// True if `val` contains a non-finite float (`inf`/`-inf`/`NaN`) anywhere,
/// including nested in arrays/objects. Used to turn jaq's silent division-by-
/// zero (a non-finite float that JSON renders as `null`) into a loud error.
fn has_nonfinite_float(val: &Val) -> bool {
    match val {
        Val::Num(Num::Float(n)) => !n.is_finite(),
        Val::Arr(arr) => arr.iter().any(has_nonfinite_float),
        Val::Obj(obj) => obj.values().any(has_nonfinite_float),
        _ => false,
    }
}

/// Convert a jaq `Val` to a `serde_json::Value` for `.data`.
///
/// Reuses jaq's own (correct) compact serialization, then parses it back into
/// serde_json — so number formatting and object key order in `.data` match
/// stdout exactly, without hand-matching the `Val` enum. A non-UTF-8 byte
/// string is the only value jaq emits that isn't valid JSON; fall back to its
/// lossy string form there rather than dropping the datum.
fn val_to_json(val: &Val) -> serde_json::Value {
    let compact = render_value(val, false, true);
    serde_json::from_str(&compact).unwrap_or(serde_json::Value::String(compact))
}

#[async_trait]
impl Tool for JqNative {
    fn name(&self) -> &str {
        "jq"
    }

    fn schema(&self) -> ToolSchema {
        // Start from clap reflection, then layer the required `filter`
        // positional on top — clap doesn't model the schema's
        // "required positional" concept for our sink. Also override the
        // `_arg`/`_argjson` params: clap parses them as 1-value (because
        // to_argv joins NAME+VALUE as a single string after `=`), but the
        // kernel's pre-parse needs `consumes=2` to grab both POSIX-style
        // tokens.
        let mut schema = schema_from_clap(
            &JqArgs::command(),
            "jq",
            "JSON query processor — built into kaish (native jaq, no external binary). \
             The canonical way to extract fields from JSON: pipe data in, read \
             from a variable via `jq '.field' <<< \"$VAR\"`, or bind kaish \
             variables into the filter with `--arg` / `--argjson` plus `-n`.",
            [
                ("Extract a field", "cat data.json | jq '.name'"),
                ("Output raw strings", "cat data.json | jq -r '.version'"),
                ("Filter an array", "cat items.json | jq '.[] | select(.active)'"),
                ("Read JSON from a variable", r#"jq -r '.name' <<< "$RESULT""#),
                (
                    "Bind a kaish variable into the filter",
                    r#"R='{"x":42}'; jq -n --argjson r "$R" '$r.x'"#,
                ),
                (
                    "Slurp a document stream into one array",
                    "cat results.jsonl | jq -s 'length'",
                ),
            ],
        );
        // Bump consumes=2 for arg/argjson: clap layer parses them as
        // single-value (the kernel joins NAME+VALUE as `--arg=NAME VALUE`
        // before clap sees it), but the kernel's POSIX-style pre-parse
        // needs to grab both tokens.
        for p in schema.params.iter_mut() {
            if matches!(p.name.as_str(), "arg" | "argjson") {
                p.consumes = 2;
            }
        }
        // Re-insert the required `filter` positional that clap can't model
        // (jq's filter expression is its first positional arg).
        schema = schema.param(
            ParamSchema::required("filter", "string", "jq filter expression").positional(),
        );
        schema
        .with_typed_substitution()
    }

    fn validate(&self, args: &ToolArgs) -> Vec<ValidationIssue> {
        let mut issues = validate_against_schema(args, &self.schema());

        // Get the filter positional (index 0). If it is the `<dynamic>` marker
        // (variable, `$(cmd)`, or glob), we cannot inspect it statically — skip.
        let filter_str = match args.get_string("filter", 0) {
            Some(f) => f,
            None => return issues,
        };
        if filter_str == "<dynamic>" {
            return issues;
        }

        // If any `--arg` / `--argjson` binding is present we cannot know the full
        // set of `$var` names that will be in scope at runtime, so skip the
        // compile-step check to avoid false "undefined variable" errors on valid
        // filters like `.foo + $x` used with `--arg x 1`.
        //
        // Schema-aware validation binds the legal `--arg NAME VALUE` space form
        // into `named["arg"]` as the same `consumes=2` Array-of-*pairs* the execute
        // path sees (each occurrence an inner 2-element array). Its presence means
        // extra `$var`s are in scope, so skip the compile check. The illegal
        // `--arg=NAME` equals form instead binds an Array-of-*scalars* — real jq
        // rejects that as "Unknown option", so we deliberately do NOT skip on it,
        // letting the filter compile reject the undefined `$NAME` loudly. A malformed
        // `--arg` with nothing to consume falls back to `flags`.
        if has_arg_pair_binding(args, "arg")
            || has_arg_pair_binding(args, "argjson")
            || args.flags.contains("arg")
            || args.flags.contains("argjson")
        {
            return issues;
        }

        if let Err(msg) = validate_filter(&filter_str, &[]) {
            issues.push(
                ValidationIssue::error(
                    IssueCode::InvalidJqFilter,
                    format!("jq: {msg}"),
                )
                .with_suggestion("check jq filter syntax: https://jqlang.org/manual/"),
            );
        }

        issues
    }

    async fn execute(&self, args: ToolArgs, ctx: &mut dyn ToolCtx) -> ExecResult {
        let Some(ctx) = ctx.as_any_mut().downcast_mut::<ExecContext>() else {
            return ExecResult::failure(1, "internal error: kernel builtin requires ExecContext");
        };
        let argv = match args.to_argv() {
            Ok(v) => v,
            Err(e) => return ExecResult::failure(2, format!("jq: {e}")),
        };
        let parsed = match JqArgs::try_parse_from(
            std::iter::once("jq".to_string()).chain(argv),
        ) {
            Ok(p) => p,
            Err(e) => return ExecResult::failure(2, format!("jq: {e}")),
        };
        parsed.global.apply(ctx);

        // Get filter (required, positional 0)
        let filter_str = match args.get_string("filter", 0) {
            Some(f) => f,
            None => return ExecResult::failure(1, "jq: filter expression required"),
        };

        // Collect `--arg NAME VALUE` (string) and `--argjson NAME VALUE` (JSON)
        // bindings in declaration order. jaq needs the names at compile time
        // and the values at run time — same order on both sides.
        let (global_var_names, global_var_values) = match collect_bindings(&args) {
            Ok(pair) => pair,
            Err(e) => return ExecResult::failure(1, e),
        };

        // Compile filter (validates at execution time)
        let filter = match compile_filter(&filter_str, &global_var_names) {
            Ok(f) => f,
            Err(e) => return ExecResult::failure(1, e),
        };

        let raw_output = parsed.raw || args.has_flag("raw") || args.has_flag("r");
        let compact = parsed.compact || args.has_flag("compact") || args.has_flag("c");
        let null_input =
            parsed.null_input || args.has_flag("null-input") || args.has_flag("n");
        let slurp = parsed.slurp || args.has_flag("slurp") || args.has_flag("s");

        // Get input JSON. `-n` / `--null-input` skips stdin entirely and feeds
        // `null` to the filter — same as real jq. Otherwise: fast path through
        // pre-parsed structured data, then file, then stdin text.
        let input_json: serde_json::Value = if null_input {
            // -n/--null-input feeds the filter a synthetic `null` "document"
            // in place of stdin. Real jq's -s/--slurp still wraps that
            // synthetic document in a one-element array like it wraps any
            // other input stream (GH #111: `jq -n -s '.'` -> `[null]`) —
            // this branch bypasses resolve_stdin_json entirely, so it needs
            // its own slurp handling rather than inheriting one.
            if slurp {
                serde_json::Value::Array(vec![serde_json::Value::Null])
            } else {
                serde_json::Value::Null
            }
        } else {
            // A binary `path` operand goes loud rather than silently falling
            // through to the stdin branches below.
            match get_path_string(&args, "path", 1) {
                Ok(Some(path)) if !path.is_empty() => {
                    // Read from backend - path takes precedence over stdin
                    let resolved = ctx.resolve_path(&path);
                    match ctx.backend.read(Path::new(&resolved), None).await {
                        Ok(bytes) => {
                            // Decode strictly: a non-UTF-8 file is a loud error, not
                            // a U+FFFD mangle that then fails JSON parsing confusingly.
                            let text = match String::from_utf8(bytes) {
                                Ok(t) => t,
                                Err(_) => {
                                    return ExecResult::failure(
                                        1,
                                        format!("jq: {}: invalid UTF-8", path),
                                    )
                                }
                            };
                            if slurp {
                                match parse_document_stream(&text) {
                                    Ok(docs) => serde_json::Value::Array(docs),
                                    Err(e) => {
                                        return ExecResult::failure(1, format!("jq: -s: {}: {}", path, e))
                                    }
                                }
                            } else {
                                match serde_json::from_str(&text) {
                                    Ok(json) => json,
                                    Err(e) => {
                                        let hint =
                                            jsonl_hint_for_trailing_error(&text, &e).unwrap_or_default();
                                        return ExecResult::failure(
                                            1,
                                            format!("jq: invalid JSON in {}: {}{}", path, e, hint),
                                        );
                                    }
                                }
                            }
                        }
                        Err(e) => {
                            return ExecResult::failure(1, format!("jq: failed to read {}: {}", path, e))
                        }
                    }
                }
                Ok(_) => match resolve_stdin_json(ctx, slurp).await {
                    Ok(json) => json,
                    Err((code, msg)) => return ExecResult::failure(code, msg),
                },
                Err(e) => return ExecResult::failure(1, format!("jq: {e}")),
            }
        };

        // Execute filter with the JSON input
        match execute_filter_json(&filter, input_json, raw_output, compact, global_var_values) {
            Ok(run) => build_exec_result(run),
            Err(e) => ExecResult::failure(1, e),
        }
    }
}

/// Pull `--arg` / `--argjson` pairs out of `ToolArgs` and turn them into
/// (jaq var name, jaq value) tuples in declaration order.
///
/// Values are returned as `serde_json::Value` (which is `Send`) and
/// converted to jaq's `Val` inside `execute_filter_json`. Holding the
/// `Rc`-full `Val` across an `.await` would make the async body `!Send`.
fn collect_bindings(args: &ToolArgs) -> Result<(Vec<String>, Vec<serde_json::Value>), String> {
    let mut names: Vec<String> = Vec::new();
    let mut values: Vec<serde_json::Value> = Vec::new();

    // Walk --arg pairs: Value::Json(Array([Array([NAME, VALUE]), ...]))
    if let Some(Value::Json(serde_json::Value::Array(occurrences))) = args.named.get("arg") {
        for occ in occurrences {
            let (name, raw) = extract_pair(occ, "arg")?;
            names.push(format!("${name}"));
            values.push(serde_json::Value::String(raw));
        }
    }

    // Walk --argjson pairs: same shape, but the value is parsed as JSON.
    if let Some(Value::Json(serde_json::Value::Array(occurrences))) = args.named.get("argjson") {
        for occ in occurrences {
            let (name, raw) = extract_pair(occ, "argjson")?;
            let parsed: serde_json::Value = serde_json::from_str(&raw)
                .map_err(|e| format!("jq: --argjson {name}: invalid JSON: {e}"))?;
            names.push(format!("${name}"));
            values.push(parsed);
        }
    }

    Ok((names, values))
}

/// Pull a (name, value) pair out of a serde_json 2-element array stored by
/// the kernel's `consumes=2` flag-collection path.
fn extract_pair(occ: &serde_json::Value, flag: &str) -> Result<(String, String), String> {
    match occ {
        serde_json::Value::Array(pair) if pair.len() == 2 => {
            let name = value_as_string(&pair[0])
                .ok_or_else(|| format!("jq: --{flag} NAME must be a string"))?;
            let value = value_as_string(&pair[1])
                .ok_or_else(|| format!("jq: --{flag} VALUE must be a string"))?;
            Ok((name, value))
        }
        other => Err(format!(
            "jq: internal error: --{flag} expected 2-element pair, got {other}"
        )),
    }
}

/// Coerce a serde_json value into the string form jaq expects. Agents
/// routinely pass numbers without quoting (`--argjson x 42`) and kaish
/// evaluates the positional to `Value::Int(42)` → `json::Number`, so we
/// stringify it here.
fn value_as_string(v: &serde_json::Value) -> Option<String> {
    match v {
        serde_json::Value::String(s) => Some(s.clone()),
        serde_json::Value::Number(n) => Some(n.to_string()),
        serde_json::Value::Bool(b) => Some(b.to_string()),
        // Objects / arrays / null aren't sensible for --arg NAMEs and also
        // aren't usually what you want for raw --argjson values — but if a
        // shell expands something exotic, round-trip via serialisation.
        _ => Some(v.to_string()),
    }
}

/// Build an ExecResult from a JqRun, carrying the filter output stream
/// into `.data` so for-loop command substitution can iterate per-value.
///
/// - 0 values → empty text, no `.data`.
/// - 1 value → render normally; promote the value to `.data` so `kaish-last`
///   and single-value iteration are consistent across `-r` and JSON modes.
/// - 2+ values → render normally; set `.data = Json(Array([…]))` so the
///   CommandSubst arm in the kernel hands a JSON array to the for-loop
///   regardless of the `-r` / `-c` rendering flags.
fn build_exec_result(run: JqRun) -> ExecResult {
    use crate::interpreter::json_to_value_no_envelope;
    let JqRun { text, mut values } = run;
    if values.is_empty() {
        return ExecResult::with_output(OutputData::text(text));
    }
    // Single-value output: keep `.data` as the underlying scalar/object so
    // `jq '.name'` continues to expose a non-array value via `kaish-last`.
    // Envelope-free: jq operates on external JSON, so an envelope-shaped object
    // stays a plain record and is never silently re-decoded to Value::Bytes
    // (matches the 2+-value path below, which hands raw serde values through).
    if values.len() == 1 {
        if let Some(only) = values.pop() {
            return ExecResult::success_with_data(text, json_to_value_no_envelope(only));
        }
    }
    ExecResult::success_with_data(text, Value::Json(serde_json::Value::Array(values)))
}

/// Parse a whitespace-separated stream of JSON documents (real-jq slurp
/// framing — `serde_json`'s `StreamDeserializer` tolerates the pretty-printed
/// multi-line case, unlike a strict line-oriented split).
///
/// Used both by `-s`/`--slurp` (GH #80) and by the JSONL-hint diagnosis
/// below, which needs to know how many documents actually parse.
fn parse_document_stream(text: &str) -> Result<Vec<serde_json::Value>, String> {
    let mut docs = Vec::new();
    for item in serde_json::Deserializer::from_str(text).into_iter::<serde_json::Value>() {
        match item {
            Ok(v) => docs.push(v),
            Err(e) => return Err(format!("invalid JSON in document stream: {e}")),
        }
    }
    Ok(docs)
}

/// After a single-document parse failure, check whether the remainder is
/// actually more JSON documents (JSONL-shaped input) rather than plain
/// garbage. kaish jq is one-document-only (GH #80) — this turns the
/// confusing raw "trailing characters" error into a pointer at the real fix:
/// `fromjsonl` upstream, or `jq -s` in place. Returns `None` (no hint) for
/// any other parse error, or when the stream doesn't fully parse (genuine
/// garbage, not a document stream).
fn jsonl_hint_for_trailing_error(text: &str, err: &serde_json::Error) -> Option<String> {
    if !err.to_string().contains("trailing characters") {
        return None;
    }
    let docs = parse_document_stream(text).ok()?;
    (docs.len() >= 2).then(|| {
        format!(
            " — input looks like JSONL ({} documents) — kaish jq takes one document; \
             use fromjsonl upstream, or jq -s",
            docs.len()
        )
    })
}

/// Resolve the pipeline's stdin (no `--path` file) into the JSON fed to the
/// filter, shared by both the bare-stdin and empty-`--path=` branches of
/// `execute`. Returns `(exit_code, message)` on failure so the caller can
/// preserve the existing 1-vs-2 exit-code split (parse/runtime vs. plumbing).
async fn resolve_stdin_json(ctx: &mut ExecContext, slurp: bool) -> Result<serde_json::Value, (i64, String)> {
    let (data, text) = ctx.resolve_stdin().await.map_err(|e| (2, format!("jq: {e}")))?;
    if let Some(data) = data {
        // `.data` path: the upstream stage (scatter/gather, fromjson, …)
        // already handed over one structured value — that's the single
        // "document" real jq would have read. `-s`/`--slurp` still wraps it
        // in a one-element array, exactly like `jq -s` always wraps its
        // input regardless of document count (see GH #93 item 2 — a bare
        // pass-through here was a kaish-only divergence from real jq).
        let json = ast_value_to_json(&data);
        return Ok(if slurp {
            serde_json::Value::Array(vec![json])
        } else {
            json
        });
    }
    if text.is_empty() {
        return if slurp {
            // A stream of zero documents slurps to an empty array (matches
            // real jq: `printf '' | jq -s .` → `[]`).
            Ok(serde_json::Value::Array(Vec::new()))
        } else {
            Err((1, "jq: no input provided".to_string()))
        };
    }
    if slurp {
        parse_document_stream(&text)
            .map(serde_json::Value::Array)
            .map_err(|e| (1, format!("jq: -s: {e}")))
    } else {
        serde_json::from_str(&text).map_err(|e| {
            let hint = jsonl_hint_for_trailing_error(&text, &e).unwrap_or_default();
            (1, format!("jq: invalid JSON input: {e}{hint}"))
        })
    }
}

/// Validate a jq filter expression without executing it.
///
/// Compiles the filter with the supplied `var_names` so that filters using
/// `--arg`/`--argjson` bindings (e.g. `.foo + $x` compiled with `["$x"]`)
/// are not falsely rejected. Pass an empty slice when no bindings are known.
///
/// Returns Ok(()) if valid, Err(message) if invalid.
fn validate_filter(filter: &str, var_names: &[String]) -> Result<(), String> {
    compile_filter(filter, var_names)?;
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::backend::WriteMode;
    use crate::vfs::{MemoryFs, VfsRouter};
    use std::sync::Arc;

    fn make_ctx() -> ExecContext {
        let mut vfs = VfsRouter::new();
        vfs.mount("/", MemoryFs::new());
        ExecContext::new(Arc::new(vfs))
    }

    #[test]
    fn test_validate_filter_valid() {
        assert!(validate_filter(".name", &[]).is_ok());
        assert!(validate_filter(".items[]", &[]).is_ok());
        assert!(validate_filter(".[] | select(.active)", &[]).is_ok());
        assert!(validate_filter("map(.x + 1)", &[]).is_ok());
    }

    #[test]
    fn test_validate_filter_invalid() {
        assert!(validate_filter(".[[[invalid", &[]).is_err());
        assert!(validate_filter(".foo | | bar", &[]).is_err());
    }

    #[tokio::test]
    async fn test_jq_native_simple_filter() {
        let mut ctx = make_ctx();
        ctx.set_stdin(r#"{"name": "Alice"}"#.to_string());

        let mut args = ToolArgs::new();
        args.positional.push(Value::String(".name".into()));

        let result = JqNative.execute(args, &mut ctx).await;
        assert!(result.ok(), "jq failed: {}", result.err);
        assert_eq!(result.text_out().trim(), "\"Alice\"");
    }

    #[tokio::test]
    async fn test_jq_native_raw_output() {
        let mut ctx = make_ctx();
        ctx.set_stdin(r#"{"name": "Alice"}"#.to_string());

        let mut args = ToolArgs::new();
        args.positional.push(Value::String(".name".into()));
        args.flags.insert("r".to_string());

        let result = JqNative.execute(args, &mut ctx).await;
        assert!(result.ok(), "jq failed: {}", result.err);
        assert_eq!(result.text_out().trim(), "Alice");
    }

    #[tokio::test]
    async fn test_jq_native_array_iteration() {
        let mut ctx = make_ctx();
        ctx.set_stdin(r#"[1, 2, 3]"#.to_string());

        let mut args = ToolArgs::new();
        args.positional.push(Value::String(".[]".into()));

        let result = JqNative.execute(args, &mut ctx).await;
        assert!(result.ok(), "jq failed: {}", result.err);
        assert_eq!(result.text_out().trim(), "1\n2\n3");
    }

    #[tokio::test]
    async fn test_jq_native_invalid_json() {
        let mut ctx = make_ctx();
        ctx.set_stdin("not valid json".to_string());

        let mut args = ToolArgs::new();
        args.positional.push(Value::String(".".into()));

        let result = JqNative.execute(args, &mut ctx).await;
        assert!(!result.ok());
        assert!(result.err.contains("invalid JSON"));
    }

    #[tokio::test]
    async fn test_jq_native_invalid_filter() {
        let mut ctx = make_ctx();
        ctx.set_stdin(r#"{"a": 1}"#.to_string());

        let mut args = ToolArgs::new();
        args.positional.push(Value::String(".[[[invalid".into()));

        let result = JqNative.execute(args, &mut ctx).await;
        assert!(!result.ok());
    }

    #[tokio::test]
    async fn test_jq_native_no_input() {
        let mut ctx = make_ctx();
        let mut args = ToolArgs::new();
        args.positional.push(Value::String(".".into()));

        let result = JqNative.execute(args, &mut ctx).await;
        assert!(!result.ok());
        assert!(result.err.contains("no input"));
    }

    #[tokio::test]
    async fn test_jq_native_from_vfs_file() {
        let mut ctx = make_ctx();

        // Write test data to backend
        ctx.backend
            .write(Path::new("/test.json"), br#"{"value": 42}"#, WriteMode::Overwrite)
            .await
            .expect("failed to write test file");

        let mut args = ToolArgs::new();
        args.positional.push(Value::String(".value".into()));
        args.named
            .insert("path".to_string(), Value::String("/test.json".into()));

        let result = JqNative.execute(args, &mut ctx).await;
        assert!(result.ok(), "jq failed: {}", result.err);
        assert_eq!(result.text_out().trim(), "42");
    }

    // ============================================================================
    // Real-world model output compatibility tests
    // Source: Claude Code session output, 2026-01-25
    // These test standard jq invocation patterns that models generate
    // ============================================================================

    #[tokio::test]
    async fn test_jq_positional_file_argument() {
        // Real-world: jq '.result[]' /tmp/file.json
        let mut ctx = make_ctx();
        ctx.backend
            .write(
                Path::new("/tmp/data.json"),
                br#"{"result": [1, 2, 3]}"#,
                WriteMode::Overwrite,
            )
            .await
            .unwrap();

        let mut args = ToolArgs::new();
        args.positional.push(Value::String(".result[]".into()));
        args.positional.push(Value::String("/tmp/data.json".into()));

        let result = JqNative.execute(args, &mut ctx).await;
        assert!(result.ok(), "jq positional file failed: {}", result.err);
        assert_eq!(result.text_out().trim(), "1\n2\n3");
    }

    #[tokio::test]
    async fn test_jq_select_with_positional_file() {
        // Real-world: jq '[.[] | select(.active)] | length' file.json
        let mut ctx = make_ctx();
        ctx.backend
            .write(
                Path::new("/query.json"),
                br#"[{"id": 1, "active": true}, {"id": 2, "active": false}, {"id": 3, "active": true}]"#,
                WriteMode::Overwrite,
            )
            .await
            .unwrap();

        let mut args = ToolArgs::new();
        args.positional
            .push(Value::String("[.[] | select(.active)] | length".into()));
        args.positional.push(Value::String("/query.json".into()));

        let result = JqNative.execute(args, &mut ctx).await;
        assert!(result.ok(), "jq select failed: {}", result.err);
        assert_eq!(result.text_out().trim(), "2");
    }

    #[tokio::test]
    async fn test_jq_type_check_positional_file() {
        // Real-world: jq 'type' /tmp/file.json
        let mut ctx = make_ctx();
        ctx.backend
            .write(Path::new("/check.json"), br#"[1, 2, 3]"#, WriteMode::Overwrite)
            .await
            .unwrap();

        let mut args = ToolArgs::new();
        args.positional.push(Value::String("type".into()));
        args.positional.push(Value::String("/check.json".into()));

        let result = JqNative.execute(args, &mut ctx).await;
        assert!(result.ok(), "jq type check failed: {}", result.err);
        assert_eq!(result.text_out().trim(), "\"array\"");
    }

    #[tokio::test]
    async fn test_jq_index_access_positional_file() {
        // Real-world: jq '.[0]' /tmp/file.json
        let mut ctx = make_ctx();
        ctx.backend
            .write(
                Path::new("/arr.json"),
                br#"[{"name": "first"}, {"name": "second"}]"#,
                WriteMode::Overwrite,
            )
            .await
            .unwrap();

        let mut args = ToolArgs::new();
        args.positional.push(Value::String(".[0]".into()));
        args.positional.push(Value::String("/arr.json".into()));

        let result = JqNative.execute(args, &mut ctx).await;
        assert!(result.ok(), "jq index failed: {}", result.err);
        assert!(result.text_out().contains("\"name\": \"first\""));
    }

    #[tokio::test]
    async fn test_jq_raw_output_with_positional_file() {
        // Real-world: jq -r '.name' /tmp/file.json
        let mut ctx = make_ctx();
        ctx.backend
            .write(
                Path::new("/person.json"),
                br#"{"name": "Alice"}"#,
                WriteMode::Overwrite,
            )
            .await
            .unwrap();

        let mut args = ToolArgs::new();
        args.positional.push(Value::String(".name".into()));
        args.positional.push(Value::String("/person.json".into()));
        args.flags.insert("r".to_string());

        let result = JqNative.execute(args, &mut ctx).await;
        assert!(result.ok(), "jq -r failed: {}", result.err);
        assert_eq!(result.text_out().trim(), "Alice"); // No quotes with -r
    }

    #[tokio::test]
    async fn test_jq_stdin_still_works() {
        // Ensure stdin path still works when no file argument
        let mut ctx = make_ctx();
        ctx.set_stdin(r#"{"value": 42}"#.to_string());

        let mut args = ToolArgs::new();
        args.positional.push(Value::String(".value".into()));

        let result = JqNative.execute(args, &mut ctx).await;
        assert!(result.ok(), "jq stdin failed: {}", result.err);
        assert_eq!(result.text_out().trim(), "42");
    }

    #[tokio::test]
    async fn test_jq_named_path_still_works() {
        // Ensure path= named argument still works
        let mut ctx = make_ctx();
        ctx.backend
            .write(Path::new("/named.json"), br#"{"x": 99}"#, WriteMode::Overwrite)
            .await
            .unwrap();

        let mut args = ToolArgs::new();
        args.positional.push(Value::String(".x".into()));
        args.named
            .insert("path".to_string(), Value::String("/named.json".into()));

        let result = JqNative.execute(args, &mut ctx).await;
        assert!(result.ok(), "jq path= failed: {}", result.err);
        assert_eq!(result.text_out().trim(), "99");
    }
}