jsonl-tui 0.1.1

//! Loading, flattening and schema accumulation for JSONL data.
//!
//! Memory layout is optimised for large files:
//! - the original JSON text is **not** kept in memory; each record only
//!   stores its byte offset/length in the source, and the detail view /
//!   export re-read lines on demand;
//! - field paths are interned once globally (`u32` ids) instead of being
//!   re-allocated per record;
//! - flattened leaf values are stored in a compact enum ([`CompactValue`])
//!   inside a sorted boxed slice instead of a `BTreeMap<String, Value>`.

use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::fs::File;
use std::io::{BufRead, BufReader, Read, Seek, SeekFrom};
use std::path::{Path, PathBuf};

use anyhow::{anyhow, bail, Context, Result};
use serde_json::Value;

/// When an array contains objects, only the first N elements are expanded
/// into `path[i].field` entries to avoid path explosion.
pub const MAX_ARRAY_OBJECTS: usize = 5;

/// Compact in-memory form of a flattened leaf value.
#[derive(Debug, Clone, PartialEq)]
pub enum CompactValue {
    Null,
    Bool(bool),
    Int(i64),
    UInt(u64),
    Float(f64),
    Str(Box<str>),
    /// Arrays and objects kept whole, stored as compact JSON text.
    Json(Box<str>),
}

impl CompactValue {
    pub fn from_value(v: &Value) -> CompactValue {
        match v {
            Value::Null => CompactValue::Null,
            Value::Bool(b) => CompactValue::Bool(*b),
            Value::Number(n) => {
                if let Some(i) = n.as_i64() {
                    CompactValue::Int(i)
                } else if let Some(u) = n.as_u64() {
                    CompactValue::UInt(u)
                } else {
                    CompactValue::Float(n.as_f64().unwrap_or(0.0))
                }
            }
            Value::String(s) => CompactValue::Str(s.as_str().into()),
            other => CompactValue::Json(other.to_string().into_boxed_str()),
        }
    }

    pub fn is_null(&self) -> bool {
        matches!(self, CompactValue::Null)
    }

    /// Numeric view for native JSON numbers (not for numeric strings).
    pub fn as_number(&self) -> Option<f64> {
        match self {
            CompactValue::Int(i) => Some(*i as f64),
            CompactValue::UInt(u) => Some(*u as f64),
            CompactValue::Float(f) => Some(*f),
            _ => None,
        }
    }

    /// Render the value the way filter/search/sort/table see it:
    /// strings unquoted, everything else compact JSON.
    pub fn display(&self) -> String {
        match self {
            CompactValue::Null => "null".to_string(),
            CompactValue::Bool(b) => b.to_string(),
            CompactValue::Int(i) => i.to_string(),
            CompactValue::UInt(u) => u.to_string(),
            CompactValue::Float(f) => serde_json::Number::from_f64(*f)
                .map(|n| n.to_string())
                .unwrap_or_else(|| f.to_string()),
            CompactValue::Str(s) => s.to_string(),
            CompactValue::Json(s) => s.to_string(),
        }
    }
}

/// A record's flattened fields: `(path id, value)` pairs sorted by path id.
#[derive(Debug)]
pub struct FlatRecord {
    entries: Box<[(u32, CompactValue)]>,
}

impl FlatRecord {
    pub fn get(&self, id: u32) -> Option<&CompactValue> {
        self.entries
            .binary_search_by_key(&id, |e| e.0)
            .ok()
            .map(|i| &self.entries[i].1)
    }

    #[cfg_attr(not(test), allow(dead_code))]
    pub fn len(&self) -> usize {
        self.entries.len()
    }

    #[allow(dead_code)]
    pub fn is_empty(&self) -> bool {
        self.entries.is_empty()
    }
}

/// One loaded JSONL record: where its line lives in the source (for the
/// detail view and export) plus its compact flattened fields (for
/// columns/search/filter/sort).
#[derive(Debug)]
pub struct Record {
    /// Byte offset of the (trimmed) JSON line in the source.
    pub offset: u64,
    /// Byte length of the (trimmed) JSON line.
    pub len: u32,
    pub flat: FlatRecord,
}

/// Per-field-path statistics accumulated while loading.
#[derive(Debug, Default, Clone)]
pub struct FieldInfo {
    /// Number of records in which this path is present.
    pub count: usize,
    /// The set of JSON types observed at this path.
    pub types: BTreeSet<&'static str>,
}

/// Global path interner: dotted path -> stable `u32` id, plus per-path stats.
#[derive(Debug, Default)]
pub struct PathInterner {
    ids: HashMap<String, u32>,
    paths: Vec<String>,
    infos: Vec<FieldInfo>,
}

impl PathInterner {
    fn intern(&mut self, path: &str) -> u32 {
        if let Some(&id) = self.ids.get(path) {
            return id;
        }
        let id = self.paths.len() as u32;
        self.ids.insert(path.to_string(), id);
        self.paths.push(path.to_string());
        self.infos.push(FieldInfo::default());
        id
    }

    pub fn id(&self, path: &str) -> Option<u32> {
        self.ids.get(path).copied()
    }

    pub fn len(&self) -> usize {
        self.paths.len()
    }

    #[allow(dead_code)]
    pub fn is_empty(&self) -> bool {
        self.paths.is_empty()
    }
}

/// Where the raw JSON lines can be re-read from.
#[derive(Debug)]
pub enum Source {
    File(PathBuf),
    /// The whole input retained in memory (tests, future stdin support).
    #[cfg_attr(not(test), allow(dead_code))]
    Memory(Box<[u8]>),
}

/// Random-access reader for original record lines (detail view, export).
pub struct LineFetcher<'a> {
    source: &'a Source,
    file: Option<File>,
    buf: Vec<u8>,
}

impl LineFetcher<'_> {
    /// The raw (trimmed) JSON line of a record.
    pub fn line(&mut self, rec: &Record) -> Result<&[u8]> {
        match self.source {
            Source::Memory(bytes) => {
                let start = rec.offset as usize;
                let end = start + rec.len as usize;
                bytes
                    .get(start..end)
                    .ok_or_else(|| anyhow!("record range out of bounds"))
            }
            Source::File(path) => {
                if self.file.is_none() {
                    self.file = Some(File::open(path).with_context(|| {
                        format!("cannot re-open source file '{}'", path.display())
                    })?);
                }
                let f = self.file.as_mut().unwrap();
                f.seek(SeekFrom::Start(rec.offset))
                    .context("failed to seek in source file")?;
                self.buf.resize(rec.len as usize, 0);
                f.read_exact(&mut self.buf)
                    .context("failed to read record from source file (file changed?)")?;
                Ok(&self.buf)
            }
        }
    }

    /// Re-parse a record's original JSON value.
    pub fn value(&mut self, rec: &Record) -> Result<Value> {
        let bytes = self.line(rec)?;
        serde_json::from_slice(bytes)
            .context("record is no longer valid JSON (source file changed?)")
    }
}

/// The fully loaded dataset.
#[derive(Debug)]
pub struct Dataset {
    pub records: Vec<Record>,
    /// Discovered schema: dotted path -> stats. BTreeMap keeps paths sorted,
    /// which naturally groups nested fields under their parents.
    pub schema: BTreeMap<String, FieldInfo>,
    /// Path interner shared by all records' flat maps.
    pub interner: PathInterner,
    /// Where original lines can be re-read from.
    source: Source,
    /// Count of lines that failed to parse as JSON (skipped, not fatal).
    pub parse_errors: usize,
    /// Total non-blank lines examined.
    pub total_lines: usize,
}

impl Dataset {
    /// Resolve a dotted path to its interned id (None when unknown).
    pub fn path_id(&self, path: &str) -> Option<u32> {
        self.interner.id(path)
    }

    /// A reusable random-access reader for original record lines.
    pub fn fetcher(&self) -> LineFetcher<'_> {
        LineFetcher {
            source: &self.source,
            file: None,
            buf: Vec::new(),
        }
    }
}

/// Short type name for a JSON value, used in schema annotations.
pub fn type_name(v: &Value) -> &'static str {
    match v {
        Value::Null => "null",
        Value::Bool(_) => "bool",
        Value::Number(_) => "num",
        Value::String(_) => "str",
        Value::Array(_) => "arr",
        Value::Object(_) => "obj",
    }
}

/// Walk a JSON value, invoking `f(path, leaf)` for every flattened leaf.
///
/// Rules:
/// - nested objects recurse: `user.id`, `user.name`
/// - arrays of scalars are kept whole under the parent path (e.g. `tags`)
/// - arrays containing objects expand the first [`MAX_ARRAY_OBJECTS`]
///   elements as `items[0].id`, `items[1].id`, ...
/// - a non-object root is reported under the pseudo-path `$`
pub fn flatten_visit<F: FnMut(&str, &Value)>(value: &Value, f: &mut F) {
    match value {
        Value::Object(map) if map.is_empty() => {}
        Value::Object(_) => {
            let mut path = String::with_capacity(64);
            visit(value, &mut path, f);
        }
        other => f("$", other),
    }
}

fn visit<F: FnMut(&str, &Value)>(value: &Value, path: &mut String, f: &mut F) {
    match value {
        Value::Object(map) => {
            if map.is_empty() {
                f(path, value);
                return;
            }
            let base = path.len();
            for (k, v) in map {
                if base > 0 {
                    path.push('.');
                }
                path.push_str(k);
                visit(v, path, f);
                path.truncate(base);
            }
        }
        Value::Array(items) => {
            if items.iter().any(Value::is_object) {
                use std::fmt::Write;
                let base = path.len();
                for (i, item) in items.iter().take(MAX_ARRAY_OBJECTS).enumerate() {
                    let _ = write!(path, "[{i}]");
                    visit(item, path, f);
                    path.truncate(base);
                }
            } else {
                f(path, value);
            }
        }
        scalar => f(path, scalar),
    }
}

/// Flatten a JSON value into a map of dotted paths to leaf values.
/// Kept for tests and small one-off uses; the loader uses
/// [`flatten_compact`], which avoids per-record path allocations.
#[cfg_attr(not(test), allow(dead_code))]
pub fn flatten(value: &Value) -> BTreeMap<String, Value> {
    let mut out = BTreeMap::new();
    flatten_visit(value, &mut |path, v| {
        out.insert(path.to_string(), v.clone());
    });
    out
}

/// Flatten a JSON value into a compact [`FlatRecord`], interning paths and
/// accumulating per-path stats in the shared interner.
pub fn flatten_compact(value: &Value, interner: &mut PathInterner) -> FlatRecord {
    let mut entries: Vec<(u32, CompactValue)> = Vec::new();
    flatten_visit(value, &mut |path, v| {
        let id = interner.intern(path);
        let info = &mut interner.infos[id as usize];
        info.count += 1;
        info.types.insert(type_name(v));
        entries.push((id, CompactValue::from_value(v)));
    });
    entries.sort_unstable_by_key(|e| e.0);
    FlatRecord {
        entries: entries.into_boxed_slice(),
    }
}

/// Load a JSONL dataset from a file path. Only offsets and flattened
/// compact values are kept in memory; original lines are re-read on demand.
pub fn load_jsonl(path: &Path, max_lines: Option<usize>) -> Result<Dataset> {
    let file =
        File::open(path).with_context(|| format!("cannot open file '{}'", path.display()))?;
    let mut loader = Loader::default();
    loader
        .read(BufReader::with_capacity(1 << 20, file), max_lines)
        .with_context(|| format!("while loading '{}'", path.display()))?;
    loader.finish(Source::File(path.to_path_buf()))
}

/// Load a JSONL dataset from any buffered reader (used directly by tests).
/// The input is retained in memory so original lines stay accessible.
#[cfg_attr(not(test), allow(dead_code))]
pub fn load_from_reader<R: BufRead>(mut reader: R, max_lines: Option<usize>) -> Result<Dataset> {
    let mut bytes = Vec::new();
    reader
        .read_to_end(&mut bytes)
        .context("failed to read input")?;
    let bytes = bytes.into_boxed_slice();
    let mut loader = Loader::default();
    loader.read(&bytes[..], max_lines)?;
    loader.finish(Source::Memory(bytes))
}

#[derive(Default)]
struct Loader {
    records: Vec<Record>,
    interner: PathInterner,
    parse_errors: usize,
    total_lines: usize,
}

impl Loader {
    fn read<R: BufRead>(&mut self, mut reader: R, max_lines: Option<usize>) -> Result<()> {
        let mut offset: u64 = 0;
        let mut buf: Vec<u8> = Vec::with_capacity(16 * 1024);
        loop {
            if let Some(max) = max_lines {
                if self.records.len() >= max {
                    break;
                }
            }
            buf.clear();
            let n = reader
                .read_until(b'\n', &mut buf)
                .context("failed to read line")?;
            if n == 0 {
                break;
            }
            let line_offset = offset;
            offset += n as u64;

            // Trim surrounding whitespace, keeping track of the byte range.
            let mut start = 0usize;
            let mut end = buf.len();
            while start < end && buf[start].is_ascii_whitespace() {
                start += 1;
            }
            while end > start && buf[end - 1].is_ascii_whitespace() {
                end -= 1;
            }
            if start == end {
                continue;
            }
            self.total_lines += 1;
            if end - start > u32::MAX as usize {
                self.parse_errors += 1;
                continue;
            }
            match serde_json::from_slice::<Value>(&buf[start..end]) {
                Ok(v) => {
                    let flat = flatten_compact(&v, &mut self.interner);
                    self.records.push(Record {
                        offset: line_offset + start as u64,
                        len: (end - start) as u32,
                        flat,
                    });
                }
                Err(_) => self.parse_errors += 1,
            }
        }
        Ok(())
    }

    fn finish(mut self, source: Source) -> Result<Dataset> {
        if self.records.is_empty() {
            if self.total_lines == 0 {
                bail!("the file contains no JSON lines (empty file)");
            }
            bail!(
                "no valid JSON records found ({} malformed line(s) out of {})",
                self.parse_errors,
                self.total_lines
            );
        }
        self.records.shrink_to_fit();
        let schema: BTreeMap<String, FieldInfo> = self
            .interner
            .paths
            .iter()
            .cloned()
            .zip(self.interner.infos.iter().cloned())
            .collect();
        Ok(Dataset {
            records: self.records,
            schema,
            interner: self.interner,
            source,
            parse_errors: self.parse_errors,
            total_lines: self.total_lines,
        })
    }
}

/// Deterministic hash of the sorted set of discovered field paths (FNV-1a),
/// used to auto-match a saved view config to files of the same "shape".
pub fn shape_hash<'a, I: Iterator<Item = &'a String>>(paths: I) -> String {
    const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325;
    const FNV_PRIME: u64 = 0x0000_0100_0000_01b3;
    let mut hash = FNV_OFFSET;
    for p in paths {
        for b in p.as_bytes() {
            hash ^= u64::from(*b);
            hash = hash.wrapping_mul(FNV_PRIME);
        }
        // path separator so ["ab","c"] != ["a","bc"]
        hash ^= 0x1e;
        hash = hash.wrapping_mul(FNV_PRIME);
    }
    format!("{hash:016x}")
}

#[cfg(test)]
mod tests {
    use super::*;
    use serde_json::json;
    use std::io::Cursor;

    #[test]
    fn flatten_nested_objects() {
        let v = json!({"user": {"id": 1, "name": "ada", "address": {"city": "x"}}, "type": "t"});
        let flat = flatten(&v);
        assert_eq!(flat.get("user.id"), Some(&json!(1)));
        assert_eq!(flat.get("user.name"), Some(&json!("ada")));
        assert_eq!(flat.get("user.address.city"), Some(&json!("x")));
        assert_eq!(flat.get("type"), Some(&json!("t")));
        assert!(!flat.contains_key("user"));
    }

    #[test]
    fn flatten_scalar_array_kept_whole() {
        let v = json!({"tags": ["a", "b", 3]});
        let flat = flatten(&v);
        assert_eq!(flat.get("tags"), Some(&json!(["a", "b", 3])));
        assert!(!flat.contains_key("tags[0]"));
    }

    #[test]
    fn flatten_object_array_expands_capped() {
        let items: Vec<Value> = (0..8).map(|i| json!({"id": i})).collect();
        let v = json!({"items": items});
        let flat = flatten(&v);
        assert_eq!(flat.get("items[0].id"), Some(&json!(0)));
        assert_eq!(flat.get("items[4].id"), Some(&json!(4)));
        assert!(!flat.contains_key("items[5].id"), "cap at {MAX_ARRAY_OBJECTS}");
    }

    #[test]
    fn flatten_empty_and_null() {
        let v = json!({"a": {}, "b": null});
        let flat = flatten(&v);
        assert_eq!(flat.get("a"), Some(&json!({})));
        assert_eq!(flat.get("b"), Some(&Value::Null));
    }

    #[test]
    fn flatten_non_object_root() {
        let flat = flatten(&json!([1, 2]));
        assert_eq!(flat.get("$"), Some(&json!([1, 2])));
    }

    #[test]
    fn flatten_compact_matches_flatten() {
        let v = json!({"user": {"id": 1, "tags": ["a", 2]}, "z": null, "b": true, "f": 1.5});
        let mut interner = PathInterner::default();
        let fr = flatten_compact(&v, &mut interner);
        let plain = flatten(&v);
        assert_eq!(fr.len(), plain.len());
        for (path, val) in &plain {
            let id = interner.id(path).expect("interned");
            let cv = fr.get(id).expect("present");
            assert_eq!(*cv, CompactValue::from_value(val), "path {path}");
        }
    }

    #[test]
    fn compact_value_display() {
        assert_eq!(CompactValue::from_value(&json!("x")).display(), "x");
        assert_eq!(CompactValue::from_value(&json!(5)).display(), "5");
        assert_eq!(CompactValue::from_value(&json!(-7)).display(), "-7");
        assert_eq!(CompactValue::from_value(&json!(1.5)).display(), "1.5");
        assert_eq!(CompactValue::from_value(&json!(null)).display(), "null");
        assert_eq!(CompactValue::from_value(&json!(true)).display(), "true");
        assert_eq!(
            CompactValue::from_value(&json!(["a", 1])).display(),
            "[\"a\",1]"
        );
        assert_eq!(
            CompactValue::from_value(&json!({"a": 1})).display(),
            "{\"a\":1}"
        );
    }

    #[test]
    fn load_accumulates_schema_and_skips_malformed() {
        let input = "\
{\"a\": 1, \"b\": {\"c\": true}}\n\
not json at all\n\
{\"a\": \"x\"}\n\
\n\
{\"b\": {\"c\": null}}\n";
        let ds = load_from_reader(Cursor::new(input), None).unwrap();
        assert_eq!(ds.records.len(), 3);
        assert_eq!(ds.parse_errors, 1);
        assert_eq!(ds.total_lines, 4);

        let a = &ds.schema["a"];
        assert_eq!(a.count, 2);
        assert!(a.types.contains("num") && a.types.contains("str"));

        let bc = &ds.schema["b.c"];
        assert_eq!(bc.count, 2);
        assert!(bc.types.contains("bool") && bc.types.contains("null"));
    }

    #[test]
    fn load_fetches_original_lines() {
        let input = "  {\"a\": 1}  \nbad\n{\"b\":\"x\"}\n";
        let ds = load_from_reader(Cursor::new(input), None).unwrap();
        let mut fetcher = ds.fetcher();
        assert_eq!(fetcher.line(&ds.records[0]).unwrap(), b"{\"a\": 1}");
        assert_eq!(fetcher.value(&ds.records[1]).unwrap(), json!({"b": "x"}));
    }

    #[test]
    fn load_respects_max_lines() {
        let input = "{\"a\":1}\n{\"a\":2}\n{\"a\":3}\n";
        let ds = load_from_reader(Cursor::new(input), Some(2)).unwrap();
        assert_eq!(ds.records.len(), 2);
    }

    #[test]
    fn load_all_malformed_is_error() {
        let err = load_from_reader(Cursor::new("nope\nstill nope\n"), None).unwrap_err();
        assert!(err.to_string().contains("no valid JSON records"));
    }

    #[test]
    fn load_empty_is_error() {
        let err = load_from_reader(Cursor::new(""), None).unwrap_err();
        assert!(err.to_string().contains("empty"));
    }

    #[test]
    fn shape_hash_is_deterministic_and_order_sensitive() {
        let a = ["a".to_string(), "b".to_string()];
        let h1 = shape_hash(a.iter());
        let h2 = shape_hash(a.iter());
        assert_eq!(h1, h2);
        let b = ["ab".to_string()];
        assert_ne!(h1, shape_hash(b.iter()));
    }
}