jsonl-tui 0.1.0

//! Loading, flattening and schema accumulation for JSONL data.

use std::collections::{BTreeMap, BTreeSet};
use std::fs::File;
use std::io::{BufRead, BufReader};
use std::path::Path;

use anyhow::{bail, Context, Result};
use serde_json::Value;

/// When an array contains objects, only the first N elements are expanded
/// into `path[i].field` entries to avoid path explosion.
pub const MAX_ARRAY_OBJECTS: usize = 5;

/// One loaded JSONL record: the original value (for the detail view and
/// export) plus its flattened dotted-path map (for columns/search/filter/sort).
#[derive(Debug)]
pub struct Record {
    pub original: Value,
    pub flat: BTreeMap<String, Value>,
}

/// Per-field-path statistics accumulated while loading.
#[derive(Debug, Default, Clone)]
pub struct FieldInfo {
    /// Number of records in which this path is present.
    pub count: usize,
    /// The set of JSON types observed at this path.
    pub types: BTreeSet<&'static str>,
}

/// The fully loaded dataset.
#[derive(Debug)]
pub struct Dataset {
    pub records: Vec<Record>,
    /// Discovered schema: dotted path -> stats. BTreeMap keeps paths sorted,
    /// which naturally groups nested fields under their parents.
    pub schema: BTreeMap<String, FieldInfo>,
    /// Count of lines that failed to parse as JSON (skipped, not fatal).
    pub parse_errors: usize,
    /// Total non-blank lines examined.
    pub total_lines: usize,
}

/// Short type name for a JSON value, used in schema annotations.
pub fn type_name(v: &Value) -> &'static str {
    match v {
        Value::Null => "null",
        Value::Bool(_) => "bool",
        Value::Number(_) => "num",
        Value::String(_) => "str",
        Value::Array(_) => "arr",
        Value::Object(_) => "obj",
    }
}

/// Flatten a JSON value into a map of dotted paths to leaf values.
///
/// Rules:
/// - nested objects recurse: `user.id`, `user.name`
/// - arrays of scalars are kept whole under the parent path (e.g. `tags`)
/// - arrays containing objects expand the first [`MAX_ARRAY_OBJECTS`]
///   elements as `items[0].id`, `items[1].id`, ...
/// - a non-object root is stored under the pseudo-path `$`
pub fn flatten(value: &Value) -> BTreeMap<String, Value> {
    let mut out = BTreeMap::new();
    match value {
        Value::Object(_) => flatten_into(value, String::new(), &mut out),
        other => {
            out.insert("$".to_string(), other.clone());
        }
    }
    out
}

fn flatten_into(value: &Value, prefix: String, out: &mut BTreeMap<String, Value>) {
    match value {
        Value::Object(map) => {
            if map.is_empty() {
                if !prefix.is_empty() {
                    out.insert(prefix, value.clone());
                }
                return;
            }
            for (k, v) in map {
                let path = if prefix.is_empty() {
                    k.clone()
                } else {
                    format!("{prefix}.{k}")
                };
                flatten_into(v, path, out);
            }
        }
        Value::Array(items) => {
            let has_objects = items.iter().any(Value::is_object);
            if has_objects {
                for (i, item) in items.iter().take(MAX_ARRAY_OBJECTS).enumerate() {
                    flatten_into(item, format!("{prefix}[{i}]"), out);
                }
            } else {
                out.insert(prefix, value.clone());
            }
        }
        scalar => {
            out.insert(prefix, scalar.clone());
        }
    }
}

/// Load a JSONL dataset from a file path.
pub fn load_jsonl(path: &Path, max_lines: Option<usize>) -> Result<Dataset> {
    let file =
        File::open(path).with_context(|| format!("cannot open file '{}'", path.display()))?;
    load_from_reader(BufReader::new(file), max_lines)
        .with_context(|| format!("while loading '{}'", path.display()))
}

/// Load a JSONL dataset from any buffered reader (used directly by tests).
pub fn load_from_reader<R: BufRead>(reader: R, max_lines: Option<usize>) -> Result<Dataset> {
    let mut records = Vec::new();
    let mut schema: BTreeMap<String, FieldInfo> = BTreeMap::new();
    let mut parse_errors = 0usize;
    let mut total_lines = 0usize;

    for line in reader.lines() {
        if let Some(max) = max_lines {
            if records.len() >= max {
                break;
            }
        }
        let line = line.context("failed to read line")?;
        let trimmed = line.trim();
        if trimmed.is_empty() {
            continue;
        }
        total_lines += 1;
        match serde_json::from_str::<Value>(trimmed) {
            Ok(v) => {
                let flat = flatten(&v);
                for (path, val) in &flat {
                    let info = schema.entry(path.clone()).or_default();
                    info.count += 1;
                    info.types.insert(type_name(val));
                }
                records.push(Record { original: v, flat });
            }
            Err(_) => parse_errors += 1,
        }
    }

    if records.is_empty() {
        if total_lines == 0 {
            bail!("the file contains no JSON lines (empty file)");
        }
        bail!(
            "no valid JSON records found ({parse_errors} malformed line(s) out of {total_lines})"
        );
    }

    Ok(Dataset {
        records,
        schema,
        parse_errors,
        total_lines,
    })
}

/// Deterministic hash of the sorted set of discovered field paths (FNV-1a),
/// used to auto-match a saved view config to files of the same "shape".
pub fn shape_hash<'a, I: Iterator<Item = &'a String>>(paths: I) -> String {
    const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325;
    const FNV_PRIME: u64 = 0x0000_0100_0000_01b3;
    let mut hash = FNV_OFFSET;
    for p in paths {
        for b in p.as_bytes() {
            hash ^= u64::from(*b);
            hash = hash.wrapping_mul(FNV_PRIME);
        }
        // path separator so ["ab","c"] != ["a","bc"]
        hash ^= 0x1e;
        hash = hash.wrapping_mul(FNV_PRIME);
    }
    format!("{hash:016x}")
}

#[cfg(test)]
mod tests {
    use super::*;
    use serde_json::json;
    use std::io::Cursor;

    #[test]
    fn flatten_nested_objects() {
        let v = json!({"user": {"id": 1, "name": "ada", "address": {"city": "x"}}, "type": "t"});
        let flat = flatten(&v);
        assert_eq!(flat.get("user.id"), Some(&json!(1)));
        assert_eq!(flat.get("user.name"), Some(&json!("ada")));
        assert_eq!(flat.get("user.address.city"), Some(&json!("x")));
        assert_eq!(flat.get("type"), Some(&json!("t")));
        assert!(!flat.contains_key("user"));
    }

    #[test]
    fn flatten_scalar_array_kept_whole() {
        let v = json!({"tags": ["a", "b", 3]});
        let flat = flatten(&v);
        assert_eq!(flat.get("tags"), Some(&json!(["a", "b", 3])));
        assert!(!flat.contains_key("tags[0]"));
    }

    #[test]
    fn flatten_object_array_expands_capped() {
        let items: Vec<Value> = (0..8).map(|i| json!({"id": i})).collect();
        let v = json!({"items": items});
        let flat = flatten(&v);
        assert_eq!(flat.get("items[0].id"), Some(&json!(0)));
        assert_eq!(flat.get("items[4].id"), Some(&json!(4)));
        assert!(!flat.contains_key("items[5].id"), "cap at {MAX_ARRAY_OBJECTS}");
    }

    #[test]
    fn flatten_empty_and_null() {
        let v = json!({"a": {}, "b": null});
        let flat = flatten(&v);
        assert_eq!(flat.get("a"), Some(&json!({})));
        assert_eq!(flat.get("b"), Some(&Value::Null));
    }

    #[test]
    fn flatten_non_object_root() {
        let flat = flatten(&json!([1, 2]));
        assert_eq!(flat.get("$"), Some(&json!([1, 2])));
    }

    #[test]
    fn load_accumulates_schema_and_skips_malformed() {
        let input = "\
{\"a\": 1, \"b\": {\"c\": true}}\n\
not json at all\n\
{\"a\": \"x\"}\n\
\n\
{\"b\": {\"c\": null}}\n";
        let ds = load_from_reader(Cursor::new(input), None).unwrap();
        assert_eq!(ds.records.len(), 3);
        assert_eq!(ds.parse_errors, 1);
        assert_eq!(ds.total_lines, 4);

        let a = &ds.schema["a"];
        assert_eq!(a.count, 2);
        assert!(a.types.contains("num") && a.types.contains("str"));

        let bc = &ds.schema["b.c"];
        assert_eq!(bc.count, 2);
        assert!(bc.types.contains("bool") && bc.types.contains("null"));
    }

    #[test]
    fn load_respects_max_lines() {
        let input = "{\"a\":1}\n{\"a\":2}\n{\"a\":3}\n";
        let ds = load_from_reader(Cursor::new(input), Some(2)).unwrap();
        assert_eq!(ds.records.len(), 2);
    }

    #[test]
    fn load_all_malformed_is_error() {
        let err = load_from_reader(Cursor::new("nope\nstill nope\n"), None).unwrap_err();
        assert!(err.to_string().contains("no valid JSON records"));
    }

    #[test]
    fn load_empty_is_error() {
        let err = load_from_reader(Cursor::new(""), None).unwrap_err();
        assert!(err.to_string().contains("empty"));
    }

    #[test]
    fn shape_hash_is_deterministic_and_order_sensitive() {
        let a = ["a".to_string(), "b".to_string()];
        let h1 = shape_hash(a.iter());
        let h2 = shape_hash(a.iter());
        assert_eq!(h1, h2);
        let b = ["ab".to_string()];
        assert_ne!(h1, shape_hash(b.iter()));
    }
}