tftio-kb 2.5.3

//! Diagnostic walker for `.org` corpora.
//!
//! Walks a directory recursively, parses every `.org` file via
//! [`crate::parser`], and reports:
//!
//! * parse failures;
//! * a byte-for-byte round-trip metric per file — whether
//!   `generate(parse(f))` reproduces the source exactly;
//! * parser residue per file — source lines no block claimed, i.e.
//!   dropped content (the deduplication safety gate);
//! * exact-variant duplicate groups, keyed by the canonical-AST
//!   s-expression encoding and gated on content-completeness.
//!
//! Read-only: no DB, no network.
//!
//! Originally a port of Haskell `app/kb-parse-survey/Main.hs`. The
//! round-trip and duplicate-detection sections are Rust-side additions
//! with no Haskell counterpart; Haskell parity is no longer maintained.

use std::ffi::OsStr;
use std::fs;
use std::path::{Path, PathBuf};

use crate::ast::{Block, Document};
use crate::canonical::canonicalize;
use crate::generator::generate;
use crate::parser::parse_document_with_residue;
use crate::sexp;

/// Per-file outcome.
#[derive(Debug)]
pub enum Outcome {
    /// Parsed successfully.
    Ok {
        /// The parsed document.
        document: Document,
        /// `true` when `generate(document)` reproduces the source text
        /// byte-for-byte. A round-trip metric only; cosmetic reformatting
        /// (re-wrapped paragraphs, whitespace) makes this `false` without
        /// any content being lost.
        lossless: bool,
        /// First line where `generate(document)` diverges from the source:
        /// `(line number, source line, generated line)`. `None` when the
        /// file round-trips.
        first_diff: Option<(usize, String, String)>,
        /// Source lines that no block claimed. Empty residue means the
        /// parser dropped no content — the gate for safe deduplication.
        residue: Vec<String>,
        /// Canonical-AST s-expression encoding. Two files share a key
        /// iff their documents are exact structural variants.
        canonical_key: String,
    },
    /// Parse or read failure; the string is the error message.
    Failed(String),
}

/// Result of processing one file.
#[derive(Debug)]
pub struct FileResult {
    /// Path of the processed file.
    pub path: PathBuf,
    /// What happened when the file was read and parsed.
    pub outcome: Outcome,
}

/// `true` when `r` parsed and round-tripped byte-for-byte.
#[must_use]
pub const fn is_lossless(r: &FileResult) -> bool {
    matches!(&r.outcome, Outcome::Ok { lossless: true, .. })
}

/// `true` when `r` parsed and the parser claimed every source line —
/// no content was dropped. This is the deduplication safety gate.
#[must_use]
pub const fn is_content_complete(r: &FileResult) -> bool {
    matches!(&r.outcome, Outcome::Ok { residue, .. } if residue.is_empty())
}

/// First line at which `generated` diverges from `src`, as
/// `(line number, source line, generated line)`. `<EOF>` stands in for a
/// side that ran out of lines. `None` when the two are line-identical.
#[must_use]
pub fn first_line_divergence(src: &str, generated: &str) -> Option<(usize, String, String)> {
    let mut src_lines = src.lines();
    let mut gen_lines = generated.lines();
    let mut line_no = 0;
    loop {
        let (src_line, gen_line) = (src_lines.next(), gen_lines.next());
        if src_line == gen_line {
            src_line?;
            line_no += 1;
        } else {
            return Some((
                line_no,
                src_line.unwrap_or("<EOF>").to_string(),
                gen_line.unwrap_or("<EOF>").to_string(),
            ));
        }
    }
}

/// Walk `root` recursively, returning every `.org` file sorted.
///
/// Unreadable directories are silently skipped (matching Haskell
/// `findOrgFiles`); unreadable files are returned and surfaced later as
/// `Failed` results during processing.
#[must_use]
pub fn find_org_files(root: &Path) -> Vec<PathBuf> {
    let mut out = Vec::new();
    walk(root, &mut out);
    out.sort();
    out
}

fn walk(path: &Path, out: &mut Vec<PathBuf>) {
    if path.is_dir() {
        let Ok(entries) = fs::read_dir(path) else {
            return;
        };
        for entry in entries.flatten() {
            walk(&entry.path(), out);
        }
    } else if path.extension().and_then(OsStr::to_str) == Some("org") {
        out.push(path.to_path_buf());
    }
}

/// Read and parse one file. Always returns a [`FileResult`] — errors
/// become `Failed` outcomes rather than propagating.
///
/// For files that parse, computes the byte-for-byte round-trip metric
/// (`generate(parse(f)) == f`), the parser residue (unclaimed source
/// lines), and the canonical-AST dedup key.
#[must_use]
pub fn process_file(path: &Path) -> FileResult {
    match fs::read_to_string(path) {
        Err(e) => FileResult {
            path: path.to_path_buf(),
            outcome: Outcome::Failed(format!("unreadable: {e}")),
        },
        Ok(txt) => match parse_document_with_residue(&txt) {
            Ok((document, residue)) => {
                let generated = generate(&document);
                let lossless = generated == txt;
                let first_diff = first_line_divergence(&txt, &generated);
                let canonical_key = sexp::encode_document(&canonicalize(&document));
                FileResult {
                    path: path.to_path_buf(),
                    outcome: Outcome::Ok {
                        document,
                        lossless,
                        first_diff,
                        residue,
                        canonical_key,
                    },
                }
            }
            Err(e) => FileResult {
                path: path.to_path_buf(),
                outcome: Outcome::Failed(e.to_string()),
            },
        },
    }
}

/// Constructor name for a [`Block`], used in the verbose AST count.
#[must_use]
pub const fn block_name(b: &Block) -> &'static str {
    match b {
        Block::Heading { .. } => "Heading",
        Block::Paragraph { .. } => "Paragraph",
        Block::SrcBlock { .. } => "SrcBlock",
        Block::ExampleBlock { .. } => "ExampleBlock",
        Block::QuoteBlock { .. } => "QuoteBlock",
        Block::List { .. } => "List",
        Block::Table { .. } => "Table",
        Block::PropertyDrawer { .. } => "PropertyDrawer",
        Block::LogbookDrawer { .. } => "LogbookDrawer",
        Block::Planning { .. } => "Planning",
        Block::Comment { .. } => "Comment",
        Block::Keyword { .. } => "Keyword",
        Block::BlankLine => "BlankLine",
        Block::HorizontalRule => "HorizontalRule",
    }
}

fn rjust(width: usize, s: &str) -> String {
    let pad = width.saturating_sub(s.chars().count());
    let mut out = String::with_capacity(pad + s.len());
    for _ in 0..pad {
        out.push(' ');
    }
    out.push_str(s);
    out
}

/// Group successfully-parsed files by canonical-AST key. Returns only
/// groups with more than one member — i.e. exact-variant duplicate sets.
/// Group order follows first appearance; members keep input order.
fn duplicate_groups<'a>(oks: &[&'a FileResult]) -> Vec<Vec<&'a FileResult>> {
    let mut groups: Vec<(&str, Vec<&'a FileResult>)> = Vec::new();
    for r in oks {
        if let Outcome::Ok { canonical_key, .. } = &r.outcome {
            if let Some((_, g)) = groups.iter_mut().find(|(k, _)| *k == canonical_key) {
                g.push(r);
            } else {
                groups.push((canonical_key, vec![r]));
            }
        }
    }
    groups
        .into_iter()
        .map(|(_, g)| g)
        .filter(|g| g.len() > 1)
        .collect()
}

/// Render the survey report for `dir` over the given `results`.
#[must_use]
pub fn render_report(dir: &str, results: &[FileResult], verbose: bool) -> String {
    let total = results.len();
    let mut oks = Vec::new();
    let mut fails = Vec::new();
    let mut lossy = Vec::new();
    let mut dropped = Vec::new();
    let mut residue_line_total = 0usize;
    for r in results {
        match &r.outcome {
            Outcome::Ok {
                lossless, residue, ..
            } => {
                oks.push(r);
                if !lossless {
                    lossy.push(r);
                }
                if !residue.is_empty() {
                    dropped.push(r);
                    residue_line_total += residue.len();
                }
            }
            Outcome::Failed(_) => fails.push(r),
        }
    }
    let lossless_count = oks.len() - lossy.len();
    let complete_count = oks.len() - dropped.len();
    let dup_groups = duplicate_groups(&oks);

    let mut out = String::new();
    out.push_str(&format!(
        "kb-parse-survey: {total} .org files under {dir}\n"
    ));
    out.push('\n');
    out.push_str(&format!("=== summary {}\n", "=".repeat(50)));
    out.push_str(&format!(
        "{} parsed cleanly\n",
        rjust(6, &oks.len().to_string())
    ));
    out.push_str(&format!(
        "{}   round-trip clean (generate reproduces source byte-for-byte)\n",
        rjust(6, &lossless_count.to_string())
    ));
    out.push_str(&format!(
        "{}   round-trip lossy (cosmetic reformatting, not dropped content)\n",
        rjust(6, &lossy.len().to_string())
    ));
    out.push_str(&format!(
        "{}   content-complete (parser claimed every input line)\n",
        rjust(6, &complete_count.to_string())
    ));
    out.push_str(&format!(
        "{}   dropped content ({residue_line_total} residue line(s) unclaimed)\n",
        rjust(6, &dropped.len().to_string())
    ));
    out.push_str(&format!(
        "{} failed to parse\n",
        rjust(6, &fails.len().to_string())
    ));
    out.push_str(&format!(
        "{} duplicate group(s) (exact structural variants)\n",
        rjust(6, &dup_groups.len().to_string())
    ));
    out.push('\n');

    if !fails.is_empty() {
        out.push_str(&format!(
            "=== failures ({}) {}\n",
            fails.len(),
            "=".repeat(38)
        ));
        for r in &fails {
            out.push_str(&format!("FAIL  {}\n", r.path.display()));
            if let Outcome::Failed(err) = &r.outcome {
                for line in err.lines() {
                    out.push_str(&format!("  {line}\n"));
                }
            }
        }
        out.push('\n');
    }

    if !lossy.is_empty() {
        out.push_str(&format!(
            "=== lossy files ({}) {}\n",
            lossy.len(),
            "=".repeat(38)
        ));
        for r in &lossy {
            out.push_str(&format!("LOSSY  {}\n", r.path.display()));
            if let Outcome::Ok {
                first_diff: Some((n, src, out_line)),
                ..
            } = &r.outcome
            {
                out.push_str(&format!("  first diff at line {n}:\n"));
                out.push_str(&format!("    source:    {src}\n"));
                out.push_str(&format!("    generated: {out_line}\n"));
            }
        }
        out.push('\n');
    }

    if !dropped.is_empty() {
        out.push_str(&format!(
            "=== dropped content ({}) {}\n",
            dropped.len(),
            "=".repeat(36)
        ));
        for r in &dropped {
            out.push_str(&format!("DROPPED  {}\n", r.path.display()));
            if let Outcome::Ok { residue, .. } = &r.outcome {
                for line in residue {
                    out.push_str(&format!("  unclaimed: {line}\n"));
                }
            }
        }
        out.push('\n');
    }

    if !dup_groups.is_empty() {
        out.push_str(&format!(
            "=== duplicate groups ({}) {}\n",
            dup_groups.len(),
            "=".repeat(34)
        ));
        for group in &dup_groups {
            // A collision among files that dropped content may be
            // spurious: the parser can collapse files that differ only
            // in the dropped lines. Only groups where every member is
            // content-complete are confirmed duplicates.
            let confirmed = group.iter().all(|r| is_content_complete(r));
            if confirmed {
                out.push_str(&format!(
                    "confirmed duplicates (group of {}):\n",
                    group.len()
                ));
            } else {
                out.push_str(&format!(
                    "review candidates — key collision with ≥1 file that dropped content (group of {}):\n",
                    group.len()
                ));
            }
            for r in group {
                out.push_str(&format!("  {}\n", r.path.display()));
            }
        }
        out.push('\n');
    }

    if verbose {
        let mut counts: Vec<(&'static str, usize)> = Vec::new();
        for r in &oks {
            if let Outcome::Ok { document, .. } = &r.outcome {
                for b in &document.blocks {
                    let name = block_name(b);
                    if let Some((_, n)) = counts.iter_mut().find(|(m, _)| *m == name) {
                        *n += 1;
                    } else {
                        counts.push((name, 1));
                    }
                }
            }
        }
        counts.sort_by_key(|b| std::cmp::Reverse(b.1));
        out.push_str(&format!(
            "=== AST constructor counts (across {} files) {}\n",
            oks.len(),
            "=".repeat(12)
        ));
        for (name, n) in &counts {
            out.push_str(&format!("{}  {}\n", rjust(6, &n.to_string()), name));
        }
        out.push('\n');
    }

    out
}

/// Walk `dir`, process every `.org` file, and return the report text.
#[must_use]
pub fn run(dir: &str, verbose: bool) -> String {
    let files = find_org_files(Path::new(dir));
    let results: Vec<FileResult> = files.iter().map(|p| process_file(p)).collect();
    render_report(dir, &results, verbose)
}