fuzzy_parser/

lib.rs

//! Fuzzy JSON repair for LLM-generated DSL
//!
//! This crate provides generic fuzzy matching and automatic correction for
//! JSON that may contain typos (common when generated by LLMs).
//!
//! # Design
//!
//! This crate provides **generic repair APIs** - the schema definitions
//! live in the calling crate (e.g., your application defines the schema).
//!
//! The repair pipeline has three independent stages:
//!
//! 1. **Extract** ([`extract_json`]) — locate the JSON payload inside raw
//!    LLM output (Markdown code fences, surrounding prose, multiple blocks)
//! 2. **Sanitize** ([`sanitize_json`]) — fix syntax errors (trailing
//!    commas, missing/mismatched closing delimiters, unclosed strings)
//! 3. **Repair** ([`repair_tagged_enum_json`]) — fix typos and coerce
//!    value types, guided by a caller-provided schema
//!
//! # Features
//!
//! - **JSON extraction**: Pull JSON out of code fences and prose
//! - **JSON sanitization**: Fix syntax errors (trailing commas, missing braces)
//! - **Tagged enum repair**: Fix type discriminator typos (e.g., `"AddDeriv"` → `"AddDerive"`)
//! - **Field name repair**: Fix field name typos (e.g., `"taget"` → `"target"`)
//! - **Enum value repair**: Fix string values constrained to closed sets
//!   (e.g., `["Debg"]` → `["Debug"]`)
//! - **Recursive schemas**: Nested objects, arrays of objects, and nested
//!   tagged enums (e.g. arrays of DSL intents) are repaired to any depth
//!   ([`FieldKind`])
//! - **Type coercion**: Fix string-encoded scalars (e.g., `"42"` → `42`)
//! - **Dynamic schemas**: Schemas own their data and can be built at runtime
//! - **JSON Schema import**: Derive repair schemas from JSON Schema
//!   documents ([`TaggedEnumSchema::from_json_schema`]) or, with the
//!   `schemars` feature, directly from `#[derive(JsonSchema)]` types
//! - **Transparency**: Every applied change is a [`Correction`]; every
//!   collision-skipped rename is a [`SkippedCorrection`]
//! - **Multiple algorithm support**: Jaro-Winkler, Levenshtein, Damerau-Levenshtein
//! - **Configurable similarity threshold**
//!
//! # Example
//!
//! ````
//! use fuzzy_parser::{
//!     extract_json, sanitize_json, repair_tagged_enum_json, TaggedEnumSchema, FuzzyOptions,
//! };
//!
//! // Define schema with enum arrays and nested objects
//! let schema = TaggedEnumSchema::new(
//!     "type",  // tag field
//!     &["AddDerive", "RemoveDerive"],  // valid types
//!     |tag| match tag {
//!         "AddDerive" | "RemoveDerive" => Some(&["target", "derives", "config"][..]),
//!         _ => None,
//!     },
//! )
//! .with_enum_array("derives", &["Debug", "Clone", "Serialize", "Default"])
//! .with_nested_object("config", &["timeout", "retries"]);
//!
//! // Raw LLM output: prose + code fence + syntax errors + typos
//! let raw = r#"Here is the change you asked for:
//!
//! ```json
//! {"type": "AddDeriv", "taget": "User", "derives": ["Debg",], "config": {"timout": 30,}}
//! ```
//! "#;
//!
//! // Step 0: Extract (strip prose and code fences)
//! let payload = extract_json(raw).unwrap();
//!
//! // Step 1: Sanitize (fix syntax errors)
//! let sanitized = sanitize_json(payload);
//!
//! // Step 2: Repair (fix typos)
//! let result = repair_tagged_enum_json(&sanitized, &schema, &FuzzyOptions::default()).unwrap();
//!
//! assert_eq!(result.repaired["type"], "AddDerive");
//! assert!(result.repaired.get("target").is_some());
//! assert_eq!(result.repaired["derives"][0], "Debug");
//! assert!(result.repaired["config"].get("timeout").is_some());
//! ````

#![warn(missing_docs)]

pub mod distance;
pub mod error;
pub mod extract;
pub mod import;
pub mod repair;
pub mod sanitize;
pub mod schema;

// Re-export main types
pub use distance::{Algorithm, Match};
pub use error::FuzzyError;
pub use extract::{extract_json, extract_json_blocks, strip_code_fences};
pub use import::{ImportWarning, SchemaImport};
pub use repair::{
    repair_enum_array, repair_fields_with_list, repair_object_fields, repair_tagged_enum,
    repair_tagged_enum_array, repair_tagged_enum_json, Correction, FuzzyOptions, RepairLog,
    RepairResult, SkipReason, SkippedCorrection,
};
pub use sanitize::sanitize_json;
pub use schema::{FieldDef, FieldKind, ObjectSchema, TaggedEnumSchema};

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_full_workflow() {
        // Define a simple schema
        let schema =
            TaggedEnumSchema::new("type", &["AddDerive", "RenameIdent"], |tag| match tag {
                "AddDerive" => Some(&["target", "derives"][..]),
                "RenameIdent" => Some(&["from", "to", "kind"][..]),
                _ => None,
            });

        // Simulate LLM output with typos
        let llm_output = r#"{
            "type": "AddDeriv",
            "taget": "DatabaseConfig",
            "derives": ["Debug", "Clone"]
        }"#;

        let result =
            repair_tagged_enum_json(llm_output, &schema, &FuzzyOptions::default()).unwrap();

        // Verify corrections
        assert_eq!(result.repaired["type"], "AddDerive");
        assert_eq!(result.repaired["target"], "DatabaseConfig");
        assert_eq!(result.corrections.len(), 2);
    }

    #[test]
    fn test_sanitize_then_repair_workflow() {
        // Define schema with enum arrays
        let schema =
            TaggedEnumSchema::new("type", &["AddDerive"], |_| Some(&["target", "derives"][..]))
                .with_enum_array("derives", ["Debug", "Clone", "Serialize"]);

        // LLM output with BOTH syntax errors AND typos
        let malformed_llm_output = r#"{
            "type": "AddDeriv",
            "taget": "User",
            "derives": ["Debg", "Clne",],
        }"#;

        // Step 1: Sanitize
        let sanitized = sanitize_json(malformed_llm_output);

        // Verify sanitization worked
        assert!(!sanitized.contains(",]"));
        assert!(!sanitized.contains(",}"));

        // Step 2: Repair
        let result =
            repair_tagged_enum_json(&sanitized, &schema, &FuzzyOptions::default()).unwrap();

        // Verify all corrections
        assert_eq!(result.repaired["type"], "AddDerive");
        assert!(result.repaired.get("target").is_some());
        assert_eq!(result.repaired["derives"][0], "Debug");
        assert_eq!(result.repaired["derives"][1], "Clone");
    }

    #[test]
    fn test_sanitize_then_repair_missing_brace() {
        let schema = TaggedEnumSchema::new("type", &["Action"], |_| Some(&["name"][..]));

        // LLM truncated output
        let truncated = r#"{"type": "Action", "name": "test"#;

        let sanitized = sanitize_json(truncated);
        let result =
            repair_tagged_enum_json(&sanitized, &schema, &FuzzyOptions::default()).unwrap();

        assert_eq!(result.repaired["type"], "Action");
        assert_eq!(result.repaired["name"], "test");
    }

    #[test]
    fn test_extract_sanitize_repair_full_pipeline() {
        // Raw LLM output: prose + fence + trailing comma + typos + truncation
        let raw = "Sure, here it is:\n\n```json\n{\"type\": \"AddDeriv\", \"taget\": \"User\", \"derives\": [\"Debg\",]\n```";

        let schema =
            TaggedEnumSchema::new("type", &["AddDerive"], |_| Some(&["target", "derives"][..]))
                .with_enum_array("derives", ["Debug", "Clone"]);

        let payload = extract_json(raw).unwrap();
        let sanitized = sanitize_json(payload);
        let result =
            repair_tagged_enum_json(&sanitized, &schema, &FuzzyOptions::default()).unwrap();

        assert_eq!(result.repaired["type"], "AddDerive");
        assert_eq!(result.repaired["target"], "User");
        assert_eq!(result.repaired["derives"][0], "Debug");
    }
}