fuzzy_parser/
lib.rs

1//! Fuzzy JSON repair for LLM-generated DSL
2//!
3//! This crate provides generic fuzzy matching and automatic correction for
4//! JSON that may contain typos (common when generated by LLMs).
5//!
6//! # Design
7//!
8//! This crate provides **generic repair APIs** - the schema definitions
9//! live in the calling crate (e.g., your application defines the schema).
10//!
11//! # Features
12//!
13//! - **JSON sanitization**: Fix syntax errors (trailing commas, missing braces)
14//! - **Tagged enum repair**: Fix type discriminator typos (e.g., `"AddDeriv"` → `"AddDerive"`)
15//! - **Field name repair**: Fix field name typos (e.g., `"taget"` → `"target"`)
16//! - **Enum array repair**: Fix values in enum arrays (e.g., `["Debg"]` → `["Debug"]`)
17//! - **Nested object repair**: Fix field names in nested objects
18//! - **Multiple algorithm support**: Jaro-Winkler, Levenshtein, Damerau-Levenshtein
19//! - **Configurable similarity threshold**
20//!
21//! # Example
22//!
23//! ```
24//! use fuzzy_parser::{
25//!     sanitize_json, repair_tagged_enum_json, TaggedEnumSchema, FuzzyOptions,
26//! };
27//!
28//! // Define schema with enum arrays and nested objects
29//! let schema = TaggedEnumSchema::new(
30//!     "type",  // tag field
31//!     &["AddDerive", "RemoveDerive"],  // valid types
32//!     |tag| match tag {
33//!         "AddDerive" | "RemoveDerive" => Some(&["target", "derives", "config"][..]),
34//!         _ => None,
35//!     },
36//! )
37//! .with_enum_array("derives", &["Debug", "Clone", "Serialize", "Default"])
38//! .with_nested_object("config", &["timeout", "retries"]);
39//!
40//! // LLM output with syntax errors AND typos
41//! let malformed = r#"{"type": "AddDeriv", "taget": "User", "derives": ["Debg",], "config": {"timout": 30,}}"#;
42//!
43//! // Step 1: Sanitize (fix syntax errors)
44//! let sanitized = sanitize_json(malformed);
45//!
46//! // Step 2: Repair (fix typos)
47//! let result = repair_tagged_enum_json(&sanitized, &schema, &FuzzyOptions::default()).unwrap();
48//!
49//! assert_eq!(result.repaired["type"], "AddDerive");
50//! assert!(result.repaired.get("target").is_some());
51//! assert_eq!(result.repaired["derives"][0], "Debug");
52//! assert!(result.repaired["config"].get("timeout").is_some());
53//! ```
54
55#![warn(missing_docs)]
56
57pub mod distance;
58pub mod error;
59pub mod repair;
60pub mod sanitize;
61pub mod schema;
62
63// Re-export main types
64pub use distance::{Algorithm, Match};
65pub use error::FuzzyError;
66pub use repair::{
67    repair_enum_array, repair_fields_with_list, repair_object_fields, repair_tagged_enum,
68    repair_tagged_enum_array, repair_tagged_enum_json, Correction, FuzzyOptions, RepairResult,
69};
70pub use sanitize::sanitize_json;
71pub use schema::{ObjectSchema, TaggedEnumSchema};
72
73#[cfg(test)]
74mod tests {
75    use super::*;
76
77    #[test]
78    fn test_full_workflow() {
79        // Define a simple schema
80        let schema =
81            TaggedEnumSchema::new("type", &["AddDerive", "RenameIdent"], |tag| match tag {
82                "AddDerive" => Some(&["target", "derives"][..]),
83                "RenameIdent" => Some(&["from", "to", "kind"][..]),
84                _ => None,
85            });
86
87        // Simulate LLM output with typos
88        let llm_output = r#"{
89            "type": "AddDeriv",
90            "taget": "DatabaseConfig",
91            "derives": ["Debug", "Clone"]
92        }"#;
93
94        let result =
95            repair_tagged_enum_json(llm_output, &schema, &FuzzyOptions::default()).unwrap();
96
97        // Verify corrections
98        assert_eq!(result.repaired["type"], "AddDerive");
99        assert_eq!(result.repaired["target"], "DatabaseConfig");
100        assert_eq!(result.corrections.len(), 2);
101    }
102
103    #[test]
104    fn test_sanitize_then_repair_workflow() {
105        // Define schema with enum arrays
106        let schema =
107            TaggedEnumSchema::new("type", &["AddDerive"], |_| Some(&["target", "derives"][..]))
108                .with_enum_array("derives", &["Debug", "Clone", "Serialize"]);
109
110        // LLM output with BOTH syntax errors AND typos
111        let malformed_llm_output = r#"{
112            "type": "AddDeriv",
113            "taget": "User",
114            "derives": ["Debg", "Clne",],
115        }"#;
116
117        // Step 1: Sanitize
118        let sanitized = sanitize_json(malformed_llm_output);
119
120        // Verify sanitization worked
121        assert!(!sanitized.contains(",]"));
122        assert!(!sanitized.contains(",}"));
123
124        // Step 2: Repair
125        let result =
126            repair_tagged_enum_json(&sanitized, &schema, &FuzzyOptions::default()).unwrap();
127
128        // Verify all corrections
129        assert_eq!(result.repaired["type"], "AddDerive");
130        assert!(result.repaired.get("target").is_some());
131        assert_eq!(result.repaired["derives"][0], "Debug");
132        assert_eq!(result.repaired["derives"][1], "Clone");
133    }
134
135    #[test]
136    fn test_sanitize_then_repair_missing_brace() {
137        let schema = TaggedEnumSchema::new("type", &["Action"], |_| Some(&["name"][..]));
138
139        // LLM truncated output
140        let truncated = r#"{"type": "Action", "name": "test"#;
141
142        let sanitized = sanitize_json(truncated);
143        let result =
144            repair_tagged_enum_json(&sanitized, &schema, &FuzzyOptions::default()).unwrap();
145
146        assert_eq!(result.repaired["type"], "Action");
147        assert_eq!(result.repaired["name"], "test");
148    }
149}
fuzzy_parser/lib.rs

fuzzy_parser/
lib.rs
