json_partial 0.2.1

Json fixing parser for imperfect json given by LLMs
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

// use baml_types::BamlMap;
// use bstd::dedent;

use crate::jsonish::Value;

#[derive(Debug)]
pub enum JsonCollection {
    // Key, Value
    Object(Vec<String>, Vec<Value>),
    Array(Vec<Value>),
    QuotedString(String),
    TripleQuotedString(String),
    SingleQuotedString(String),
    // edge cases that need handling:
    // - triple backticks in a triple backtick string
    // - will the LLM terminate a triple backtick with a single backtick? probably not
    // - do we give the language specifier out? no
    // - what if the triple backtick block contains both a lang and path specifier? e.g. ```tsx path/to/file.tsx
    //   should we hand back the path?
    // - do we dedent the output?
    // - is it an acceptable heuristic to discard the first line of a triple backtick block?
    TripleBacktickString {
        lang: Option<String>,
        path: Option<String>,
        content: String,
    },
    BacktickString(String),
    // Handles numbers, booleans, null, and unquoted strings
    UnquotedString(String),
    // Starting with // or #
    TrailingComment(String),
    // Content between /* and */
    BlockComment(String),
}

impl JsonCollection {
    pub fn name(&self) -> &'static str {
        match self {
            JsonCollection::Object(_, _) => "Object",
            JsonCollection::Array(_) => "Array",
            JsonCollection::QuotedString(_) => "String",
            JsonCollection::SingleQuotedString(_) => "String",
            JsonCollection::TripleBacktickString { .. } => "TripleBacktickString",
            JsonCollection::BacktickString(_) => "String",
            JsonCollection::TripleQuotedString(_) => "TripleQuotedString",
            JsonCollection::UnquotedString(_) => "UnquotedString",
            JsonCollection::TrailingComment(_) => "Comment",
            JsonCollection::BlockComment(_) => "Comment",
        }
    }
}

impl From<JsonCollection> for Option<Value> {
    fn from(collection: JsonCollection) -> Option<Value> {
        Some(match collection {
            JsonCollection::TrailingComment(_) | JsonCollection::BlockComment(_) => return None,
            JsonCollection::Object(keys, values) => {
                // log::debug!("keys: {:?}", keys);
                let mut object = Vec::new();
                for (key, value) in keys.into_iter().zip(values.into_iter()) {
                    object.push((key, value));
                }
                Value::Object(object)
            }
            JsonCollection::Array(values) => Value::Array(values),
            JsonCollection::QuotedString(s) => Value::String(s),
            JsonCollection::TripleQuotedString(s) => Value::String(s),
            JsonCollection::SingleQuotedString(s) => Value::String(s),
            JsonCollection::TripleBacktickString { content, .. } => {
                let Some((fenced_codeblock_info, codeblock_contents)) = content.split_once("\n")
                else {
                    return Some(Value::String(content));
                };

                Value::String(codeblock_contents.into())
                // todo - dedent the codeblock look at engine/bstd/src/dedent.rs
                // Value::String(dedent(codeblock_contents).content)
            }
            JsonCollection::BacktickString(s) => Value::String(s),
            JsonCollection::UnquotedString(s) => {
                let s = s.trim();
                if s == "true" {
                    Value::Boolean(true)
                } else if s == "false" {
                    Value::Boolean(false)
                } else if s == "null" {
                    Value::Null
                } else if let Ok(n) = s.parse::<i64>() {
                    Value::Number(n.into())
                } else if let Ok(n) = s.parse::<u64>() {
                    Value::Number(n.into())
                } else if let Ok(n) = s.parse::<f64>() {
                    match serde_json::Number::from_f64(n) {
                        Some(n) => Value::Number(n),
                        None => Value::String(s.into()),
                    }
                } else {
                    Value::String(s.into())
                }
            }
        })
    }
}