harn_parser/
ast_json.rs

1//! Stable JSON projections for parser-facing tooling.
2//!
3//! The core AST keeps its Rust-native shape for compiler consumers.
4//! This module owns the CLI/tooling projection so `harn parse --json`
5//! and `harn tokens --json` can evolve under explicit schema versions.
6
7use harn_lexer::{Span, Token, TokenKind};
8use serde::Serialize;
9use serde_json::{Map, Value};
10
11use crate::interpolation;
12use crate::operators::OperatorGrammar;
13use crate::SNode;
14
15pub const AST_JSON_SCHEMA_VERSION: u32 = 1;
16pub const TOKEN_JSON_SCHEMA_VERSION: u32 = 1;
17
18#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
19pub struct AstJsonProgram {
20    pub kind: &'static str,
21    #[serde(rename = "schemaVersion")]
22    pub schema_version: u32,
23    pub body: Vec<Value>,
24}
25
26#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
27pub struct TokenJson {
28    pub kind: &'static str,
29    pub lexeme: String,
30    pub start: usize,
31    pub end: usize,
32    pub line: usize,
33    pub column: usize,
34}
35
36pub fn program_to_json(nodes: &[SNode]) -> AstJsonProgram {
37    AstJsonProgram {
38        kind: "Program",
39        schema_version: AST_JSON_SCHEMA_VERSION,
40        body: nodes.iter().map(node_to_json).collect(),
41    }
42}
43
44pub fn node_to_json(node: &SNode) -> Value {
45    let raw = serde_json::to_value(node).expect("AST node should serialize to JSON");
46    normalize_value(raw)
47}
48
49pub fn tokens_to_json(source: &str, tokens: &[Token]) -> Vec<TokenJson> {
50    tokens
51        .iter()
52        .map(|token| token_to_json(source, token))
53        .collect()
54}
55
56pub fn token_to_json(source: &str, token: &Token) -> TokenJson {
57    TokenJson {
58        kind: token_kind_name(&token.kind),
59        lexeme: lexeme_for_span(source, token.span),
60        start: token.span.start,
61        end: token.span.end,
62        line: token.span.line,
63        column: token.span.column,
64    }
65}
66
67pub fn token_kind_name(kind: &TokenKind) -> &'static str {
68    match kind {
69        TokenKind::Pipeline => "Pipeline",
70        TokenKind::Extends => "Extends",
71        TokenKind::Override => "Override",
72        TokenKind::Let => "Let",
73        TokenKind::Const => "Const",
74        TokenKind::Var => "Var",
75        TokenKind::If => "If",
76        TokenKind::Else => "Else",
77        TokenKind::For => "For",
78        TokenKind::In => "In",
79        TokenKind::Match => "Match",
80        TokenKind::Retry => "Retry",
81        TokenKind::Parallel => "Parallel",
82        TokenKind::Return => "Return",
83        TokenKind::Import => "Import",
84        TokenKind::True => "True",
85        TokenKind::False => "False",
86        TokenKind::Nil => "Nil",
87        TokenKind::Try => "Try",
88        TokenKind::Catch => "Catch",
89        TokenKind::Throw => "Throw",
90        TokenKind::Finally => "Finally",
91        TokenKind::Fn => "Fn",
92        TokenKind::Spawn => "Spawn",
93        TokenKind::While => "While",
94        TokenKind::TypeKw => "TypeKw",
95        TokenKind::Enum => "Enum",
96        TokenKind::Struct => "Struct",
97        TokenKind::Interface => "Interface",
98        TokenKind::Pub => "Pub",
99        TokenKind::Guard => "Guard",
100        TokenKind::Defer => "Defer",
101        TokenKind::Yield => "Yield",
102        TokenKind::Break => "Break",
103        TokenKind::Continue => "Continue",
104        TokenKind::Impl => "Impl",
105        TokenKind::Throws => "Throws",
106        TokenKind::Identifier(_) => "Identifier",
107        TokenKind::StringLiteral(_) => "StringLiteral",
108        TokenKind::InterpolatedString(_) => "InterpolatedString",
109        TokenKind::RawStringLiteral(_) => "RawStringLiteral",
110        TokenKind::IntLiteral(_) => "IntLiteral",
111        TokenKind::FloatLiteral(_) => "FloatLiteral",
112        TokenKind::DurationLiteral(_) => "DurationLiteral",
113        TokenKind::Eq => "Eq",
114        TokenKind::Neq => "Neq",
115        TokenKind::And => "And",
116        TokenKind::Or => "Or",
117        TokenKind::Pipe => "Pipe",
118        TokenKind::NilCoal => "NilCoal",
119        TokenKind::Pow => "Pow",
120        TokenKind::DotDot => "DotDot",
121        TokenKind::DotDotEq => "DotDotEq",
122        TokenKind::QuestionDot => "QuestionDot",
123        TokenKind::Arrow => "Arrow",
124        TokenKind::Lte => "Lte",
125        TokenKind::Gte => "Gte",
126        TokenKind::PlusAssign => "PlusAssign",
127        TokenKind::MinusAssign => "MinusAssign",
128        TokenKind::StarAssign => "StarAssign",
129        TokenKind::SlashAssign => "SlashAssign",
130        TokenKind::PercentAssign => "PercentAssign",
131        TokenKind::Assign => "Assign",
132        TokenKind::Not => "Not",
133        TokenKind::Dot => "Dot",
134        TokenKind::Plus => "Plus",
135        TokenKind::Minus => "Minus",
136        TokenKind::Star => "Star",
137        TokenKind::Slash => "Slash",
138        TokenKind::Percent => "Percent",
139        TokenKind::Lt => "Lt",
140        TokenKind::Gt => "Gt",
141        TokenKind::Question => "Question",
142        TokenKind::Bar => "Bar",
143        TokenKind::Amp => "Amp",
144        TokenKind::LBrace => "LBrace",
145        TokenKind::RBrace => "RBrace",
146        TokenKind::LParen => "LParen",
147        TokenKind::RParen => "RParen",
148        TokenKind::LBracket => "LBracket",
149        TokenKind::RBracket => "RBracket",
150        TokenKind::Comma => "Comma",
151        TokenKind::Colon => "Colon",
152        TokenKind::Semicolon => "Semicolon",
153        TokenKind::At => "At",
154        TokenKind::LineComment { .. } => "LineComment",
155        TokenKind::BlockComment { .. } => "BlockComment",
156        TokenKind::Newline => "Newline",
157        TokenKind::Eof => "Eof",
158    }
159}
160
161fn lexeme_for_span(source: &str, span: Span) -> String {
162    source
163        .get(span.start..span.end)
164        .unwrap_or_default()
165        .to_string()
166}
167
168fn normalize_value(value: Value) -> Value {
169    match value {
170        Value::Array(values) => Value::Array(values.into_iter().map(normalize_value).collect()),
171        Value::Object(map) => normalize_object(map),
172        other => other,
173    }
174}
175
176fn normalize_object(mut map: Map<String, Value>) -> Value {
177    if is_span_map(&map) {
178        return normalize_span_map(map);
179    }
180
181    if is_spanned_map(&map) {
182        return normalize_spanned_map(map);
183    }
184
185    if map.len() == 1 {
186        let key = map.keys().next().cloned().expect("map has one key");
187        if is_variant_name(&key) {
188            let fields = map.remove(&key).expect("variant payload is present");
189            return tagged_value(key, normalize_value(fields));
190        }
191    }
192
193    for value in map.values_mut() {
194        let raw = std::mem::take(value);
195        *value = normalize_value(raw);
196    }
197    Value::Object(map)
198}
199
200fn normalize_spanned_map(mut map: Map<String, Value>) -> Value {
201    let node = map.remove("node").expect("spanned node is present");
202    let span = map.remove("span").expect("spanned span is present");
203    let mut normalized = normalize_enum_value(node);
204    if let Value::Object(object) = &mut normalized {
205        object.insert("span".to_string(), normalize_value(span));
206    }
207    normalized
208}
209
210fn normalize_enum_value(value: Value) -> Value {
211    match value {
212        Value::Object(mut map) if map.len() == 1 => {
213            let key = map.keys().next().cloned().expect("map has one key");
214            let fields = map.remove(&key).expect("variant payload is present");
215            tagged_value(key, normalize_value(fields))
216        }
217        Value::String(kind) if is_variant_name(&kind) => tagged_value(kind, Value::Null),
218        other => normalize_value(other),
219    }
220}
221
222fn tagged_value(kind: String, fields: Value) -> Value {
223    let mut object = Map::new();
224    object.insert("kind".to_string(), Value::String(kind));
225    object.insert("fields".to_string(), fields);
226    Value::Object(object)
227}
228
229fn normalize_span_map(mut map: Map<String, Value>) -> Value {
230    if let Some(end_line) = map.remove("end_line") {
231        map.insert("endLine".to_string(), end_line);
232    }
233    Value::Object(map)
234}
235
236fn is_spanned_map(map: &Map<String, Value>) -> bool {
237    map.len() == 2 && map.contains_key("node") && map.contains_key("span")
238}
239
240fn is_span_map(map: &Map<String, Value>) -> bool {
241    map.len() == 5
242        && map.contains_key("start")
243        && map.contains_key("end")
244        && map.contains_key("line")
245        && map.contains_key("column")
246        && map.contains_key("end_line")
247}
248
249fn is_variant_name(name: &str) -> bool {
250    name.chars()
251        .next()
252        .is_some_and(|ch| ch.is_ascii_uppercase())
253}
254
255/// A span-free projection of `program` with every interpolation hole replaced
256/// by its parsed tree, so two parses compare by structure alone. `grammar`
257/// is the operator grammar the holes are parsed with.
258pub fn structural_projection(program: &[SNode], grammar: OperatorGrammar) -> Value {
259    Value::Array(
260        program
261            .iter()
262            .map(|node| normalized(serde_json::to_value(node).unwrap_or(Value::Null), grammar))
263            .collect(),
264    )
265}
266
267fn normalized(value: Value, grammar: OperatorGrammar) -> Value {
268    match value {
269        Value::Object(map) => {
270            if let Some(Value::Array(segments)) = map.get("InterpolatedString") {
271                let holes = segments
272                    .iter()
273                    .map(|segment| normalized_segment(segment, grammar))
274                    .collect();
275                return Value::Array(holes);
276            }
277            // Positions differ between two parses of equivalent source. Any
278            // serialized `Span`, whatever field holds it, is dropped.
279            if is_span_map(&map) {
280                return Value::Null;
281            }
282            Value::Object(
283                map.into_iter()
284                    .map(|(key, value)| (key, normalized(value, grammar)))
285                    .collect(),
286            )
287        }
288        Value::Array(items) => Value::Array(
289            items
290                .into_iter()
291                .map(|item| normalized(item, grammar))
292                .collect(),
293        ),
294        other => other,
295    }
296}
297
298fn normalized_segment(segment: &Value, grammar: OperatorGrammar) -> Value {
299    let Some(Value::Array(parts)) = segment.get("Expression") else {
300        return segment.clone();
301    };
302    let Some(text) = parts.first().and_then(Value::as_str) else {
303        return segment.clone();
304    };
305    interpolation::parse_expression_with_grammar(grammar, None, text, 1, 1).map_or_else(
306        || Value::String(text.to_string()),
307        |expression| structural_projection(std::slice::from_ref(&expression), grammar),
308    )
309}
310
311#[cfg(test)]
312mod tests {
313    use harn_lexer::Lexer;
314
315    use super::*;
316    use crate::parse_source;
317
318    #[test]
319    fn program_json_projects_tagged_nodes_with_spans() {
320        let source = "pipeline main(task) {\n  return 1\n}\n";
321        let program = parse_source(source).expect("parse");
322
323        let json = program_to_json(&program);
324
325        assert_eq!(json.kind, "Program");
326        assert_eq!(json.schema_version, AST_JSON_SCHEMA_VERSION);
327        let pipeline = &json.body[0];
328        assert_eq!(pipeline["kind"], "Pipeline");
329        assert_eq!(pipeline["span"]["start"], 0);
330        assert_eq!(pipeline["span"]["line"], 1);
331        assert_eq!(pipeline["fields"]["name"], "main");
332        assert_eq!(pipeline["fields"]["body"][0]["kind"], "ReturnStmt");
333        assert_eq!(
334            pipeline["fields"]["body"][0]["fields"]["value"]["kind"],
335            "IntLiteral"
336        );
337    }
338
339    #[test]
340    fn token_json_preserves_byte_spans_and_lexemes() {
341        let source = "let x = \"é\"\n";
342        let mut lexer = Lexer::new(source);
343        let tokens = lexer.tokenize_with_comments().expect("tokenize");
344
345        let json = tokens_to_json(source, &tokens);
346        let string = json
347            .iter()
348            .find(|token| token.kind == "StringLiteral")
349            .expect("string literal token");
350
351        let quote = source.find('"').expect("opening quote");
352        assert_eq!(string.lexeme, "\"é\"");
353        assert_eq!(string.start, quote);
354        assert_eq!(string.end, quote + "\"é\"".len());
355        assert_eq!(string.line, 1);
356        assert_eq!(string.column, 9);
357    }
358}
harn_parser/ast_json.rs

harn_parser/
ast_json.rs
