sim_codec_python/
lower.rs

1//! Stable `python/*` forms and the three codec lanes.
2
3use sim_codec::{DecodeBudget, Input, Output, ReadCx};
4use sim_kernel::{
5    CodecId, Error, Expr, LocatedExpr, LocatedExprTree, Origin, Result, SourceId,
6    Span as KernelSpan, Symbol,
7};
8
9use crate::{Limits, Node, NodeKind, SyntaxTree, Token, TokenKind, parse_module_with_limits};
10
11const FALLBACK_HEAD: &str = "__sim_expr__(";
12
13/// Lower a complete concrete tree to the stable `python/module`,
14/// `python/statement`, and `python/token` vocabulary. Parser coverage and
15/// executable support are deliberately distinct: each token carries a final
16/// boolean saying whether the future Python runtime profile may execute it.
17pub fn lower_python(tree: &SyntaxTree) -> Expr {
18    call(
19        "module",
20        tree.root
21            .children
22            .iter()
23            .map(|node| lower_node(tree, node))
24            .collect(),
25    )
26}
27
28fn lower_node(tree: &SyntaxTree, node: &Node) -> Expr {
29    let head = match node.kind {
30        NodeKind::Module => "module",
31        NodeKind::Statement => "statement",
32        NodeKind::Suite => "suite",
33        NodeKind::Group => "group",
34        NodeKind::Expression => "expression",
35    };
36    let tokens = node
37        .tokens
38        .clone()
39        .map(|index| lower_token(tree, &tree.tokens[index]));
40    call(
41        head,
42        tokens
43            .chain(node.children.iter().map(|child| {
44                let child_head = match child.kind {
45                    NodeKind::Module => "module",
46                    NodeKind::Statement => "statement",
47                    NodeKind::Suite => "suite",
48                    NodeKind::Group => "group",
49                    NodeKind::Expression => "expression",
50                };
51                call(child_head, Vec::new())
52            }))
53            .collect(),
54    )
55}
56
57fn lower_token(tree: &SyntaxTree, token: &Token) -> Expr {
58    let text = tree.source()[token.span.start..token.span.end].to_owned();
59    call(
60        "token",
61        vec![
62            Expr::Symbol(Symbol::new(token_kind_name(&token.kind))),
63            Expr::String(text),
64            Expr::Bool(executable_token(&token.kind)),
65        ],
66    )
67}
68
69fn token_kind_name(kind: &TokenKind) -> &'static str {
70    match kind {
71        TokenKind::Name => "name",
72        TokenKind::Keyword => "keyword",
73        TokenKind::Number => "number",
74        TokenKind::String => "string",
75        TokenKind::FString => "f-string",
76        TokenKind::TemplateString => "template-string",
77        TokenKind::Operator => "operator",
78        TokenKind::Newline => "newline",
79        TokenKind::Indent => "indent",
80        TokenKind::Dedent => "dedent",
81        TokenKind::Trivia => "trivia",
82        TokenKind::End => "end",
83    }
84}
85
86fn executable_token(kind: &TokenKind) -> bool {
87    matches!(
88        kind,
89        TokenKind::Name | TokenKind::Number | TokenKind::String | TokenKind::Operator
90    )
91}
92
93/// Decode Python source with the shared codec budget.
94pub fn decode_python(cx: &mut ReadCx<'_>, source: &str, budget: &mut DecodeBudget) -> Result<Expr> {
95    budget.check_input_bytes(cx.codec, source.len())?;
96    if source.starts_with(FALLBACK_HEAD) {
97        return decode_fallback(cx.codec, source, budget);
98    }
99    let tree = parse_module_with_limits(source, parser_limits(budget))
100        .map_err(|error| codec_error(cx.codec, error.to_string()))?;
101    budget.check_tokens(cx.codec, tree.tokens.len())?;
102    Ok(lower_python(&tree))
103}
104
105/// Decode into a root-located expression.
106pub fn decode_python_located(
107    cx: &mut ReadCx<'_>,
108    source_id: impl Into<String>,
109    input: Input,
110) -> Result<LocatedExpr> {
111    let source = input_text(cx.codec, input)?;
112    let source_id = SourceId(source_id.into());
113    cx.cx.sources_mut().intern_text(source_id.clone(), &source);
114    let mut budget = DecodeBudget::new(cx.limits);
115    let expr = decode_python(cx, &source, &mut budget)?;
116    Ok(LocatedExpr {
117        expr,
118        origin: Some(origin(cx.codec, source_id, 0, source.len())),
119    })
120}
121
122/// Decode into a source-origin tree. The root owns the file, statements own
123/// their exact ranges, and token children retain the leaf spans.
124pub fn decode_python_tree(
125    cx: &mut ReadCx<'_>,
126    source_id: impl Into<String>,
127    input: Input,
128) -> Result<LocatedExprTree> {
129    let source = input_text(cx.codec, input)?;
130    let source_id = SourceId(source_id.into());
131    cx.cx.sources_mut().intern_text(source_id.clone(), &source);
132    let mut budget = DecodeBudget::new(cx.limits);
133    if source.starts_with(FALLBACK_HEAD) {
134        let expr = decode_python(cx, &source, &mut budget)?;
135        let mut tree = LocatedExprTree::from_expr_recursive(expr);
136        tree.origin = Some(origin(cx.codec, source_id, 0, source.len()));
137        return Ok(tree);
138    }
139    let parsed = parse_module_with_limits(&source, parser_limits(&budget))
140        .map_err(|error| codec_error(cx.codec, error.to_string()))?;
141    budget.check_tokens(cx.codec, parsed.tokens.len())?;
142    let expr = lower_python(&parsed);
143    let mut root = LocatedExprTree::from_expr_recursive(expr);
144    root.origin = Some(origin(cx.codec, source_id.clone(), 0, source.len()));
145    for (child, statement) in root.children.iter_mut().skip(1).zip(&parsed.root.children) {
146        if let Some((start, end)) = node_bytes(&parsed, statement) {
147            child.origin = Some(origin(cx.codec, source_id.clone(), start, end));
148            for (leaf, token_index) in child
149                .children
150                .iter_mut()
151                .skip(1)
152                .zip(statement.tokens.clone())
153            {
154                let span = parsed.tokens[token_index].span;
155                leaf.origin = Some(origin(cx.codec, source_id.clone(), span.start, span.end));
156            }
157        }
158    }
159    Ok(root)
160}
161
162/// Canonically encode lowered Python forms. Expressions outside the stable
163/// vocabulary use the established canonical tagged JSON projection inside a
164/// reserved Python call, keeping `codec/python` general-purpose.
165pub fn encode_python(expr: &Expr) -> Result<Output> {
166    if python_call(expr).is_some() {
167        let mut out = String::new();
168        encode_form(expr, &mut out)?;
169        let reparsed = parse_module_with_limits(&out, Limits::default())
170            .map_err(|error| codec_error(crate::PYTHON_CODEC_ID, error.to_string()))?;
171        if lower_python(&reparsed) != *expr {
172            return Err(codec_error(
173                crate::PYTHON_CODEC_ID,
174                "python form is not the canonical lowering of its source",
175            ));
176        }
177        return Ok(Output::Text(out));
178    }
179    let json = sim_codec_json::expr_to_json(expr);
180    Ok(Output::Text(format!("{FALLBACK_HEAD}{json})")))
181}
182
183fn encode_form(expr: &Expr, out: &mut String) -> Result<()> {
184    let Some((head, args)) = python_call(expr) else {
185        return Err(codec_error(crate::PYTHON_CODEC_ID, "malformed python form"));
186    };
187    match head {
188        "module" | "statement" | "suite" | "group" | "expression" => {
189            for arg in args {
190                encode_form(arg, out)?;
191            }
192            Ok(())
193        }
194        "token" if args.len() == 3 => match (&args[0], &args[1], &args[2]) {
195            (Expr::Symbol(_), Expr::String(text), Expr::Bool(_)) => {
196                out.push_str(text);
197                Ok(())
198            }
199            _ => Err(codec_error(
200                crate::PYTHON_CODEC_ID,
201                "python/token expects kind, text, executable",
202            )),
203        },
204        _ => Err(codec_error(
205            crate::PYTHON_CODEC_ID,
206            format!("unknown python form python/{head}"),
207        )),
208    }
209}
210
211fn decode_fallback(codec: CodecId, source: &str, budget: &mut DecodeBudget) -> Result<Expr> {
212    let Some(json) = source
213        .strip_prefix(FALLBACK_HEAD)
214        .and_then(|rest| rest.strip_suffix(')'))
215    else {
216        return Err(codec_error(codec, "malformed __sim_expr__ fallback"));
217    };
218    let value = serde_json::from_str(json)
219        .map_err(|error| codec_error(codec, format!("malformed tagged fallback: {error}")))?;
220    sim_codec_json::json_to_expr(codec, &value, budget, 0)
221}
222
223fn parser_limits(budget: &DecodeBudget) -> Limits {
224    let limits = budget.limits();
225    Limits {
226        max_bytes: limits.max_input_bytes,
227        max_tokens: limits.max_tokens,
228        max_nesting: limits.max_depth,
229        max_lines: limits.max_tokens,
230    }
231}
232
233fn node_bytes(tree: &SyntaxTree, node: &Node) -> Option<(usize, usize)> {
234    let tokens = &tree.tokens[node.tokens.clone()];
235    Some((tokens.first()?.span.start, tokens.last()?.span.end))
236}
237
238fn python_call(expr: &Expr) -> Option<(&str, &[Expr])> {
239    let Expr::Call { operator, args } = expr else {
240        return None;
241    };
242    let Expr::Symbol(symbol) = operator.as_ref() else {
243        return None;
244    };
245    (symbol.namespace.as_deref().map(AsRef::as_ref) == Some("python"))
246        .then_some((symbol.name.as_ref(), args))
247}
248
249fn call(name: &str, args: Vec<Expr>) -> Expr {
250    Expr::Call {
251        operator: Box::new(Expr::Symbol(Symbol::qualified("python", name))),
252        args,
253    }
254}
255
256fn input_text(codec: CodecId, input: Input) -> Result<String> {
257    match input {
258        Input::Text(text) => Ok(text),
259        Input::Bytes(bytes) => String::from_utf8(bytes).map_err(|error| {
260            codec_error(codec, format!("codec input is not valid UTF-8: {error}"))
261        }),
262    }
263}
264
265fn origin(codec: CodecId, source: SourceId, start: usize, end: usize) -> Origin {
266    Origin {
267        codec,
268        source,
269        span: KernelSpan { start, end },
270        trivia: Vec::new(),
271    }
272}
273
274fn codec_error(codec: CodecId, message: impl Into<String>) -> Error {
275    Error::CodecError {
276        codec,
277        message: message.into(),
278    }
279}
sim_codec_python/lower.rs

sim_codec_python/
lower.rs
