command_stream/fy/
source.rs

1//! Lossless source migration: parse, formalize typed links, substitute rules.
2use super::{
3    engine::{Mode, RuleEngine},
4    formalizer::formalize_shell,
5    Translation,
6};
7use meta_language::link_network::{LinkId, LinkMetadata, LinkNetwork, LinkType};
8use meta_language::query::LinkQuery;
9use meta_language::translation_rules::{TranslationRule, TranslationRuleSet};
10use std::collections::HashMap;
11
12/// Source selection and generated module options shared by the CLI and API.
13#[derive(Clone, Debug)]
14pub struct SourceOptions {
15    pub from: String,
16    pub filename: String,
17    pub module_name: String,
18    pub shell: Option<String>,
19    pub name: String,
20    pub shebang: bool,
21    pub use_m: bool,
22    pub commonjs: bool,
23}
24impl Default for SourceOptions {
25    fn default() -> Self {
26        Self {
27            from: "auto".into(),
28            filename: String::new(),
29            module_name: "command-stream".into(),
30            shell: None,
31            name: "translated.sh".into(),
32            shebang: true,
33            use_m: true,
34            commonjs: false,
35        }
36    }
37}
38
39fn syntax(network: &mut LinkNetwork, language: &str, term: &str, children: &[LinkId]) -> LinkId {
40    network.insert_dynamic_link(
41        children,
42        LinkMetadata::new()
43            .with_link_type(LinkType::Syntax)
44            .with_language(language)
45            .with_term(term),
46    )
47}
48fn rule(language: &str, term: &str, text: &str, captures: &[(&str, usize)]) -> TranslationRule {
49    let mut rule = TranslationRule::new(
50        term,
51        LinkQuery::new()
52            .with_link_type(LinkType::Syntax)
53            .with_language(language)
54            .with_term(term),
55    )
56    .with_template("JavaScript", text);
57    for (name, index) in captures {
58        rule = rule.with_reference_capture(*name, *index);
59    }
60    rule
61}
62fn json(text: &str) -> String {
63    serde_json::to_string(text).expect("string serialization")
64}
65fn result(code: String, diagnostics: Vec<String>) -> Translation {
66    Translation {
67        body: code.clone(),
68        code,
69        preamble: String::new(),
70        diagnostics,
71    }
72}
73
74/// Translate a complete shell program without changing its interpreter semantics.
75pub fn translate_shell_program(source: &str, options: &SourceOptions) -> Translation {
76    let mut formal = formalize_shell(source);
77    let inferred = source
78        .lines()
79        .next()
80        .filter(|line| line.starts_with("#!"))
81        .and_then(|line| line.split_whitespace().last())
82        .and_then(|word| word.rsplit('/').next())
83        .filter(|word| ["sh", "bash", "dash", "zsh"].contains(word));
84    let shell = options.shell.as_deref().or(inferred).unwrap_or("sh");
85    let refs: Vec<_> = [source, shell, &options.name, &options.module_name]
86        .iter()
87        .map(|value| formal.network.insert_source_token("Shell", value))
88        .chain(std::iter::once(formal.root))
89        .collect();
90    let root = syntax(&mut formal.network, "Shell", "shell-program", &refs);
91    let header = if options.shebang {
92        "#!/usr/bin/env node\n"
93    } else {
94        ""
95    };
96    let load = if options.use_m {
97        "import {{ use }} from 'use-m';\nconst {{ command }} = await use({module:json});\n"
98    } else {
99        "import {{ command }} from {module:json};\n"
100    };
101    let template = format!("{header}{load}const source = {{source:json}};\nconst result = await command({{shell:json}}, '-c', source, {{name:json}}, ...process.argv.slice(2))\n  .with({{{{ mirror: true, capture: false, stdin: 'inherit' }}}}).run();\nprocess.exitCode = result.code;\n");
102    let rules = TranslationRuleSet::new("source-to-command-stream").with_rule(rule(
103        "Shell",
104        "shell-program",
105        &template,
106        &[("source", 0), ("shell", 1), ("name", 2), ("module", 3)],
107    ));
108    let modes = HashMap::from([(
109        "json".into(),
110        Mode {
111            language: None,
112            transform: Some(json),
113        },
114    )]);
115    let mut engine = RuleEngine::new(&formal.network, &rules, "JavaScript", HashMap::new(), modes);
116    let code = engine.render_root(root);
117    result(code, engine.report())
118}
119fn target(module: &str) -> Option<&str> {
120    match module {
121        "zx" | "zx/core" | "zx/globals" | "zx/cli" | "execa" | "shelljs" | "shelljs/plugin"
122        | "cross-spawn" | "bun" => Some(module),
123        _ => None,
124    }
125}
126
127/// Find import/export/dynamic-import/require arguments through a JavaScript CST.
128fn module_references(
129    source: &str,
130    commonjs: bool,
131    diagnostics: &mut Vec<String>,
132) -> Result<Vec<(usize, usize, String)>, String> {
133    let mut parser = tree_sitter::Parser::new();
134    parser
135        .set_language(&tree_sitter_javascript::LANGUAGE.into())
136        .map_err(|error| error.to_string())?;
137    let tree = parser
138        .parse(source, None)
139        .ok_or("JavaScript parsing cancelled")?;
140    if tree.root_node().has_error() {
141        return Err("Invalid JavaScript source".into());
142    }
143    let mut pending = vec![tree.root_node()];
144    let mut references = Vec::new();
145    let mut requires = Vec::new();
146    let mut bound_require = false;
147    while let Some(node) = pending.pop() {
148        let binding = match node.kind() {
149            "variable_declarator"
150            | "function_declaration"
151            | "function_expression"
152            | "class_declaration" => node.child_by_field_name("name"),
153            "catch_clause" => node.child_by_field_name("parameter"),
154            "formal_parameters" | "import_clause" => Some(node),
155            "arrow_function" => node.child_by_field_name("parameter"),
156            _ => None,
157        };
158        if let Some(binding) = binding {
159            let mut bindings = vec![binding];
160            while let Some(binding) = bindings.pop() {
161                if binding.kind() == "identifier" && &source[binding.byte_range()] == "require" {
162                    bound_require = true;
163                }
164                let mut cursor = binding.walk();
165                bindings.extend(binding.named_children(&mut cursor));
166            }
167        }
168        let child = match node.kind() {
169            "import_statement" | "export_statement" => node.child_by_field_name("source"),
170            "call_expression" => {
171                let function = node.child_by_field_name("function");
172                let name = function.map(|node| &source[node.byte_range()]);
173                if matches!(name, Some("import" | "require")) {
174                    node.child_by_field_name("arguments")
175                        .and_then(|args| args.named_child(0))
176                } else {
177                    None
178                }
179            }
180            "return_statement"
181                if !commonjs
182                    && node
183                        .parent()
184                        .is_some_and(|parent| parent.kind() == "program") =>
185            {
186                return Err("Top-level return requires CommonJS mode".into())
187            }
188            _ => None,
189        };
190        if let Some(child) = child.filter(|child| child.kind() == "string") {
191            let raw = &source[child.byte_range()];
192            let reference = (child.start_byte(), child.end_byte(), decode_string(raw)?);
193            if node.kind() == "call_expression"
194                && node
195                    .child_by_field_name("function")
196                    .is_some_and(|function| &source[function.byte_range()] == "require")
197            {
198                requires.push(reference);
199            } else {
200                references.push(reference);
201            }
202        }
203        let mut cursor = node.walk();
204        pending.extend(node.named_children(&mut cursor));
205    }
206    if bound_require && !requires.is_empty() {
207        diagnostics.push(
208            "locally bound require: require calls retained; import declarations still migrate"
209                .into(),
210        );
211    } else {
212        references.extend(requires);
213    }
214    references.sort_by_key(|reference| reference.0);
215    Ok(references)
216}
217
218// Decode the escape forms JavaScript permits in either quote style. Parsing
219// already validated the literal; do not confuse JSON's narrower grammar with JS.
220fn decode_string(raw: &str) -> Result<String, String> {
221    let mut chars = raw[1..raw.len() - 1].chars().peekable();
222    let mut result = String::new();
223    while let Some(ch) = chars.next() {
224        if ch != '\\' {
225            result.push(ch);
226            continue;
227        }
228        let ch = chars.next().ok_or("Invalid string escape")?;
229        match ch {
230            '\n' => {}
231            '\r' => {
232                if chars.peek() == Some(&'\n') {
233                    chars.next();
234                }
235            }
236            'n' => result.push('\n'),
237            'r' => result.push('\r'),
238            't' => result.push('\t'),
239            'b' => result.push('\u{8}'),
240            'f' => result.push('\u{c}'),
241            'v' => result.push('\u{b}'),
242            '0' => result.push('\0'),
243            'x' | 'u' => {
244                let digits = if ch == 'u' && chars.peek() == Some(&'{') {
245                    chars.next();
246                    chars.by_ref().take_while(|c| *c != '}').collect::<String>()
247                } else {
248                    chars.by_ref().take(if ch == 'x' { 2 } else { 4 }).collect()
249                };
250                let code = u32::from_str_radix(&digits, 16).map_err(|error| error.to_string())?;
251                // Isolated UTF-16 surrogates cannot name one of our ASCII
252                // competitor packages. Preserve them as non-matching text.
253                result.push(char::from_u32(code).unwrap_or('\u{fffd}'));
254            }
255            other => result.push(other),
256        }
257    }
258    Ok(result)
259}
260
261/// Migrate source to matching command-stream APIs, retaining all other bytes.
262pub fn translate_competitor_source(
263    source: &str,
264    options: &SourceOptions,
265) -> Result<Translation, String> {
266    let commonjs = options.commonjs || options.filename.to_ascii_lowercase().ends_with(".cjs");
267    let mut network = LinkNetwork::new();
268    let mut children = Vec::new();
269    let mut rules = vec![rule("JavaScript", "source-program", "{*.:raw}", &[])];
270    let mut diagnostics = Vec::new();
271    let mut end = 0;
272    for (start, stop, module) in module_references(source, commonjs, &mut diagnostics)? {
273        let Some(suffix) = target(&module) else {
274            if ["zx/", "execa/", "shelljs/", "cross-spawn/"]
275                .iter()
276                .any(|prefix| module.starts_with(prefix))
277            {
278                diagnostics.push(format!("unsupported competitor entry point: {module}"));
279            }
280            continue;
281        };
282        children.push(network.insert_source_token("JavaScript", &source[end..start]));
283        let original = network.insert_source_token("JavaScript", &source[start..stop]);
284        let term = format!("module-{module}");
285        children.push(syntax(&mut network, "JavaScript", &term, &[original]));
286        rules.push(rule(
287            "JavaScript",
288            &term,
289            &json(&format!("{}/{suffix}", options.module_name))
290                .replace('{', "{{")
291                .replace('}', "}}"),
292            &[],
293        ));
294        end = stop;
295    }
296    children.push(network.insert_source_token("JavaScript", &source[end..]));
297    let root = syntax(&mut network, "JavaScript", "source-program", &children);
298    let mut rule_set = TranslationRuleSet::new("competitors-to-command-stream");
299    for rule in rules {
300        rule_set.add_rule(rule);
301    }
302    let rules = rule_set;
303    let modes = HashMap::from([(
304        "raw".into(),
305        Mode {
306            language: Some("JavaScript".into()),
307            transform: None,
308        },
309    )]);
310    let mut engine = RuleEngine::new(&network, &rules, "JavaScript", HashMap::new(), modes);
311    engine.preserve_layout = true;
312    let mut code = engine.render_root(root);
313    diagnostics.extend(engine.report());
314    if commonjs {
315        code = commonjs_module(&code);
316    }
317    if !options.shebang && code.starts_with("#!") {
318        code = code
319            .split_once('\n')
320            .map_or("", |(_, body)| body)
321            .to_owned();
322    }
323    Ok(result(code, diagnostics))
324}
325fn commonjs_module(source: &str) -> String {
326    let body = if source.starts_with("#!") {
327        source.split_once('\n').map_or("", |(_, body)| body)
328    } else {
329        source
330    };
331    format!("#!/usr/bin/env node\nimport {{ createRequire as __fyRequire }} from 'node:module';\nimport {{ fileURLToPath as __fyPath }} from 'node:url';\nimport {{ dirname as __fyDir }} from 'node:path';\nconst __fyModule = {{ exports: {{}} }};\n(function (require, module, exports, __filename, __dirname) {{\n{body}\n}}).call(__fyModule.exports, __fyRequire(import.meta.url), __fyModule, __fyModule.exports, __fyPath(import.meta.url), __fyDir(__fyPath(import.meta.url)));\nexport default __fyModule.exports;\n")
332}
333
334/// Select a source front end and translate through its typed links and rules.
335pub fn translate_source(source: &str, options: &SourceOptions) -> Result<Translation, String> {
336    if source.contains('\0') {
337        return Err("Source must not contain NUL".into());
338    }
339    let from = if options.from == "auto" {
340        if [".js", ".mjs", ".cjs"]
341            .iter()
342            .any(|ext| options.filename.to_ascii_lowercase().ends_with(ext))
343        {
344            "javascript"
345        } else {
346            "shell"
347        }
348    } else {
349        &options.from
350    };
351    match from {
352        "shell" | "sh" | "bash" | "dash" | "zsh" => {
353            let mut options = options.clone();
354            if from != "shell" && options.shell.is_none() {
355                options.shell = Some(from.into());
356            }
357            Ok(translate_shell_program(source, &options))
358        }
359        "javascript" | "js" | "zx" | "execa" | "shelljs" | "cross-spawn" | "bun" => {
360            translate_competitor_source(source, options)
361        }
362        _ => Err(format!("Unsupported source language: {from}")),
363    }
364}
command_stream/fy/source.rs

command_stream/fy/
source.rs
