flowrclib/compiler/
compile_wasm.rs

1#[cfg(feature = "debugger")]
2use std::collections::BTreeMap;
3use std::fs;
4use std::path::{Path, PathBuf};
5use std::process::Command;
6use std::process::Stdio;
7
8use log::{debug, info, warn};
9use simpath::{FileType, FoundType, Simpath};
10use tempdir::TempDir;
11#[cfg(feature = "debugger")]
12use url::Url;
13
14use flowcore::model::function_definition::FunctionDefinition;
15
16use crate::compiler::cargo_build;
17use crate::errors::*;
18
19/// Compile a function's implementation to wasm and modify implementation to point to the wasm file
20/// Checks the timestamps of the source and wasm files and only recompiles if wasm file is out of date
21#[allow(clippy::too_many_arguments)]
22pub fn compile_implementation(
23    out_dir: &Path,
24    cargo_target_dir: PathBuf, // where the binary will be built by cargo
25    wasm_destination: &Path,
26    implementation_source_path: &Path,
27    function: &mut FunctionDefinition,
28    native_only: bool,
29    optimize: bool,
30    #[cfg(feature = "debugger")]
31    source_urls: &mut BTreeMap<String, Url>,
32) -> Result<bool> {
33    let mut built = false;
34
35    let wasm_relative_path = wasm_destination.strip_prefix(out_dir)
36        .map_err(|_| "Could not strip_prefix from wasm location path")?;
37
38    let (missing, out_of_date) = out_of_date(implementation_source_path, wasm_destination)?;
39
40    if missing || out_of_date {
41        if native_only {
42            if missing {
43                warn!("Implementation '{}' is missing and you have \
44                selected to skip compiling to 'wasm', so flows relying on this implementation will not \
45                execute correctly.", wasm_destination.display());
46            }
47            if out_of_date {
48                info!(
49                    "Implementation '{}' is out of date with source at '{}'",
50                    wasm_destination.display(),
51                    implementation_source_path.display()
52                );
53            }
54        } else {
55            match function.build_type.as_str() {
56                "rust" => {
57                    cargo_build::run(implementation_source_path, cargo_target_dir,
58                                     wasm_destination, optimize)?;
59                },
60                _ => bail!(
61                    "Unknown build type '{}' for function at '{}'",
62                    implementation_source_path.display(),
63                    function.build_type),
64            }
65
66            if optimize {
67                optimize_wasm_file_size(wasm_destination)?;
68            }
69            built = true;
70        }
71    } else {
72        debug!(
73            "wasm at '{}' is up-to-date with source at '{}'",
74            wasm_destination.display(),
75            implementation_source_path.display()
76        );
77    }
78
79    let function_source_url = Url::from_file_path(implementation_source_path)
80        .map_err(|_| "Could not create Url from source path")?;
81    source_urls.insert(wasm_relative_path.to_string_lossy().to_string(),
82                       function_source_url);
83    function.set_implementation(
84        wasm_destination
85            .to_str()
86            .ok_or("Could not convert path to string")?,
87    );
88
89    Ok(built)
90}
91
92/*
93   Try and run a command that may or may not be installed in the system thus:
94   - create a temporary directory where the output file will be created
95   - run the command: $command $wasm_path $args $temp_file
96   - copy the resulting $temp_file to the desired output path (possibly across file systems)
97*/
98fn run_optional_command(wasm_path: &Path, command: &str, args: Vec<&str>) -> Result<()> {
99    if let Ok(FoundType::File(command_path)) =
100        Simpath::new("PATH").find_type(command, FileType::File)
101    {
102        let tmp_dir = TempDir::new_in(
103            wasm_path
104                .parent()
105                .ok_or("Could not get destination directory to create TempDir in")?,
106            "wasm-opt",
107        )?;
108        let temp_file_path = tmp_dir
109            .path()
110            .join(wasm_path.file_name().ok_or("Could not get wasm filename")?);
111        let mut command = Command::new(&command_path);
112        command.arg(wasm_path);
113        command.args(&args);
114        command.arg(&temp_file_path);
115        let child = command
116            .stdin(Stdio::inherit())
117            .stdout(Stdio::inherit())
118            .stderr(Stdio::inherit());
119
120        let output = child.output()?;
121
122        match output.status.code() {
123            Some(0) | None => {
124                fs::copy(&temp_file_path, wasm_path)?;
125                fs::remove_file(&temp_file_path)?;
126            },
127            Some(_) => bail!(format!(
128                "{} exited with non-zero status code",
129                command_path.to_string_lossy()
130            )),
131        }
132
133        fs::remove_dir_all(&tmp_dir)?;
134    }
135
136    Ok(()) // No error if the command was not present
137}
138
139/*
140   Optimize a wasm file's size using external tools that maybe installed on user's system
141*/
142fn optimize_wasm_file_size(wasm_path: &Path) -> Result<()> {
143    run_optional_command(wasm_path, "wasm-snip", vec!["-o"])?;
144    run_optional_command(wasm_path, "wasm-strip", vec!["-o"])?;
145    run_optional_command(wasm_path, "wasm-opt", vec!["-O4", "--dce", "-o"],
146    )
147}
148
149/*
150    Determine if one file that is derived from another source is missing and if not missing
151    if it is out of date (source is newer that derived)
152    Returns: (out_of_date, missing)
153    out_of_date
154        true - source file has been modified since the derived file was last modified or is missing
155        false - source has not been modified since derived file was last modified
156    missing
157        true - the derived file does no exist
158        false - the derived file does exist
159*/
160fn out_of_date(source: &Path, derived: &Path) -> Result<(bool, bool)> {
161    let source_last_modified = fs::metadata(source)
162        .chain_err(|| format!("Could not get metadata for file: '{}'", source.display()))?
163        .modified()?;
164
165    if derived.exists() {
166        let derived_last_modified = fs::metadata(derived)
167            .chain_err(|| format!("Could not get metadata for file: '{}'", derived.display()))?
168            .modified()?;
169        Ok(((source_last_modified > derived_last_modified), false))
170    } else {
171        Ok((true, true))
172    }
173}
174
175#[cfg(test)]
176mod test {
177    use std::collections::BTreeMap;
178    use std::env;
179    use std::fs::{File, remove_file, write};
180    use std::path::Path;
181    use std::time::Duration;
182
183    use tempdir::TempDir;
184    use url::Url;
185
186    use flowcore::model::datatype::STRING_TYPE;
187    use flowcore::model::function_definition::FunctionDefinition;
188    use flowcore::model::io::IO;
189    use flowcore::model::output_connection::{OutputConnection, Source};
190    use flowcore::model::route::Route;
191
192    use crate::compiler::compile;
193
194    use super::out_of_date;
195    use super::run_optional_command;
196
197    #[test]
198    fn test_run_optional_non_existent() {
199        let _ = run_optional_command(Path::new("/tmp"), "foo", vec!["bar"]);
200    }
201
202    #[test]
203    fn test_run_optional_exists() {
204        let temp_dir = TempDir::new("flow-tests").expect("Could not get temp dir");
205        let temp_file_path = temp_dir.path().join("from.test");
206        File::create(&temp_file_path).expect("Could not create test file");
207        let _ = run_optional_command(temp_file_path.as_path(), "cp", vec![]);
208        assert!(temp_file_path.exists());
209    }
210
211    #[test]
212    fn test_run_optional_exists_fail() {
213        let temp_dir = TempDir::new("flow-tests").expect("Could not get temp dir");
214        let temp_file_path = temp_dir.path().join("from.test");
215        File::create(&temp_file_path).expect("Could not create test file");
216        let _ = run_optional_command(
217            temp_file_path.as_path(),
218            "cp",
219            vec!["--no-such-flag"],
220        );
221        assert!(temp_file_path.exists());
222    }
223
224    #[test]
225    fn out_of_date_test() {
226        let output_dir = TempDir::new("flow")
227            .expect("Could not create TempDir during testing")
228            .into_path();
229
230        // make older file
231        let derived = output_dir.join("older");
232        write(&derived, "older").expect("Could not write to file during testing");
233
234        std::thread::sleep(Duration::from_secs(1));
235
236        // make second/newer file
237        let source = output_dir.join("newer");
238        write(&source, "newer").expect("Could not write to file during testing");
239
240        assert!(
241            out_of_date(&source, &derived)
242                .expect("Error in 'out__of_date'")
243                .0
244        );
245    }
246
247    #[test]
248    fn not_out_of_date_test() {
249        let output_dir = TempDir::new("flow")
250            .expect("Could not create TempDir during testing")
251            .into_path();
252
253        // make older file
254        let source = output_dir.join("older");
255        write(&source, "older").expect("Could not write to file {} during testing");
256
257        // make second/newer file
258        let derived = output_dir.join("newer");
259        write(&derived, "newer").expect("Could not write to file {} during testing");
260
261        assert!(
262            !out_of_date(&source, &derived)
263                .expect("Error in 'out_of_date'")
264                .0
265        );
266    }
267
268    #[test]
269    fn out_of_date_missing_test() {
270        let output_dir = TempDir::new("flow")
271            .expect("Could not create TempDir during testing")
272            .into_path();
273
274        // make older file
275        let source = output_dir.join("older");
276        write(&source, "older").expect("Could not write to file {} during testing");
277
278        // make second/newer file
279        let derived = output_dir.join("newer");
280        write(&derived, "newer").expect("Could not write to file {} during testing");
281
282        remove_file(&derived).unwrap_or_else(|_| panic!("Error in 'remove_file' during testing"));
283
284        assert!(
285            out_of_date(&source, &derived)
286                .expect("Error in 'out__of_date'")
287                .1
288        );
289    }
290
291    fn test_function() -> FunctionDefinition {
292        FunctionDefinition::new(
293            "Stdout".into(),
294            false,
295            "test.rs".to_string(),
296            "print".into(),
297            vec![IO::new(vec!(STRING_TYPE.into()), Route::default())],
298            vec![IO::new(vec!(STRING_TYPE.into()), Route::default())],
299            Url::parse(&format!(
300                "file://{}/{}",
301                env!("CARGO_MANIFEST_DIR"),
302                "tests/test-functions/test/test"
303            ))
304            .expect("Could not create source Url"),
305            Route::from("/flow0/stdout"),
306            Some(Url::parse("lib::/tests/test-functions/test/test")
307                .expect("Could not parse Url")),
308            None,
309            vec![OutputConnection::new(
310                Source::default(),
311                1,
312                0,
313                0,
314                String::default(),
315                #[cfg(feature = "debugger")]
316                String::default(),
317            )],
318            0,
319            0,
320        )
321    }
322
323    #[test]
324    fn test_compile_implementation_skip_missing() {
325        let mut function = test_function();
326
327        let wasm_output_dir = TempDir::new("flow")
328            .expect("Could not create TempDir during testing")
329            .into_path();
330        let expected_output_wasm = wasm_output_dir.join("test.wasm");
331        let _ = remove_file(&expected_output_wasm);
332
333        let (implementation_source_path, wasm_destination) = compile::get_paths(&wasm_output_dir, &function)
334            .expect("Could not get paths for compiling");
335        assert_eq!(wasm_destination, expected_output_wasm);
336
337        let mut cargo_target_dir = implementation_source_path.parent()
338            .ok_or("Could not get directory where Cargo.toml resides")
339            .expect("Could not get source directory").to_path_buf();
340        cargo_target_dir.push("target");
341
342        let mut source_urls = BTreeMap::<String, Url>::new();
343
344        let built = super::compile_implementation(
345            wasm_output_dir.as_path(),
346            cargo_target_dir,
347            &wasm_output_dir,
348            &implementation_source_path,
349            &mut function,
350            true,
351            false,
352            #[cfg(feature = "debugger")]
353            &mut source_urls
354        )
355        .expect("compile_implementation() failed");
356
357        assert!(!built);
358    }
359
360    #[test]
361    fn test_compile_implementation_not_needed() {
362        let mut function = test_function();
363
364        let wasm_output_dir = TempDir::new("flow")
365            .expect("Could not create TempDir during testing")
366            .into_path();
367        let expected_output_wasm = wasm_output_dir.join("test.wasm");
368
369        let _ = remove_file(&expected_output_wasm);
370        write(&expected_output_wasm, b"file touched during testing")
371            .expect("Could not write to file during testing");
372
373        let (implementation_source_path, wasm_destination) = compile::get_paths(&wasm_output_dir, &function)
374            .expect("Could not get paths for compiling");
375        assert_eq!(wasm_destination, expected_output_wasm);
376
377        let mut cargo_target_dir = implementation_source_path.parent()
378            .ok_or("Could not get directory where Cargo.toml resides")
379            .expect("Could not get source directory").to_path_buf();
380        cargo_target_dir.push("target");
381
382        let mut source_urls = BTreeMap::<String, Url>::new();
383
384        let built = super::compile_implementation(
385            wasm_output_dir.as_path(),
386            cargo_target_dir,
387            &wasm_output_dir,
388            &implementation_source_path,
389            &mut function,
390            false,
391            false,
392            #[cfg(feature = "debugger")]
393            &mut source_urls
394        )
395        .expect("compile_implementation() failed");
396
397        assert!(!built); // destination newer than source so should not have been built
398    }
399
400    #[test]
401    fn test_compile_implementation_skip() {
402        let mut function = test_function();
403
404        let wasm_output_dir = TempDir::new("flow")
405            .expect("Could not create TempDir during testing")
406            .into_path();
407        let expected_output_wasm = wasm_output_dir.join("test.wasm");
408
409        let (implementation_source_path, wasm_destination) = compile::get_paths(&wasm_output_dir, &function)
410            .expect("Could not get paths for compiling");
411        assert_eq!(expected_output_wasm, wasm_destination);
412        let mut cargo_target_dir = implementation_source_path.parent()
413            .ok_or("Could not get directory where Cargo.toml resides")
414            .expect("Could not get source directory").to_path_buf();
415        cargo_target_dir.push("target");
416
417        let mut source_urls = BTreeMap::<String, Url>::new();
418
419        let built = super::compile_implementation(
420            wasm_output_dir.as_path(),
421            cargo_target_dir,
422            &wasm_output_dir,
423            &implementation_source_path,
424            &mut function,
425            true,
426            false,
427            #[cfg(feature = "debugger")]
428            &mut source_urls
429        ).expect("compile_implementation() failed");
430
431        assert!(!built);
432    }
433
434    #[test]
435    fn test_compile_implementation_invalid_paths() {
436        let mut function = test_function();
437        function.set_source("does_not_exist");
438
439        let wasm_output_dir = TempDir::new("flow")
440            .expect("Could not create TempDir during testing")
441            .into_path();
442
443        let (implementation_source_path, _wasm_destination) = compile::get_paths(&wasm_output_dir, &function)
444            .expect("Could not get paths for compiling");
445        let mut cargo_target_dir = implementation_source_path.parent()
446            .ok_or("Could not get directory where Cargo.toml resides")
447            .expect("Could not get source directory").to_path_buf();
448        cargo_target_dir.push("target");
449
450        let mut source_urls = BTreeMap::<String, Url>::new();
451
452        assert!(super::compile_implementation(
453            wasm_output_dir.as_path(),
454            cargo_target_dir,
455            &wasm_output_dir,
456            &implementation_source_path,
457            &mut function,
458            true,
459            false,
460            #[cfg(feature = "debugger")]
461            &mut source_urls
462        )
463            .is_err());
464    }
465
466    #[test]
467    fn test_compile_implementation() {
468        let mut function = test_function();
469        function.build_type = "rust".into();
470
471        let wasm_output_dir = TempDir::new("flow")
472            .expect("Could not create TempDir during testing")
473            .into_path();
474        let expected_output_wasm = wasm_output_dir.join("test.wasm");
475        let _ = remove_file(&expected_output_wasm);
476
477        let (implementation_source_path, wasm_destination) =
478            compile::get_paths(&wasm_output_dir, &function)
479            .expect("Could not get paths for compiling");
480        assert_eq!(wasm_destination, expected_output_wasm);
481
482        let mut cargo_target_dir = implementation_source_path.parent()
483            .ok_or("Could not get directory where Cargo.toml resides")
484            .expect("Could not get source directory").to_path_buf();
485        cargo_target_dir.push("target/wasm32-unknown-unknown/debug");
486
487        let mut source_urls = BTreeMap::<String, Url>::new();
488
489        let built = super::compile_implementation(
490            wasm_output_dir.as_path(),
491            cargo_target_dir,
492            &wasm_destination,
493            &implementation_source_path,
494            &mut function,
495            false,
496            false,
497            #[cfg(feature = "debugger")]
498            &mut source_urls
499        )
500            .expect("compile_implementation() failed");
501
502        assert!(built);
503    }
504}