makepad_hub/
hubbuilder.rs

1use crate::process::*;
2use crate::hubmsg::*;
3use crate::hubrouter::*;
4use crate::hubclient::*;
5use crate::httpserver::*;
6use crate::wasmstrip::*;
7
8use serde::{Deserialize};
9use std::sync::{Arc, Mutex};
10use std::fs;
11use std::sync::{mpsc};
12use std::sync::mpsc::RecvTimeoutError;
13use toml::Value;
14use std::collections::HashMap;
15use std::net::{SocketAddr};
16
17pub struct HubBuilder {
18    pub route_send: HubRouteSend,
19    pub http_server: Arc<Mutex<Option<HttpServer>>>,
20    pub workspaces: Arc<Mutex<HashMap<String, String>>>,
21    pub builder: String,
22    pub abs_cwd_path: String,
23    pub processes: Arc<Mutex<Vec<HubProcess>>>,
24}
25
26pub struct HubWorkspace {
27    pub name: String,
28    pub abs_path: String
29}
30
31pub struct HubProcess {
32    uid: HubUid,
33    process: Process,
34}
35
36pub enum HubWsError {
37    Error(String),
38    LocErrors(Vec<LocMessage>)
39}
40
41const INCLUDED_FILES: &[&'static str] = &[".json", ".toml", ".js", ".rs", ".txt", ".text", ".ron", ".html"];
42const EXCLUDED_FILES: &[&'static str] = &["key.ron", "makepad_state.ron"];
43const EXCLUDED_DIRS: &[&'static str] = &["target", ".git", ".github", "edit_repo"];
44
45impl HubBuilder {
46    
47    pub fn run_builder_direct<F>(builder: &str, hub_router: &mut HubRouter, event_handler: F) -> HubRouteSend
48    where F: Fn(&mut HubBuilder, FromHubMsg) -> Result<(), HubWsError> + Clone + Send + 'static {
49        let workspaces = Arc::new(Mutex::new(HashMap::<String, String>::new()));
50        let http_server = Arc::new(Mutex::new(None));
51        let abs_cwd_path = format!("{}", std::env::current_dir().unwrap().display());
52        let processes = Arc::new(Mutex::new(Vec::<HubProcess>::new()));
53        
54        let (tx_write, rx_write) = mpsc::channel::<FromHubMsg>();
55        
56        let route_send = hub_router.connect_direct(HubRouteType::Builder(builder.to_string()), tx_write);
57        // mode direct
58        let _thread = {
59            let builder = builder.to_string();
60            let route_send = route_send.clone();
61            let event_handler = event_handler.clone();
62            std::thread::spawn(move || {
63
64                route_send.send(ToHubMsg {
65                    to: HubMsgTo::All,
66                    msg: HubMsg::ConnectBuilder(builder.to_string())
67                });
68
69                while let Ok(htc) = rx_write.recv() {
70                    let is_blocking = htc.msg.is_blocking();
71                    let thread = {
72                        let event_handler = event_handler.clone();
73                        let mut hub_builder = HubBuilder {
74                            route_send: route_send.clone(),
75                            http_server: Arc::clone(&http_server),
76                            workspaces: Arc::clone(&workspaces),
77                            processes: Arc::clone(&processes),
78                            builder: builder.to_string(),
79                            abs_cwd_path: abs_cwd_path.clone(),
80                        };
81                        std::thread::spawn(move || {
82                            let is_build_uid = if let HubMsg::Build {uid, ..} = &htc.msg {Some(*uid)}else {None};
83                            
84                            let result = event_handler(&mut hub_builder, htc);
85                            
86                            if let Some(is_build_uid) = is_build_uid {
87                                if result.is_ok() {
88                                    hub_builder.route_send.send(ToHubMsg {
89                                        to: HubMsgTo::UI,
90                                        msg: HubMsg::BuildFailure {uid: is_build_uid}
91                                    });
92                                }
93                                else {
94                                    hub_builder.route_send.send(ToHubMsg {
95                                        to: HubMsgTo::UI,
96                                        msg: HubMsg::BuildSuccess {uid: is_build_uid}
97                                    });
98                                }
99                            }
100                        })
101                    };
102                    if is_blocking {
103                        let _ = thread.join();
104                    }
105                }
106            })
107        };
108        route_send.clone()
109    }
110    
111    pub fn run_builder_networked<F>(digest: Digest, in_address: Option<SocketAddr>, builder: &str, hub_log: HubLog, event_handler: F)
112    where F: Fn(&mut HubBuilder, FromHubMsg) -> Result<(), HubWsError> + Clone + Send + 'static {
113        
114        let workspaces = Arc::new(Mutex::new(HashMap::<String, String>::new()));
115        let http_server = Arc::new(Mutex::new(None));
116        let abs_cwd_path = format!("{}", std::env::current_dir().unwrap().display());
117        let processes = Arc::new(Mutex::new(Vec::<HubProcess>::new()));
118        
119        loop {
120            let address = if let Some(address) = in_address {
121                address
122            }
123            else {
124                hub_log.msg("Builder waiting for hub announcement..", &builder);
125
126                HubClient::wait_for_announce(digest.clone()).expect("cannot wait for announce")
127            };
128            
129            hub_log.msg("Builder connecting to {:?}", &address);
130
131            let mut hub_client = if let Ok(hub_client) = HubClient::connect_to_server(digest.clone(), address, hub_log.clone()) {
132                hub_client
133            }
134            else {
135                println!("Builder cannot connect to to {:?}, retrying", address);
136                std::thread::sleep(std::time::Duration::from_millis(500));
137                continue;
138            };
139            
140            println!("Builder connected to {:?}", hub_client.own_addr);
141            
142            let route_send = hub_client.get_route_send();
143            
144            route_send.send(ToHubMsg {
145                to: HubMsgTo::All,
146                msg: HubMsg::ConnectBuilder(builder.to_string())
147            });
148            
149            let rx_read = hub_client.rx_read.take().unwrap();
150            
151            while let Ok(htc) = rx_read.recv() {
152                match &htc.msg {
153                    HubMsg::ConnectionError(_e) => {
154                        println!("Got connection error, need to restart loop TODO kill all processes!");
155                        break;
156                    },
157                    _ => ()
158                }
159                let is_blocking = htc.msg.is_blocking();
160                let thread = {
161                    let event_handler = event_handler.clone();
162                    let mut hub_builder = HubBuilder {
163                        route_send: route_send.clone(),
164                        http_server: Arc::clone(&http_server),
165                        workspaces: Arc::clone(&workspaces),
166                        processes: Arc::clone(&processes),
167                        builder: builder.to_string(),
168                        abs_cwd_path: abs_cwd_path.clone(),
169                    };
170                    std::thread::spawn(move || {
171                        let is_build_uid = if let HubMsg::Build {uid, ..} = &htc.msg {Some(*uid)}else {None};
172                        
173                        let result = event_handler(&mut hub_builder, htc);
174                        
175                        if let Some(is_build_uid) = is_build_uid {
176                            if result.is_ok() {
177                                hub_builder.route_send.send(ToHubMsg {
178                                    to: HubMsgTo::UI,
179                                    msg: HubMsg::BuildFailure {uid: is_build_uid}
180                                });
181                            }
182                            else {
183                                hub_builder.route_send.send(ToHubMsg {
184                                    to: HubMsgTo::UI,
185                                    msg: HubMsg::BuildSuccess {uid: is_build_uid}
186                                });
187                            }
188                        }
189                    })
190                };
191                if is_blocking {
192                    let _ = thread.join();
193                }
194            }
195        }
196    }
197    
198    pub fn run_builder_commandline<F>(args: Vec<String>, event_handler: F)
199    where F: Fn(&mut HubBuilder, FromHubMsg) -> Result<(), HubWsError> + Clone + Send + 'static {
200        
201        fn print_help() {
202            println!("----- Builder commandline interface -----");
203            println!("Connect to a specific hub server:");
204            println!("cargo run -p builder -- connect <ip>:<port> <key.ron> <workspace>");
205            println!("example: cargo run -p builder -- connect 127.0.0.1:7243 key.ron windows");
206            println!("");
207            println!("Listen to hub server announce");
208            println!("cargo run -p builder -- listen <key.ron> <workspace>");
209            println!("example: cargo run -p builder -- listen key.ron windows");
210            println!("");
211            println!("Build a specific package");
212            println!("cargo run -p builder -- build <path> <package> <config>");
213            println!("example: cargo run -p builder -- build edit_repo makepad release");
214            println!("");
215            println!("List packages");
216            println!("cargo run -p builder -- list <path>");
217            println!("example: cargo run -p builder -- list edit_repo");
218            println!("");
219            println!("Build index.ron");
220            println!("cargo run -p builder -- index <path>");
221            println!("example: cargo run -p workspace -- index edit_repo");
222        }
223        
224        if args.len()<2 {
225            return print_help();
226        }
227        
228        let (message, path) = match args[1].as_ref() {
229            "listen" => {
230                if args.len() != 4 {
231                    return print_help();
232                }
233                let key_file = args[2].to_string();
234                let builder = args[3].to_string();
235                let utf8_data = std::fs::read_to_string(key_file).expect("Can't read key file");
236                let digest: Digest = ron::de::from_str(&utf8_data).expect("Can't load key file");
237                println!("Starting workspace listening to announce");
238                Self::run_builder_networked(digest, None, &builder, HubLog::None, event_handler);
239                return
240            },
241            "connect" => {
242                if args.len() != 5 {
243                    return print_help();
244                }
245                let addr = args[2].parse().expect("cant parse address");
246                let key_file = args[3].to_string();
247                let builder = args[4].to_string();
248                let utf8_data = std::fs::read_to_string(key_file).expect("Can't read key file");
249                let digest: Digest = ron::de::from_str(&utf8_data).expect("Can't load key file");
250                println!("Starting workspace connecting to ip");
251                Self::run_builder_networked(digest, Some(addr), &builder, HubLog::None, event_handler);
252                return
253            },
254            "list" => {
255                if args.len() != 3 {
256                    return print_help();
257                }
258                (HubMsg::ListPackagesRequest {
259                    uid: HubUid::zero()
260                }, args[2].clone())
261            },
262            "build" => {
263                if args.len() != 5 {
264                    return print_help();
265                }
266                (HubMsg::Build {
267                    uid: HubUid::zero(),
268                    workspace: "main".to_string(),
269                    package: args[3].clone(),
270                    config: args[4].clone()
271                }, args[2].clone())
272            },
273            "index" => {
274                if args.len() != 3 {
275                    return print_help();
276                }
277                (HubMsg::BuilderFileTreeRequest {
278                    uid: HubUid::zero(),
279                    create_digest: false,
280                }, args[2].clone())
281            },
282            _ => {
283                return print_help();
284            }
285        };
286        
287        let workspaces = Arc::new(Mutex::new(HashMap::<String, String>::new()));
288        let http_server = Arc::new(Mutex::new(None));
289        let processes = Arc::new(Mutex::new(Vec::<HubProcess>::new()));
290        let abs_cwd_path = format!("{}", std::env::current_dir().unwrap().display());
291        
292        if let Ok(mut workspaces) = workspaces.lock() {
293            workspaces.insert(
294                "main".to_string(),
295                rel_to_abs_path(&abs_cwd_path, &path)
296            );
297        };
298        
299        let (tx_write, rx_write) = mpsc::channel::<(HubAddr, ToHubMsg)>();
300        
301        let mut hub_builder = HubBuilder {
302            route_send: HubRouteSend::Direct {
303                uid_alloc: Arc::new(Mutex::new(0)),
304                tx_pump: tx_write.clone(),
305                own_addr: HubAddr::None
306            },
307            http_server: Arc::clone(&http_server),
308            builder: "".to_string(),
309            processes: Arc::clone(&processes),
310            workspaces: Arc::clone(&workspaces),
311            abs_cwd_path: abs_cwd_path.clone()
312        };
313        
314        // lets check our command. its either build or list
315        let thread = std::thread::spawn(move || {
316            while let Ok((_addr, htc)) = rx_write.recv() {
317                match htc.msg {
318                    HubMsg::BuilderFileTreeResponse {tree, ..} => {
319                        //write index.ron
320                        if let BuilderFileTreeNode::Folder {folder, ..} = tree {
321                            let ron = ron::ser::to_string_pretty(&folder[0], ron::ser::PrettyConfig::default()).expect("cannot serialize settings");
322                            fs::write("index.ron", ron).expect("cannot write index.ron");
323                            println!("Written index.ron")
324                        }
325                        return
326                    },
327                    HubMsg::ListPackagesResponse {packages, ..} => {
328                        println!("{:?}", packages);
329                    },
330                    HubMsg::LogItem {item, ..} => {
331                        println!("{:?}", item)
332                    },
333                    HubMsg::CargoArtifact {package_id, ..} => {
334                        println!("{}", package_id)
335                    },
336                    HubMsg::CargoEnd {build_result, ..} => {
337                        println!("CargoEnd {:?}", build_result);
338                    }
339                    HubMsg::BuildSuccess {..} => {
340                        println!("Success!");
341                        return
342                    },
343                    HubMsg::BuildFailure {..} => {
344                        println!("Failure!");
345                        return
346                    },
347                    _ => ()
348                }
349            }
350        });
351        
352        let result = event_handler(&mut hub_builder, FromHubMsg {
353            from: HubAddr::None,
354            msg: message
355        });
356        if result.is_ok() {
357            let _ = tx_write.send((HubAddr::None, ToHubMsg {
358                to: HubMsgTo::All,
359                msg: HubMsg::BuildSuccess {uid: HubUid {id: 0, addr: HubAddr::None}}
360            }));
361        }
362        else {
363            let _ = tx_write.send((HubAddr::None, ToHubMsg {
364                to: HubMsgTo::All,
365                msg: HubMsg::BuildFailure {uid: HubUid {id: 0, addr: HubAddr::None}}
366            }));
367        }
368        let _ = thread.join();
369    }
370    
371    pub fn set_config(&mut self, _uid: HubUid, config: HubBuilderConfig) -> Result<(), HubWsError> {
372        // if we have a http server. just shut it down
373        if let Ok(mut workspaces) = self.workspaces.lock() {
374            *workspaces = config.workspaces;
375            for (_, rel_path) in workspaces.iter_mut() {
376                *rel_path = rel_to_abs_path(&self.abs_cwd_path, &rel_path)
377            }
378        };
379        
380        let workspaces = Arc::clone(&self.workspaces);
381        
382        if let Ok(mut http_server) = self.http_server.lock() {
383            if let Some(http_server) = &mut *http_server {
384                http_server.terminate();
385            }
386            
387            *http_server = HttpServer::start_http_server(&config.http_server, workspaces);
388        }
389        
390        
391        Ok(())
392    }
393    
394    pub fn default(&mut self, htc: FromHubMsg) -> Result<(), HubWsError> {
395        let ws = self;
396        match htc.msg {
397            HubMsg::BuilderConfig {uid, config} => {
398                ws.set_config(uid, config)
399            },
400            HubMsg::BuilderFileTreeRequest {uid, create_digest} => {
401                let tree = ws.workspace_file_tree(
402                    create_digest,
403                    INCLUDED_FILES,
404                    EXCLUDED_FILES,
405                    EXCLUDED_DIRS
406                );
407                ws.route_send.send(ToHubMsg {
408                    to: HubMsgTo::Client(htc.from),
409                    msg: HubMsg::BuilderFileTreeResponse {
410                        uid: uid,
411                        tree: tree
412                    }
413                });
414                Ok(())
415            },
416            HubMsg::FileReadRequest {uid, path} => {
417                ws.file_read(htc.from, uid, &path);
418                Ok(())
419            },
420            HubMsg::FileWriteRequest {uid, path, data} => {
421                ws.file_write(htc.from, uid, &path, data);
422                Ok(())
423            },
424            HubMsg::BuildKill {uid} => {
425                ws.process_kill(uid);
426                Ok(())
427            },
428            HubMsg::ProgramKill {uid} => {
429                ws.process_kill(uid);
430                Ok(())
431            },
432            HubMsg::ProgramRun {uid, path, args} => {
433                let v: Vec<&str> = args.iter().map( | v | v.as_ref()).collect();
434                ws.program_run(uid, &path, &v) ?;
435                Ok(())
436            },
437            _ => Ok(())
438        }
439    }
440    
441    pub fn process_kill(&mut self, uid: HubUid) {
442        if let Ok(mut procs) = self.processes.lock() {
443            for proc in procs.iter_mut() {
444                if proc.uid == uid {
445                    proc.process.kill();
446                }
447            }
448        };
449    }
450    
451    pub fn workspace_split_from_path(&mut self, uid: HubUid, path: &str) -> Result<(String, String, String), HubWsError> {
452        if let Some(workspace_pos) = path.find("/") {
453            let (workspace, rest) = path.split_at(workspace_pos);
454            let (_, rest) = rest.split_at(1);
455            let abs_dir = self.get_workspace_abs(uid, workspace) ?;
456            return Ok((abs_dir.to_string(), workspace.to_string(), rest.to_string()));
457        }
458        Err(
459            self.error(uid, format!("Builder {} path {} incorrent", self.builder, path))
460        )
461    }
462    
463    pub fn get_workspace_abs(&mut self, uid: HubUid, workspace: &str) -> Result<String, HubWsError> {
464        if let Ok(workspaces) = self.workspaces.lock() {
465            if let Some(abs_dir) = workspaces.get(workspace) {
466                return Ok(abs_dir.to_string())
467            }
468        }
469        Err(
470            self.error(uid, format!("Builder {} workspace {} not found", self.builder, workspace))
471        )
472    }
473    
474    pub fn program_run(&mut self, uid: HubUid, path: &str, args: &[&str]) -> Result<(), HubWsError> {
475
476        let (abs_dir, workspace, sub_path) = self.workspace_split_from_path(uid, path) ?;
477
478        let process = Process::start(&sub_path, args, &abs_dir, &[("RUST_BACKTRACE", "full")]);
479        if let Err(e) = process {
480            return Err(
481                self.error(uid, format!("Builder {} program run {} {} not found {:?}", self.builder, abs_dir, sub_path, e))
482            );
483        }
484        let mut process = process.unwrap();
485
486        let route_mode = self.route_send.clone();
487        
488        let rx_line = process.rx_line.take().unwrap();
489        
490        if let Ok(mut processes) = self.processes.lock() {
491            processes.push(HubProcess {
492                uid: uid,
493                process: process,
494            });
495        };
496        
497        let builder = self.builder.clone();
498        route_mode.send(ToHubMsg {
499            to: HubMsgTo::UI,
500            msg: HubMsg::ProgramBegin {uid: uid}
501        });
502        
503        fn starts_with_digit(val: &str) -> bool {
504            if val.len() <= 1 {
505                return false;
506            }
507            let c1 = val.chars().next().unwrap();
508            c1 >= '0' && c1 <= '9'
509        }
510        
511        let mut stderr: Vec<String> = Vec::new();
512        
513        fn try_parse_stderr(uid: HubUid, builder: &str, workspace: &str, stderr: &Vec<String>, route_send: &HubRouteSend) {
514            
515            let mut tracing_panic = false;
516            let mut panic_stack = Vec::new();
517            for line in stderr {
518                let mut trimmed = line.trim_start().to_string();
519                trimmed.retain( | c | c != '\0');
520                if tracing_panic == false && (trimmed.starts_with("thread '") || trimmed.starts_with("0:")) { // this is how we recognise a stacktrace start..Very sturdy.
521                    tracing_panic = true;
522                    panic_stack.truncate(0);
523                }
524                if tracing_panic {
525                    panic_stack.push(trimmed);
526                }
527                else {
528                    route_send.send(ToHubMsg {
529                        to: HubMsgTo::UI,
530                        msg: HubMsg::LogItem {
531                            uid: uid,
532                            item: HubLogItem::Error(trimmed.clone())
533                        }
534                    });
535                }
536            }
537            
538            let mut path = None;
539            let mut row = 0;
540            let mut rendered = Vec::new();
541            if panic_stack.len()>0 {rendered.push(panic_stack[0].clone())};
542            let mut last_fn_name = String::new();
543            for panic_line in &panic_stack {
544                if panic_line.starts_with("at ")
545                    && !panic_line.starts_with("at /")
546                    && !panic_line.starts_with("at src/libstd/")
547                    && !panic_line.starts_with("at src/libcore/")
548                    && !panic_line.starts_with("at src/libpanic_unwind/") {
549                    if let Some(end) = panic_line.find(":") {
550                        let proc_path = format!("{}/{}/{}", builder, workspace, panic_line.get(3..end).unwrap().to_string());
551                        let proc_row = panic_line.get((end + 1)..(panic_line.len() - 1)).unwrap().parse::<usize>().unwrap();
552                        
553                        rendered.push(format!("{}:{} - {}", proc_path, proc_row, last_fn_name));
554                        if path.is_none() {
555                            path = Some(proc_path);
556                            row = proc_row
557                        }
558                    }
559                }
560                else if starts_with_digit(panic_line) {
561                    if let Some(pos) = panic_line.find(" 0x") {
562                        last_fn_name = panic_line.get((pos + 15)..).unwrap().to_string();
563                    }
564                }
565            }
566            rendered.push("\n".to_string());
567            
568            if panic_stack.len()<3 {
569                return;
570            }
571            
572            route_send.send(ToHubMsg {
573                to: HubMsgTo::UI,
574                msg: HubMsg::LogItem {
575                    uid: uid,
576                    item: HubLogItem::LocPanic(LocMessage {
577                        path: if let Some(path) = path {path}else {"".to_string()},
578                        row: row,
579                        col: 1,
580                        range: None,
581                        body: rendered[0].clone(),
582                        rendered: Some(rendered.join("")),
583                        explanation: Some(panic_stack[1..].join("")),
584                    })
585                }
586            });
587        }
588        
589        loop {
590            let result = rx_line.recv_timeout(std::time::Duration::from_millis(100));
591            match result {
592                Ok(line) => {
593                    if let Some((is_stderr, line)) = line {
594                        if is_stderr { // start collecting stderr
595                            stderr.push(line);
596                        }
597                        else {
598                            if stderr.len() > 0 {
599                                try_parse_stderr(uid, &builder, &workspace, &stderr, &route_mode);
600                                stderr.truncate(0);
601                            }
602
603                            route_mode.send(ToHubMsg {
604                                to: HubMsgTo::UI,
605                                msg: HubMsg::LogItem {
606                                    uid: uid,
607                                    item: HubLogItem::Message(line.clone())
608                                }
609                            });
610                        }
611                    }
612                },
613                Err(err) => {
614                    if stderr.len() > 0 {
615                        try_parse_stderr(uid, &builder, &workspace, &stderr, &route_mode);
616                        stderr.truncate(0);
617                    }
618                    if let RecvTimeoutError::Disconnected = err {
619                        break
620                    }
621                }
622            }
623        }
624        
625        route_mode.send(ToHubMsg {
626            to: HubMsgTo::UI,
627            msg: HubMsg::ProgramEnd {
628                uid: uid,
629            }
630        });
631        
632        if let Ok(mut processes) = self.processes.lock() {
633            if let Some(index) = processes.iter().position( | p | p.uid == uid) {
634                processes.remove(index);
635            }
636        };
637        
638        Ok(())
639    }
640    
641    
642    pub fn cannot_find_build(&mut self, uid: HubUid, package: &str, target: &str) -> Result<(), HubWsError> {
643        Err(
644            self.error(uid, format!("Builder {} Cannot find package {} and target {}", self.builder, package, target))
645        )
646    }
647    
648    pub fn cargo(&mut self, uid: HubUid, workspace: &str, args: &[&str], env: &[(&str, &str)]) -> Result<BuildResult, HubWsError> {
649        
650        if let Ok(mut http_server) = self.http_server.lock() {
651            if let Some(http_server) = &mut *http_server {
652                http_server.send_build_start();
653            }
654        };
655        
656        let abs_root_path = self.get_workspace_abs(uid, workspace) ?;
657
658        let mut extargs = args.to_vec();
659        extargs.push("--message-format=json");
660        let mut process = Process::start("cargo", &extargs, &abs_root_path, env).expect("Cannot start process");
661
662        let route_send = self.route_send.clone();
663        
664        let rx_line = process.rx_line.take().unwrap();
665        
666        if let Ok(mut processes) = self.processes.lock() {
667            processes.push(HubProcess {
668                uid: uid,
669                process: process,
670            });
671        };
672        
673        route_send.send(ToHubMsg {
674            to: HubMsgTo::UI,
675            msg: HubMsg::CargoBegin {uid: uid}
676        });
677        
678        let builder = self.builder.clone();
679        
680        let mut errors = Vec::new();
681        let mut build_result = BuildResult::NoOutput;
682        while let Ok(line) = rx_line.recv() {
683            if let Some((is_stderr, line)) = line {
684                if is_stderr && line != "\n"
685                    && !line.contains("Finished")
686                    && !line.contains("Blocking")
687                    && !line.contains("Compiling")
688                    && !line.contains("--verbose") {
689                    route_send.send(ToHubMsg {
690                        to: HubMsgTo::UI,
691                        msg: HubMsg::LogItem {
692                            uid: uid,
693                            item: HubLogItem::Error(line.clone())
694                        }
695                    });
696                }
697
698                let mut parsed: serde_json::Result<RustcCompilerMessage> = serde_json::from_str(&line);
699                match &mut parsed {
700                    Err(_) => (), //self.hub_log.log(&format!("Json Parse Error {:?} {}", err, line)),
701                    Ok(parsed) => {
702                        if let Some(message) = &mut parsed.message { //.spans;
703                            let spans = &message.spans;
704                            for i in 0..spans.len() {
705                                let span = spans[i].clone();
706                                if !span.is_primary {
707                                    continue
708                                }
709                                
710                                let mut msg = message.message.clone();
711                                
712                                for child in &message.children {
713                                    msg.push_str(" - ");
714                                    msg.push_str(&child.message);
715                                }
716                                msg = msg.replace("\n", "");
717                                // lets try to pull path out of rendered, this fixes some rust bugs
718                                let mut path = span.file_name;
719                                let row = span.line_start as usize;
720                                let col = span.column_start as usize;
721                                if let Some(rendered) = &message.rendered {
722                                    let lines: Vec<&str> = rendered.split('\n').collect();
723                                    if lines.len() >= 1 {
724                                        if let Some(start) = lines[1].find("--> ") {
725                                            if let Some(end) = lines[1].find(":") {
726                                                path = lines[1].get((start + 4)..end).unwrap().to_string();
727                                                // TODO parse row/col from this line
728                                                
729                                            }
730                                        }
731                                    }
732                                }
733                                let loc_message = LocMessage {
734                                    path: format!("{}/{}/{}", builder, workspace, de_relativize_path(&path)).replace("\\", "/"),
735                                    row: row,
736                                    col: col,
737                                    range: Some((span.byte_start as usize, span.byte_end as usize)),
738                                    body: msg,
739                                    rendered: message.rendered.clone(),
740                                    explanation: if let Some(code) = &message.code {code.explanation.clone()}else {None},
741                                };
742                                let item = match message.level.as_ref() {
743                                    "error" => {
744                                        errors.push(loc_message.clone());
745                                        HubLogItem::LocError(loc_message)
746                                    },
747                                    _ => HubLogItem::LocWarning(loc_message),
748                                };
749                                
750                                route_send.send(ToHubMsg {
751                                    to: HubMsgTo::UI,
752                                    msg: HubMsg::LogItem {
753                                        uid: uid,
754                                        item: item
755                                    }
756                                });
757                            }
758                        }
759                        else {
760                            route_send.send(ToHubMsg {
761                                to: HubMsgTo::UI,
762                                msg: HubMsg::CargoArtifact {
763                                    uid: uid,
764                                    package_id: parsed.package_id.clone(),
765                                    fresh: if let Some(fresh) = parsed.fresh {fresh}else {false}
766                                }
767                            });
768                            if errors.len() == 0 {
769                                build_result = BuildResult::NoOutput;
770                                if let Some(executable) = &parsed.executable {
771                                    if !executable.ends_with(".rmeta") && abs_root_path.len() + 1 < executable.len() {
772                                        let last = executable.clone().split_off(abs_root_path.len() + 1);
773                                        
774                                        build_result = BuildResult::Executable {path: format!("{}/{}", workspace, last).replace("\\", "/")};
775                                    }
776                                }
777                                // detect wasm files being build and tell the http server
778                                if let Some(filenames) = &parsed.filenames {
779                                    for filename in filenames {
780                                        if filename.ends_with(".wasm") && abs_root_path.len() + 1 < filename.len() {
781                                            let last = filename.clone().split_off(abs_root_path.len() + 1);
782                                            let path = format!("{}/{}", workspace, last).replace("\\", "/");
783                                            if let Ok(mut http_server) = self.http_server.lock() {
784                                                if let Some(http_server) = &mut *http_server {
785                                                    http_server.send_file_change(&path);
786                                                }
787                                            };
788                                            build_result = BuildResult::Wasm {path: path};
789                                        }
790                                    }
791                                }
792                            }
793                            else {
794                                build_result = BuildResult::Error;
795                            }
796                        }
797                    }
798                }
799            }
800            else { // process terminated
801                break;
802            }
803        }
804        
805        // process ends as well
806        route_send.send(ToHubMsg {
807            to: HubMsgTo::UI,
808            msg: HubMsg::CargoEnd {
809                uid: uid,
810                build_result: build_result.clone()
811            }
812        });
813        
814        // remove process from process list
815        if let Ok(mut processes) = self.processes.lock() {
816            if let Some(index) = processes.iter().position( | p | p.uid == uid) {
817                processes[index].process.wait();
818                processes.remove(index);
819            }
820        };
821        if let BuildResult::Error = build_result {
822            return Err(HubWsError::LocErrors(errors))
823        }
824        return Ok(build_result);
825    }
826    
827    pub fn packages_response(&mut self, from: HubAddr, uid: HubUid, packages: Vec<HubPackage>) {
828        
829        self.route_send.send(ToHubMsg {
830            to: HubMsgTo::Client(from),
831            msg: HubMsg::ListPackagesResponse {
832                uid: uid,
833                packages: packages
834            }
835        });
836    }
837    
838    pub fn error(&mut self, uid: HubUid, msg: String) -> HubWsError {
839        self.route_send.send(ToHubMsg {
840            to: HubMsgTo::UI,
841            msg: HubMsg::LogItem {uid: uid, item: HubLogItem::Error(msg.clone())}
842        });
843        
844        return HubWsError::Error(msg)
845    }
846    
847    pub fn message(&mut self, uid: HubUid, msg: String) {
848        self.route_send.send(ToHubMsg {
849            to: HubMsgTo::UI,
850            msg: HubMsg::LogItem {uid: uid, item: HubLogItem::Message(msg.clone())}
851        });
852    }
853    
854    pub fn wasm_strip_debug(&mut self, uid: HubUid, path: &str) -> Result<BuildResult, HubWsError> {
855        
856        let (abs_root_path, _project, sub_path) = self.workspace_split_from_path(uid, path) ?;
857        
858        let filepath = format!("{}/{}", abs_root_path, sub_path);
859
860        if let Ok(data) = fs::read(&filepath) {
861            if let Ok(strip) = wasm_strip_debug(&data) {
862                let uncomp_len = strip.len();
863                let mut enc = snap::Encoder::new();
864                let comp_len = if let Ok(compressed) = enc.compress_vec(&strip) {compressed.len()}else {0};
865                
866                if let Err(_) = fs::write(&filepath, strip) {
867                    return Err(self.error(uid, format!("Cannot write stripped wasm {}", filepath)));
868                }
869                else {
870                    self.message(uid, format!("Wasm file stripped size: {} kb uncompressed {} kb with snap", uncomp_len>>10, comp_len>>10));
871                    return Ok(BuildResult::Wasm {path: path.to_string()})
872                }
873            }
874            else {
875                return Err(self.error(uid, format!("Cannot parse wasm {}", filepath)));
876            }
877        }
878        Err(self.error(uid, format!("Cannot read wasm {}", filepath)))
879    }
880    
881    pub fn read_packages(&mut self, uid: HubUid) -> Vec<(String, String)> {
882        let mut packages = Vec::new();
883        let workspaces = Arc::clone(&self.workspaces);
884        if let Ok(workspaces) = workspaces.lock() {
885            for (workspace, abs_path) in workspaces.iter() {
886                let vis_path = format!("{}/{}/Cargo.toml", self.builder, workspace);
887                let root_cargo = match std::fs::read_to_string(format!("{}/Cargo.toml", abs_path)) {
888                    Err(_) => {
889                        self.error(uid, format!("Cannot read cargo {}", vis_path));
890                        continue;
891                    },
892                    Ok(v) => v
893                };
894                let value = match root_cargo.parse::<Value>() {
895                    Err(e) => {
896                        self.error(uid, format!("Cannot parse {} {:?}", vis_path, e));
897                        continue;
898                    },
899                    Ok(v) => v
900                };
901                let mut ws_members = Vec::new();
902                if let Value::Table(table) = &value {
903                    if let Some(table) = table.get("workspace") {
904                        if let Value::Table(table) = table {
905                            if let Some(members) = table.get("members") {
906                                if let Value::Array(members) = members {
907                                    for member in members {
908                                        if let Value::String(member) = member {
909                                            ws_members.push(member);
910                                        }
911                                    }
912                                }
913                            }
914                        }
915                    }
916                    else if let Some(table) = table.get("package") {
917                        if let Value::Table(table) = table {
918                            if let Some(name) = table.get("name") {
919                                if let Value::String(name) = name {
920                                    packages.push((workspace.clone(), name.clone()));
921                                }
922                            }
923                        }
924                    }
925                }
926                for member in ws_members {
927                    let file_path = format!("{}/{}/Cargo.toml", abs_path, member);
928                    let vis_path = format!("{}/{}/{}/Cargo.toml", self.builder, workspace, member);
929                    let cargo = match std::fs::read_to_string(&file_path) {
930                        Err(_) => {
931                            self.error(uid, format!("Cannot read cargo {}", vis_path));
932                            continue;
933                        },
934                        Ok(v) => v
935                    };
936                    let value = match cargo.parse::<Value>() {
937                        Err(e) => {
938                            self.error(uid, format!("Cannot parse cargo {} {:?}", vis_path, e));
939                            continue;
940                        },
941                        Ok(v) => v
942                    };
943                    if let Value::Table(table) = &value {
944                        if let Some(table) = table.get("package") {
945                            if let Value::Table(table) = table {
946                                if let Some(name) = table.get("name") {
947                                    if let Value::String(name) = name {
948                                        packages.push((workspace.clone(), name.clone()));
949                                    }
950                                }
951                            }
952                        }
953                    }
954                }
955            }
956        }
957        return packages
958    }
959    
960    pub fn file_read(&mut self, from: HubAddr, uid: HubUid, path: &str) {
961
962        if let Ok((abs_dir, _workspace, sub_path)) = self.workspace_split_from_path(uid, path) {
963            
964            if let Some(_) = sub_path.find("..") {
965                self.error(uid, format!("file_read got relative path, ignoring {}", path));
966                return
967            }
968            if sub_path.ends_with("key.ron") {
969                self.error(uid, format!("Ends with key.ron, ignoring {}", path));
970                return
971            }
972            
973            let data = if let Ok(data) = std::fs::read(format!("{}/{}", abs_dir, sub_path)) {
974                Some(data)
975            }
976            else {
977                None
978            };
979            
980            self.route_send.send(ToHubMsg {
981                to: HubMsgTo::Client(from),
982                msg: HubMsg::FileReadResponse {
983                    uid: uid,
984                    path: path.to_string(),
985                    data: data
986                }
987            });
988        }
989    }
990    
991    pub fn file_write(&mut self, from: HubAddr, uid: HubUid, path: &str, data: Vec<u8>) {
992        if let Ok((abs_dir, _project, sub_path)) = self.workspace_split_from_path(uid, path) {
993            
994            if path.contains("..") {
995                self.error(uid, format!("file_write got relative path, ignoring {}", path));
996                println!("file_read got relative path, ignoring {}", path);
997                return
998            }
999            
1000            let done = std::fs::write(format!("{}/{}", abs_dir, sub_path), &data).is_ok();
1001
1002            if let Ok(mut http_server) = self.http_server.lock() {
1003                if let Some(http_server) = &mut *http_server {
1004                    http_server.send_file_change(path);
1005                }
1006            };
1007            
1008            self.route_send.send(ToHubMsg {
1009                to: HubMsgTo::Client(from),
1010                msg: HubMsg::FileWriteResponse {
1011                    uid: uid,
1012                    path: path.to_string(),
1013                    done: done
1014                }
1015            });
1016        }
1017    }
1018    
1019    pub fn workspace_file_tree(&mut self, create_digest: bool, ext_inc: &[&str], file_ex: &[&str], dir_ex: &[&str]) -> BuilderFileTreeNode {
1020        fn digest_folder(create_digest: bool, name: &str, folder: &Vec<BuilderFileTreeNode>) -> Option<Box<Digest>> {
1021            if !create_digest {
1022                return None;
1023            }
1024            let mut digest_out = Digest::default();
1025            for item in folder {
1026                match item {
1027                    BuilderFileTreeNode::File {digest, ..} => {
1028                        if let Some(digest) = digest {
1029                            digest_out.digest_other(&*digest);
1030                        }
1031                    },
1032                    BuilderFileTreeNode::Folder {digest, ..} => {
1033                        if let Some(digest) = digest {
1034                            digest_out.digest_other(&*digest);
1035                        }
1036                    },
1037                }
1038            }
1039            digest_out.digest_buffer(name.as_bytes());
1040            Some(Box::new(digest_out))
1041        }
1042        
1043        fn read_recur(path: &str, create_digest: bool, ext_inc: &Vec<String>, file_ex: &Vec<String>, dir_ex: &Vec<String>) -> Vec<BuilderFileTreeNode> {
1044            let mut ret = Vec::new();
1045            if let Ok(read_dir) = fs::read_dir(path) {
1046                for entry in read_dir {
1047                    if let Ok(entry) = entry {
1048                        if let Ok(ty) = entry.file_type() {
1049                            if let Ok(name) = entry.file_name().into_string() {
1050                                if ty.is_dir() {
1051                                    if dir_ex.iter().find( | dir | **dir == name).is_some() {
1052                                        continue
1053                                    }
1054                                    let folder = read_recur(&format!("{}/{}", path, name), create_digest, ext_inc, file_ex, dir_ex);
1055                                    ret.push(BuilderFileTreeNode::Folder {
1056                                        name: name.clone(),
1057                                        digest: digest_folder(create_digest, &name, &folder),
1058                                        folder: folder
1059                                    });
1060                                }
1061                                else {
1062                                    if file_ex.iter().find( | file | **file == name).is_some() {
1063                                        continue
1064                                    }
1065                                    if ext_inc.iter().find(|ext| name.ends_with(*ext)).is_some(){
1066                                        if create_digest {
1067                                            
1068                                        }
1069                                        ret.push(BuilderFileTreeNode::File {
1070                                            digest: None,
1071                                            name: name
1072                                        });
1073                                    }
1074                                }
1075                            }
1076                        }
1077                    }
1078                }
1079            }
1080            ret.sort();
1081            
1082            ret
1083        }
1084        
1085        let ext_inc: Vec<String> = ext_inc.to_vec().iter().map( | v | v.to_string()).collect();
1086        let file_ex: Vec<String> = file_ex.to_vec().iter().map( | v | v.to_string()).collect();
1087        let dir_ex: Vec<String> = dir_ex.to_vec().iter().map( | v | v.to_string()).collect();
1088        
1089        let mut root_folder = Vec::new();
1090        
1091        if let Ok(workspaces) = self.workspaces.lock() {
1092            for (project, abs_path) in workspaces.iter() {
1093                
1094                let folder = read_recur(&abs_path, create_digest, &ext_inc, &file_ex, &dir_ex);
1095                let tree = BuilderFileTreeNode::Folder {
1096                    name: project.clone(),
1097                    digest: digest_folder(create_digest, &project, &folder),
1098                    folder: folder
1099                };
1100                root_folder.push(tree);
1101            }
1102        }
1103        let root = BuilderFileTreeNode::Folder {
1104            name: self.builder.clone(),
1105            digest: digest_folder(create_digest, &self.builder, &root_folder),
1106            folder: root_folder
1107        };
1108        root
1109    }
1110}
1111
1112fn rel_to_abs_path(abs_root: &str, path: &str) -> String {
1113    if path.starts_with("/") {
1114        return path.to_string();
1115    }
1116    de_relativize_path(&format!("{}/{}", abs_root, path))
1117}
1118
1119fn de_relativize_path(path: &str) -> String {
1120    let splits: Vec<&str> = path.split("/").collect();
1121    let mut out = Vec::new();
1122    for split in splits {
1123        if split == ".." && out.len()>0 {
1124            out.pop();
1125        }
1126        else if split != "." {
1127            out.push(split);
1128        }
1129    }
1130    out.join("/")
1131}
1132
1133
1134// rust compiler output json structs
1135#[derive(Clone, Deserialize, Default)]
1136pub struct RustcTarget {
1137    kind: Vec<String>,
1138    crate_types: Vec<String>,
1139    name: String,
1140    src_path: String,
1141    edition: String
1142}
1143
1144#[derive(Clone, Deserialize, Default)]
1145pub struct RustcText {
1146    text: String,
1147    highlight_start: u32,
1148    highlight_end: u32
1149}
1150
1151#[derive(Clone, Deserialize, Default)]
1152pub struct RustcSpan {
1153    file_name: String,
1154    byte_start: u32,
1155    byte_end: u32,
1156    line_start: u32,
1157    line_end: u32,
1158    column_start: u32,
1159    column_end: u32,
1160    is_primary: bool,
1161    text: Vec<RustcText>,
1162    label: Option<String>,
1163    suggested_replacement: Option<String>,
1164    sugggested_applicability: Option<String>,
1165    expansion: Option<Box<RustcExpansion>>,
1166    level: Option<String>
1167}
1168
1169#[derive(Clone, Deserialize, Default)]
1170pub struct RustcExpansion {
1171    span: Option<RustcSpan>,
1172    macro_decl_name: String,
1173    def_site_span: Option<RustcSpan>
1174}
1175
1176#[derive(Clone, Deserialize, Default)]
1177pub struct RustcCode {
1178    code: String,
1179    explanation: Option<String>
1180}
1181
1182#[derive(Clone, Deserialize, Default)]
1183pub struct RustcMessage {
1184    message: String,
1185    code: Option<RustcCode>,
1186    level: String,
1187    spans: Vec<RustcSpan>,
1188    children: Vec<RustcMessage>,
1189    rendered: Option<String>
1190}
1191
1192#[derive(Clone, Deserialize, Default)]
1193pub struct RustcProfile {
1194    opt_level: String,
1195    debuginfo: Option<u32>,
1196    debug_assertions: bool,
1197    overflow_checks: bool,
1198    test: bool
1199}
1200
1201#[derive(Clone, Deserialize, Default)]
1202pub struct RustcCompilerMessage {
1203    reason: String,
1204    package_id: String,
1205    target: Option<RustcTarget>,
1206    message: Option<RustcMessage>,
1207    profile: Option<RustcProfile>,
1208    features: Option<Vec<String>>,
1209    filenames: Option<Vec<String>>,
1210    executable: Option<String>,
1211    fresh: Option<bool>
1212}
makepad_hub/hubbuilder.rs

makepad_hub/
hubbuilder.rs
