makepad_file_server/
file_server.rs

1use {
2    makepad_shell::*,
3    crate::{
4        makepad_file_protocol::*,
5    },
6    std::{
7        time::Instant,
8        thread,
9        cmp::Ordering,
10        fmt,
11        fs,
12        str,
13        time::Duration,
14        path::{Path, PathBuf},
15        sync::{Arc, RwLock, Mutex},
16    },
17};
18
19pub struct FileServer {
20    // The id for the next connection
21    next_connection_id: usize,
22    // State that is shared between every connection
23    shared: Arc<RwLock<Shared >>,
24}
25
26impl FileServer {
27    /// Creates a new collab server rooted at the given path.
28    pub fn new(roots: FileSystemRoots) -> FileServer {
29        FileServer {
30            next_connection_id: 0,
31            shared: Arc::new(RwLock::new(Shared {
32                roots,
33            })),
34        }
35    }
36    
37    /// Creates a new connection to this collab server, and returns a handle for the connection.
38    ///
39    /// The given `notification_sender` is called whenever the server wants to send a notification
40    /// for this connection. The embedder is responsible for sending the notification.
41    pub fn connect(&mut self, notification_sender: Box<dyn NotificationSender>) -> FileServerConnection {
42        let connection_id = ConnectionId(self.next_connection_id);
43        self.next_connection_id += 1;
44        FileServerConnection {
45            _connection_id:connection_id,
46            shared: self.shared.clone(),
47            _notification_sender: notification_sender,
48            open_files: Default::default(),
49            stop_observation: Default::default()
50        }
51    }
52}
53
54/// A connection to a collab server.
55pub struct FileServerConnection {
56    // The id for this connection.
57    _connection_id: ConnectionId,
58    // State is shared between every connection.
59    shared: Arc<RwLock<Shared >>,
60    // Used to send notifications for this connection.
61    _notification_sender: Box<dyn NotificationSender>,
62    open_files: Arc<Mutex<Vec<(String, u64, Vec<u8>)>>>,
63    
64    stop_observation: Arc<Mutex<bool>>,
65}
66
67impl FileServerConnection {
68    /// Handles the given `request` for this connection, and returns the corresponding response.
69    ///
70    /// The embedder is responsible for receiving requests, calling this method to handle them, and
71    /// sending back the response.
72    pub fn handle_request(&self, request: FileRequest) -> FileResponse {
73        match request {
74            FileRequest::Search{set, id}=>{
75                self.search_start(set, id);
76                FileResponse::SearchInProgress(id)
77            }
78            FileRequest::LoadFileTree {with_data} => FileResponse::LoadFileTree(self.load_file_tree(with_data)),
79            FileRequest::OpenFile{path,id} => FileResponse::OpenFile(self.open_file(path, id)),
80            FileRequest::SaveFile{path, data, id, patch} => FileResponse::SaveFile(self.save_file(path, data, id, patch)),
81            FileRequest::LoadSnapshotImage{root, hash}=>FileResponse::LoadSnapshotImage(self.load_snapshot_image(root, hash)),
82            FileRequest::SaveSnapshotImage{root, hash, data}=>FileResponse::SaveSnapshotImage(self.save_snapshot_image(root, hash, data)),
83            FileRequest::CreateSnapshot{root, message}=>FileResponse::CreateSnapshot(self.create_snapshot(root, message)),
84            FileRequest::LoadSnapshot{root, hash}=>FileResponse::LoadSnapshot(self.load_snapshot(root, hash)),
85                                    
86        }
87    }
88    
89    fn search_start(&self, what:Vec<SearchItem>, id:u64) {
90        let mut sender = self._notification_sender.clone();
91        let roots = self.shared.read().unwrap().roots.clone();
92        thread::spawn(move || {
93            
94            // A recursive helper function for traversing the entries of a directory and creating the
95            // data structures that describe them.
96            fn search_files(id: u64, set:&Vec<SearchItem>, path: &Path, string_path:&str, sender: &mut Box<dyn NotificationSender>, last_send: &mut Instant, results: &mut Vec<SearchResult>) {
97                if let Ok(entries) = fs::read_dir(path){
98                    for entry in entries{
99                        if let Ok(entry) = entry{
100                            let entry_path = entry.path();
101                            let name = entry.file_name();
102                            if let Ok(name) = name.into_string() {
103                                if entry_path.is_file() && !name.ends_with(".rs") || entry_path.is_dir() && name == "target"
104                                || name.starts_with('.') {
105                                    continue;
106                                }
107                            }
108                            else {
109                                // Skip over entries with a non UTF-8 file name.
110                                continue;
111                            }
112                            
113                            let entry_string_name = entry.file_name().to_string_lossy().to_string();
114                            let entry_string_path = if string_path != ""{
115                                format!("{}/{}", string_path, entry_string_name)
116                            }else {
117                                entry_string_name
118                            };
119                            
120                            if entry_path.is_dir() {
121                                search_files(id, set, &entry_path, &entry_string_path, sender, last_send, results);
122                            }
123                            else if entry_path.is_file() {
124                                let mut rk_results = Vec::new();
125                                if let Ok(bytes) = fs::read(&entry_path){
126                                    // lets look for what in bytes
127                                    // if we find thigns we emit it on the notification send
128                                    fn is_word_char(b: u8)->bool{
129                                        b == b'_' || b == b':' || b >= b'0' && b<= b'9' || b >= b'A' && b <= b'Z' || b >= b'a' && b <= b'z' || b>126
130                                    }
131                                    for item in set{
132                                        let needle_bytes = item.needle.as_bytes();
133                                        if needle_bytes.len()==0{
134                                            continue;
135                                        }
136                                        makepad_rabin_karp::search(&bytes, &needle_bytes, &mut rk_results);
137                                        for result in &rk_results{
138                                            
139                                            if item.pre_word_boundary && result.byte > 0 && is_word_char(bytes[result.byte-1]){
140                                                continue
141                                            }
142                                            if item.post_word_boundary && result.byte + needle_bytes.len() < bytes.len() && is_word_char(bytes[result.byte + needle_bytes.len()]){
143                                                continue
144                                            }
145                                            if let Some(prefixes) = &item.prefixes{
146                                                // alright so prefixes as_bytes should be right before the match
147                                                if !prefixes.iter().any(|prefix|{
148                                                    let pb = prefix.as_bytes();
149                                                    if result.byte > pb.len(){
150                                                        if &bytes[result.byte - pb.len()..result.byte] == pb{
151                                                            return true
152                                                        }
153                                                    }
154                                                    false
155                                                }){
156                                                    continue
157                                                }
158                                            }
159                                             
160                                            let mut line_count = 0;
161                                            for i in result.new_line_byte..bytes.len()+1{
162                                                if i < bytes.len() && bytes[i] == b'\n'{
163                                                    line_count += 1;
164                                                }
165                                                if i == bytes.len() || bytes[i] == b'\n' && line_count == 4{
166                                                    if let Ok(result_line) = str::from_utf8(&bytes[result.new_line_byte..i]){
167                                                        // lets output it to our results
168                                                       results.push(SearchResult{
169                                                            file_name: entry_string_path.clone(),
170                                                            line: result.line,
171                                                            column_byte: result.column_byte,
172                                                            result_line: result_line.to_string()
173                                                        });
174                                                    }
175                                                    break;
176                                                }
177                                            }
178                                        }
179                                        rk_results.clear();
180                                    }
181                                }
182                            }
183                        }
184                        // lets compare time
185                        if last_send.elapsed().as_secs_f64()>0.1{
186                            *last_send = Instant::now();
187                            let  mut new_results = Vec::new();
188                            std::mem::swap(results, &mut new_results);
189                            sender.send_notification(FileNotification::SearchResults{
190                                id,
191                                results: new_results
192                            });
193                            
194                        }
195                    }
196                }
197            }
198            let mut last_send = Instant::now();
199            let mut results = Vec::new();
200            for (root_name, root_path) in roots.roots{
201                search_files(id, &what, &root_path, &root_name, &mut sender, &mut last_send, &mut results);
202            }
203            if results.len()>0{
204                sender.send_notification(FileNotification::SearchResults{
205                    id,
206                    results
207                });
208            }
209        });
210    }
211    
212    fn create_snapshot(&self, root:String, message:String) -> Result<CreateSnapshotResponse, CreateSnapshotError> {
213        let root_path = self.shared.read().unwrap().roots.find_root(&root).map_err(|error|{
214            CreateSnapshotError{error:format!("{:?}",error), root:root.clone()}
215        })?;
216        
217        match shell_env_cap(&[], &root_path, "git", &["commit", "-a",&format!("-m {message}")]) {
218            Ok(_) => {
219                match shell_env_cap(&[], &root_path, "git", &["log", "--pretty=format:%H","--max-count=1"]) {
220                    Ok(stdout) => {
221                        // ok we have the last commit hash, return that
222                        Ok(CreateSnapshotResponse{
223                            root,
224                            hash: stdout.trim().to_string()
225                        })
226                    }
227                    // we expect it on stderr
228                    Err(e) => {
229                        Err(CreateSnapshotError{root, error:e})
230                    }
231                }
232            }
233            // we expect it on stderr
234            Err(e) => {
235                Err(CreateSnapshotError{root, error:e})
236            }
237        }
238    }
239    
240        
241    fn load_snapshot(&self, root:String, hash:String) -> Result<LoadSnapshotResponse, LoadSnapshotError> {
242        
243        let root_path = self.shared.read().unwrap().roots.find_root(&root).map_err(|error|{
244            LoadSnapshotError{error:format!("{:?}",error), root:root.clone()}
245        })?;
246                
247        match shell_env_cap(&[], &root_path, "git", &["checkout", &hash]) {
248            Ok(_) => {
249                Ok(LoadSnapshotResponse{root, hash})
250            }
251            // we expect it on stderr
252            Err(e) => {
253                Err(LoadSnapshotError{root, error:e})
254            }
255        }
256    }
257
258    // Handles a `LoadFileTree` request.
259    fn load_file_tree(&self, with_data: bool) -> Result<FileTreeData, FileError> {
260        // A recursive helper function for traversing the entries of a directory and creating the
261        // data structures that describe them.
262        fn get_directory_entries(path: &Path, with_data: bool) -> Result<Vec<DirectoryEntry>, FileError> {
263            let mut entries = Vec::new();
264            for entry in fs::read_dir(path).map_err( | error | FileError::Unknown(error.to_string())) ? {
265                // We can't get the entry for some unknown reason. Raise an error.
266                let entry = entry.map_err( | error | FileError::Unknown(error.to_string())) ?;
267                // Get the path for the entry.
268                let entry_path = entry.path();
269                // Get the file name for the entry.
270                let name = entry.file_name();
271                if let Ok(name_string) = name.into_string() {
272                    if entry_path.is_dir() && name_string == "target"
273                        || name_string.starts_with('.') {
274                        // Skip over directories called "target". This is sort of a hack. The reason
275                        // it's here is that the "target" directory for Rust projects is huge, and
276                        // our current implementation of the file tree widget is not yet fast enough
277                        // to display vast numbers of nodes. We paper over this by pretending the
278                        // "target" directory does not exist.
279                        continue;
280                    }
281                }
282                else {
283                    // Skip over entries with a non UTF-8 file name.
284                    continue;
285                }
286                // Create a `DirectoryEntry` for this entry and add it to the list of entries.
287                entries.push(DirectoryEntry {
288                    name: entry.file_name().to_string_lossy().to_string(),
289                    node: if entry_path.is_dir() {
290                        // If this entry is a subdirectory, recursively create `DirectoryEntry`'s
291                        // for its entries as well.
292                        FileNodeData::Directory {
293                            git_log: None,
294                            entries: get_directory_entries(&entry_path, with_data) ?,
295                        }
296                    } else if entry_path.is_file() {
297                        if with_data {
298                            let bytes: Vec<u8> = fs::read(&entry_path).map_err(
299                                | error | FileError::Unknown(error.to_string())
300                            ) ?;
301                            FileNodeData::File {data: Some(bytes)}
302                        }
303                        else {
304                            FileNodeData::File {data: None}
305                        }
306                    }
307                    else {
308                        // If this entry is neither a directory or a file, skip it. This ignores
309                        // things such as symlinks, for which we are not yet sure how we want to
310                        // handle them.
311                        continue
312                    },
313                });
314            }
315            
316            // Sort all the entries by name, directories first, and files second.
317            entries.sort_by( | entry_0, entry_1 | {
318                match &entry_0.node {
319                    FileNodeData::Directory {..} => match &entry_1.node {
320                        FileNodeData::Directory {..} => entry_0.name.cmp(&entry_1.name),
321                        FileNodeData::File {..} => Ordering::Less
322                    }
323                    FileNodeData::File {..} => match &entry_1.node {
324                        FileNodeData::Directory {..} => Ordering::Greater,
325                        FileNodeData::File {..} => entry_0.name.cmp(&entry_1.name)
326                    }
327                }
328            });
329            Ok(entries)
330        }
331        
332        let roots = self.shared.read().unwrap().roots.clone();
333        let mut entries = Vec::new();
334        for (root_name, root_path) in roots.roots{
335            let mut commits = Vec::new();
336            match shell_env_cap(&[], &root_path, "git", &["log", "--pretty=format:%H %s"]) {
337                Ok(stdout) => {
338                    for line in stdout.split("\n"){
339                        let mut parts = line.splitn(2," ");
340                        if let Some(hash) = parts.next(){
341                            if let Some(message) = parts.next(){
342                                if hash.len() == 40{
343                                    // we have something
344                                    commits.push(GitCommit{
345                                        hash: hash.to_string(),
346                                        message: message.to_string()
347                                    })
348                                }
349                            }
350                        }
351                    }
352                }
353                // we expect it on stderr
354                Err(_e) => {}
355            }
356            
357            entries.push(DirectoryEntry{
358                name: root_name.clone(),
359                node: FileNodeData::Directory {
360                    git_log: Some(GitLog{
361                        root: root_name,
362                        commits
363                    }),
364                    entries: get_directory_entries(&root_path, with_data) ?,
365                }
366            });
367        }
368        Ok(FileTreeData {root_path: "".into(), root:FileNodeData::Directory {
369            git_log: None,
370            entries,
371        }})
372    }
373    
374    
375    fn start_observation(&self) {
376        let open_files = self.open_files.clone();
377        let shared = self.shared.clone();
378        let notification_sender = self._notification_sender.clone();
379        let stop_observation = self.stop_observation.clone();
380        thread::spawn(move || {
381            while !*stop_observation.lock().unwrap(){
382                if let Ok(mut files) = open_files.lock(){
383                    for (path, file_id, last_content) in files.iter_mut() {
384                        let full_path = {
385                            shared.read().unwrap().roots.make_full_path(&path)
386                        }.unwrap();
387                        if let Ok(bytes) = fs::read(&full_path) {
388                            if bytes.len() > 0 && bytes != *last_content {
389                                let new_data = String::from_utf8_lossy(&bytes);
390                                let old_data = String::from_utf8_lossy(&last_content);
391                                // Send notification of external file change.
392                                notification_sender
393                                .send_notification(FileNotification::FileChangedOnDisk(
394                                    SaveFileResponse{
395                                        path: path.to_string(),
396                                        new_data: new_data.to_string(),
397                                        old_data: old_data.to_string(),
398                                        kind: SaveKind::Observation,
399                                        id: *file_id
400                                    }
401                                ));
402                                *last_content = bytes;
403                            }
404                        }
405                    }
406                }
407                // Sleep for 500ms.
408                thread::sleep(Duration::from_millis(100));
409            }
410        });
411    }
412    
413    fn load_snapshot_image(&self, root: String, hash:String) -> Result<LoadSnapshotImageResponse, LoadSnapshotImageError> {
414        // alright letrs find the root
415        let root_path = self.shared.read().unwrap().roots.find_root(&root).map_err(|error|{
416            LoadSnapshotImageError{error, root:root.clone(), hash:hash.clone()}
417        })?;
418        let path = root_path.join("snapshots").join(&hash).with_extension("jpg");
419        let bytes = fs::read(&path).map_err(
420            | error | LoadSnapshotImageError{error:FileError::Unknown(error.to_string()), root:root.clone(), hash:hash.clone()}
421        ) ?;
422        
423        return Ok(LoadSnapshotImageResponse{
424            root,
425            hash,
426            data: bytes,
427        })
428    }
429    
430    fn save_snapshot_image(&self, root: String, hash:String, data:Vec<u8>) -> Result<SaveSnapshotImageResponse, FileError> {
431        // alright letrs find the root
432        let root_path = self.shared.read().unwrap().roots.find_root(&root)?;
433        let path = root_path.join("snapshots").join(&hash).with_extension("jpg");
434                
435        fs::write(&path, data).map_err(
436            | error | FileError::Unknown(error.to_string())
437        ) ?;
438                
439        return Ok(SaveSnapshotImageResponse{
440            root,
441            hash,
442        })
443    }
444    
445    // Handles an `OpenFile` request.
446    fn open_file(&self, child_path: String, id:u64) -> Result<OpenFileResponse, FileError> {
447        let path = self.shared.read().unwrap().roots.make_full_path(&child_path)?;
448        
449        let bytes = fs::read(&path).map_err(
450            | error | FileError::Unknown(error.to_string())
451        ) ?;
452        
453        let mut open_files = self.open_files.lock().unwrap();
454        
455        if open_files.iter().find(|(cp,_,_)| *cp == child_path).is_none(){
456            open_files.push((child_path.clone(), id, bytes.clone()));
457        }
458        
459        if open_files.len() == 1 {
460            self.start_observation();
461        }
462        // Converts the file contents to a `Text`. This is necessarily a lossy conversion
463        // because `Text` assumes everything is UTF-8 encoded, and this isn't always the
464        // case for files on disk (is this a problem?)
465        /*let text: Text = Text::from_lines(String::from_utf8_lossy(&bytes)
466            .lines()
467            .map( | line | line.chars().collect::<Vec<_ >> ())
468            .collect::<Vec<_ >>());*/
469        
470        let text = String::from_utf8_lossy(&bytes);
471        Ok(OpenFileResponse{
472            path: child_path,
473            data: text.to_string(),
474            id
475        })
476    }
477    
478    // Handles an `ApplyDelta` request.
479    fn save_file(
480        &self,
481        child_path: String,
482        new_data: String,
483        id: u64,
484        patch: bool
485    ) -> Result<SaveFileResponse, FileError> {
486        let mut open_files = self.open_files.lock().unwrap();
487                
488        if let Some(of) = open_files.iter_mut().find(|(cp,_,_)| *cp == child_path){
489            of.2 =  new_data.as_bytes().to_vec();
490        }
491        else{
492            open_files.push((child_path.clone(), id, new_data.as_bytes().to_vec()));
493        }
494        
495        let path = self.shared.read().unwrap().roots.make_full_path(&child_path)?;
496        
497        let old_data = String::from_utf8_lossy(&fs::read(&path).map_err(
498            | error | FileError::Unknown(error.to_string())
499        ) ?).to_string();
500
501        fs::write(&path, &new_data).map_err(
502            | error | FileError::Unknown(error.to_string())
503        ) ?;
504        
505        Ok(SaveFileResponse{
506            path: child_path, 
507            old_data,
508            new_data,
509            id,
510            kind: if patch{SaveKind::Patch}else{SaveKind::Save}
511        })
512    }
513}
514
515/// A trait for sending notifications over a connection.
516pub trait NotificationSender: Send {
517    /// This method is necessary to create clones of boxed trait objects.
518    fn box_clone(&self) -> Box<dyn NotificationSender>;
519    
520    /// This method is called to send a notification over the corresponding connection.
521    fn send_notification(&self, notification: FileNotification);
522}
523
524impl<F: Clone + Fn(FileNotification) + Send + 'static> NotificationSender for F {
525    fn box_clone(&self) -> Box<dyn NotificationSender> {
526        Box::new(self.clone())
527    }
528    
529    fn send_notification(&self, notification: FileNotification) {
530        self (notification)
531    }
532}
533
534impl Clone for Box<dyn NotificationSender> {
535    fn clone(&self) -> Self {
536        self.box_clone()
537    }
538}
539
540impl fmt::Debug for dyn NotificationSender {
541    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
542        write!(f, "NotificationSender")
543    }
544}
545
546#[derive(Debug, Clone, Default)]
547pub struct FileSystemRoots{
548    pub roots: Vec<(String, PathBuf)>
549}
550
551impl FileSystemRoots{
552    pub fn map_path(&self, possible_root:&str, what:&str)->String{
553
554        let what_path = Path::new(what);
555        if what_path.is_absolute(){
556            for (root_name, root_path) in &self.roots{
557                if let Ok(end) = what_path.strip_prefix(root_path){
558                    if let Ok(end) = end.to_path_buf().into_os_string().into_string(){
559                        return format!("{root_name}/{end}");
560                    }
561                }
562            }
563            return what.to_string()
564        }
565        else{
566            if possible_root.len() == 0{
567                what.to_string()
568            }
569            else{
570                format!("{possible_root}/{}",  what)
571            }
572        }
573    }
574    
575    pub fn find_root(&self, root:&str)->Result<PathBuf,FileError>{
576        if let Some(p) = self.roots.iter().find(|v| v.0 == root){
577            Ok(p.1.clone())
578        }
579        else{
580            Err(FileError::RootNotFound(root.to_string()))
581        }
582    }
583    
584    fn make_full_path(&self, child_path:&String)->Result<PathBuf,FileError>{
585        let mut parts = child_path.splitn(2,"/");
586        let root = parts.next().unwrap();
587        let file = parts.next().unwrap();
588        for (root_name, root_path) in &self.roots{
589            // lets split off the first directory
590            if root_name == root{
591                let mut path = root_path.clone();
592                path.push(file);
593                return Ok(path)
594            }
595        }
596        return Err(FileError::RootNotFound(child_path.clone()))
597    }
598}
599
600// State that is shared between every connection.
601#[derive(Debug)]
602struct Shared {
603    roots: FileSystemRoots
604}
605
606impl Shared{
607        
608}
609
610/// An identifier for a connection.
611#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
612struct ConnectionId(usize);
613
makepad_file_server/file_server.rs

makepad_file_server/
file_server.rs
