semora_cli/
bundle.rs

1//! Packing a folder into a tar for a bundle push.
2
3use std::collections::BTreeMap;
4use std::path::Path;
5
6use anyhow::Context;
7
8/// Extensions the server accepts in a bundle.
9pub fn allowed(path: &str) -> bool {
10    let ext = path.rsplit('.').next().unwrap_or("").to_ascii_lowercase();
11    path.contains('.')
12        && matches!(
13            ext.as_str(),
14            "md" | "markdown"
15                | "html"
16                | "htm"
17                | "css"
18                | "js"
19                | "mjs"
20                | "json"
21                | "svg"
22                | "png"
23                | "jpg"
24                | "jpeg"
25                | "gif"
26                | "webp"
27                | "avif"
28                | "bmp"
29                | "ico"
30                | "woff"
31                | "woff2"
32                | "ttf"
33                | "otf"
34                | "zip"
35                | "7z"
36                | "gz"
37                | "rar"
38        )
39}
40
41pub fn is_page(path: &str) -> bool {
42    let p = path.to_ascii_lowercase();
43    p.ends_with(".md") || p.ends_with(".markdown") || p.ends_with(".html") || p.ends_with(".htm")
44}
45
46/// Folders never worth uploading.
47fn skip_dir(name: &str) -> bool {
48    name.starts_with('.') || name == "node_modules"
49}
50
51/// Files left out on purpose, with the reason.
52pub type Skipped = Vec<(String, &'static str)>;
53
54pub struct Packed {
55    pub tar: Vec<u8>,
56    pub files: Vec<String>,
57    /// The sha256 of each packed file, by its path in the folder.
58    pub hashes: BTreeMap<String, String>,
59    pub skipped: Skipped,
60}
61
62/// The files a push of `root` would send (sorted), and those it would leave out.
63pub fn list(root: &Path) -> anyhow::Result<(Vec<String>, Skipped)> {
64    let mut files = Vec::new();
65    let mut skipped = Vec::new();
66    walk(root, "", &mut files, &mut skipped)?;
67    files.sort();
68    Ok((files, skipped))
69}
70
71pub fn pack_dir(root: &Path) -> anyhow::Result<Packed> {
72    let (files, skipped) = list(root)?;
73    let mut hashes = BTreeMap::new();
74    let mut builder = tar::Builder::new(Vec::new());
75    for rel in &files {
76        let data = std::fs::read(root.join(rel)).with_context(|| format!("reading {rel}"))?;
77        hashes.insert(rel.clone(), crate::checkout::sha256(&data));
78        let mut h = tar::Header::new_gnu();
79        h.set_size(data.len() as u64);
80        h.set_mode(0o644);
81        h.set_entry_type(tar::EntryType::Regular);
82        h.set_mtime(0);
83        builder
84            .append_data(&mut h, rel, data.as_slice())
85            .with_context(|| format!("adding {rel}"))?;
86    }
87    let tar = builder.into_inner()?;
88    Ok(Packed {
89        tar,
90        files,
91        hashes,
92        skipped,
93    })
94}
95
96fn walk(
97    root: &Path,
98    rel: &str,
99    files: &mut Vec<String>,
100    skipped: &mut Vec<(String, &'static str)>,
101) -> anyhow::Result<()> {
102    let dir = if rel.is_empty() {
103        root.to_path_buf()
104    } else {
105        root.join(rel)
106    };
107    let mut entries: Vec<_> = std::fs::read_dir(&dir)
108        .with_context(|| format!("reading {}", dir.display()))?
109        .collect::<Result<_, _>>()?;
110    entries.sort_by_key(|e| e.file_name());
111    for e in entries {
112        // A tar path and a bundle path are text. A name that is not valid UTF-8 cannot be
113        // sent as it is, and its lossy form names no file here, so it is left out with a reason.
114        let name = match e.file_name().into_string() {
115            Ok(name) => name,
116            Err(raw) => {
117                let shown = raw.to_string_lossy();
118                let path = if rel.is_empty() {
119                    shown.into_owned()
120                } else {
121                    format!("{rel}/{shown}")
122                };
123                skipped.push((path, "the name is not valid UTF-8"));
124                continue;
125            }
126        };
127        let path = if rel.is_empty() {
128            name.clone()
129        } else {
130            format!("{rel}/{name}")
131        };
132        let ft = e.file_type()?;
133        let kind = if ft.is_symlink() {
134            Kind::Link
135        } else if ft.is_dir() {
136            Kind::Dir
137        } else {
138            Kind::File
139        };
140        match sort_entry(kind, &name, &path) {
141            Entry::Pack => files.push(path),
142            Entry::Enter => walk(root, &path, files, skipped)?,
143            Entry::Skip(why) => skipped.push((path, why)),
144            Entry::Ignore => {}
145        }
146    }
147    Ok(())
148}
149
150/// What a directory entry is, as far as the walk cares.
151#[derive(Clone, Copy)]
152enum Kind {
153    Link,
154    Dir,
155    File,
156}
157
158/// What the walk does with one directory entry.
159#[derive(Debug, PartialEq)]
160enum Entry {
161    /// A file that goes into the bundle.
162    Pack,
163    /// A folder to walk into.
164    Enter,
165    /// Left out on purpose, and the user is told why.
166    Skip(&'static str),
167    /// Left out without a word: hidden, or never worth uploading.
168    Ignore,
169}
170
171/// Decides one entry from its `name` and its `path` below the pushed folder. Takes no file
172/// system, so a name the file system here cannot hold can still be put to it.
173fn sort_entry(kind: Kind, name: &str, path: &str) -> Entry {
174    match kind {
175        Kind::Link => Entry::Skip("symbolic link"),
176        Kind::Dir if skip_dir(name) => Entry::Ignore,
177        // The server refuses a colon in any bundle path, and one such path would fail the
178        // whole push. A folder goes as a whole, with one warning.
179        Kind::Dir if name.contains(':') => {
180            Entry::Skip("a colon in the folder name; everything in it is skipped")
181        }
182        Kind::Dir => match unservable(name, path) {
183            Some(why) => Entry::Skip(why),
184            None => Entry::Enter,
185        },
186        // Hidden files stay local.
187        Kind::File if name.starts_with('.') => Entry::Ignore,
188        Kind::File if name.contains(':') => Entry::Skip("a colon in the file name"),
189        Kind::File => match unservable(name, path) {
190            Some(why) => Entry::Skip(why),
191            None if !allowed(path) => Entry::Skip("file type not served"),
192            None => Entry::Pack,
193        },
194    }
195}
196
197/// The longest bundle path the server takes, in bytes.
198const MAX_PATH_BYTES: usize = 512;
199
200/// Why the server would refuse this name, for the rules besides the colon: one such path fails
201/// the whole push, so the walk leaves it out and says why. A file system on Unix can hold all
202/// of these.
203fn unservable(name: &str, path: &str) -> Option<&'static str> {
204    if name.contains('\\') {
205        Some("a backslash in the name")
206    } else if name.chars().any(char::is_control) {
207        Some("a control character in the name")
208    } else if path.len() > MAX_PATH_BYTES {
209        Some("the path is over 512 bytes")
210    } else {
211        None
212    }
213}
214
215/// Relative links from a Markdown file to other local pages (for the bundle hint).
216pub fn local_page_links(md: &str) -> Vec<String> {
217    let mut out = Vec::new();
218    let mut rest = md;
219    while let Some(i) = rest.find("](") {
220        rest = &rest[i + 2..];
221        let end = rest.find([')', ' ', '\n']).unwrap_or(rest.len());
222        let href = rest[..end].trim_matches(['<', '>']);
223        let path = href.split(['#', '?']).next().unwrap_or("");
224        let external = href.contains("://")
225            || href.starts_with("mailto:")
226            || href.starts_with('#')
227            || href.starts_with('/');
228        if !external && is_page(path) {
229            out.push(href.to_string());
230        }
231        rest = &rest[end..];
232    }
233    out
234}
235
236#[cfg(test)]
237mod tests {
238    use super::*;
239
240    #[test]
241    fn packs_and_skips() {
242        let dir = tempfile::tempdir().unwrap();
243        let w = |p: &str| {
244            let full = dir.path().join(p);
245            std::fs::create_dir_all(full.parent().unwrap()).unwrap();
246            std::fs::write(full, "x").unwrap();
247        };
248        for p in [
249            "index.md",
250            "a/b.png",
251            ".env",
252            ".git/HEAD",
253            "node_modules/m/i.js",
254            "notes.txt",
255            "a/.hidden.md",
256        ] {
257            w(p);
258        }
259        let packed = pack_dir(dir.path()).unwrap();
260        assert_eq!(packed.files, vec!["a/b.png", "index.md"]);
261        assert_eq!(
262            packed.skipped,
263            vec![("notes.txt".to_string(), "file type not served")]
264        );
265        let mut archive = tar::Archive::new(packed.tar.as_slice());
266        let names: Vec<String> = archive
267            .entries()
268            .unwrap()
269            .map(|e| {
270                e.unwrap()
271                    .path()
272                    .unwrap()
273                    .to_string_lossy()
274                    .replace('\\', "/")
275            })
276            .collect();
277        assert_eq!(names, vec!["a/b.png", "index.md"]);
278    }
279
280    /// Colons cannot be in a file name here, so the walk is fed names, not files.
281    #[test]
282    fn a_name_with_a_colon_is_skipped_with_a_reason() {
283        let file = Entry::Skip("a colon in the file name");
284        assert_eq!(sort_entry(Kind::File, "a:b.md", "a:b.md"), file);
285        assert_eq!(sort_entry(Kind::File, "a:b.md", "docs/a:b.md"), file);
286        assert_eq!(sort_entry(Kind::File, "notes:", "notes:"), file);
287        let dir = Entry::Skip("a colon in the folder name; everything in it is skipped");
288        assert_eq!(sort_entry(Kind::Dir, "v1:final", "v1:final"), dir);
289        assert_eq!(sort_entry(Kind::Dir, "v1:final", "docs/v1:final"), dir);
290        // Names without a colon are sorted as before.
291        assert_eq!(sort_entry(Kind::File, "a.md", "a.md"), Entry::Pack);
292        assert_eq!(
293            sort_entry(Kind::File, "a;b-c_d.md", "a;b-c_d.md"),
294            Entry::Pack
295        );
296        assert_eq!(sort_entry(Kind::Dir, "docs", "docs"), Entry::Enter);
297        assert_eq!(
298            sort_entry(Kind::File, "a.txt", "a.txt"),
299            Entry::Skip("file type not served")
300        );
301        assert_eq!(
302            sort_entry(Kind::Link, "a.md", "a.md"),
303            Entry::Skip("symbolic link")
304        );
305        // The server's other rules for a path: each is a reason of its own.
306        assert_eq!(
307            sort_entry(Kind::File, "a\\b.md", "a\\b.md"),
308            Entry::Skip("a backslash in the name")
309        );
310        assert_eq!(
311            sort_entry(Kind::Dir, "tab\there", "docs/tab\there"),
312            Entry::Skip("a control character in the name")
313        );
314        let long = format!("{}.md", "a".repeat(200));
315        let deep = format!("{0}/{0}/{long}", "d".repeat(160));
316        assert_eq!(sort_entry(Kind::File, &long, &long), Entry::Pack);
317        assert_eq!(
318            sort_entry(Kind::File, &long, &deep),
319            Entry::Skip("the path is over 512 bytes")
320        );
321        // What stays local without a word does so with a colon too, rather than warn.
322        assert_eq!(sort_entry(Kind::File, ".a:b", ".a:b"), Entry::Ignore);
323        assert_eq!(sort_entry(Kind::Dir, ".a:b", ".a:b"), Entry::Ignore);
324        assert_eq!(
325            sort_entry(Kind::Dir, "node_modules", "node_modules"),
326            Entry::Ignore
327        );
328    }
329
330    /// Where the file system can hold the names, the whole walk leaves them out.
331    #[cfg(unix)]
332    #[test]
333    fn a_folder_with_colon_names_packs_the_rest() {
334        let dir = tempfile::tempdir().unwrap();
335        let w = |p: &str| {
336            let full = dir.path().join(p);
337            std::fs::create_dir_all(full.parent().unwrap()).unwrap();
338            std::fs::write(full, "x").unwrap();
339        };
340        for p in [
341            "index.md",
342            "a:b.md",
343            "docs/guide.md",
344            "docs/c:d.png",
345            "v1:final/one.md",
346            "v1:final/deeper/two.md",
347        ] {
348            w(p);
349        }
350        let packed = pack_dir(dir.path()).unwrap();
351        assert_eq!(packed.files, vec!["docs/guide.md", "index.md"]);
352        let skipped: Vec<&str> = packed.skipped.iter().map(|(p, _)| p.as_str()).collect();
353        assert_eq!(skipped, vec!["a:b.md", "docs/c:d.png", "v1:final"]);
354        let mut archive = tar::Archive::new(packed.tar.as_slice());
355        let names: Vec<String> = archive
356            .entries()
357            .unwrap()
358            .map(|e| e.unwrap().path().unwrap().to_string_lossy().into_owned())
359            .collect();
360        assert_eq!(names, vec!["docs/guide.md", "index.md"]);
361    }
362
363    /// Windows names are UTF-16 and macOS refuses such a name, so this runs on Linux.
364    #[cfg(target_os = "linux")]
365    #[test]
366    fn a_name_that_is_not_utf8_is_skipped_and_the_rest_is_packed() {
367        use std::ffi::OsStr;
368        use std::os::unix::ffi::OsStrExt;
369        let dir = tempfile::tempdir().unwrap();
370        std::fs::write(dir.path().join("index.md"), "x").unwrap();
371        std::fs::write(dir.path().join(OsStr::from_bytes(b"caf\xe9.md")), "x").unwrap();
372        let bad_dir = dir.path().join(OsStr::from_bytes(b"d\xff"));
373        std::fs::create_dir(&bad_dir).unwrap();
374        std::fs::write(bad_dir.join("inside.md"), "x").unwrap();
375        let packed = pack_dir(dir.path()).unwrap();
376        assert_eq!(packed.files, vec!["index.md"]);
377        assert_eq!(
378            packed.skipped,
379            vec![
380                ("caf\u{fffd}.md".to_string(), "the name is not valid UTF-8"),
381                ("d\u{fffd}".to_string(), "the name is not valid UTF-8"),
382            ]
383        );
384    }
385
386    #[test]
387    fn finds_local_page_links() {
388        let md = "[a](guide.md) [b](https://x/y.md) [c](img.png) [d](sub/page.html#top) [e](#here)";
389        assert_eq!(local_page_links(md), vec!["guide.md", "sub/page.html#top"]);
390    }
391}
semora_cli/bundle.rs

semora_cli/
bundle.rs
