semora_cli/
bundle.rs

1//! Packing a folder into a tar for a bundle push.
2
3use std::collections::BTreeMap;
4use std::path::Path;
5
6use anyhow::Context;
7
8/// Extensions the server accepts in a bundle.
9pub fn allowed(path: &str) -> bool {
10    let ext = path.rsplit('.').next().unwrap_or("").to_ascii_lowercase();
11    path.contains('.')
12        && matches!(
13            ext.as_str(),
14            "md" | "markdown"
15                | "html"
16                | "htm"
17                | "css"
18                | "js"
19                | "mjs"
20                | "json"
21                | "svg"
22                | "png"
23                | "jpg"
24                | "jpeg"
25                | "gif"
26                | "webp"
27                | "avif"
28                | "bmp"
29                | "ico"
30                | "woff"
31                | "woff2"
32                | "ttf"
33                | "otf"
34                | "zip"
35                | "7z"
36                | "gz"
37                | "rar"
38        )
39}
40
41pub fn is_page(path: &str) -> bool {
42    let p = path.to_ascii_lowercase();
43    p.ends_with(".md") || p.ends_with(".markdown") || p.ends_with(".html") || p.ends_with(".htm")
44}
45
46/// Folders never worth uploading.
47fn skip_dir(name: &str) -> bool {
48    name.starts_with('.') || name == "node_modules"
49}
50
51/// Files left out on purpose, with the reason.
52pub type Skipped = Vec<(String, &'static str)>;
53
54pub struct Packed {
55    pub tar: Vec<u8>,
56    pub files: Vec<String>,
57    /// The sha256 of each packed file, by its path in the folder.
58    pub hashes: BTreeMap<String, String>,
59    pub skipped: Skipped,
60}
61
62/// The files a push of `root` would send (sorted), and those it would leave out.
63pub fn list(root: &Path) -> anyhow::Result<(Vec<String>, Skipped)> {
64    let mut files = Vec::new();
65    let mut skipped = Vec::new();
66    walk(root, "", &mut files, &mut skipped)?;
67    files.sort();
68    Ok((files, skipped))
69}
70
71pub fn pack_dir(root: &Path) -> anyhow::Result<Packed> {
72    let (files, skipped) = list(root)?;
73    let mut hashes = BTreeMap::new();
74    let mut builder = tar::Builder::new(Vec::new());
75    for rel in &files {
76        let data = std::fs::read(root.join(rel)).with_context(|| format!("reading {rel}"))?;
77        hashes.insert(rel.clone(), crate::checkout::sha256(&data));
78        let mut h = tar::Header::new_gnu();
79        h.set_size(data.len() as u64);
80        h.set_mode(0o644);
81        h.set_entry_type(tar::EntryType::Regular);
82        h.set_mtime(0);
83        builder
84            .append_data(&mut h, rel, data.as_slice())
85            .with_context(|| format!("adding {rel}"))?;
86    }
87    let tar = builder.into_inner()?;
88    Ok(Packed {
89        tar,
90        files,
91        hashes,
92        skipped,
93    })
94}
95
96fn walk(
97    root: &Path,
98    rel: &str,
99    files: &mut Vec<String>,
100    skipped: &mut Vec<(String, &'static str)>,
101) -> anyhow::Result<()> {
102    let dir = if rel.is_empty() {
103        root.to_path_buf()
104    } else {
105        root.join(rel)
106    };
107    let mut entries: Vec<_> = std::fs::read_dir(&dir)
108        .with_context(|| format!("reading {}", dir.display()))?
109        .collect::<Result<_, _>>()?;
110    entries.sort_by_key(|e| e.file_name());
111    for e in entries {
112        let name = e.file_name().to_string_lossy().into_owned();
113        let path = if rel.is_empty() {
114            name.clone()
115        } else {
116            format!("{rel}/{name}")
117        };
118        let ft = e.file_type()?;
119        let kind = if ft.is_symlink() {
120            Kind::Link
121        } else if ft.is_dir() {
122            Kind::Dir
123        } else {
124            Kind::File
125        };
126        match sort_entry(kind, &name, &path) {
127            Entry::Pack => files.push(path),
128            Entry::Enter => walk(root, &path, files, skipped)?,
129            Entry::Skip(why) => skipped.push((path, why)),
130            Entry::Ignore => {}
131        }
132    }
133    Ok(())
134}
135
136/// What a directory entry is, as far as the walk cares.
137#[derive(Clone, Copy)]
138enum Kind {
139    Link,
140    Dir,
141    File,
142}
143
144/// What the walk does with one directory entry.
145#[derive(Debug, PartialEq)]
146enum Entry {
147    /// A file that goes into the bundle.
148    Pack,
149    /// A folder to walk into.
150    Enter,
151    /// Left out on purpose, and the user is told why.
152    Skip(&'static str),
153    /// Left out without a word: hidden, or never worth uploading.
154    Ignore,
155}
156
157/// Decides one entry from its `name` and its `path` below the pushed folder. Takes no file
158/// system, so a name the file system here cannot hold can still be put to it.
159fn sort_entry(kind: Kind, name: &str, path: &str) -> Entry {
160    match kind {
161        Kind::Link => Entry::Skip("symbolic link"),
162        Kind::Dir if skip_dir(name) => Entry::Ignore,
163        // The server refuses a colon in any bundle path, and one such path would fail the
164        // whole push. A folder goes as a whole, with one warning.
165        Kind::Dir if name.contains(':') => {
166            Entry::Skip("a colon in the folder name; everything in it is skipped")
167        }
168        Kind::Dir => Entry::Enter,
169        // Hidden files stay local.
170        Kind::File if name.starts_with('.') => Entry::Ignore,
171        Kind::File if name.contains(':') => Entry::Skip("a colon in the file name"),
172        Kind::File if !allowed(path) => Entry::Skip("file type not served"),
173        Kind::File => Entry::Pack,
174    }
175}
176
177/// Relative links from a Markdown file to other local pages (for the bundle hint).
178pub fn local_page_links(md: &str) -> Vec<String> {
179    let mut out = Vec::new();
180    let mut rest = md;
181    while let Some(i) = rest.find("](") {
182        rest = &rest[i + 2..];
183        let end = rest.find([')', ' ', '\n']).unwrap_or(rest.len());
184        let href = rest[..end].trim_matches(['<', '>']);
185        let path = href.split(['#', '?']).next().unwrap_or("");
186        let external = href.contains("://")
187            || href.starts_with("mailto:")
188            || href.starts_with('#')
189            || href.starts_with('/');
190        if !external && is_page(path) {
191            out.push(href.to_string());
192        }
193        rest = &rest[end..];
194    }
195    out
196}
197
198#[cfg(test)]
199mod tests {
200    use super::*;
201
202    #[test]
203    fn packs_and_skips() {
204        let dir = tempfile::tempdir().unwrap();
205        let w = |p: &str| {
206            let full = dir.path().join(p);
207            std::fs::create_dir_all(full.parent().unwrap()).unwrap();
208            std::fs::write(full, "x").unwrap();
209        };
210        for p in [
211            "index.md",
212            "a/b.png",
213            ".env",
214            ".git/HEAD",
215            "node_modules/m/i.js",
216            "notes.txt",
217            "a/.hidden.md",
218        ] {
219            w(p);
220        }
221        let packed = pack_dir(dir.path()).unwrap();
222        assert_eq!(packed.files, vec!["a/b.png", "index.md"]);
223        assert_eq!(
224            packed.skipped,
225            vec![("notes.txt".to_string(), "file type not served")]
226        );
227        let mut archive = tar::Archive::new(packed.tar.as_slice());
228        let names: Vec<String> = archive
229            .entries()
230            .unwrap()
231            .map(|e| {
232                e.unwrap()
233                    .path()
234                    .unwrap()
235                    .to_string_lossy()
236                    .replace('\\', "/")
237            })
238            .collect();
239        assert_eq!(names, vec!["a/b.png", "index.md"]);
240    }
241
242    /// Colons cannot be in a file name here, so the walk is fed names, not files.
243    #[test]
244    fn a_name_with_a_colon_is_skipped_with_a_reason() {
245        let file = Entry::Skip("a colon in the file name");
246        assert_eq!(sort_entry(Kind::File, "a:b.md", "a:b.md"), file);
247        assert_eq!(sort_entry(Kind::File, "a:b.md", "docs/a:b.md"), file);
248        assert_eq!(sort_entry(Kind::File, "notes:", "notes:"), file);
249        let dir = Entry::Skip("a colon in the folder name; everything in it is skipped");
250        assert_eq!(sort_entry(Kind::Dir, "v1:final", "v1:final"), dir);
251        assert_eq!(sort_entry(Kind::Dir, "v1:final", "docs/v1:final"), dir);
252        // Names without a colon are sorted as before.
253        assert_eq!(sort_entry(Kind::File, "a.md", "a.md"), Entry::Pack);
254        assert_eq!(
255            sort_entry(Kind::File, "a;b-c_d.md", "a;b-c_d.md"),
256            Entry::Pack
257        );
258        assert_eq!(sort_entry(Kind::Dir, "docs", "docs"), Entry::Enter);
259        assert_eq!(
260            sort_entry(Kind::File, "a.txt", "a.txt"),
261            Entry::Skip("file type not served")
262        );
263        assert_eq!(
264            sort_entry(Kind::Link, "a.md", "a.md"),
265            Entry::Skip("symbolic link")
266        );
267        // What stays local without a word does so with a colon too, rather than warn.
268        assert_eq!(sort_entry(Kind::File, ".a:b", ".a:b"), Entry::Ignore);
269        assert_eq!(sort_entry(Kind::Dir, ".a:b", ".a:b"), Entry::Ignore);
270        assert_eq!(
271            sort_entry(Kind::Dir, "node_modules", "node_modules"),
272            Entry::Ignore
273        );
274    }
275
276    /// Where the file system can hold the names, the whole walk leaves them out.
277    #[cfg(unix)]
278    #[test]
279    fn a_folder_with_colon_names_packs_the_rest() {
280        let dir = tempfile::tempdir().unwrap();
281        let w = |p: &str| {
282            let full = dir.path().join(p);
283            std::fs::create_dir_all(full.parent().unwrap()).unwrap();
284            std::fs::write(full, "x").unwrap();
285        };
286        for p in [
287            "index.md",
288            "a:b.md",
289            "docs/guide.md",
290            "docs/c:d.png",
291            "v1:final/one.md",
292            "v1:final/deeper/two.md",
293        ] {
294            w(p);
295        }
296        let packed = pack_dir(dir.path()).unwrap();
297        assert_eq!(packed.files, vec!["docs/guide.md", "index.md"]);
298        let skipped: Vec<&str> = packed.skipped.iter().map(|(p, _)| p.as_str()).collect();
299        assert_eq!(skipped, vec!["a:b.md", "docs/c:d.png", "v1:final"]);
300        let mut archive = tar::Archive::new(packed.tar.as_slice());
301        let names: Vec<String> = archive
302            .entries()
303            .unwrap()
304            .map(|e| e.unwrap().path().unwrap().to_string_lossy().into_owned())
305            .collect();
306        assert_eq!(names, vec!["docs/guide.md", "index.md"]);
307    }
308
309    #[test]
310    fn finds_local_page_links() {
311        let md = "[a](guide.md) [b](https://x/y.md) [c](img.png) [d](sub/page.html#top) [e](#here)";
312        assert_eq!(local_page_links(md), vec!["guide.md", "sub/page.html#top"]);
313    }
314}
semora_cli/bundle.rs

semora_cli/
bundle.rs
