semora-cli 1.1.0

//! Packing a folder into a tar for a bundle push.

use std::collections::BTreeMap;
use std::path::Path;

use anyhow::Context;

/// Extensions the server accepts in a bundle.
pub fn allowed(path: &str) -> bool {
    let ext = path.rsplit('.').next().unwrap_or("").to_ascii_lowercase();
    path.contains('.')
        && matches!(
            ext.as_str(),
            "md" | "markdown"
                | "html"
                | "htm"
                | "css"
                | "js"
                | "mjs"
                | "json"
                | "svg"
                | "png"
                | "jpg"
                | "jpeg"
                | "gif"
                | "webp"
                | "avif"
                | "bmp"
                | "ico"
                | "woff"
                | "woff2"
                | "ttf"
                | "otf"
                | "zip"
                | "7z"
                | "gz"
                | "rar"
        )
}

pub fn is_page(path: &str) -> bool {
    let p = path.to_ascii_lowercase();
    p.ends_with(".md") || p.ends_with(".markdown") || p.ends_with(".html") || p.ends_with(".htm")
}

/// Folders never worth uploading.
fn skip_dir(name: &str) -> bool {
    name.starts_with('.') || name == "node_modules"
}

/// Files left out on purpose, with the reason.
pub type Skipped = Vec<(String, &'static str)>;

pub struct Packed {
    pub tar: Vec<u8>,
    pub files: Vec<String>,
    /// The sha256 of each packed file, by its path in the folder.
    pub hashes: BTreeMap<String, String>,
    pub skipped: Skipped,
}

/// The files a push of `root` would send (sorted), and those it would leave out.
pub fn list(root: &Path) -> anyhow::Result<(Vec<String>, Skipped)> {
    let mut files = Vec::new();
    let mut skipped = Vec::new();
    walk(root, "", &mut files, &mut skipped)?;
    files.sort();
    Ok((files, skipped))
}

pub fn pack_dir(root: &Path) -> anyhow::Result<Packed> {
    let (files, skipped) = list(root)?;
    let mut hashes = BTreeMap::new();
    let mut builder = tar::Builder::new(Vec::new());
    for rel in &files {
        let data = std::fs::read(root.join(rel)).with_context(|| format!("reading {rel}"))?;
        hashes.insert(rel.clone(), crate::checkout::sha256(&data));
        let mut h = tar::Header::new_gnu();
        h.set_size(data.len() as u64);
        h.set_mode(0o644);
        h.set_entry_type(tar::EntryType::Regular);
        h.set_mtime(0);
        builder
            .append_data(&mut h, rel, data.as_slice())
            .with_context(|| format!("adding {rel}"))?;
    }
    let tar = builder.into_inner()?;
    Ok(Packed {
        tar,
        files,
        hashes,
        skipped,
    })
}

fn walk(
    root: &Path,
    rel: &str,
    files: &mut Vec<String>,
    skipped: &mut Vec<(String, &'static str)>,
) -> anyhow::Result<()> {
    let dir = if rel.is_empty() {
        root.to_path_buf()
    } else {
        root.join(rel)
    };
    let mut entries: Vec<_> = std::fs::read_dir(&dir)
        .with_context(|| format!("reading {}", dir.display()))?
        .collect::<Result<_, _>>()?;
    entries.sort_by_key(|e| e.file_name());
    for e in entries {
        // A tar path and a bundle path are text. A name that is not valid UTF-8 cannot be
        // sent as it is, and its lossy form names no file here, so it is left out with a reason.
        let name = match e.file_name().into_string() {
            Ok(name) => name,
            Err(raw) => {
                let shown = raw.to_string_lossy();
                let path = if rel.is_empty() {
                    shown.into_owned()
                } else {
                    format!("{rel}/{shown}")
                };
                skipped.push((path, "the name is not valid UTF-8"));
                continue;
            }
        };
        let path = if rel.is_empty() {
            name.clone()
        } else {
            format!("{rel}/{name}")
        };
        let ft = e.file_type()?;
        let kind = if ft.is_symlink() {
            Kind::Link
        } else if ft.is_dir() {
            Kind::Dir
        } else {
            Kind::File
        };
        match sort_entry(kind, &name, &path) {
            Entry::Pack => files.push(path),
            Entry::Enter => walk(root, &path, files, skipped)?,
            Entry::Skip(why) => skipped.push((path, why)),
            Entry::Ignore => {}
        }
    }
    Ok(())
}

/// What a directory entry is, as far as the walk cares.
#[derive(Clone, Copy)]
enum Kind {
    Link,
    Dir,
    File,
}

/// What the walk does with one directory entry.
#[derive(Debug, PartialEq)]
enum Entry {
    /// A file that goes into the bundle.
    Pack,
    /// A folder to walk into.
    Enter,
    /// Left out on purpose, and the user is told why.
    Skip(&'static str),
    /// Left out without a word: hidden, or never worth uploading.
    Ignore,
}

/// Decides one entry from its `name` and its `path` below the pushed folder. Takes no file
/// system, so a name the file system here cannot hold can still be put to it.
fn sort_entry(kind: Kind, name: &str, path: &str) -> Entry {
    match kind {
        Kind::Link => Entry::Skip("symbolic link"),
        Kind::Dir if skip_dir(name) => Entry::Ignore,
        // The server refuses a colon in any bundle path, and one such path would fail the
        // whole push. A folder goes as a whole, with one warning.
        Kind::Dir if name.contains(':') => {
            Entry::Skip("a colon in the folder name; everything in it is skipped")
        }
        Kind::Dir => match unservable(name, path) {
            Some(why) => Entry::Skip(why),
            None => Entry::Enter,
        },
        // Hidden files stay local.
        Kind::File if name.starts_with('.') => Entry::Ignore,
        Kind::File if name.contains(':') => Entry::Skip("a colon in the file name"),
        Kind::File => match unservable(name, path) {
            Some(why) => Entry::Skip(why),
            None if !allowed(path) => Entry::Skip("file type not served"),
            None => Entry::Pack,
        },
    }
}

/// The longest bundle path the server takes, in bytes.
const MAX_PATH_BYTES: usize = 512;

/// Why the server would refuse this name, for the rules besides the colon: one such path fails
/// the whole push, so the walk leaves it out and says why. A file system on Unix can hold all
/// of these.
fn unservable(name: &str, path: &str) -> Option<&'static str> {
    if name.contains('\\') {
        Some("a backslash in the name")
    } else if name.chars().any(char::is_control) {
        Some("a control character in the name")
    } else if path.len() > MAX_PATH_BYTES {
        Some("the path is over 512 bytes")
    } else {
        None
    }
}

/// Relative links from a Markdown file to other local pages (for the bundle hint).
pub fn local_page_links(md: &str) -> Vec<String> {
    let mut out = Vec::new();
    let mut rest = md;
    while let Some(i) = rest.find("](") {
        rest = &rest[i + 2..];
        let end = rest.find([')', ' ', '\n']).unwrap_or(rest.len());
        let href = rest[..end].trim_matches(['<', '>']);
        let path = href.split(['#', '?']).next().unwrap_or("");
        let external = href.contains("://")
            || href.starts_with("mailto:")
            || href.starts_with('#')
            || href.starts_with('/');
        if !external && is_page(path) {
            out.push(href.to_string());
        }
        rest = &rest[end..];
    }
    out
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn packs_and_skips() {
        let dir = tempfile::tempdir().unwrap();
        let w = |p: &str| {
            let full = dir.path().join(p);
            std::fs::create_dir_all(full.parent().unwrap()).unwrap();
            std::fs::write(full, "x").unwrap();
        };
        for p in [
            "index.md",
            "a/b.png",
            ".env",
            ".git/HEAD",
            "node_modules/m/i.js",
            "notes.txt",
            "a/.hidden.md",
        ] {
            w(p);
        }
        let packed = pack_dir(dir.path()).unwrap();
        assert_eq!(packed.files, vec!["a/b.png", "index.md"]);
        assert_eq!(
            packed.skipped,
            vec![("notes.txt".to_string(), "file type not served")]
        );
        let mut archive = tar::Archive::new(packed.tar.as_slice());
        let names: Vec<String> = archive
            .entries()
            .unwrap()
            .map(|e| {
                e.unwrap()
                    .path()
                    .unwrap()
                    .to_string_lossy()
                    .replace('\\', "/")
            })
            .collect();
        assert_eq!(names, vec!["a/b.png", "index.md"]);
    }

    /// Colons cannot be in a file name here, so the walk is fed names, not files.
    #[test]
    fn a_name_with_a_colon_is_skipped_with_a_reason() {
        let file = Entry::Skip("a colon in the file name");
        assert_eq!(sort_entry(Kind::File, "a:b.md", "a:b.md"), file);
        assert_eq!(sort_entry(Kind::File, "a:b.md", "docs/a:b.md"), file);
        assert_eq!(sort_entry(Kind::File, "notes:", "notes:"), file);
        let dir = Entry::Skip("a colon in the folder name; everything in it is skipped");
        assert_eq!(sort_entry(Kind::Dir, "v1:final", "v1:final"), dir);
        assert_eq!(sort_entry(Kind::Dir, "v1:final", "docs/v1:final"), dir);
        // Names without a colon are sorted as before.
        assert_eq!(sort_entry(Kind::File, "a.md", "a.md"), Entry::Pack);
        assert_eq!(
            sort_entry(Kind::File, "a;b-c_d.md", "a;b-c_d.md"),
            Entry::Pack
        );
        assert_eq!(sort_entry(Kind::Dir, "docs", "docs"), Entry::Enter);
        assert_eq!(
            sort_entry(Kind::File, "a.txt", "a.txt"),
            Entry::Skip("file type not served")
        );
        assert_eq!(
            sort_entry(Kind::Link, "a.md", "a.md"),
            Entry::Skip("symbolic link")
        );
        // The server's other rules for a path: each is a reason of its own.
        assert_eq!(
            sort_entry(Kind::File, "a\\b.md", "a\\b.md"),
            Entry::Skip("a backslash in the name")
        );
        assert_eq!(
            sort_entry(Kind::Dir, "tab\there", "docs/tab\there"),
            Entry::Skip("a control character in the name")
        );
        let long = format!("{}.md", "a".repeat(200));
        let deep = format!("{0}/{0}/{long}", "d".repeat(160));
        assert_eq!(sort_entry(Kind::File, &long, &long), Entry::Pack);
        assert_eq!(
            sort_entry(Kind::File, &long, &deep),
            Entry::Skip("the path is over 512 bytes")
        );
        // What stays local without a word does so with a colon too, rather than warn.
        assert_eq!(sort_entry(Kind::File, ".a:b", ".a:b"), Entry::Ignore);
        assert_eq!(sort_entry(Kind::Dir, ".a:b", ".a:b"), Entry::Ignore);
        assert_eq!(
            sort_entry(Kind::Dir, "node_modules", "node_modules"),
            Entry::Ignore
        );
    }

    /// Where the file system can hold the names, the whole walk leaves them out.
    #[cfg(unix)]
    #[test]
    fn a_folder_with_colon_names_packs_the_rest() {
        let dir = tempfile::tempdir().unwrap();
        let w = |p: &str| {
            let full = dir.path().join(p);
            std::fs::create_dir_all(full.parent().unwrap()).unwrap();
            std::fs::write(full, "x").unwrap();
        };
        for p in [
            "index.md",
            "a:b.md",
            "docs/guide.md",
            "docs/c:d.png",
            "v1:final/one.md",
            "v1:final/deeper/two.md",
        ] {
            w(p);
        }
        let packed = pack_dir(dir.path()).unwrap();
        assert_eq!(packed.files, vec!["docs/guide.md", "index.md"]);
        let skipped: Vec<&str> = packed.skipped.iter().map(|(p, _)| p.as_str()).collect();
        assert_eq!(skipped, vec!["a:b.md", "docs/c:d.png", "v1:final"]);
        let mut archive = tar::Archive::new(packed.tar.as_slice());
        let names: Vec<String> = archive
            .entries()
            .unwrap()
            .map(|e| e.unwrap().path().unwrap().to_string_lossy().into_owned())
            .collect();
        assert_eq!(names, vec!["docs/guide.md", "index.md"]);
    }

    /// Windows names are UTF-16 and macOS refuses such a name, so this runs on Linux.
    #[cfg(target_os = "linux")]
    #[test]
    fn a_name_that_is_not_utf8_is_skipped_and_the_rest_is_packed() {
        use std::ffi::OsStr;
        use std::os::unix::ffi::OsStrExt;
        let dir = tempfile::tempdir().unwrap();
        std::fs::write(dir.path().join("index.md"), "x").unwrap();
        std::fs::write(dir.path().join(OsStr::from_bytes(b"caf\xe9.md")), "x").unwrap();
        let bad_dir = dir.path().join(OsStr::from_bytes(b"d\xff"));
        std::fs::create_dir(&bad_dir).unwrap();
        std::fs::write(bad_dir.join("inside.md"), "x").unwrap();
        let packed = pack_dir(dir.path()).unwrap();
        assert_eq!(packed.files, vec!["index.md"]);
        assert_eq!(
            packed.skipped,
            vec![
                ("caf\u{fffd}.md".to_string(), "the name is not valid UTF-8"),
                ("d\u{fffd}".to_string(), "the name is not valid UTF-8"),
            ]
        );
    }

    #[test]
    fn finds_local_page_links() {
        let md = "[a](guide.md) [b](https://x/y.md) [c](img.png) [d](sub/page.html#top) [e](#here)";
        assert_eq!(local_page_links(md), vec!["guide.md", "sub/page.html#top"]);
    }
}