static_filez/
lib.rs

1extern crate clap_port_flag;
2extern crate deflate;
3extern crate fst;
4extern crate futures;
5extern crate hyper;
6extern crate memmap;
7extern crate mime_guess;
8extern crate quicli;
9extern crate tokio;
10extern crate walkdir;
11#[cfg(test)]
12#[macro_use]
13extern crate proptest;
14
15use std::fs::File;
16use std::path::Path;
17use std::result::Result;
18
19use quicli::prelude::*;
20use walkdir::WalkDir;
21
22mod slice;
23
24pub use server::serve;
25mod server;
26
27mod site;
28pub use site::Site;
29use std::path::PathBuf;
30
31pub fn build(src: &Path, target: &Path) -> Result<(), Error> {
32    info!(
33        "trying to build an index and archive from `{}`",
34        src.display()
35    );
36    let src = src
37        .canonicalize()
38        .with_context(|_| format!("Cannot canonicalize path `{}`", src.display()))?;
39
40    let src = Box::new(src.to_path_buf());
41    let src = &*Box::leak(src);
42
43    ensure!(src.is_dir(), "Directory `{}` doesn't exist", src.display());
44
45    use std::io::{BufWriter, Write};
46    let index_path = target.with_extension("index");
47    let index = BufWriter::new(
48        File::create(&index_path)
49            .with_context(|e| format!("couldn't create file `{}`: {}", target.display(), e))?,
50    );
51    let mut index = fst::MapBuilder::new(index)
52        .with_context(|e| format!("couldn't create index file `{}`: {}", target.display(), e))?;
53    info!("will write index to `{}`", index_path.display());
54
55    let archive_path = target.with_extension("archive");
56    let mut archive = BufWriter::new(
57        File::create(&archive_path)
58            .with_context(|e| format!("couldn't create file `{}`: {}", target.display(), e))?,
59    );
60    info!("will write archive to `{}`", archive_path.display());
61
62    let mut archive_index = 0;
63
64    let mut files = WalkDir::new(src)
65        .into_iter()
66        .par_bridge()
67        .flat_map(|entry| entry.map_err(|e| warn!("Couldn't read dir entry {}", e)))
68        .filter(|f| f.path().is_file())
69        .collect::<Vec<_>>();
70
71    ensure!(
72        !files.is_empty(),
73        "Would create empty archive. Is the `{}` directory empty?",
74        src.display()
75    );
76    info!("found {} files", files.len());
77
78    // fst map requires keys to be inserted in lexicographic order _represented as bytes_
79    fn rel_as_bytes(path: &Path) -> Vec<u8> {
80        path.to_string_lossy().to_string().into_bytes()
81    }
82    files.par_sort_by(move |a, b| rel_as_bytes(a.path()).cmp(&rel_as_bytes(b.path())));
83    info!("sorted {} files", files.len());
84
85    info!(
86        "now building archive {} as well as index {}",
87        archive_path.display(),
88        index_path.display()
89    );
90
91    let files = files
92        .chunks(2 << 8)
93        .flat_map(|chunk| -> Result<Vec<(PathBuf, Vec<u8>)>, ()> {
94            let files: Result<Vec<(PathBuf, Vec<u8>)>, Error> = chunk
95                .par_iter()
96                .map(|entry| -> Result<(PathBuf, Vec<u8>), Error> {
97                    let path = entry.path();
98                    let file_content = get_compressed_content(&path).with_context(|_| {
99                        format!("Could not read/compress content of {}", path.display())
100                    })?;
101                    let rel_path = path
102                        .strip_prefix(src)
103                        .with_context(|_| {
104                            format!("Couldn't get relative path for `{:?}`", path.display())
105                        })?.to_path_buf();
106                    Ok((rel_path, file_content))
107                }).collect();
108            let mut files = files.map_err(|e| warn!("{}", e))?;
109
110            files.par_sort_by(move |a, b| rel_as_bytes(&a.0).cmp(&rel_as_bytes(&b.0)));
111            Ok(files)
112        }).flat_map(|xs| xs);
113
114    for (rel_path, file_content) in files {
115        archive.write_all(&file_content).with_context(|_| {
116            format!(
117                "Could not write compressed content to {}",
118                archive_path.display()
119            )
120        })?;
121
122        index
123            .insert(
124                rel_path.to_string_lossy().as_bytes(),
125                slice::pack_in_u64(archive_index, file_content.len()),
126            ).with_context(|_| format!("Could not insert file {} into index", rel_path.display()))?;
127        archive_index += file_content.len();
128    }
129    info!("wrote all files");
130
131    index
132        .finish()
133        .with_context(|e| format!("Could not finish building index: {}", e))?;
134    info!("finished index");
135
136    Ok(())
137}
138
139fn get_compressed_content(path: &Path) -> Result<Vec<u8>, Error> {
140    use std::fs::read;
141    use std::io::Write;
142
143    use deflate::write::GzEncoder;
144    use deflate::Compression;
145
146    let data =
147        read(path).with_context(|e| format!("Couldn't read file {}: {}", path.display(), e))?;
148
149    let mut encoder = GzEncoder::new(Vec::new(), Compression::Best);
150    encoder.write_all(&data)?;
151    let compressed_data = encoder.finish()?;
152
153    Ok(compressed_data)
154}
static_filez/lib.rs

static_filez/
lib.rs
