logoxide_import/
lib.rs

1/*!
2logoxide-import: packages from other publishers' downloads, so a Bible that isn't a logoxide
3package can still be installed: its files with a `logoxide.toml` around them, which a project
4uses as an `import` dependency kept in the library (logoxide-app's `importer`), or as a path
5package (or which can be pushed anywhere as a git repository).
6
7- **eBible.org** ([`ebible`]): a translation's `<id>_usfm.zip`, one USFM file per book, with its
8  copyright in `copr.htm` (the package's license file)
9- **fetch.bible** ([`fetch`]): a translation's USX, a file per book, with its attribution in
10  `COPYRIGHT.txt`
11- **CCEL** ([`ccel`]): a Bible's ThML, the whole Bible in one file, described by its own head
12  (title, language, rights, version), with its rights in `COPYRIGHT.txt`
13- **Door43** ([`door43`]): a Bible's release, its repository's archive at the tag, a resource
14  container whose `manifest.yaml` describes it, its USFM a file per book
15- **SWORD** ([`sword`]): a module's zip from its repository, read with sword-modules from
16  memory: a Bible's or commentary's entries in their own markup (OSIS, ThML, or GBF), a file per
17  book, or a dictionary's as `entries/<module>.xml`, with its conf's copyright and license in
18  `COPYRIGHT.txt`
19- **Bundles** ([`burrito`]): a Scripture Burrito (`metadata.json`) or a Digital Bible Library
20  text release bundle (`metadata.xml`) a translator or publisher hands over, by URL or as a local
21  file or folder, its USX, USFM, or USJ checked by the MD5s its metadata lists
22
23Each publisher's list is read into a logoxide [`Catalog`](logoxide::catalog::Catalog) ([`catalog::parse`]), every entry a
24[`CatalogSource::External`] naming its import (`ebible:eng-asv`), the download it's made from,
25and what to fetch ([`downloads`]); [`build`] makes the package from the bytes. A bundle has no
26list: its import names where it is, and [`named_catalog`] makes its entry from that. The package
27manager keeps only that, and knows no publisher.
28
29A new import is version 1.0.0 ([`IMPORT_VERSION`]), logoxide's own: the publisher's version (an
30update date, a module's `Version`) is kept in `[package.upstream]`, and each update raises the
31package's version by what it changes ([`update::compare_import`]), so `lx add` can ask for `^1`
32and updates cross the publisher's years.
33*/
34
35pub mod burrito;
36pub mod catalog;
37pub mod ccel;
38pub mod door43;
39pub mod ebible;
40pub mod fetch;
41pub mod license;
42pub mod sword;
43pub mod update;
44
45use logoxide::{
46    LogoxideError, Result,
47    catalog::{Catalog, CatalogEntry, CatalogSource, Download},
48    manifest::{Format, FormatEntry, Keys, MANIFEST, Manifest, Package as Info, Upstream},
49    package::{Features, Package},
50    storage::{MemoryStorage, Storage},
51};
52
53pub use update::{
54    ImportUpdate, apply_import, compare_import, compare_publisher_versions, import_is_current,
55    import_removals, set_version,
56};
57
58/// Every publisher's import prefix: what `ebible:<id>` and `fetch:<id>` start with
59pub const PREFIXES: [&str; 7] = [
60    "ebible", "fetch", "ccel", "sword", "door43", "burrito", "dbl",
61];
62
63/// The version a new import has: logoxide's own, which updates raise by what they change
64pub const IMPORT_VERSION: semver::Version = semver::Version::new(1, 0, 0);
65
66/// Each file in a `.zip`, by path (folders left out), within logoxide-text's limits
67pub fn unzip(bytes: &[u8]) -> Result<Vec<(String, Vec<u8>)>> {
68    Ok(logoxide_text::archive::unzip(bytes)?)
69}
70
71/**
72What importing `entry` downloads, so every host fetches the same files the same way: eBible.org's
73zip, fetch.bible's books (each named by its code), CCEL's ThML, a SWORD module's zip, or a
74Door43 release's archive (its URLs tried in turn). Packages from git or a registry aren't imported, which is an error
75*/
76pub fn downloads(entry: &CatalogEntry) -> Result<Vec<Download>> {
77    match &entry.source {
78        CatalogSource::External { downloads, .. } => Ok(downloads.clone()),
79        CatalogSource::Git { .. } | CatalogSource::Registry { .. } => Err(LogoxideError::Import(
80            format!("{} isn't imported: add it", entry.id),
81        )),
82    }
83}
84
85/// The package `entry`'s import makes, written into `dest`, from the bytes of each of its
86/// [`downloads`] (by name), whichever publisher it comes from
87pub fn build(entry: &CatalogEntry, files: &[(String, Vec<u8>)], dest: &dyn Storage) -> Result<()> {
88    let named = |name: &str| {
89        (files.iter())
90            .find(|(n, _)| n == name)
91            .map(|(_, bytes)| bytes.as_slice())
92            .ok_or_else(|| LogoxideError::Import(format!("{}: {name} wasn't downloaded", entry.id)))
93    };
94    match entry.source.prefix() {
95        Some("ebible") => ebible::import(entry, named("zip")?, dest),
96        Some("fetch") => fetch::import(entry, files, dest),
97        Some("ccel") => ccel::import(entry, named("thml")?, dest),
98        Some("sword") => sword::import(entry, named("zip")?, dest),
99        Some("door43") => door43::import(entry, named("archive")?, dest),
100        Some("burrito" | "dbl") => burrito::import(entry, files, dest),
101        Some(other) => Err(LogoxideError::Import(format!(
102            "{}: no import reads `{other}:` downloads",
103            entry.id
104        ))),
105        None => Err(LogoxideError::Import(format!(
106            "{} isn't imported: add it",
107            entry.id
108        ))),
109    }
110}
111
112/**
113What an import of `entry` writes as its package's `[package.upstream]`: the download it was made
114from, its import name, and the publisher's version the entry lists, so an imported package names
115the catalog entry it came from ([`Catalog::find_upstream`](logoxide::catalog::Catalog::find_upstream)); none for packages that aren't
116imported
117*/
118pub fn upstream(entry: &CatalogEntry) -> Option<Upstream> {
119    let CatalogSource::External {
120        import, upstream, ..
121    } = &entry.source
122    else {
123        return None;
124    };
125    Some(Upstream {
126        url: upstream.clone(),
127        import: Some(import.clone()),
128        version: entry.version.clone(),
129        // A SWORD module's entries are made into OSIS or ThML files; the other publishers'
130        // files are kept as they are
131        converted: entry.source.prefix() == Some("sword"),
132        note: None,
133    })
134}
135
136/**
137A catalog of the one entry an import names when no publisher's list is needed to find it: a CCEL
138work by its author and name (`ccel:henry/mhc1`), or a bundle by where it is
139(`burrito:https://example.org/web.zip`); none for another
140*/
141pub fn named_catalog(import: &str) -> Option<Catalog> {
142    ccel::named_catalog(import).or_else(|| burrito::named_catalog(import))
143}
144
145/// The entry's [`CatalogSource::External`] fields for an import of `prefix:id`, with its
146/// download(s): what each publisher's list is read into
147pub(crate) fn external(
148    prefix: &str,
149    id: &str,
150    tracks_versions: bool,
151    downloads: Vec<Download>,
152) -> CatalogSource {
153    CatalogSource::External {
154        import: format!("{prefix}:{id}"),
155        upstream: (downloads.first()).and_then(|d| d.urls.first().cloned()),
156        tracks_versions,
157        downloads,
158    }
159}
160
161/// A package id from a publisher's (`eng-asv`, `engkjv2006`, `KJV`): lowercase letters, digits,
162/// `-`, `_`
163pub(crate) fn package_id(id: &str) -> String {
164    id.to_lowercase()
165        .chars()
166        .map(|c| match c {
167            'a'..='z' | '0'..='9' | '-' | '_' => c,
168            _ => '-',
169        })
170        .collect()
171}
172
173/// A publisher's id after its import prefix (`eng-asv` of `ebible:eng-asv`), checked to be the
174/// publisher's
175pub(crate) fn import_id<'a>(
176    entry: &'a CatalogEntry,
177    prefix: &str,
178    publisher: &str,
179) -> Result<&'a str> {
180    match entry.source.prefix() == Some(prefix) {
181        true => entry
182            .source
183            .import_id()
184            .ok_or_else(|| LogoxideError::Import(format!("{}: no import id", entry.id))),
185        false => Err(LogoxideError::Import(format!(
186            "{} isn't from {publisher}",
187            entry.id
188        ))),
189    }
190}
191
192/// How an import made its package
193pub(crate) struct Made {
194    /// The format its books are in, in a folder of that name
195    pub(crate) format: &'static str,
196    pub(crate) license_file: Option<String>,
197    /// Where it came from (`eBible.org`): the package's publisher, unless `publisher_name` says
198    pub(crate) publisher: &'static str,
199    /// The package's publisher, when the download names it (a bundle's rights holder)
200    pub(crate) publisher_name: Option<String>,
201    /// None when empty
202    pub(crate) homepage: String,
203    /// The versification the publisher names (SWORD's conf), else the package's `own`
204    pub(crate) versification: Option<String>,
205    /// For a dictionary, what its entries are keyed by
206    pub(crate) keys: Option<Keys>,
207    /// Where the text came from, in words (`[package.upstream] note`)
208    pub(crate) note: Option<String>,
209}
210
211/// Writes the package's `logoxide.toml`, checks it opens with books (a Bible's or a commentary's)
212/// or entries (a dictionary's; a book needs neither), and copies it into `dest`
213pub(crate) fn finish(
214    entry: &CatalogEntry,
215    made: Made,
216    staged: MemoryStorage,
217    dest: &dyn Storage,
218) -> Result<()> {
219    use logoxide::language::{script_direction, script_of_tag};
220    // The direction is written only where the script doesn't say it
221    let script = (entry.script.clone()).or_else(|| script_of_tag(&entry.language));
222    let implied = script_direction(script.as_deref());
223    let info = Info {
224        id: entry.id.clone(),
225        name: entry.name.clone(),
226        abbreviation: entry.abbreviation.clone(),
227        authors: vec![],
228        description: entry.description.clone(),
229        aliases: entry.aliases.clone(),
230        version: IMPORT_VERSION,
231        kind: entry.kind,
232        language: entry.language.clone(),
233        language_name: entry.language_name.clone(),
234        script: entry.script.clone(),
235        direction: (entry.direction != implied).then_some(entry.direction),
236        // A license SPDX can't read (a publisher's odd version) is the publisher's own
237        license: (entry.license.clone()).map(|license| {
238            match logoxide::license::check_license(&license) {
239                Ok(()) => license,
240                Err(_) => format!("LicenseRef-{}", made.publisher.replace(' ', "-")),
241            }
242        }),
243        rights: (!entry.rights.is_empty()).then_some(entry.rights),
244        license_file: made.license_file,
245        publisher: Some(
246            made.publisher_name
247                .unwrap_or_else(|| made.publisher.to_string()),
248        ),
249        homepage: (!made.homepage.is_empty()).then_some(made.homepage),
250        repository: None,
251        documentation: None,
252        readme: None,
253        keywords: entry.keywords.clone(),
254        categories: entry.categories.clone(),
255        about_books: entry.about_books.clone(),
256        // The work logoxide's table knows the import to be, never guessed from its title
257        work: (entry.work.clone()).or_else(|| logoxide::works::of_source(&entry.source)),
258        edition: entry.edition.clone(),
259        edition_year: entry.edition_year,
260        upstream: upstream(entry).map(|u| Upstream {
261            note: made.note,
262            ..u
263        }),
264        versification: made.versification.unwrap_or_else(|| String::from("own")),
265        maps_to: None,
266        keys: made.keys,
267        include: vec![],
268    };
269    // A dictionary's files are in `entries/`; every other format's folder is named for it
270    let folder = match Format::from_name(made.format) {
271        Some(Format::Dictionary) => "entries",
272        _ => made.format,
273    };
274    let manifest = Manifest {
275        package: Some(info),
276        formats: [(
277            made.format.to_string(),
278            FormatEntry {
279                path: format!("{folder}/"),
280                feature: None,
281            },
282        )]
283        .into(),
284        ..Manifest::default()
285    };
286    let text = toml::to_string(&manifest).map_err(|e| LogoxideError::Import(e.to_string()))?;
287    let header = format!("# Imported from {} by logoxide\n", made.publisher);
288    staged.write(MANIFEST, format!("{header}{text}").as_bytes())?;
289    let staged = std::sync::Arc::new(staged);
290    let package = Package::open(staged.clone(), Features::default())?;
291    let verses = (package.text_format()).is_some_and(|(f, _)| f.is_verse_text());
292    // A book (CCEL's ThML) isn't read as verses or entries: its references are found in it
293    let book = entry.kind == logoxide::manifest::PackageKind::Book;
294    if verses && package.books().is_empty() {
295        return Err(LogoxideError::Import(format!("{}: no books", entry.id)));
296    }
297    if !verses && !book && package.source().keys().is_empty() {
298        return Err(LogoxideError::Import(format!("{}: no entries", entry.id)));
299    }
300    for path in staged.walk("")? {
301        dest.write(&path, &staged.read(&path)?)?;
302    }
303    Ok(())
304}
305
306#[cfg(test)]
307pub(crate) mod tests {
308    use std::io::Write;
309
310    /// A zip of these files, as a publisher's download
311    pub(crate) fn zip(files: &[(&str, &str)]) -> Vec<u8> {
312        zip_bytes(
313            &(files.iter())
314                .map(|(path, text)| (*path, text.as_bytes()))
315                .collect::<Vec<_>>(),
316        )
317    }
318
319    pub(crate) fn zip_bytes(files: &[(&str, &[u8])]) -> Vec<u8> {
320        let mut writer = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
321        for (path, bytes) in files {
322            writer
323                .start_file(*path, zip::write::SimpleFileOptions::default())
324                .unwrap();
325            writer.write_all(bytes).unwrap();
326        }
327        writer.finish().unwrap().into_inner()
328    }
329
330    /**
331    Builds `entry`'s import into `dest`, and again elsewhere: the two must be the same bytes, so
332    a locked import made again on another machine has the checksum it was locked with
333    */
334    pub(crate) fn build_twice(
335        entry: &logoxide::catalog::CatalogEntry,
336        files: &[(String, Vec<u8>)],
337        dest: &dyn logoxide::storage::Storage,
338    ) {
339        super::build(entry, files, dest).unwrap();
340        use logoxide::storage::Storage;
341        let again = super::MemoryStorage::new();
342        super::build(entry, files, &again).unwrap();
343        assert_eq!(dest.walk("").unwrap(), again.walk("").unwrap());
344        for path in again.walk("").unwrap() {
345            assert_eq!(
346                dest.read(&path).unwrap(),
347                again.read(&path).unwrap(),
348                "{path}"
349            );
350        }
351    }
352
353    #[test]
354    fn what_isnt_imported_is_said() {
355        let catalog = logoxide::catalog::Catalog::parse_toml(
356            "[[package]]\nid = \"kjv\"\nname = \"KJV\"\nabbreviation = \"KJV\"\nlanguage = \"en\"\ngit = \"https://example.org/kjv\"\n",
357        )
358        .unwrap();
359        let entry = &catalog.entries[0];
360        assert!(super::downloads(entry).is_err());
361        assert!(super::build(entry, &[], &super::MemoryStorage::new()).is_err());
362        assert_eq!(super::upstream(entry), None);
363        assert_eq!(super::package_id("eng-ASV 1901"), "eng-asv-1901");
364    }
365}
logoxide_import/lib.rs

logoxide_import/
lib.rs
