logoxide_import/
lib.rs

1/*!
2logoxide-import: packages from other publishers' downloads, so a Bible that isn't a logoxide
3package can still be installed: its files with a `logoxide.toml` around them, which a project
4uses as an `import` dependency kept in the library (logoxide-app's `importer`), or as a path
5package (or which can be pushed anywhere as a git repository).
6
7- **eBible.org** ([`ebible`]): a translation's `<id>_usfm.zip`, one USFM file per book, with its
8  copyright in `copr.htm` (the package's license file)
9- **fetch.bible** ([`fetch`]): a translation's USX, a file per book, with its attribution in
10  `COPYRIGHT.txt`
11- **CCEL** ([`ccel`]): a Bible's ThML, the whole Bible in one file, described by its own head
12  (title, language, rights, version), with its rights in `COPYRIGHT.txt`
13- **Door43** ([`door43`]): a Bible's release, its repository's archive at the tag, a resource
14  container whose `manifest.yaml` describes it, its USFM a file per book
15- **SWORD** ([`sword`]): a module's zip from its repository, read with sword-modules from
16  memory: a Bible's or commentary's entries in their own markup (OSIS, ThML, or GBF), a file per
17  book, or a dictionary's as `entries/<module>.xml`, with its conf's copyright and license in
18  `COPYRIGHT.txt`
19
20Each publisher's list is read into a logoxide [`Catalog`](logoxide::catalog::Catalog) ([`catalog::parse`]), every entry a
21[`CatalogSource::External`] naming its import (`ebible:eng-asv`), the download it's made from,
22and what to fetch ([`downloads`]); [`build`] makes the package from the bytes. The package
23manager keeps only that, and knows no publisher.
24
25A new import is version 1.0.0 ([`IMPORT_VERSION`]), logoxide's own: the publisher's version (an
26update date, a module's `Version`) is kept in `[package.upstream]`, and each update raises the
27package's version by what it changes ([`update::compare_import`]), so `lx add` can ask for `^1`
28and updates cross the publisher's years.
29*/
30
31pub mod catalog;
32pub mod ccel;
33pub mod door43;
34pub mod ebible;
35pub mod fetch;
36pub mod license;
37pub mod sword;
38pub mod update;
39
40use logoxide::{
41    LogoxideError, Result,
42    catalog::{CatalogEntry, CatalogSource, Download},
43    manifest::{Format, FormatEntry, Keys, MANIFEST, Manifest, Package as Info, Upstream},
44    package::{Features, Package},
45    storage::{MemoryStorage, Storage},
46};
47
48pub use update::{
49    ImportUpdate, apply_import, compare_import, compare_publisher_versions, import_is_current,
50    import_removals, set_version,
51};
52
53/// Every publisher's import prefix: what `ebible:<id>` and `fetch:<id>` start with
54pub const PREFIXES: [&str; 5] = ["ebible", "fetch", "ccel", "sword", "door43"];
55
56/// The version a new import has: logoxide's own, which updates raise by what they change
57pub const IMPORT_VERSION: semver::Version = semver::Version::new(1, 0, 0);
58
59/// Each file in a `.zip`, by path (folders left out), within logoxide-text's limits
60pub fn unzip(bytes: &[u8]) -> Result<Vec<(String, Vec<u8>)>> {
61    Ok(logoxide_text::archive::unzip(bytes)?)
62}
63
64/**
65What importing `entry` downloads, so every host fetches the same files the same way: eBible.org's
66zip, fetch.bible's books (each named by its code), CCEL's ThML, a SWORD module's zip, or a
67Door43 release's archive (its URLs tried in turn). Packages from git or a registry aren't imported, which is an error
68*/
69pub fn downloads(entry: &CatalogEntry) -> Result<Vec<Download>> {
70    match &entry.source {
71        CatalogSource::External { downloads, .. } => Ok(downloads.clone()),
72        CatalogSource::Git { .. } | CatalogSource::Registry { .. } => Err(LogoxideError::Import(
73            format!("{} isn't imported: add it", entry.id),
74        )),
75    }
76}
77
78/// The package `entry`'s import makes, written into `dest`, from the bytes of each of its
79/// [`downloads`] (by name), whichever publisher it comes from
80pub fn build(entry: &CatalogEntry, files: &[(String, Vec<u8>)], dest: &dyn Storage) -> Result<()> {
81    let named = |name: &str| {
82        (files.iter())
83            .find(|(n, _)| n == name)
84            .map(|(_, bytes)| bytes.as_slice())
85            .ok_or_else(|| LogoxideError::Import(format!("{}: {name} wasn't downloaded", entry.id)))
86    };
87    match entry.source.prefix() {
88        Some("ebible") => ebible::import(entry, named("zip")?, dest),
89        Some("fetch") => fetch::import(entry, files, dest),
90        Some("ccel") => ccel::import(entry, named("thml")?, dest),
91        Some("sword") => sword::import(entry, named("zip")?, dest),
92        Some("door43") => door43::import(entry, named("archive")?, dest),
93        Some(other) => Err(LogoxideError::Import(format!(
94            "{}: no import reads `{other}:` downloads",
95            entry.id
96        ))),
97        None => Err(LogoxideError::Import(format!(
98            "{} isn't imported: add it",
99            entry.id
100        ))),
101    }
102}
103
104/**
105What an import of `entry` writes as its package's `[package.upstream]`: the download it was made
106from, its import name, and the publisher's version the entry lists, so an imported package names
107the catalog entry it came from ([`Catalog::find_upstream`](logoxide::catalog::Catalog::find_upstream)); none for packages that aren't
108imported
109*/
110pub fn upstream(entry: &CatalogEntry) -> Option<Upstream> {
111    let CatalogSource::External {
112        import, upstream, ..
113    } = &entry.source
114    else {
115        return None;
116    };
117    Some(Upstream {
118        url: upstream.clone(),
119        import: Some(import.clone()),
120        version: entry.version.clone(),
121        // A SWORD module's entries are made into OSIS or ThML files; the other publishers'
122        // files are kept as they are
123        converted: entry.source.prefix() == Some("sword"),
124        note: None,
125    })
126}
127
128/// The entry's [`CatalogSource::External`] fields for an import of `prefix:id`, with its
129/// download(s): what each publisher's list is read into
130pub(crate) fn external(
131    prefix: &str,
132    id: &str,
133    tracks_versions: bool,
134    downloads: Vec<Download>,
135) -> CatalogSource {
136    CatalogSource::External {
137        import: format!("{prefix}:{id}"),
138        upstream: (downloads.first()).and_then(|d| d.urls.first().cloned()),
139        tracks_versions,
140        downloads,
141    }
142}
143
144/// A package id from a publisher's (`eng-asv`, `engkjv2006`, `KJV`): lowercase letters, digits,
145/// `-`, `_`
146pub(crate) fn package_id(id: &str) -> String {
147    id.to_lowercase()
148        .chars()
149        .map(|c| match c {
150            'a'..='z' | '0'..='9' | '-' | '_' => c,
151            _ => '-',
152        })
153        .collect()
154}
155
156/// A publisher's id after its import prefix (`eng-asv` of `ebible:eng-asv`), checked to be the
157/// publisher's
158pub(crate) fn import_id<'a>(
159    entry: &'a CatalogEntry,
160    prefix: &str,
161    publisher: &str,
162) -> Result<&'a str> {
163    match entry.source.prefix() == Some(prefix) {
164        true => entry
165            .source
166            .import_id()
167            .ok_or_else(|| LogoxideError::Import(format!("{}: no import id", entry.id))),
168        false => Err(LogoxideError::Import(format!(
169            "{} isn't from {publisher}",
170            entry.id
171        ))),
172    }
173}
174
175/// How an import made its package
176pub(crate) struct Made {
177    /// The format its books are in, in a folder of that name
178    pub(crate) format: &'static str,
179    pub(crate) license_file: Option<String>,
180    pub(crate) publisher: &'static str,
181    pub(crate) homepage: String,
182    /// The versification the publisher names (SWORD's conf), else the package's `own`
183    pub(crate) versification: Option<String>,
184    /// For a dictionary, what its entries are keyed by
185    pub(crate) keys: Option<Keys>,
186}
187
188/// Writes the package's `logoxide.toml`, checks it opens with books (a Bible's or a commentary's)
189/// or entries (a dictionary's; a book needs neither), and copies it into `dest`
190pub(crate) fn finish(
191    entry: &CatalogEntry,
192    made: Made,
193    staged: MemoryStorage,
194    dest: &dyn Storage,
195) -> Result<()> {
196    use logoxide::language::{script_direction, script_of_tag};
197    // The direction is written only where the script doesn't say it
198    let script = (entry.script.clone()).or_else(|| script_of_tag(&entry.language));
199    let implied = script_direction(script.as_deref());
200    let info = Info {
201        id: entry.id.clone(),
202        name: entry.name.clone(),
203        abbreviation: entry.abbreviation.clone(),
204        authors: vec![],
205        description: entry.description.clone(),
206        aliases: entry.aliases.clone(),
207        version: IMPORT_VERSION,
208        kind: entry.kind,
209        language: entry.language.clone(),
210        language_name: entry.language_name.clone(),
211        script: entry.script.clone(),
212        direction: (entry.direction != implied).then_some(entry.direction),
213        // A license SPDX can't read (a publisher's odd version) is the publisher's own
214        license: (entry.license.clone()).map(|license| {
215            match logoxide::license::check_license(&license) {
216                Ok(()) => license,
217                Err(_) => format!("LicenseRef-{}", made.publisher.replace(' ', "-")),
218            }
219        }),
220        rights: (!entry.rights.is_empty()).then_some(entry.rights),
221        license_file: made.license_file,
222        publisher: Some(made.publisher.to_string()),
223        homepage: Some(made.homepage),
224        repository: None,
225        documentation: None,
226        readme: None,
227        keywords: entry.keywords.clone(),
228        categories: entry.categories.clone(),
229        about_books: entry.about_books.clone(),
230        // The work logoxide's table knows the import to be, never guessed from its title
231        work: (entry.work.clone()).or_else(|| logoxide::works::of_source(&entry.source)),
232        edition: entry.edition.clone(),
233        edition_year: entry.edition_year,
234        upstream: upstream(entry),
235        versification: made.versification.unwrap_or_else(|| String::from("own")),
236        maps_to: None,
237        keys: made.keys,
238        include: vec![],
239    };
240    // A dictionary's files are in `entries/`; every other format's folder is named for it
241    let folder = match Format::from_name(made.format) {
242        Some(Format::Dictionary) => "entries",
243        _ => made.format,
244    };
245    let manifest = Manifest {
246        package: Some(info),
247        formats: [(
248            made.format.to_string(),
249            FormatEntry {
250                path: format!("{folder}/"),
251                feature: None,
252            },
253        )]
254        .into(),
255        ..Manifest::default()
256    };
257    let text = toml::to_string(&manifest).map_err(|e| LogoxideError::Import(e.to_string()))?;
258    let header = format!("# Imported from {} by logoxide\n", made.publisher);
259    staged.write(MANIFEST, format!("{header}{text}").as_bytes())?;
260    let staged = std::sync::Arc::new(staged);
261    let package = Package::open(staged.clone(), Features::default())?;
262    let verses = (package.text_format()).is_some_and(|(f, _)| f.is_verse_text());
263    // A book (CCEL's ThML) isn't read as verses or entries: its references are found in it
264    let book = entry.kind == logoxide::manifest::PackageKind::Book;
265    if verses && package.books().is_empty() {
266        return Err(LogoxideError::Import(format!("{}: no books", entry.id)));
267    }
268    if !verses && !book && package.source().keys().is_empty() {
269        return Err(LogoxideError::Import(format!("{}: no entries", entry.id)));
270    }
271    for path in staged.walk("")? {
272        dest.write(&path, &staged.read(&path)?)?;
273    }
274    Ok(())
275}
276
277#[cfg(test)]
278pub(crate) mod tests {
279    use std::io::Write;
280
281    /// A zip of these files, as a publisher's download
282    pub(crate) fn zip(files: &[(&str, &str)]) -> Vec<u8> {
283        zip_bytes(
284            &(files.iter())
285                .map(|(path, text)| (*path, text.as_bytes()))
286                .collect::<Vec<_>>(),
287        )
288    }
289
290    pub(crate) fn zip_bytes(files: &[(&str, &[u8])]) -> Vec<u8> {
291        let mut writer = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
292        for (path, bytes) in files {
293            writer
294                .start_file(*path, zip::write::SimpleFileOptions::default())
295                .unwrap();
296            writer.write_all(bytes).unwrap();
297        }
298        writer.finish().unwrap().into_inner()
299    }
300
301    /**
302    Builds `entry`'s import into `dest`, and again elsewhere: the two must be the same bytes, so
303    a locked import made again on another machine has the checksum it was locked with
304    */
305    pub(crate) fn build_twice(
306        entry: &logoxide::catalog::CatalogEntry,
307        files: &[(String, Vec<u8>)],
308        dest: &dyn logoxide::storage::Storage,
309    ) {
310        super::build(entry, files, dest).unwrap();
311        use logoxide::storage::Storage;
312        let again = super::MemoryStorage::new();
313        super::build(entry, files, &again).unwrap();
314        assert_eq!(dest.walk("").unwrap(), again.walk("").unwrap());
315        for path in again.walk("").unwrap() {
316            assert_eq!(
317                dest.read(&path).unwrap(),
318                again.read(&path).unwrap(),
319                "{path}"
320            );
321        }
322    }
323
324    #[test]
325    fn what_isnt_imported_is_said() {
326        let catalog = logoxide::catalog::Catalog::parse_toml(
327            "[[package]]\nid = \"kjv\"\nname = \"KJV\"\nabbreviation = \"KJV\"\nlanguage = \"en\"\ngit = \"https://example.org/kjv\"\n",
328        )
329        .unwrap();
330        let entry = &catalog.entries[0];
331        assert!(super::downloads(entry).is_err());
332        assert!(super::build(entry, &[], &super::MemoryStorage::new()).is_err());
333        assert_eq!(super::upstream(entry), None);
334        assert_eq!(super::package_id("eng-ASV 1901"), "eng-asv-1901");
335    }
336}
logoxide_import/lib.rs

logoxide_import/
lib.rs
