logoxide_import/
fetch.rs

1/*!
2fetch.bible: its manifest of translations ([`catalog`], `manifest.json`) and a translation's USX
3([`import`]), a file per book ([`book_url`]), with its attribution in `COPYRIGHT.txt`. Its
4versions are years, which don't follow its updates: an update is found by downloading
5*/
6
7use logoxide::{
8    LogoxideError, Result,
9    catalog::{Catalog, CatalogEntry, Download},
10    language,
11    license::{Derivatives, Rights},
12    manifest::{Direction, PackageKind},
13    storage::{MemoryStorage, Storage},
14};
15
16use crate::{Made, external, finish, import_id, package_id};
17
18/// Where fetch.bible serves a translation (its books are below it)
19pub fn url(id: &str) -> String {
20    format!("https://v1.fetch.bible/bibles/{id}/")
21}
22
23/// Where fetch.bible serves a translation's book as USX (`code` as its catalog lists it: `jhn`)
24pub fn book_url(id: &str, code: &str) -> String {
25    format!(
26        "https://v1.fetch.bible/bibles/{id}/usx/{}.usx",
27        code.to_ascii_lowercase()
28    )
29}
30
31/// fetch.bible's `manifest.json`: its translations, with their books and licenses, each a
32/// `fetch:<id>` import downloading every book's USX (named by its lowercase code)
33pub fn catalog(text: &str) -> Result<Catalog> {
34    use serde_json::Value;
35    let bad = |message: &str| LogoxideError::Catalog(format!("manifest.json: {message}"));
36    let json: Value = serde_json::from_str(text).map_err(|e| bad(&e.to_string()))?;
37    let bibles = json
38        .get("bibles")
39        .and_then(Value::as_object)
40        .ok_or_else(|| bad("no bibles"))?;
41    let ordered: Vec<String> = (json.get("books_ordered").and_then(Value::as_array))
42        .map(|books| {
43            books
44                .iter()
45                .filter_map(|b| Some(b.as_str()?.to_string()))
46                .collect()
47        })
48        .unwrap_or_default();
49    // The first 39 are the Old Testament's
50    let (ot, nt) = ordered.split_at(ordered.len().min(39));
51    let languages = json.get("languages");
52    let text_of = |v: Option<&Value>| {
53        v.and_then(Value::as_str)
54            .map(str::trim)
55            .filter(|s| !s.is_empty())
56            .map(String::from)
57    };
58    let books_of = |v: Option<&Value>, all: &[String]| match v {
59        Some(Value::Bool(true)) => all.to_vec(),
60        Some(Value::Array(list)) => list
61            .iter()
62            .filter_map(|b| Some(b.as_str()?.to_ascii_lowercase()))
63            .collect(),
64        _ => vec![],
65    };
66    let mut entries = vec![];
67    for (id, bible) in bibles {
68        let mut books = books_of(bible.get("books_ot"), ot);
69        books.extend(books_of(bible.get("books_nt"), nt));
70        if books.is_empty() {
71            continue;
72        }
73        let names = bible.get("name");
74        let name_of = |key: &str| text_of(names.and_then(|n| n.get(key)));
75        let code = id.split('_').next().unwrap_or(id).to_string();
76        let language_entry = languages.and_then(|l| l.get(&code));
77        let (tag, raw) = language::normalize_language(&code);
78        let copyright = bible.get("copyright");
79        let (license, rights) = (copyright.and_then(|c| c.get("licenses")))
80            .and_then(Value::as_array)
81            .and_then(|l| l.first())
82            .map(|l| license_of(l.get("license"), text_of(l.get("url"))))
83            .map_or((None, Default::default()), |(l, r)| (Some(l), r));
84        let abbreviation = name_of("english_abbrev")
85            .or_else(|| name_of("local_abbrev"))
86            .unwrap_or_else(|| id.to_uppercase());
87        let english = name_of("english");
88        let local = name_of("local");
89        // Every book's USX, each download named by its code; the upstream URL is the
90        // translation's folder, which they're below
91        let downloads: Vec<Download> = (books.iter())
92            .map(|code| Download {
93                name: code.clone(),
94                urls: vec![book_url(id, code)],
95            })
96            .collect();
97        let source = match external("fetch", id, false, downloads) {
98            logoxide::catalog::CatalogSource::External {
99                import,
100                tracks_versions,
101                downloads,
102                ..
103            } => logoxide::catalog::CatalogSource::External {
104                import,
105                upstream: Some(url(id)),
106                tracks_versions,
107                downloads,
108            },
109            other => other,
110        };
111        entries.push(CatalogEntry {
112            // Prefixed, as every import's id is: fetch.bible's ids are other catalogs' too
113            id: format!("fetch-{}", package_id(id)),
114            name: english
115                .clone()
116                .or_else(|| local.clone())
117                .unwrap_or_else(|| id.clone()),
118            aliases: local
119                .filter(|l| Some(l) != english.as_ref())
120                .into_iter()
121                .collect(),
122            abbreviation: Some(abbreviation),
123            language_name: text_of(language_entry.and_then(|l| l.get("english"))).or(raw),
124            script: language::script_of_tag(&tag),
125            language: tag,
126            description: None,
127            license,
128            rights,
129            copyright: text_of(copyright.and_then(|c| c.get("attribution"))),
130            version: (bible.get("year").and_then(Value::as_u64)).map(|y| y.to_string()),
131            kind: PackageKind::Bible,
132            direction: match bible.get("direction").and_then(Value::as_str) {
133                Some("rtl") => Direction::Rtl,
134                _ => Direction::Ltr,
135            },
136            source,
137            path: None,
138            releases: vec![],
139            keywords: vec![],
140            categories: vec![],
141            about_books: vec![],
142            work: None,
143            edition: None,
144            edition_year: None,
145            keys: None,
146        });
147    }
148    entries.sort_by(|a, b| a.id.cmp(&b.id));
149    logoxide::catalog::known_works(&mut entries);
150    Ok(Catalog {
151        name: String::from("fetch.bible"),
152        entries,
153        works: vec![],
154        warnings: vec![],
155    })
156}
157
158/**
159A fetch.bible license as SPDX: `cc-by-sa` with its URL's version is `CC-BY-SA-4.0`, `public` is
160public domain, and its own terms (an object of restrictions: `forbid_commercial`,
161`forbid_derivatives` (`true`, or `same-license`), `forbid_attributionless`) are
162`LicenseRef-fetch.bible`, with the rights they give
163*/
164fn license_of(license: Option<&serde_json::Value>, url: Option<String>) -> (String, Rights) {
165    let license = match license {
166        Some(serde_json::Value::Object(terms)) => {
167            let forbids = |key: &str| terms.get(key).and_then(serde_json::Value::as_bool);
168            let derivatives = match terms.get("forbid_derivatives") {
169                Some(serde_json::Value::Bool(true)) => Some(Derivatives::None),
170                Some(serde_json::Value::Bool(false)) => Some(Derivatives::Allowed),
171                Some(serde_json::Value::String(s)) if s == "same-license" => {
172                    Some(Derivatives::ShareAlike)
173                }
174                _ => None,
175            };
176            let rights = Rights {
177                // fetch.bible publishes only what may be shared
178                redistributable: Some(true),
179                derivatives,
180                commercial: forbids("forbid_commercial").map(|f| !f),
181                attribution: forbids("forbid_attributionless"),
182            };
183            return (String::from("LicenseRef-fetch.bible"), rights);
184        }
185        Some(license) => license.as_str(),
186        None => None,
187    };
188    let license = match license {
189        Some("public") => String::from("LicenseRef-PublicDomain"),
190        Some(cc) if cc.starts_with("cc-") => {
191            let version = (url.as_deref())
192                .and_then(|u| u.trim_end_matches('/').rsplit('/').next())
193                .filter(|v| v.chars().all(|c| c.is_ascii_digit() || c == '.'))
194                .unwrap_or("4.0")
195                .to_string();
196            format!("{}-{version}", cc.to_uppercase())
197        }
198        _ => String::from("LicenseRef-fetch.bible"),
199    };
200    (license, Rights::default())
201}
202
203/**
204A fetch.bible translation as a package in `dest`, from each of its books' USX (`books`: each
205book's code, as the entry's downloads name them, and the bytes at [`book_url`]): `logoxide.toml`,
206`usx/`, and `COPYRIGHT.txt` (its attribution and license, the license file)
207*/
208pub fn import(entry: &CatalogEntry, books: &[(String, Vec<u8>)], dest: &dyn Storage) -> Result<()> {
209    let id = import_id(entry, "fetch", "fetch.bible")?;
210    let staged = MemoryStorage::new();
211    for (code, data) in books {
212        if !data.starts_with(b"<?xml")
213            && !data.starts_with(b"<usx")
214            && !data.starts_with(b"\xef\xbb\xbf")
215        {
216            return Err(LogoxideError::Import(format!(
217                "{id}: {code} isn't USX (was it downloaded from {}?)",
218                book_url(id, code)
219            )));
220        }
221        staged.write(&format!("usx/{}.usx", code.to_ascii_uppercase()), data)?;
222    }
223    let mut license_file = None;
224    let notice = [entry.copyright.as_deref(), entry.license.as_deref()];
225    if notice.iter().any(Option::is_some) {
226        let text: Vec<&str> = notice.into_iter().flatten().collect();
227        staged.write("COPYRIGHT.txt", format!("{}\n", text.join("\n")).as_bytes())?;
228        license_file = Some(String::from("COPYRIGHT.txt"));
229    }
230    let made = Made {
231        format: "usx",
232        license_file,
233        publisher: "fetch.bible",
234        publisher_name: None,
235        homepage: String::from("https://fetch.bible/"),
236        versification: None,
237        keys: None,
238        note: None,
239    };
240    finish(entry, made, staged, dest)
241}
242
243#[cfg(test)]
244mod tests {
245    use super::*;
246    use logoxide::{
247        catalog::FETCH_BIBLE_CATALOG,
248        package::{Features, Package},
249    };
250
251    #[test]
252    fn reads_fetch_bibles_manifest() {
253        let json = r#"{"bibles": {
254            "eng_bsb": {"name": {"local": "Berean Standard Bible", "local_abbrev": "BSB", "english": "Berean Standard Bible", "english_abbrev": "BSB"},
255                "year": 2024, "direction": "ltr", "copyright": {"licenses": [{"license": "public", "url": ""}], "attribution": "BSB Publishing, LLC"},
256                "books_ot": true, "books_nt": true},
257            "spa_x": {"name": {"local": "Biblia", "local_abbrev": "", "english": "A Bible", "english_abbrev": "AB"},
258                "year": 2020, "direction": "ltr", "copyright": {"licenses": [{"license": "cc-by-sa", "url": "https://creativecommons.org/licenses/by-sa/3.0/"}], "attribution": "Someone"},
259                "books_ot": [], "books_nt": ["jhn", "rom"]},
260            "heb_y": {"name": {"local": "", "local_abbrev": "", "english": "Restricted", "english_abbrev": "R"},
261                "year": 2020, "direction": "rtl", "copyright": {"licenses": [{"license": {"forbid_commercial": true}, "url": ""}], "attribution": ""},
262                "books_ot": ["gen"], "books_nt": false},
263            "none_z": {"name": {"english": "Nothing"}, "books_ot": [], "books_nt": false}
264        },
265        "languages": {"eng": {"english": "English"}, "spa": {"english": "Spanish"}},
266        "books_ordered": ["gen","exo","lev","num","deu","jos","jdg","rut","1sa","2sa","1ki","2ki","1ch","2ch","ezr","neh","est","job","psa","pro","ecc","sng","isa","jer","lam","ezk","dan","hos","jol","amo","oba","jon","mic","nam","hab","zep","hag","zec","mal","mat","mrk","luk","jhn","act","rom","1co","2co","gal","eph","php","col","1th","2th","1ti","2ti","tit","phm","heb","jas","1pe","2pe","1jn","2jn","3jn","jud","rev"]}"#;
267        let catalog = crate::catalog::parse_text(FETCH_BIBLE_CATALOG, json).unwrap();
268        assert_eq!(
269            catalog.entries.len(),
270            3,
271            "a translation without books is left out"
272        );
273        let bsb = &catalog.entries[0];
274        assert_eq!(
275            (bsb.id.as_str(), bsb.short_name(), bsb.language.as_str()),
276            ("fetch-eng_bsb", "BSB", "en")
277        );
278        assert_eq!(bsb.license.as_deref(), Some("LicenseRef-PublicDomain"));
279        assert_eq!(bsb.source.import_name(), Some("fetch:eng_bsb"));
280        assert_eq!(
281            bsb.source.upstream_url(),
282            Some("https://v1.fetch.bible/bibles/eng_bsb/")
283        );
284        assert!(!bsb.source.versions_track_updates());
285        // Every book, by its code
286        let downloads = bsb.source.downloads();
287        assert_eq!((downloads.len(), downloads[42].name.as_str()), (66, "jhn"));
288        assert_eq!(
289            downloads[42].urls,
290            ["https://v1.fetch.bible/bibles/eng_bsb/usx/jhn.usx"]
291        );
292        // The work logoxide's table knows the import to be
293        assert_eq!(bsb.work.as_deref(), Some("bsb"));
294        let x = catalog
295            .entries
296            .iter()
297            .find(|e| e.id == "fetch-spa_x")
298            .unwrap();
299        assert_eq!(x.license.as_deref(), Some("CC-BY-SA-3.0"));
300        assert_eq!(x.aliases, ["Biblia"]);
301        assert_eq!(x.language_name.as_deref(), Some("Spanish"));
302        let y = catalog
303            .entries
304            .iter()
305            .find(|e| e.id == "fetch-heb_y")
306            .unwrap();
307        assert_eq!(
308            (y.license.as_deref(), y.direction),
309            (Some("LicenseRef-fetch.bible"), Direction::Rtl)
310        );
311        // Its own terms, as rights
312        assert_eq!(
313            y.rights,
314            Rights {
315                redistributable: Some(true),
316                derivatives: None,
317                commercial: Some(false),
318                attribution: None
319            }
320        );
321        assert_eq!(x.text_rights().derivatives, Some(Derivatives::ShareAlike));
322    }
323
324    #[test]
325    fn imports_fetch_bibles_books() {
326        let json = r#"{"bibles": {"eng_asv": {"name": {"english": "American Standard Version", "english_abbrev": "ASV"},
327            "year": 1901, "copyright": {"licenses": [{"license": "public", "url": ""}], "attribution": "Someone"},
328            "books_ot": [], "books_nt": ["jhn"]}},
329            "languages": {"eng": {"english": "English"}},
330            "books_ordered": ["gen","exo","lev","num","deu","jos","jdg","rut","1sa","2sa","1ki","2ki","1ch","2ch","ezr","neh","est","job","psa","pro","ecc","sng","isa","jer","lam","ezk","dan","hos","jol","amo","oba","jon","mic","nam","hab","zep","hag","zec","mal","mat","mrk","luk","jhn"]}"#;
331        let catalog = catalog(json).unwrap();
332        let entry = &catalog.entries[0];
333        let usx = r#"<?xml version="1.0" encoding="utf-8"?><usx version="3.0"><book code="JHN" style="id"/><chapter number="3" style="c" sid="JHN 3"/><para style="p"><verse number="16" style="v" sid="JHN 3:16"/>For God so loved the world.<verse eid="JHN 3:16"/></para></usx>"#;
334        let dest = MemoryStorage::new();
335        crate::tests::build_twice(entry, &[("jhn".into(), usx.as_bytes().to_vec())], &dest);
336        assert_eq!(
337            dest.walk("").unwrap(),
338            ["COPYRIGHT.txt", "logoxide.toml", "usx/JHN.usx"]
339        );
340        let package = Package::open(std::sync::Arc::new(dest), Features::default()).unwrap();
341        assert_eq!(package.books().len(), 1);
342        let info = package.info();
343        assert_eq!(info.work.as_deref(), Some("asv"));
344        let upstream = info.upstream.as_ref().unwrap();
345        assert_eq!(
346            (upstream.import.as_deref(), upstream.url.as_deref()),
347            (
348                Some("fetch:eng_asv"),
349                Some("https://v1.fetch.bible/bibles/eng_asv/")
350            )
351        );
352        assert!(!upstream.converted);
353        // A page in place of a book (fetch.bible answers unknown paths with its site)
354        let page = [("jhn".to_string(), b"<!DOCTYPE html>".to_vec())];
355        let error = import(entry, &page, &MemoryStorage::new()).unwrap_err();
356        assert!(error.to_string().contains("isn't USX"), "{error}");
357    }
358}
logoxide_import/fetch.rs

logoxide_import/
fetch.rs
