logoxide_import/
door43.rs

1/*!
2Door43 (unfoldingWord's Door43 Content Service, git.door43.org): its catalog of Bibles
3([`catalog`], the catalog API's `search?subject=Bible&stage=prod`) and a Bible's release
4([`import`]): the repository's `archive/<tag>.tar.gz`, a resource container whose
5`manifest.yaml` describes it (`dublin_core`: title, language, rights, version) and whose `.usfm`
6files are its books.
7
8Written without the service answering (it timed out from the owner's machine in October 2026):
9the `manifest.yaml` shape is the resource container specification's, but the catalog API's JSON
10is as remembered (`tests/fixtures/door43/README.md`), so [`catalog`] reads it loosely: every field
11but `owner` and `name` may be missing.
12*/
13
14use logoxide::{
15    LogoxideError, Result,
16    catalog::{Catalog, CatalogEntry, Download},
17    language,
18    manifest::{Direction, PackageKind},
19    storage::{MemoryStorage, Storage},
20};
21use serde::Deserialize;
22
23use crate::{Made, external, finish, import_id, package_id};
24
25/// Door43's catalog of Bibles in production, as the catalog API lists them
26pub const CATALOG: &str = "https://git.door43.org/api/v1/catalog/search?subject=Bible&stage=prod";
27
28/// Where Door43 serves a repository's files at a tag, as a `.tar.gz`
29pub fn archive_url(repository: &str, tag: &str) -> String {
30    format!("https://git.door43.org/{repository}/archive/{tag}.tar.gz")
31}
32
33/// Whether `location` is Door43's catalog API (a search of it)
34pub fn is_catalog(location: &str) -> bool {
35    location
36        .to_ascii_lowercase()
37        .contains("door43.org/api/v1/catalog")
38}
39
40/// The catalog API's answer: its entries in `data`
41#[derive(Deserialize)]
42struct Search {
43    #[serde(default)]
44    data: Vec<Listed>,
45}
46
47/// One entry of the catalog API, the fields read
48#[derive(Deserialize)]
49struct Listed {
50    owner: String,
51    name: String,
52    #[serde(default)]
53    title: Option<String>,
54    #[serde(default)]
55    language: Option<String>,
56    #[serde(default)]
57    language_title: Option<String>,
58    #[serde(default)]
59    language_direction: Option<String>,
60    #[serde(default)]
61    subject: Option<String>,
62    #[serde(default)]
63    branch_or_tag_name: Option<String>,
64    #[serde(default)]
65    release: Option<Release>,
66    #[serde(default)]
67    tarbar_url: Option<String>,
68    #[serde(default)]
69    abbreviation: Option<String>,
70}
71
72#[derive(Deserialize)]
73struct Release {
74    #[serde(default)]
75    tag_name: Option<String>,
76}
77
78/// Whether a name is one of a Door43 owner's or repository's (`Door43-Catalog`, `hi_irv`)
79fn is_repository_name(name: &str) -> bool {
80    !name.is_empty()
81        && !name.starts_with(['.', '-'])
82        && (name.bytes()).all(|b| b.is_ascii_alphanumeric() || b"_-.".contains(&b))
83}
84
85/**
86Door43's catalog ([`CATALOG`]): each Bible (`subject` "Bible") a `door43:<owner>/<repo>` import
87of its release's archive, its id `door43-<owner>-<repo>`, its version the release's tag (which a
88newer release raises, so `tracks_versions`), named, languaged, and directed as the catalog says.
89An entry without a release, or whose owner or name isn't a repository's, is passed over
90*/
91pub fn catalog(text: &str) -> Result<Catalog> {
92    let search: Search = serde_json::from_str(text)
93        .map_err(|e| LogoxideError::Catalog(format!("Door43's catalog: {e}")))?;
94    let mut entries: Vec<CatalogEntry> = vec![];
95    for listed in search.data {
96        if listed.subject.as_deref() != Some("Bible") {
97            continue;
98        }
99        if !is_repository_name(&listed.owner) || !is_repository_name(&listed.name) {
100            continue;
101        }
102        let tag = (listed.release.and_then(|r| r.tag_name))
103            .or(listed.branch_or_tag_name)
104            .filter(|t| !t.is_empty() && !t.starts_with('-'));
105        let Some(tag) = tag else { continue };
106        let repository = format!("{}/{}", listed.owner, listed.name);
107        if (entries.iter()).any(|e| e.source.import_id() == Some(repository.as_str())) {
108            continue;
109        }
110        let mut urls = vec![archive_url(&repository, &tag)];
111        urls.extend(listed.tarbar_url.filter(|u| !urls.contains(u)));
112        let downloads = vec![Download {
113            name: String::from("archive"),
114            urls,
115        }];
116        let (language, language_name) = match listed.language.as_deref() {
117            Some(code) => {
118                let (tag, raw) = language::normalize_language(code);
119                (tag, raw.or(listed.language_title))
120            }
121            None => (String::from("und"), listed.language_title),
122        };
123        let direction = match listed.language_direction.as_deref() {
124            Some("rtl") => Direction::Rtl,
125            _ => Direction::Ltr,
126        };
127        // `hi_irv` is IRV, as the catalog says
128        let abbreviation = (listed.abbreviation.filter(|a| !a.is_empty()))
129            .unwrap_or_else(|| {
130                let name = &listed.name;
131                name.split_once('_')
132                    .map_or(name.as_str(), |(_, a)| a)
133                    .to_string()
134            })
135            .to_uppercase();
136        entries.push(CatalogEntry {
137            id: format!("door43-{}", package_id(&repository.replace('/', "-"))),
138            name: listed.title.unwrap_or_else(|| listed.name.clone()),
139            abbreviation: Some(abbreviation),
140            aliases: vec![],
141            language,
142            language_name,
143            script: None,
144            description: None,
145            license: None,
146            rights: Default::default(),
147            copyright: None,
148            version: Some(tag),
149            kind: PackageKind::Bible,
150            direction,
151            source: external("door43", &repository, true, downloads),
152            path: None,
153            releases: vec![],
154            keywords: vec![],
155            categories: vec![],
156            about_books: vec![],
157            work: None,
158            edition: None,
159            edition_year: None,
160            keys: None,
161        });
162    }
163    if entries.is_empty() {
164        return Err(LogoxideError::Catalog(String::from(
165            "Door43's catalog lists no Bibles (has its API changed?)",
166        )));
167    }
168    entries.sort_by(|a, b| a.id.cmp(&b.id));
169    logoxide::catalog::known_works(&mut entries);
170    Ok(Catalog {
171        name: String::from("Door43"),
172        entries,
173        works: vec![],
174        warnings: vec![],
175    })
176}
177
178/// A resource container's `manifest.yaml`, the fields read
179#[derive(Debug, Default, Deserialize)]
180pub struct ResourceManifest {
181    #[serde(default)]
182    pub dublin_core: DublinCore,
183}
184
185/// `manifest.yaml`'s `dublin_core`
186#[derive(Debug, Default, Deserialize)]
187pub struct DublinCore {
188    #[serde(default)]
189    pub title: Option<String>,
190    #[serde(default)]
191    pub description: Option<String>,
192    #[serde(default)]
193    pub language: Option<RcLanguage>,
194    /// A license's name (`CC BY-SA 4.0`)
195    #[serde(default)]
196    pub rights: Option<String>,
197    /// The resource's version (`3`), which may be written as a number
198    #[serde(default)]
199    pub version: Option<serde_yaml::Value>,
200    #[serde(default)]
201    pub publisher: Option<String>,
202}
203
204/// `dublin_core.language`
205#[derive(Debug, Default, Deserialize)]
206pub struct RcLanguage {
207    #[serde(default)]
208    pub identifier: Option<String>,
209    #[serde(default)]
210    pub title: Option<String>,
211    #[serde(default)]
212    pub direction: Option<String>,
213}
214
215impl DublinCore {
216    /// Its version as text (`3`, `1.2`), if it gives one
217    pub fn version(&self) -> Option<String> {
218        let version = match self.version.as_ref()? {
219            serde_yaml::Value::String(s) => s.trim().to_string(),
220            serde_yaml::Value::Number(n) => n.to_string(),
221            _ => return None,
222        };
223        (!version.is_empty()).then_some(version)
224    }
225}
226
227/// The license `dublin_core.rights` names, as SPDX: Door43's short names (`CC BY-SA 4.0` is
228/// `CC-BY-SA-4.0`, `CC0 1.0`), else as a statement ([`crate::license::license_from_text`])
229fn rights_license(rights: &str) -> Option<String> {
230    let spdx = rights.split_whitespace().collect::<Vec<_>>().join("-");
231    match logoxide::license::check_license(&spdx) {
232        Ok(()) if spdx.to_ascii_uppercase().starts_with("CC") => Some(spdx),
233        _ => crate::license::license_from_text(rights),
234    }
235}
236
237/**
238A Bible Door43 publishes, from its release's archive (the bytes at [`archive_url`]), as a
239package in `dest`: `logoxide.toml`, each `.usfm` file in `usfm/`, and `LICENSE.md` (the license
240file) when the archive has one. `manifest.yaml`'s `dublin_core` names it (`title`), gives its
241language and direction, its license (`rights`), and the upstream version (`version`, else the
242release's tag). An archive without a `manifest.yaml` or USFM is an error
243*/
244pub fn import(entry: &CatalogEntry, archive: &[u8], dest: &dyn Storage) -> Result<()> {
245    let repository = import_id(entry, "door43", "Door43")?;
246    let unpacked = MemoryStorage::new();
247    logoxide::net::archive::unpack(archive, &unpacked)?;
248    let manifest = unpacked.read_string("manifest.yaml").map_err(|_| {
249        LogoxideError::Import(format!(
250            "{repository}: no manifest.yaml (is it a resource container?)"
251        ))
252    })?;
253    let manifest: ResourceManifest = serde_yaml::from_str(&manifest)
254        .map_err(|e| LogoxideError::Import(format!("{repository}: manifest.yaml: {e}")))?;
255    let core = manifest.dublin_core;
256    let staged = MemoryStorage::new();
257    let mut license_file = None;
258    for path in unpacked.walk("")? {
259        let name = path.rsplit('/').next().unwrap_or(&path);
260        let lower = name.to_ascii_lowercase();
261        if lower.ends_with(".usfm") {
262            staged.write(&format!("usfm/{name}"), &unpacked.read(&path)?)?;
263        } else if path == "LICENSE.md" {
264            staged.write("LICENSE.md", &unpacked.read(&path)?)?;
265            license_file = Some(String::from("LICENSE.md"));
266        }
267    }
268    let mut entry = entry.clone();
269    if let Some(title) = core
270        .title
271        .as_deref()
272        .map(str::trim)
273        .filter(|t| !t.is_empty())
274        && title != entry.name
275    {
276        let listed = std::mem::replace(&mut entry.name, title.to_string());
277        entry.aliases.push(listed);
278    }
279    if let Some(description) = core.description.as_deref().map(str::trim) {
280        entry.description = (!description.is_empty()).then(|| description.to_string());
281    }
282    if let Some(rc) = core.language.as_ref() {
283        if let Some(code) = rc.identifier.as_deref() {
284            let (tag, raw) = language::normalize_language(code);
285            entry.language = tag;
286            entry.language_name = raw.or_else(|| rc.title.clone()).or(entry.language_name);
287        }
288        match rc.direction.as_deref() {
289            Some("rtl") => entry.direction = Direction::Rtl,
290            Some("ltr") => entry.direction = Direction::Ltr,
291            _ => {}
292        }
293    }
294    entry.license = (core.rights.as_deref())
295        .and_then(rights_license)
296        .or(entry.license);
297    entry.version = core.version().or(entry.version);
298    let made = Made {
299        format: "usfm",
300        license_file,
301        publisher: "Door43",
302        homepage: format!("https://git.door43.org/{repository}"),
303        versification: None,
304        keys: None,
305    };
306    finish(&entry, made, staged, dest)
307}
308
309#[cfg(test)]
310mod tests {
311    use super::*;
312    use logoxide::package::{Features, Package};
313
314    /// The catalog API's answer, as remembered (see the fixture's README)
315    const SEARCH: &str = include_str!("../tests/fixtures/door43/catalog-search.json");
316
317    const MANIFEST: &str = r#"---
318dublin_core:
319    conformsto: 'rc0.2'
320    contributor:
321        - 'A Contributor'
322    creator: 'Someone Or Organization'
323    description: 'One or two sentence description of the resource.'
324    format: 'text/usfm'
325    identifier: 'irv'
326    issued: '2019-12-17'
327    language:
328        identifier: 'hi'
329        title: 'हिन्दी'
330        direction: 'ltr'
331    modified: '2019-12-22T12:01:30-05:00'
332    publisher: 'Bridge Connectivity Solutions'
333    relation:
334        - 'hi/tn'
335    rights: 'CC BY-SA 4.0'
336    subject: 'Bible'
337    title: 'Indian Revised Version'
338    type: 'bundle'
339    version: 2
340checking:
341    checking_level: '3'
342projects:
343    -
344        categories:
345            - 'bible-nt'
346        identifier: 'jhn'
347        path: './44-JHN.usfm'
348        sort: 43
349        title: 'यूहन्ना'
350        versification: 'ufw'
351"#;
352
353    /// A `.tar.gz` of these files in one top folder, as Door43 serves a tag's archive
354    fn archive(files: &[(&str, &str)]) -> Vec<u8> {
355        let mut tar = tar::Builder::new(flate2::write::GzEncoder::new(
356            Vec::new(),
357            flate2::Compression::fast(),
358        ));
359        for (path, text) in files {
360            let mut header = tar::Header::new_gnu();
361            header.set_size(text.len() as u64);
362            header.set_mode(0o644);
363            header.set_cksum();
364            tar.append_data(&mut header, format!("hi_irv/{path}"), text.as_bytes())
365                .unwrap();
366        }
367        tar.into_inner().unwrap().finish().unwrap()
368    }
369
370    #[test]
371    fn the_catalog() {
372        assert!(is_catalog(CATALOG));
373        let catalog = crate::catalog::parse_text(CATALOG, SEARCH).unwrap();
374        assert_eq!(catalog.name, "Door43");
375        // Translation notes aren't a Bible
376        let ids: Vec<&str> = catalog.entries.iter().map(|e| e.id.as_str()).collect();
377        assert_eq!(
378            ids,
379            [
380                "door43-door43-catalog-ar_avd",
381                "door43-door43-catalog-hi_irv",
382                "door43-door43-catalog-ur-deva_irv"
383            ]
384        );
385        let irv = catalog.find_import("door43:Door43-Catalog/hi_irv").unwrap();
386        assert_eq!((irv.short_name(), irv.language.as_str()), ("IRV", "hi"));
387        assert_eq!(irv.version.as_deref(), Some("v2"));
388        assert!(irv.source.versions_track_updates());
389        assert_eq!(
390            irv.source.downloads()[0].urls,
391            ["https://git.door43.org/Door43-Catalog/hi_irv/archive/v2.tar.gz"]
392        );
393        let avd = catalog.find_import("door43:Door43-Catalog/ar_avd").unwrap();
394        assert_eq!(avd.direction, Direction::Rtl);
395        assert_eq!(crate::catalog::publisher_name(&avd.source), Some("Door43"));
396        assert!(crate::catalog::parse_text(CATALOG, "{\"ok\":true,\"data\":[]}").is_err());
397    }
398
399    #[test]
400    fn imports_a_release() {
401        let catalog = catalog(SEARCH).unwrap();
402        let entry = catalog.find_import("door43:Door43-Catalog/hi_irv").unwrap();
403        let usfm = "\\id JHN\n\\h यूहन्ना\n\\c 1\n\\p\n\\v 1 आदि में वचन था।\n\\v 2 यही आदि में परमेश्‍वर के साथ था।\n";
404        let bytes = archive(&[
405            ("manifest.yaml", MANIFEST),
406            ("LICENSE.md", "# License\n\nCC BY-SA 4.0\n"),
407            ("44-JHN.usfm", usfm),
408            ("media.yaml", "projects: []\n"),
409        ]);
410        let dest = MemoryStorage::new();
411        crate::tests::build_twice(entry, &[("archive".into(), bytes)], &dest);
412        assert_eq!(
413            dest.walk("").unwrap(),
414            ["LICENSE.md", "logoxide.toml", "usfm/44-JHN.usfm"]
415        );
416        let package = Package::open(std::sync::Arc::new(dest), Features::default()).unwrap();
417        let info = package.info();
418        assert_eq!(info.name, "Indian Revised Version");
419        assert_eq!(info.aliases, ["Indian Revised Version (IRV) Hindi - 2019"]);
420        assert_eq!(info.license.as_deref(), Some("CC-BY-SA-4.0"));
421        assert_eq!(info.license_file.as_deref(), Some("LICENSE.md"));
422        assert_eq!(info.version.to_string(), "1.0.0");
423        let upstream = info.upstream.as_ref().unwrap();
424        assert_eq!(
425            (upstream.import.as_deref(), upstream.version.as_deref()),
426            (Some("door43:Door43-Catalog/hi_irv"), Some("2"))
427        );
428        assert_eq!(
429            upstream.url.as_deref(),
430            Some("https://git.door43.org/Door43-Catalog/hi_irv/archive/v2.tar.gz")
431        );
432        assert_eq!(package.books().len(), 1);
433        // Without a manifest, it isn't a resource container
434        let bytes = archive(&[("44-JHN.usfm", usfm)]);
435        let error = import(entry, &bytes, &MemoryStorage::new()).unwrap_err();
436        assert!(error.to_string().contains("manifest.yaml"), "{error}");
437        // Without a version, the release's tag
438        let bare = MANIFEST.replace("    version: 2\n", "");
439        let dest = MemoryStorage::new();
440        import(
441            entry,
442            &archive(&[("manifest.yaml", &bare), ("44-JHN.usfm", usfm)]),
443            &dest,
444        )
445        .unwrap();
446        let package = Package::open(std::sync::Arc::new(dest), Features::default()).unwrap();
447        let upstream = package.info().upstream.clone().unwrap();
448        assert_eq!(upstream.version.as_deref(), Some("v2"));
449    }
450
451    #[test]
452    fn rights_as_licenses() {
453        assert_eq!(
454            rights_license("CC BY-SA 4.0").as_deref(),
455            Some("CC-BY-SA-4.0")
456        );
457        assert_eq!(rights_license("CC BY 4.0").as_deref(), Some("CC-BY-4.0"));
458        assert_eq!(
459            rights_license("Public Domain").as_deref(),
460            Some("LicenseRef-PublicDomain")
461        );
462        assert_eq!(rights_license("All rights reserved"), None);
463    }
464}
logoxide_import/door43.rs

logoxide_import/
door43.rs
