ctranslate2_src_build_support/
submodules.rs

1use base64::Engine;
2use serde::Deserialize;
3use std::collections::HashMap;
4use std::fs::File;
5use std::io::BufReader;
6use std::path::{Path, PathBuf};
7use std::process::Command;
8
9#[derive(Debug, Deserialize)]
10struct GitRef {
11    object: GitObject,
12}
13
14#[derive(Debug, Deserialize)]
15struct GitObject {
16    sha: String,
17}
18
19#[derive(Debug, Deserialize)]
20struct Tree {
21    tree: Vec<TreeEntry>,
22}
23
24#[derive(Debug, Deserialize)]
25struct TreeEntry {
26    path: String,
27    mode: String,
28    sha: String,
29}
30
31#[derive(Debug, Deserialize)]
32struct GitModuleFile {
33    content: String,
34}
35
36pub fn get_submodules(
37    owner: &str,
38    repo: &str,
39    tag: &str,
40) -> Result<Vec<(String, String, String)>, Box<dyn std::error::Error>> {
41    let ref_url = format!(
42        "https://api.github.com/repos/{}/{}/git/ref/tags/{}",
43        owner, repo, tag
44    );
45    let git_ref: GitRef = {
46        let resp = ureq::get(&ref_url)
47            .header("User-Agent", "rust-submodule-fetcher")
48            .call()?;
49        serde_json::from_reader(BufReader::new(resp.into_body().into_reader()))?
50    };
51    let commit_sha = git_ref.object.sha;
52
53    let tree_url = format!(
54        "https://api.github.com/repos/{}/{}/git/trees/{}?recursive=1",
55        owner, repo, commit_sha
56    );
57    let tree: Tree = {
58        let resp = ureq::get(&tree_url)
59            .header("User-Agent", "rust-submodule-fetcher")
60            .call()?;
61        serde_json::from_reader(BufReader::new(resp.into_body().into_reader()))?
62    };
63
64    let gitmodules_url = format!(
65        "https://api.github.com/repos/{}/{}/contents/.gitmodules?ref={}",
66        owner, repo, tag
67    );
68    let gitmodules_file: GitModuleFile = {
69        let resp = ureq::get(&gitmodules_url)
70            .header("User-Agent", "rust-submodule-fetcher")
71            .call()?;
72        serde_json::from_reader(BufReader::new(resp.into_body().into_reader()))?
73    };
74
75    let decoded = base64::engine::general_purpose::STANDARD
76        .decode(gitmodules_file.content.replace("\n", ""))?;
77    let gitmodules_str = String::from_utf8(decoded)?;
78
79    let mut path_to_url = HashMap::new();
80    let mut current_path = String::new();
81    for line in gitmodules_str.lines() {
82        if line.trim().starts_with("[submodule") {
83            current_path.clear();
84        } else if let Some(rest) = line.trim().strip_prefix("path = ") {
85            current_path = rest.to_string();
86        } else if let Some(rest) = line.trim().strip_prefix("url = ") {
87            path_to_url.insert(current_path.clone(), rest.to_string());
88        }
89    }
90
91    let submodules = tree
92        .tree
93        .into_iter()
94        .filter(|entry| entry.mode == "160000") // gitlink indicates submodule
95        .map(|entry| {
96            let url = path_to_url.get(&entry.path).cloned().unwrap_or_default();
97            (entry.path, entry.sha, url)
98        })
99        .collect();
100
101    Ok(submodules)
102}
103
104pub fn get_submodules_helper(path: &Path, version: &str) -> Vec<PathBuf> {
105    let p = path.join(format!("CTranslate2-{version}"));
106    let f = p.join("submodules_downloaded");
107    if f.exists() {
108        return vec![];
109    }
110    let submodules = get_submodules("OpenNMT", "CTranslate2", &format!("v{version}")).unwrap();
111    let mut modules = vec![];
112    for (path, sha, url) in submodules {
113        let submodule_path = p.join(path);
114        modules.push(submodule_path.clone());
115        let status = Command::new("git")
116            .args([
117                "clone",
118                "--recurse-submodules",
119                "--no-checkout",
120                &url,
121                submodule_path.to_str().unwrap(),
122            ])
123            .status()
124            .expect("git clone failed");
125        assert!(status.success());
126
127        let status = Command::new("git")
128            .current_dir(&submodule_path)
129            .args(["checkout", &sha])
130            .status()
131            .expect("git checkout failed");
132        assert!(status.success());
133        let status = Command::new("git")
134            .current_dir(&submodule_path)
135            .args(["submodule", "update", "--init", "--recursive"])
136            .status()
137            .expect("git submodule update failed");
138        assert!(status.success());
139    }
140    File::create(f).unwrap();
141    modules
142}
ctranslate2_src_build_support/submodules.rs

ctranslate2_src_build_support/
submodules.rs
