pop_common/
git.rs

1// SPDX-License-Identifier: GPL-3.0
2
3use crate::{api::ApiClient, errors::Error, polkadot_sdk::parse_latest_tag};
4use anyhow::Result;
5use git2::{
6	build::RepoBuilder, FetchOptions, IndexAddOption, RemoteCallbacks, Repository as GitRepository,
7	ResetType,
8};
9use git2_credentials::CredentialHandler;
10use std::{fs, path::Path, sync::LazyLock};
11use url::Url;
12
13/// A helper for handling Git operations.
14pub struct Git;
15impl Git {
16	/// Clone a Git repository.
17	///
18	/// # Arguments
19	/// * `url` - the URL of the repository to clone.
20	/// * `working_dir` - the target working directory.
21	/// * `reference` - an optional reference (revision/tag).
22	pub fn clone(url: &Url, working_dir: &Path, reference: Option<&str>) -> Result<()> {
23		let mut fo = FetchOptions::new();
24		if reference.is_none() {
25			fo.depth(1);
26		}
27		let mut repo = RepoBuilder::new();
28		repo.fetch_options(fo);
29		let repo = match repo.clone(url.as_str(), working_dir) {
30			Ok(repository) => repository,
31			Err(e) => match Self::ssh_clone(url, working_dir) {
32				Ok(repository) => repository,
33				Err(_) => return Err(e.into()),
34			},
35		};
36
37		if let Some(reference) = reference {
38			let object = repo
39				.revparse_single(reference)
40				.or_else(|_| repo.revparse_single(&format!("refs/tags/{}", reference)))
41				.or_else(|_| repo.revparse_single(&format!("refs/remotes/origin/{}", reference)))?;
42			repo.checkout_tree(&object, None)?;
43			repo.set_head_detached(object.id())?;
44		}
45		Ok(())
46	}
47
48	fn ssh_clone(url: &Url, working_dir: &Path) -> Result<GitRepository> {
49		let ssh_url = GitHub::convert_to_ssh_url(url);
50		// Prepare callback and fetch options.
51		let mut fo = FetchOptions::new();
52		Self::set_up_ssh_fetch_options(&mut fo)?;
53		// Prepare builder and clone.
54		let mut repo = RepoBuilder::new();
55		repo.fetch_options(fo);
56		Ok(repo.clone(&ssh_url, working_dir)?)
57	}
58
59	/// Clone a Git repository and degit it.
60	///
61	/// # Arguments
62	///
63	/// * `url` - the URL of the repository to clone.
64	/// * `target` - location where the repository will be cloned.
65	/// * `tag_version` - the specific tag or version of the repository to use
66	pub fn clone_and_degit(
67		url: &str,
68		target: &Path,
69		tag_version: Option<String>,
70	) -> Result<Option<String>> {
71		let repo = match GitRepository::clone(url, target) {
72			Ok(repo) => repo,
73			Err(_e) => Self::ssh_clone_and_degit(Url::parse(url).map_err(Error::from)?, target)?,
74		};
75
76		if let Some(tag_version) = tag_version {
77			let (object, reference) = repo.revparse_ext(&tag_version).expect("Object not found");
78			repo.checkout_tree(&object, None).expect("Failed to checkout");
79			match reference {
80				// gref is an actual reference like branches or tags
81				Some(gref) => repo.set_head(gref.name().unwrap()),
82				// this is a commit, not a reference
83				None => repo.set_head_detached(object.id()),
84			}
85			.expect("Failed to set HEAD");
86
87			let git_dir = repo.path();
88			fs::remove_dir_all(git_dir)?;
89			return Ok(Some(tag_version));
90		}
91
92		// fetch tags from remote
93		let release = Self::fetch_latest_tag(&repo);
94
95		let git_dir = repo.path();
96		fs::remove_dir_all(git_dir)?;
97		// Or by default the last one
98		Ok(release)
99	}
100
101	/// For users that have ssh configuration for cloning repositories.
102	fn ssh_clone_and_degit(url: Url, target: &Path) -> Result<GitRepository> {
103		let ssh_url = GitHub::convert_to_ssh_url(&url);
104		// Prepare callback and fetch options.
105		let mut fo = FetchOptions::new();
106		Self::set_up_ssh_fetch_options(&mut fo)?;
107		// Prepare builder and clone.
108		let mut builder = RepoBuilder::new();
109		builder.fetch_options(fo);
110		let repo = builder.clone(&ssh_url, target)?;
111		Ok(repo)
112	}
113
114	fn set_up_ssh_fetch_options(fo: &mut FetchOptions) -> Result<()> {
115		let mut callbacks = RemoteCallbacks::new();
116		let git_config = git2::Config::open_default()
117			.map_err(|e| Error::Config(format!("Cannot open git configuration: {}", e)))?;
118		let mut ch = CredentialHandler::new(git_config);
119		callbacks.credentials(move |url, username, allowed| {
120			ch.try_next_credential(url, username, allowed)
121		});
122
123		fo.remote_callbacks(callbacks);
124		Ok(())
125	}
126
127	/// Fetch the latest release from a repository
128	fn fetch_latest_tag(repo: &GitRepository) -> Option<String> {
129		let tags = repo.tag_names(None).ok()?;
130		parse_latest_tag(tags.iter().flatten().collect::<Vec<_>>())
131	}
132
133	/// Init a new git repository.
134	///
135	/// # Arguments
136	///
137	/// * `target` - location where the parachain will be created.
138	/// * `message` - message for first commit.
139	pub fn git_init(target: &Path, message: &str) -> Result<(), git2::Error> {
140		let repo = GitRepository::init(target)?;
141		let signature = repo.signature()?;
142
143		let mut index = repo.index()?;
144		index.add_all(["*"].iter(), IndexAddOption::DEFAULT, None)?;
145		let tree_id = index.write_tree()?;
146
147		let tree = repo.find_tree(tree_id)?;
148		let commit_id = repo.commit(Some("HEAD"), &signature, &signature, message, &tree, &[])?;
149
150		let commit_object = repo.find_object(commit_id, Some(git2::ObjectType::Commit))?;
151		repo.reset(&commit_object, ResetType::Hard, None)?;
152
153		Ok(())
154	}
155}
156
157/// A client for the GitHub REST API.
158pub(crate) static GITHUB_API_CLIENT: LazyLock<ApiClient> = LazyLock::new(|| {
159	// GitHub API: unauthenticated = 60 requests per hour, authenticated = 5,000 requests per hour,
160	// GitHub Actions = 1,000 requests per hour per repository
161	ApiClient::new(1, std::env::var("GITHUB_TOKEN").ok())
162});
163
164/// A helper for handling GitHub operations.
165pub struct GitHub {
166	/// The organization name.
167	pub org: String,
168	/// The repository name
169	pub name: String,
170	api: String,
171}
172
173impl GitHub {
174	const GITHUB: &'static str = "github.com";
175
176	/// Parse URL of a GitHub repository.
177	///
178	/// # Arguments
179	///
180	/// * `url` - the URL of the repository to clone.
181	pub fn parse(url: &str) -> Result<Self> {
182		let url = Url::parse(url)?;
183		Ok(Self::new(Self::org(&url)?, Self::name(&url)?))
184	}
185
186	/// Create a new [GitHub] instance.
187	///
188	/// # Arguments
189	/// * `org` - The organization name.
190	/// * `name` - The repository name.
191	pub(crate) fn new(org: impl Into<String>, name: impl Into<String>) -> Self {
192		Self { org: org.into(), name: name.into(), api: "https://api.github.com".into() }
193	}
194
195	// Overrides the api base url for testing
196	#[cfg(test)]
197	fn with_api(mut self, api: impl Into<String>) -> Self {
198		self.api = api.into();
199		self
200	}
201
202	/// Fetch the latest releases of the GitHub repository.
203	pub async fn releases(&self) -> Result<Vec<Release>> {
204		let url = self.api_releases_url();
205		let response = GITHUB_API_CLIENT.get(url).await?;
206		let mut releases = response.json::<Vec<Release>>().await?;
207
208		// Sort releases by `published_at` in descending order
209		releases.sort_by(|a, b| b.published_at.cmp(&a.published_at));
210		Ok(releases)
211	}
212
213	/// Retrieves the commit hash associated with a specified tag in a GitHub repository.
214	pub async fn get_commit_sha_from_release(&self, tag_name: &str) -> Result<String> {
215		let response = GITHUB_API_CLIENT.get(self.api_tag_information(tag_name)).await?;
216		let value = response.json::<serde_json::Value>().await?;
217		let commit = value
218			.get("object")
219			.and_then(|v| v.get("sha"))
220			.and_then(|v| v.as_str())
221			.map(|v| v.to_owned())
222			.ok_or(Error::Git("the github release tag sha was not found".to_string()))?;
223		Ok(commit)
224	}
225
226	/// Retrieves the license from the repository.
227	pub async fn get_repo_license(&self) -> Result<String> {
228		let url = self.api_license_url();
229		let response = GITHUB_API_CLIENT.get(url).await?;
230		let value = response.json::<serde_json::Value>().await?;
231		let license = value
232			.get("license")
233			.and_then(|v| v.get("spdx_id"))
234			.and_then(|v| v.as_str())
235			.map(|v| v.to_owned())
236			.ok_or(Error::Git("Unable to find license for GitHub repo".to_string()))?;
237		Ok(license)
238	}
239
240	fn api_releases_url(&self) -> String {
241		format!("{}/repos/{}/{}/releases", self.api, self.org, self.name)
242	}
243
244	fn api_tag_information(&self, tag_name: &str) -> String {
245		format!("{}/repos/{}/{}/git/ref/tags/{}", self.api, self.org, self.name, tag_name)
246	}
247
248	fn api_license_url(&self) -> String {
249		format!("{}/repos/{}/{}/license", self.api, self.org, self.name)
250	}
251
252	fn org(repo: &Url) -> Result<&str> {
253		let path_segments = repo
254			.path_segments()
255			.map(|c| c.collect::<Vec<_>>())
256			.expect("repository must have path segments");
257		Ok(path_segments.first().ok_or(Error::Git(
258			"the organization (or user) is missing from the github url".to_string(),
259		))?)
260	}
261
262	/// Determines the name of a repository from a URL.
263	///
264	/// # Arguments
265	/// * `repo` - the URL of the repository.
266	pub fn name(repo: &Url) -> Result<&str> {
267		let path_segments = repo
268			.path_segments()
269			.map(|c| c.collect::<Vec<_>>())
270			.expect("repository must have path segments");
271		Ok(path_segments
272			.get(1)
273			.ok_or(Error::Git("the repository name is missing from the github url".to_string()))?)
274	}
275
276	#[cfg(test)]
277	pub(crate) fn release(repo: &Url, tag: &str, artifact: &str) -> String {
278		format!("{}/releases/download/{tag}/{artifact}", repo.as_str())
279	}
280
281	pub(crate) fn convert_to_ssh_url(url: &Url) -> String {
282		format!("git@{}:{}.git", url.host_str().unwrap_or(Self::GITHUB), &url.path()[1..])
283	}
284}
285
286/// Represents the data of a GitHub release.
287#[derive(Debug, PartialEq, serde::Deserialize)]
288pub struct Release {
289	/// The name of the tag.
290	pub tag_name: String,
291	/// The name of the release.
292	pub name: String,
293	/// Whether to identify the release as a prerelease or a full release.
294	pub prerelease: bool,
295	/// The commit hash for the release.
296	pub commit: Option<String>,
297	/// When the release was published.
298	pub published_at: String,
299}
300
301/// A descriptor of a remote repository.
302#[derive(Debug, PartialEq)]
303pub struct Repository {
304	/// The url of the repository.
305	pub url: Url,
306	/// If applicable, the branch or tag to be used.
307	pub reference: Option<String>,
308	/// The name of a package within the repository. Defaults to the repository name.
309	pub package: String,
310}
311
312impl Repository {
313	/// Parses a url in the form of <https://github.com/org/repository?package#tag> into its component parts.
314	///
315	/// # Arguments
316	/// * `url` - The url to be parsed.
317	pub fn parse(url: &str) -> Result<Self, Error> {
318		let url = Url::parse(url)?;
319		let package = url.query();
320		let reference = url.fragment().map(|f| f.to_string());
321
322		let mut url = url.clone();
323		url.set_query(None);
324		url.set_fragment(None);
325
326		let package = match package {
327			Some(b) => b,
328			None => GitHub::name(&url)?,
329		}
330		.to_string();
331
332		Ok(Self { url, reference, package })
333	}
334}
335
336#[cfg(test)]
337mod tests {
338	use super::*;
339	use mockito::{Mock, Server};
340
341	const BASE_PARACHAIN: &str = "https://github.com/r0gue-io/base-parachain";
342	const POLKADOT_SDK: &str = "https://github.com/paritytech/polkadot-sdk";
343
344	async fn releases_mock(mock_server: &mut Server, repo: &GitHub, payload: &str) -> Mock {
345		mock_server
346			.mock("GET", format!("/repos/{}/{}/releases", repo.org, repo.name).as_str())
347			.with_status(200)
348			.with_header("content-type", "application/json")
349			.with_body(payload)
350			.create_async()
351			.await
352	}
353
354	async fn tag_mock(mock_server: &mut Server, repo: &GitHub, tag: &str, payload: &str) -> Mock {
355		mock_server
356			.mock("GET", format!("/repos/{}/{}/git/ref/tags/{tag}", repo.org, repo.name).as_str())
357			.with_status(200)
358			.with_header("content-type", "application/json")
359			.with_body(payload)
360			.create_async()
361			.await
362	}
363
364	async fn license_mock(mock_server: &mut Server, repo: &GitHub, payload: &str) -> Mock {
365		mock_server
366			.mock("GET", format!("/repos/{}/{}/license", repo.org, repo.name).as_str())
367			.with_status(200)
368			.with_header("content-type", "application/json")
369			.with_body(payload)
370			.create_async()
371			.await
372	}
373
374	#[tokio::test]
375	async fn test_get_latest_releases() -> Result<(), Box<dyn std::error::Error>> {
376		let mut mock_server = Server::new_async().await;
377
378		let expected_payload = r#"[{
379			"tag_name": "polkadot-v1.10.0",
380			"name": "Polkadot v1.10.0",
381			"prerelease": false,
382			"published_at": "2024-01-01T00:00:00Z"
383		  },
384		  {
385			"tag_name": "polkadot-v1.11.0",
386			"name": "Polkadot v1.11.0",
387			"prerelease": false,
388			"published_at": "2023-01-01T00:00:00Z"
389		  },
390		  {
391			"tag_name": "polkadot-v1.12.0",
392			"name": "Polkadot v1.12.0",
393			"prerelease": false,
394			"published_at": "2025-01-01T00:00:00Z"
395		  }
396		]"#;
397		let repo = GitHub::parse(BASE_PARACHAIN)?.with_api(&mock_server.url());
398		let mock = releases_mock(&mut mock_server, &repo, expected_payload).await;
399		let latest_release = repo.releases().await?;
400		assert_eq!(
401			latest_release,
402			vec![
403				Release {
404					tag_name: "polkadot-v1.12.0".to_string(),
405					name: "Polkadot v1.12.0".into(),
406					prerelease: false,
407					commit: None,
408					published_at: "2025-01-01T00:00:00Z".to_string()
409				},
410				Release {
411					tag_name: "polkadot-v1.10.0".to_string(),
412					name: "Polkadot v1.10.0".into(),
413					prerelease: false,
414					commit: None,
415					published_at: "2024-01-01T00:00:00Z".to_string()
416				},
417				Release {
418					tag_name: "polkadot-v1.11.0".to_string(),
419					name: "Polkadot v1.11.0".into(),
420					prerelease: false,
421					commit: None,
422					published_at: "2023-01-01T00:00:00Z".to_string()
423				}
424			]
425		);
426		mock.assert_async().await;
427		Ok(())
428	}
429
430	#[tokio::test]
431	async fn get_releases_with_commit_sha() -> Result<(), Box<dyn std::error::Error>> {
432		let mut mock_server = Server::new_async().await;
433
434		let expected_payload = r#"{
435			"ref": "refs/tags/polkadot-v1.11.0",
436			"node_id": "REF_kwDOKDT1SrpyZWZzL3RhZ3MvcG9sa2Fkb3QtdjEuMTEuMA",
437			"url": "https://api.github.com/repos/paritytech/polkadot-sdk/git/refs/tags/polkadot-v1.11.0",
438			"object": {
439				"sha": "0bb6249268c0b77d2834640b84cb52fdd3d7e860",
440				"type": "commit",
441				"url": "https://api.github.com/repos/paritytech/polkadot-sdk/git/commits/0bb6249268c0b77d2834640b84cb52fdd3d7e860"
442			}
443		  }"#;
444		let repo = GitHub::parse(BASE_PARACHAIN)?.with_api(&mock_server.url());
445		let mock = tag_mock(&mut mock_server, &repo, "polkadot-v1.11.0", expected_payload).await;
446		let hash = repo.get_commit_sha_from_release("polkadot-v1.11.0").await?;
447		assert_eq!(hash, "0bb6249268c0b77d2834640b84cb52fdd3d7e860");
448		mock.assert_async().await;
449		Ok(())
450	}
451
452	#[tokio::test]
453	async fn get_repo_license() -> Result<(), Box<dyn std::error::Error>> {
454		let mut mock_server = Server::new_async().await;
455
456		let expected_payload = r#"{
457			"license": {
458			"key":"unlicense",
459			"name":"The Unlicense",
460			"spdx_id":"Unlicense",
461			"url":"https://api.github.com/licenses/unlicense",
462			"node_id":"MDc6TGljZW5zZTE1"
463			}
464		}"#;
465		let repo = GitHub::parse(BASE_PARACHAIN)?.with_api(&mock_server.url());
466		let mock = license_mock(&mut mock_server, &repo, expected_payload).await;
467		let license = repo.get_repo_license().await?;
468		assert_eq!(license, "Unlicense".to_string());
469		mock.assert_async().await;
470		Ok(())
471	}
472
473	#[test]
474	fn test_get_releases_api_url() -> Result<(), Box<dyn std::error::Error>> {
475		assert_eq!(
476			GitHub::parse(POLKADOT_SDK)?.api_releases_url(),
477			"https://api.github.com/repos/paritytech/polkadot-sdk/releases"
478		);
479		Ok(())
480	}
481
482	#[test]
483	fn test_url_api_tag_information() -> Result<(), Box<dyn std::error::Error>> {
484		assert_eq!(
485			GitHub::parse(POLKADOT_SDK)?.api_tag_information("polkadot-v1.11.0"),
486			"https://api.github.com/repos/paritytech/polkadot-sdk/git/ref/tags/polkadot-v1.11.0"
487		);
488		Ok(())
489	}
490
491	#[test]
492	fn test_api_license_url() -> Result<(), Box<dyn std::error::Error>> {
493		assert_eq!(
494			GitHub::parse(POLKADOT_SDK)?.api_license_url(),
495			"https://api.github.com/repos/paritytech/polkadot-sdk/license"
496		);
497		Ok(())
498	}
499
500	#[test]
501	fn test_parse_org() -> Result<(), Box<dyn std::error::Error>> {
502		assert_eq!(GitHub::parse(BASE_PARACHAIN)?.org, "r0gue-io");
503		Ok(())
504	}
505
506	#[test]
507	fn test_parse_name() -> Result<(), Box<dyn std::error::Error>> {
508		let url = Url::parse(BASE_PARACHAIN)?;
509		let name = GitHub::name(&url)?;
510		assert_eq!(name, "base-parachain");
511		Ok(())
512	}
513
514	#[test]
515	fn test_release_url() -> Result<(), Box<dyn std::error::Error>> {
516		let repo = Url::parse(POLKADOT_SDK)?;
517		let url = GitHub::release(&repo, "polkadot-v1.9.0", "polkadot");
518		assert_eq!(url, format!("{}/releases/download/polkadot-v1.9.0/polkadot", POLKADOT_SDK));
519		Ok(())
520	}
521
522	#[test]
523	fn test_convert_to_ssh_url() {
524		assert_eq!(
525			GitHub::convert_to_ssh_url(&Url::parse(BASE_PARACHAIN).expect("valid repository url")),
526			"git@github.com:r0gue-io/base-parachain.git"
527		);
528		assert_eq!(
529			GitHub::convert_to_ssh_url(
530				&Url::parse("https://github.com/paritytech/substrate-contracts-node")
531					.expect("valid repository url")
532			),
533			"git@github.com:paritytech/substrate-contracts-node.git"
534		);
535		assert_eq!(
536			GitHub::convert_to_ssh_url(
537				&Url::parse("https://github.com/paritytech/frontier-parachain-template")
538					.expect("valid repository url")
539			),
540			"git@github.com:paritytech/frontier-parachain-template.git"
541		);
542	}
543
544	mod repository {
545		use super::Error;
546		use crate::git::Repository;
547		use url::Url;
548
549		#[test]
550		fn parsing_full_url_works() {
551			assert_eq!(
552				Repository::parse("https://github.com/org/repository?package#tag").unwrap(),
553				Repository {
554					url: Url::parse("https://github.com/org/repository").unwrap(),
555					reference: Some("tag".into()),
556					package: "package".into(),
557				}
558			);
559		}
560
561		#[test]
562		fn parsing_simple_url_works() {
563			let url = "https://github.com/org/repository";
564			assert_eq!(
565				Repository::parse(url).unwrap(),
566				Repository {
567					url: Url::parse(url).unwrap(),
568					reference: None,
569					package: "repository".into(),
570				}
571			);
572		}
573
574		#[test]
575		fn parsing_invalid_url_returns_error() {
576			assert!(matches!(
577				Repository::parse("github.com/org/repository"),
578				Err(Error::ParseError(..))
579			));
580		}
581	}
582}