1pub mod file_location;
2pub mod index_file;
3
4use super::IPFSNode;
5use crate::{
6 manifest::{self, GenericManifestFile, PackageManifestFile},
7 source::{
8 self,
9 ipfs::{ipfs_client, Cid},
10 },
11};
12use anyhow::{anyhow, bail, Context};
13use file_location::{location_from_root, Namespace};
14use forc_tracing::println_action_green;
15use index_file::IndexFile;
16use serde::{Deserialize, Serialize};
17use std::{
18 fmt::Display,
19 fs,
20 path::{Path, PathBuf},
21 str::FromStr,
22 thread,
23 time::Duration,
24};
25
26pub const REG_DIR_NAME: &str = "registry";
28
29#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Deserialize, Serialize)]
31pub struct Source {
32 pub name: String,
34 pub version: semver::Version,
36 pub namespace: Namespace,
39}
40
41#[derive(Clone, Debug, Eq, Hash, PartialEq, Deserialize, Serialize)]
43pub struct Pinned {
44 pub source: Source,
46 pub cid: Cid,
48}
49
50pub struct GithubRegistryResolver {
56 repo_org: String,
58 repo_name: String,
60 chunk_size: usize,
66 namespace: Namespace,
69 branch_name: String,
71}
72
73#[derive(Clone, Debug)]
75pub enum PinnedParseError {
76 Prefix,
77 PackageName,
78 PackageVersion,
79 Cid,
80 Namespace,
81}
82
83impl GithubRegistryResolver {
84 pub const DEFAULT_GITHUB_ORG: &str = "FuelLabs";
86 pub const DEFAULT_REPO_NAME: &str = "forc.pub-index";
88 pub const DEFAULT_CHUNKING_SIZE: usize = 2;
90 const DEFAULT_BRANCH_NAME: &str = "master";
92 const DEFAULT_TIMEOUT_MS: u64 = 10000;
95
96 pub fn new(
97 repo_org: String,
98 repo_name: String,
99 chunk_size: usize,
100 namespace: Namespace,
101 branch_name: String,
102 ) -> Self {
103 Self {
104 repo_org,
105 repo_name,
106 chunk_size,
107 namespace,
108 branch_name,
109 }
110 }
111
112 pub fn with_default_github(namespace: Namespace) -> Self {
115 Self {
116 repo_org: Self::DEFAULT_GITHUB_ORG.to_string(),
117 repo_name: Self::DEFAULT_REPO_NAME.to_string(),
118 chunk_size: Self::DEFAULT_CHUNKING_SIZE,
119 namespace,
120 branch_name: Self::DEFAULT_BRANCH_NAME.to_string(),
121 }
122 }
123
124 pub fn namespace(&self) -> &Namespace {
128 &self.namespace
129 }
130
131 pub fn branch_name(&self) -> &str {
135 &self.branch_name
136 }
137
138 pub fn chunk_size(&self) -> usize {
142 self.chunk_size
143 }
144
145 pub fn repo_org(&self) -> &str {
150 &self.repo_org
151 }
152
153 pub fn repo_name(&self) -> &str {
158 &self.repo_name
159 }
160}
161
162impl Pinned {
163 pub const PREFIX: &str = "registry";
164}
165
166impl Display for Pinned {
167 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
168 write!(
170 f,
171 "{}+{}?{}#{}!{}",
172 Self::PREFIX,
173 self.source.name,
174 self.source.version,
175 self.cid.0,
176 self.source.namespace
177 )
178 }
179}
180
181impl FromStr for Pinned {
182 type Err = PinnedParseError;
183
184 fn from_str(s: &str) -> Result<Self, Self::Err> {
185 let s = s.trim();
187
188 let prefix_plus = format!("{}+", Self::PREFIX);
190 if s.find(&prefix_plus).is_some_and(|loc| loc != 0) {
191 return Err(PinnedParseError::Prefix);
192 }
193
194 let without_prefix = &s[prefix_plus.len()..];
195
196 let pkg_name = without_prefix
198 .split('?')
199 .next()
200 .ok_or(PinnedParseError::PackageName)?;
201
202 let without_package_name = &without_prefix[pkg_name.len() + "?".len()..];
203 let mut s_iter = without_package_name.split('#');
204
205 let pkg_version = s_iter.next().ok_or(PinnedParseError::PackageVersion)?;
207 let pkg_version =
208 semver::Version::from_str(pkg_version).map_err(|_| PinnedParseError::PackageVersion)?;
209
210 let cid_and_namespace = s_iter.next().ok_or(PinnedParseError::Cid)?;
212 let mut s_iter = cid_and_namespace.split('!');
213
214 let cid = s_iter.next().ok_or(PinnedParseError::Cid)?;
215 if !validate_cid(cid) {
216 return Err(PinnedParseError::Cid);
217 }
218 let cid = Cid::from_str(cid).map_err(|_| PinnedParseError::Cid)?;
219
220 let namespace = s_iter
223 .next()
224 .filter(|ns| !ns.is_empty())
225 .map_or_else(|| Namespace::Flat, |ns| Namespace::Domain(ns.to_string()));
226
227 let source = Source {
228 name: pkg_name.to_string(),
229 version: pkg_version,
230 namespace,
231 };
232
233 Ok(Self { source, cid })
234 }
235}
236
237impl Display for Source {
238 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
239 write!(f, "{}+{}", self.name, self.version)
240 }
241}
242fn registry_dir() -> PathBuf {
243 forc_util::user_forc_directory().join(REG_DIR_NAME)
244}
245
246fn registry_with_namespace_dir(namespace: &Namespace) -> PathBuf {
247 let base = registry_dir();
248 match namespace {
249 Namespace::Flat => base,
250 Namespace::Domain(ns) => base.join(ns),
251 }
252}
253
254fn registry_package_dir(
255 namespace: &Namespace,
256 pkg_name: &str,
257 pkg_version: &semver::Version,
258) -> PathBuf {
259 registry_with_namespace_dir(namespace).join(format!("{pkg_name}-{pkg_version}"))
260}
261
262fn registry_package_dir_name(name: &str, pkg_version: &semver::Version) -> String {
264 use std::hash::{Hash, Hasher};
265 fn hash_version(pkg_version: &semver::Version) -> u64 {
266 let mut hasher = std::collections::hash_map::DefaultHasher::new();
267 pkg_version.hash(&mut hasher);
268 hasher.finish()
269 }
270 let package_ver_hash = hash_version(pkg_version);
271 format!("{name}-{package_ver_hash:x}")
272}
273
274fn validate_cid(cid: &str) -> bool {
283 let cid = cid.trim();
284 let starts_with_qm = cid.starts_with("Qm");
285 starts_with_qm && cid.len() == 46
286}
287
288fn tmp_registry_package_dir(
301 fetch_id: u64,
302 name: &str,
303 version: &semver::Version,
304 namespace: &Namespace,
305) -> PathBuf {
306 let repo_dir_name = format!(
307 "{:x}-{}",
308 fetch_id,
309 registry_package_dir_name(name, version)
310 );
311 registry_with_namespace_dir(namespace)
312 .join("tmp")
313 .join(repo_dir_name)
314}
315
316impl source::Pin for Source {
317 type Pinned = Pinned;
318 fn pin(&self, ctx: source::PinCtx) -> anyhow::Result<(Self::Pinned, PathBuf)> {
319 let pkg_name = ctx.name.to_string();
320 let fetch_id = ctx.fetch_id();
321 let source = self.clone();
322 let pkg_name = pkg_name.clone();
323
324 let cid = block_on_any_runtime(async move {
325 with_tmp_fetch_index(fetch_id, &pkg_name, &source, |index_file| {
326 let version = source.version.clone();
327 let pkg_name = pkg_name.clone();
328 async move {
329 let pkg_entry = index_file
330 .get(&version)
331 .ok_or_else(|| anyhow!("No {} found for {}", version, pkg_name))?;
332 Cid::from_str(pkg_entry.source_cid()).map_err(anyhow::Error::from)
333 }
334 })
335 .await
336 })?;
337
338 let path = registry_package_dir(&self.namespace, ctx.name, &self.version);
339 let pinned = Pinned {
340 source: self.clone(),
341 cid,
342 };
343 Ok((pinned, path))
344 }
345}
346
347impl source::Fetch for Pinned {
348 fn fetch(&self, ctx: source::PinCtx, path: &Path) -> anyhow::Result<PackageManifestFile> {
349 let mut lock = forc_util::path_lock(path)?;
351 {
358 let _guard = lock.write()?;
359 if !path.exists() {
360 println_action_green(
361 "Fetching",
362 &format!(
363 "{} {}",
364 ansiterm::Style::new().bold().paint(ctx.name),
365 self.source.version
366 ),
367 );
368 let pinned = self.clone();
369 let fetch_id = ctx.fetch_id();
370 let ipfs_node = ctx.ipfs_node().clone();
371
372 block_on_any_runtime(async move {
373 let node = match ipfs_node {
377 node if node == IPFSNode::public() => IPFSNode::fuel(),
378 node => node,
379 };
380 fetch(fetch_id, &pinned, &node).await
381 })?;
382 }
383 }
384 let path = {
385 let _guard = lock.read()?;
386 manifest::find_within(path, ctx.name())
387 .ok_or_else(|| anyhow!("failed to find package `{}` in {}", ctx.name(), self))?
388 };
389 PackageManifestFile::from_file(path)
390 }
391}
392
393impl source::DepPath for Pinned {
394 fn dep_path(&self, _name: &str) -> anyhow::Result<source::DependencyPath> {
395 bail!("dep_path: registry dependencies are not yet supported");
396 }
397}
398
399impl From<Pinned> for source::Pinned {
400 fn from(p: Pinned) -> Self {
401 Self::Registry(p)
402 }
403}
404
405fn resolve_to_cid(index_file: &IndexFile, pinned: &Pinned) -> anyhow::Result<Cid> {
408 let other_versions = index_file
409 .versions()
410 .filter(|ver| **ver != pinned.source.version)
411 .map(|ver| format!("{}.{}.{}", ver.major, ver.minor, ver.patch))
412 .collect::<Vec<_>>()
413 .join(",");
414
415 let package_entry = index_file.get(&pinned.source.version).ok_or_else(|| {
416 anyhow!(
417 "Version {} not found for {}. Other available versions: [{}]",
418 pinned.source.version,
419 pinned.source.name,
420 other_versions
421 )
422 })?;
423
424 let cid = Cid::from_str(package_entry.source_cid()).with_context(|| {
425 format!(
426 "Invalid CID {}v{}: `{}`",
427 package_entry.name(),
428 package_entry.version(),
429 package_entry.source_cid()
430 )
431 })?;
432 if package_entry.yanked() {
433 bail!(
434 "Version {} of {} is yanked. Other available versions: [{}]",
435 pinned.source.version,
436 pinned.source.name,
437 other_versions
438 );
439 }
440 Ok(cid)
441}
442
443async fn fetch(fetch_id: u64, pinned: &Pinned, ipfs_node: &IPFSNode) -> anyhow::Result<PathBuf> {
444 let path = with_tmp_fetch_index(
445 fetch_id,
446 &pinned.source.name,
447 &pinned.source,
448 |index_file| async move {
449 let path = registry_package_dir(
450 &pinned.source.namespace,
451 &pinned.source.name,
452 &pinned.source.version,
453 );
454 if path.exists() {
455 let _ = fs::remove_dir_all(&path);
456 }
457 fs::create_dir_all(&path)?;
458
459 let cid = resolve_to_cid(&index_file, pinned)?;
460
461 match ipfs_node {
462 IPFSNode::Local => {
463 println_action_green("Fetching", "with local IPFS node");
464 cid.fetch_with_client(&ipfs_client(), &path).await?;
465 }
466 IPFSNode::WithUrl(gateway_url) => {
467 println_action_green(
468 "Fetching",
469 &format!("from {}. Note: This can take several minutes.", gateway_url),
470 );
471 cid.fetch_with_gateway_url(gateway_url, &path).await?;
472 }
473 }
474
475 Ok(path)
476 },
477 )
478 .await?;
479 Ok(path)
480}
481
482async fn with_tmp_fetch_index<F, O, Fut>(
483 fetch_id: u64,
484 pkg_name: &str,
485 source: &Source,
486 f: F,
487) -> anyhow::Result<O>
488where
489 F: FnOnce(IndexFile) -> Fut,
490 Fut: std::future::Future<Output = anyhow::Result<O>>,
491{
492 let tmp_dir = tmp_registry_package_dir(fetch_id, pkg_name, &source.version, &source.namespace);
493 if tmp_dir.exists() {
494 let _ = std::fs::remove_dir_all(&tmp_dir);
495 }
496
497 let _cleanup_guard = scopeguard::guard(&tmp_dir, |dir| {
500 let _ = std::fs::remove_dir_all(dir);
501 });
502
503 let github_resolver = GithubRegistryResolver::with_default_github(source.namespace.clone());
504
505 let path = location_from_root(github_resolver.chunk_size, &source.namespace, pkg_name)
506 .display()
507 .to_string();
508 let index_repo_owner = github_resolver.repo_org();
509 let index_repo_name = github_resolver.repo_name();
510 let reference = format!("refs/heads/{}", github_resolver.branch_name());
511 let github_endpoint = format!(
512 "https://raw.githubusercontent.com/{index_repo_owner}/{index_repo_name}/{reference}/{path}"
513 );
514 let client = reqwest::Client::new();
515 let timeout_duration = Duration::from_millis(GithubRegistryResolver::DEFAULT_TIMEOUT_MS);
516 let index_response = client
517 .get(github_endpoint)
518 .timeout(timeout_duration)
519 .send()
520 .await
521 .map_err(|e| {
522 anyhow!(
523 "Failed to send request to github to obtain package index file from registry {e}"
524 )
525 })?
526 .error_for_status()
527 .map_err(|_| anyhow!("Failed to fetch {pkg_name}"))?;
528
529 let contents = index_response.text().await?;
530 let index_file: IndexFile = serde_json::from_str(&contents).with_context(|| {
531 format!(
532 "Unable to deserialize a github registry lookup response. Body was: \"{}\"",
533 contents
534 )
535 })?;
536
537 let res = f(index_file).await?;
538 Ok(res)
539}
540
541pub(crate) fn block_on_any_runtime<F>(future: F) -> F::Output
547where
548 F: std::future::Future + Send + 'static,
549 F::Output: Send + 'static,
550{
551 if tokio::runtime::Handle::try_current().is_ok() {
552 thread::spawn(move || {
554 let rt = tokio::runtime::Builder::new_current_thread()
555 .enable_all()
556 .build()
557 .unwrap();
558 rt.block_on(future)
559 })
560 .join()
561 .unwrap()
562 } else {
563 let rt = tokio::runtime::Builder::new_current_thread()
565 .enable_all()
566 .build()
567 .unwrap();
568 rt.block_on(future)
569 }
570}
571
572#[cfg(test)]
573mod tests {
574 use super::{file_location::Namespace, resolve_to_cid, Pinned, Source};
575 use crate::source::{
576 ipfs::Cid,
577 reg::index_file::{IndexFile, PackageEntry},
578 };
579 use std::str::FromStr;
580
581 #[test]
582 fn parse_pinned_entry_without_namespace() {
583 let pinned_str = "registry+core?0.0.1#QmdMVqLqpba2mMB5AUjYCxubC6tLGevQFunpBkbC2UbrKS!";
584 let pinned = Pinned::from_str(pinned_str).unwrap();
585
586 let expected_source = Source {
587 name: "core".to_string(),
588 version: semver::Version::new(0, 0, 1),
589 namespace: Namespace::Flat,
590 };
591
592 let cid = Cid::from_str("QmdMVqLqpba2mMB5AUjYCxubC6tLGevQFunpBkbC2UbrKS").unwrap();
593
594 let expected_pinned = Pinned {
595 source: expected_source,
596 cid,
597 };
598
599 assert_eq!(pinned, expected_pinned)
600 }
601
602 #[test]
603 fn parse_pinned_entry_with_namespace() {
604 let pinned_str =
605 "registry+core?0.0.1#QmdMVqLqpba2mMB5AUjYCxubC6tLGevQFunpBkbC2UbrKS!fuelnamespace";
606 let pinned = Pinned::from_str(pinned_str).unwrap();
607
608 let expected_source = Source {
609 name: "core".to_string(),
610 version: semver::Version::new(0, 0, 1),
611 namespace: Namespace::Domain("fuelnamespace".to_string()),
612 };
613
614 let cid = Cid::from_str("QmdMVqLqpba2mMB5AUjYCxubC6tLGevQFunpBkbC2UbrKS").unwrap();
615
616 let expected_pinned = Pinned {
617 source: expected_source,
618 cid,
619 };
620
621 assert_eq!(pinned, expected_pinned)
622 }
623
624 #[test]
625 fn test_resolve_to_cid() {
626 let mut index_file = IndexFile::default();
627
628 let valid_cid = "QmdMVqLqpba2mMB5AUjYCxubC6tLGevQFunpBkbC2UbrKS";
630 let valid_version = semver::Version::new(1, 0, 0);
631 let valid_entry = PackageEntry::new(
632 "test_package".to_string(),
633 valid_version.clone(),
634 valid_cid.to_string(),
635 None, vec![], false, );
639 index_file.insert(valid_entry);
640
641 let yanked_cid = "QmdMVqLqpba2mMB5AUjYCxubC6tLGevQFunpBkbC2UbrKR";
643 let yanked_version = semver::Version::new(0, 9, 0);
644 let yanked_entry = PackageEntry::new(
645 "test_package".to_string(),
646 yanked_version.clone(),
647 yanked_cid.to_string(),
648 None, vec![], true, );
652 index_file.insert(yanked_entry);
653
654 let other_cid = "QmdMVqLqpba2mMB5AUjYCxubC6tLGevQFunpBkbC2UbrKT";
656 let other_version = semver::Version::new(1, 1, 0);
657 let other_entry = PackageEntry::new(
658 "test_package".to_string(),
659 other_version.clone(),
660 other_cid.to_string(),
661 None, vec![], false, );
665 index_file.insert(other_entry);
666
667 let valid_source = Source {
669 name: "test_package".to_string(),
670 version: valid_version.clone(),
671 namespace: Namespace::Flat,
672 };
673 let valid_pinned = Pinned {
674 source: valid_source,
675 cid: Cid::from_str(valid_cid).unwrap(),
676 };
677
678 let result = resolve_to_cid(&index_file, &valid_pinned);
679 assert!(result.is_ok());
680 let valid_cid = Cid::from_str(valid_cid).unwrap();
681 assert_eq!(result.unwrap(), valid_cid);
682
683 let nonexistent_version = semver::Version::new(2, 0, 0);
685 let nonexistent_source = Source {
686 name: "test_package".to_string(),
687 version: nonexistent_version,
688 namespace: Namespace::Flat,
689 };
690 let nonexistent_pinned = Pinned {
691 source: nonexistent_source,
692 cid: valid_cid,
694 };
695
696 let result = resolve_to_cid(&index_file, &nonexistent_pinned);
697 assert!(result.is_err());
698 let error_msg = result.unwrap_err().to_string();
699 assert!(error_msg.contains("Version 2.0.0 not found"));
700 assert!(
701 error_msg.contains("Other available versions: [1.1.0,0.9.0,1.0.0]")
702 || error_msg.contains("Other available versions: [0.9.0,1.0.0,1.1.0]")
703 || error_msg.contains("Other available versions: [1.0.0,0.9.0,1.1.0]")
704 || error_msg.contains("Other available versions: [0.9.0,1.1.0,1.0.0]")
705 || error_msg.contains("Other available versions: [1.0.0,1.1.0,0.9.0]")
706 || error_msg.contains("Other available versions: [1.1.0,1.0.0,0.9.0]")
707 );
708
709 let yanked_source = Source {
711 name: "test_package".to_string(),
712 version: yanked_version.clone(),
713 namespace: Namespace::Flat,
714 };
715 let yanked_pinned = Pinned {
716 source: yanked_source,
717 cid: Cid::from_str(yanked_cid).unwrap(),
718 };
719
720 let result = resolve_to_cid(&index_file, &yanked_pinned);
721 assert!(result.is_err());
722 let error_msg = result.unwrap_err().to_string();
723 assert!(error_msg.contains("Version 0.9.0 of test_package is yanked"));
724 assert!(
725 error_msg.contains("Other available versions: [1.1.0,1.0.0]")
726 || error_msg.contains("Other available versions: [1.0.0,1.1.0]")
727 );
728 }
729}