forc_pkg/source/reg/
mod.rs

1pub mod file_location;
2pub mod index_file;
3
4use super::IPFSNode;
5use crate::{
6    manifest::{self, GenericManifestFile, PackageManifestFile},
7    source::{
8        self,
9        ipfs::{ipfs_client, Cid},
10    },
11};
12use anyhow::{anyhow, bail, Context};
13use file_location::{location_from_root, Namespace};
14use forc_tracing::println_action_green;
15use index_file::IndexFile;
16use serde::{Deserialize, Serialize};
17use std::{
18    fmt::Display,
19    fs,
20    path::{Path, PathBuf},
21    str::FromStr,
22    time::Duration,
23};
24
25/// Name of the folder containing fetched registry sources.
26pub const REG_DIR_NAME: &str = "registry";
27
28/// A package from the official registry.
29#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Deserialize, Serialize)]
30pub struct Source {
31    /// The name of the specified package.
32    pub name: String,
33    /// The base version specified for the package.
34    pub version: semver::Version,
35    /// The namespace this package resides in, if no there is no namespace in
36    /// registry setup, this will be `None`.
37    pub namespace: Namespace,
38}
39
40/// A pinned instance of the registry source.
41#[derive(Clone, Debug, Eq, Hash, PartialEq, Deserialize, Serialize)]
42pub struct Pinned {
43    /// The registry package with base version.
44    pub source: Source,
45    /// The corresponding CID for this registry entry.
46    pub cid: Cid,
47}
48
49/// A resolver for registry index hosted as a github repo.
50///
51/// Given a package name and a version, a `GithubRegistryResolver` will be able
52/// to resolve, fetch, pin a package through using the index hosted on a github
53/// repository.
54pub struct GithubRegistryResolver {
55    /// Name of the github organization holding the registry index repository.
56    repo_org: String,
57    /// Name of git repository holding the registry index.
58    repo_name: String,
59    /// The number of letters used to chunk package name.
60    ///
61    /// Example:
62    /// If set to 2, and package name is "foobar", the index file location
63    /// will be ".../fo/ob/ar/foobar".
64    chunk_size: usize,
65    /// Type of the namespacing is needed to determine whether to add domain at
66    /// the beginning of the file location.
67    namespace: Namespace,
68    /// Branch name of the registry repo, the resolver is going to be using.
69    branch_name: String,
70}
71
72/// Error returned upon failed parsing of `Pinned::from_str`.
73#[derive(Clone, Debug)]
74pub enum PinnedParseError {
75    Prefix,
76    PackageName,
77    PackageVersion,
78    Cid,
79    Namespace,
80}
81
82impl GithubRegistryResolver {
83    /// Default github organization name that holds the registry git repo.
84    pub const DEFAULT_GITHUB_ORG: &str = "FuelLabs";
85    /// Default name of the repository that holds the registry git repo.
86    pub const DEFAULT_REPO_NAME: &str = "forc.pub-index";
87    /// Default chunking size of the repository that holds registry git repo.
88    pub const DEFAULT_CHUNKING_SIZE: usize = 2;
89    /// Default branch name for the repository repo.
90    const DEFAULT_BRANCH_NAME: &str = "master";
91    /// Default timeout for each github look-up request. If exceeded request is
92    /// dropped.
93    const DEFAULT_TIMEOUT_MS: u64 = 10000;
94
95    pub fn new(
96        repo_org: String,
97        repo_name: String,
98        chunk_size: usize,
99        namespace: Namespace,
100        branch_name: String,
101    ) -> Self {
102        Self {
103            repo_org,
104            repo_name,
105            chunk_size,
106            namespace,
107            branch_name,
108        }
109    }
110
111    /// Returns a `GithubRegistryResolver` that automatically uses
112    /// `Self::DEFAULT_GITHUB_ORG` and `Self::DEFAULT_REPO_NAME`.
113    pub fn with_default_github(namespace: Namespace) -> Self {
114        Self {
115            repo_org: Self::DEFAULT_GITHUB_ORG.to_string(),
116            repo_name: Self::DEFAULT_REPO_NAME.to_string(),
117            chunk_size: Self::DEFAULT_CHUNKING_SIZE,
118            namespace,
119            branch_name: Self::DEFAULT_BRANCH_NAME.to_string(),
120        }
121    }
122
123    /// Returns the namespace associated with this `GithubRegistryResolver`.
124    ///
125    /// See `[GithubRegistryResolver::namespace]` for details.
126    pub fn namespace(&self) -> &Namespace {
127        &self.namespace
128    }
129
130    /// Returns the branch name used by this `GithubRegistryResolver`.
131    ///
132    /// See `[GithubRegistryResolver::branch_name]` for details.
133    pub fn branch_name(&self) -> &str {
134        &self.branch_name
135    }
136
137    /// Returns the chunk size used by this `GithubRegistryResolver`.
138    ///
139    /// See `[GithubRegistryResolver::chunk_size]` for details.
140    pub fn chunk_size(&self) -> usize {
141        self.chunk_size
142    }
143
144    /// Returns the owner of the repo this `GithubRegistryResolver` configured
145    /// to fetch from.
146    ///
147    /// See `[GithubRegistryResolver::repo_org]` for details.
148    pub fn repo_org(&self) -> &str {
149        &self.repo_org
150    }
151
152    /// Returns the name of the repo this `GithubRegistryResolver` configured
153    /// to fetch from.
154    ///
155    /// See `[GithubRegistryResolver::repo_name]` for details.
156    pub fn repo_name(&self) -> &str {
157        &self.repo_name
158    }
159}
160
161impl Pinned {
162    pub const PREFIX: &str = "registry";
163}
164
165impl Display for Pinned {
166    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
167        // registry+<package_name>?v<version>#<cid>!namespace
168        write!(
169            f,
170            "{}+{}?{}#{}!{}",
171            Self::PREFIX,
172            self.source.name,
173            self.source.version,
174            self.cid.0,
175            self.source.namespace
176        )
177    }
178}
179
180impl FromStr for Pinned {
181    type Err = PinnedParseError;
182
183    fn from_str(s: &str) -> Result<Self, Self::Err> {
184        // registry+<package_name>?v<version>#<cid>!<namespace>
185        let s = s.trim();
186
187        // Check for "registry+" at the start.
188        let prefix_plus = format!("{}+", Self::PREFIX);
189        if s.find(&prefix_plus).is_some_and(|loc| loc != 0) {
190            return Err(PinnedParseError::Prefix);
191        }
192
193        let without_prefix = &s[prefix_plus.len()..];
194
195        // Parse the package name.
196        let pkg_name = without_prefix
197            .split('?')
198            .next()
199            .ok_or(PinnedParseError::PackageName)?;
200
201        let without_package_name = &without_prefix[pkg_name.len() + "?".len()..];
202        let mut s_iter = without_package_name.split('#');
203
204        // Parse the package version
205        let pkg_version = s_iter.next().ok_or(PinnedParseError::PackageVersion)?;
206        let pkg_version =
207            semver::Version::from_str(pkg_version).map_err(|_| PinnedParseError::PackageVersion)?;
208
209        // Parse the CID and namespace.
210        let cid_and_namespace = s_iter.next().ok_or(PinnedParseError::Cid)?;
211        let mut s_iter = cid_and_namespace.split('!');
212
213        let cid = s_iter.next().ok_or(PinnedParseError::Cid)?;
214        if !validate_cid(cid) {
215            return Err(PinnedParseError::Cid);
216        }
217        let cid = Cid::from_str(cid).map_err(|_| PinnedParseError::Cid)?;
218
219        // If there is a namespace string after ! and if it is not empty
220        // get a `Namespace::Domain` otherwise return a `Namespace::Flat`.
221        let namespace = s_iter
222            .next()
223            .filter(|ns| !ns.is_empty())
224            .map_or_else(|| Namespace::Flat, |ns| Namespace::Domain(ns.to_string()));
225
226        let source = Source {
227            name: pkg_name.to_string(),
228            version: pkg_version,
229            namespace,
230        };
231
232        Ok(Self { source, cid })
233    }
234}
235
236impl Display for Source {
237    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
238        write!(f, "{}+{}", self.name, self.version)
239    }
240}
241fn registry_dir() -> PathBuf {
242    forc_util::user_forc_directory().join(REG_DIR_NAME)
243}
244
245fn registry_with_namespace_dir(namespace: &Namespace) -> PathBuf {
246    let base = registry_dir();
247    match namespace {
248        Namespace::Flat => base,
249        Namespace::Domain(ns) => base.join(ns),
250    }
251}
252
253fn registry_package_dir(
254    namespace: &Namespace,
255    pkg_name: &str,
256    pkg_version: &semver::Version,
257) -> PathBuf {
258    registry_with_namespace_dir(namespace).join(format!("{pkg_name}-{pkg_version}"))
259}
260
261/// The name to use for a package's identifier entry under the user's forc directory.
262fn registry_package_dir_name(name: &str, pkg_version: &semver::Version) -> String {
263    use std::hash::{Hash, Hasher};
264    fn hash_version(pkg_version: &semver::Version) -> u64 {
265        let mut hasher = std::collections::hash_map::DefaultHasher::new();
266        pkg_version.hash(&mut hasher);
267        hasher.finish()
268    }
269    let package_ver_hash = hash_version(pkg_version);
270    format!("{name}-{package_ver_hash:x}")
271}
272
273/// Validates if the cid string is valid by checking the initial 2 letters and
274/// length.
275///
276/// For CIDs to be marked as valid:
277/// 1. Must start with `Qm`.
278/// 2. Must be 46 chars long.
279///
280/// For more details see: https://docs.ipfs.tech/concepts/content-addressing/#version-0-v0
281fn validate_cid(cid: &str) -> bool {
282    let cid = cid.trim();
283    let starts_with_qm = cid.starts_with("Qm");
284    starts_with_qm && cid.len() == 46
285}
286
287/// A temporary directory that we can use for cloning a registry-sourced package's index file and discovering
288/// the corresponding CID for that package.
289///
290/// The resulting directory is:
291///
292/// ```ignore
293/// $HOME/.forc/registry/cache/tmp/<fetch_id>-name-<version_hash>
294/// ```
295///
296/// A unique `fetch_id` may be specified to avoid contention over the registry directory in the
297/// case that multiple processes or threads may be building different projects that may require
298/// fetching the same dependency.
299fn tmp_registry_package_dir(
300    fetch_id: u64,
301    name: &str,
302    version: &semver::Version,
303    namespace: &Namespace,
304) -> PathBuf {
305    let repo_dir_name = format!(
306        "{:x}-{}",
307        fetch_id,
308        registry_package_dir_name(name, version)
309    );
310    registry_with_namespace_dir(namespace)
311        .join("tmp")
312        .join(repo_dir_name)
313}
314
315impl source::Pin for Source {
316    type Pinned = Pinned;
317    fn pin(&self, ctx: source::PinCtx) -> anyhow::Result<(Self::Pinned, PathBuf)> {
318        let pkg_name = ctx.name;
319        let cid = futures::executor::block_on(async {
320            with_tmp_fetch_index(ctx.fetch_id(), pkg_name, self, |index_file| async move {
321                let version = &self.version;
322                let pkg_entry = index_file
323                    .get(version)
324                    .ok_or_else(|| anyhow!("No {} found for {}", version, pkg_name))?;
325                let cid = Cid::from_str(pkg_entry.source_cid());
326                Ok(cid)
327            })
328            .await
329        })??;
330        let path = registry_package_dir(&self.namespace, pkg_name, &self.version);
331        let pinned = Pinned {
332            source: self.clone(),
333            cid,
334        };
335        Ok((pinned, path))
336    }
337}
338
339impl source::Fetch for Pinned {
340    fn fetch(&self, ctx: source::PinCtx, path: &Path) -> anyhow::Result<PackageManifestFile> {
341        // Co-ordinate access to the registry checkout directory using an advisory file lock.
342        let mut lock = forc_util::path_lock(path)?;
343        // TODO: Here we assume that if the local path already exists, that it contains the
344        // full and correct source for that registry entry and hasn't been tampered with. This is
345        // probably fine for most cases as users should never be touching these
346        // directories, however we should add some code to validate this. E.g. can we
347        // recreate the ipfs cid by hashing the directory or something along these lines?
348        // https://github.com/FuelLabs/sway/issues/7075
349        {
350            let _guard = lock.write()?;
351            if !path.exists() {
352                println_action_green(
353                    "Fetching",
354                    &format!(
355                        "{} {}",
356                        ansiterm::Style::new().bold().paint(ctx.name),
357                        self.source.version
358                    ),
359                );
360                futures::executor::block_on(async {
361                    // If the user is trying to use public IPFS node with
362                    // registry sources. Use fuel operated ipfs node
363                    // instead.
364                    let node = match ctx.ipfs_node() {
365                        node if node == &IPFSNode::public() => &IPFSNode::fuel(),
366                        node => node,
367                    };
368                    fetch(ctx.fetch_id(), self, node).await
369                })?;
370            }
371        }
372        let path = {
373            let _guard = lock.read()?;
374            manifest::find_within(path, ctx.name())
375                .ok_or_else(|| anyhow!("failed to find package `{}` in {}", ctx.name(), self))?
376        };
377        PackageManifestFile::from_file(path)
378    }
379}
380
381impl source::DepPath for Pinned {
382    fn dep_path(&self, _name: &str) -> anyhow::Result<source::DependencyPath> {
383        bail!("dep_path: registry dependencies are not yet supported");
384    }
385}
386
387impl From<Pinned> for source::Pinned {
388    fn from(p: Pinned) -> Self {
389        Self::Registry(p)
390    }
391}
392
393/// Resolve a CID from index file and pinned package. Basically goes through
394/// the index file to find corresponding entry described by the pinned instance.
395fn resolve_to_cid(index_file: &IndexFile, pinned: &Pinned) -> anyhow::Result<Cid> {
396    let other_versions = index_file
397        .versions()
398        .filter(|ver| **ver != pinned.source.version)
399        .map(|ver| format!("{}.{}.{}", ver.major, ver.minor, ver.patch))
400        .collect::<Vec<_>>()
401        .join(",");
402
403    let package_entry = index_file.get(&pinned.source.version).ok_or_else(|| {
404        anyhow!(
405            "Version {} not found for {}. Other available versions: [{}]",
406            pinned.source.version,
407            pinned.source.name,
408            other_versions
409        )
410    })?;
411
412    let cid = Cid::from_str(package_entry.source_cid()).with_context(|| {
413        format!(
414            "Invalid CID {}v{}: `{}`",
415            package_entry.name(),
416            package_entry.version(),
417            package_entry.source_cid()
418        )
419    })?;
420    if package_entry.yanked() {
421        bail!(
422            "Version {} of {} is yanked. Other avaiable versions: [{}]",
423            pinned.source.version,
424            pinned.source.name,
425            other_versions
426        );
427    }
428    Ok(cid)
429}
430
431async fn fetch(fetch_id: u64, pinned: &Pinned, ipfs_node: &IPFSNode) -> anyhow::Result<PathBuf> {
432    let path = with_tmp_fetch_index(
433        fetch_id,
434        &pinned.source.name,
435        &pinned.source,
436        |index_file| async move {
437            let path = registry_package_dir(
438                &pinned.source.namespace,
439                &pinned.source.name,
440                &pinned.source.version,
441            );
442            if path.exists() {
443                let _ = fs::remove_dir_all(&path);
444            }
445            fs::create_dir_all(&path)?;
446
447            let cid = resolve_to_cid(&index_file, pinned)?;
448
449            match ipfs_node {
450                IPFSNode::Local => {
451                    println_action_green("Fetching", "with local IPFS node");
452                    cid.fetch_with_client(&ipfs_client(), &path).await?;
453                }
454                IPFSNode::WithUrl(gateway_url) => {
455                    println_action_green(
456                        "Fetching",
457                        &format!("from {}. Note: This can take several minutes.", gateway_url),
458                    );
459                    cid.fetch_with_gateway_url(gateway_url, &path).await?;
460                }
461            }
462
463            Ok(path)
464        },
465    )
466    .await?;
467    Ok(path)
468}
469
470async fn with_tmp_fetch_index<F, O, Fut>(
471    fetch_id: u64,
472    pkg_name: &str,
473    source: &Source,
474    f: F,
475) -> anyhow::Result<O>
476where
477    F: FnOnce(IndexFile) -> Fut,
478    Fut: std::future::Future<Output = anyhow::Result<O>>,
479{
480    let tmp_dir = tmp_registry_package_dir(fetch_id, pkg_name, &source.version, &source.namespace);
481    if tmp_dir.exists() {
482        let _ = std::fs::remove_dir_all(&tmp_dir);
483    }
484
485    // Add a guard to ensure cleanup happens if we got out of scope whether by
486    // returning or panicking.
487    let _cleanup_guard = scopeguard::guard(&tmp_dir, |dir| {
488        let _ = std::fs::remove_dir_all(dir);
489    });
490
491    let github_resolver = GithubRegistryResolver::with_default_github(source.namespace.clone());
492
493    let path = location_from_root(github_resolver.chunk_size, &source.namespace, pkg_name)
494        .display()
495        .to_string();
496    let index_repo_owner = github_resolver.repo_org();
497    let index_repo_name = github_resolver.repo_name();
498    let reference = format!("refs/heads/{}", github_resolver.branch_name());
499    let github_endpoint = format!(
500        "https://raw.githubusercontent.com/{index_repo_owner}/{index_repo_name}/{reference}/{path}"
501    );
502    let client = reqwest::Client::new();
503    let timeout_duration = Duration::from_millis(GithubRegistryResolver::DEFAULT_TIMEOUT_MS);
504    let index_response = client
505        .get(github_endpoint)
506        .timeout(timeout_duration)
507        .send()
508        .await
509        .map_err(|e| {
510            anyhow!(
511                "Failed to send request to github to obtain package index file from registry {e}"
512            )
513        })?
514        .error_for_status()
515        .map_err(|_| anyhow!("Failed to fetch {pkg_name}"))?;
516
517    let contents = index_response.text().await?;
518    let index_file: IndexFile = serde_json::from_str(&contents).with_context(|| {
519        format!(
520            "Unable to deserialize a github registry lookup response. Body was: \"{}\"",
521            contents
522        )
523    })?;
524
525    let res = f(index_file).await?;
526    Ok(res)
527}
528
529#[cfg(test)]
530mod tests {
531    use super::{file_location::Namespace, resolve_to_cid, Pinned, Source};
532    use crate::source::{
533        ipfs::Cid,
534        reg::index_file::{IndexFile, PackageEntry},
535    };
536    use std::str::FromStr;
537
538    #[test]
539    fn parse_pinned_entry_without_namespace() {
540        let pinned_str = "registry+core?0.0.1#QmdMVqLqpba2mMB5AUjYCxubC6tLGevQFunpBkbC2UbrKS!";
541        let pinned = Pinned::from_str(pinned_str).unwrap();
542
543        let expected_source = Source {
544            name: "core".to_string(),
545            version: semver::Version::new(0, 0, 1),
546            namespace: Namespace::Flat,
547        };
548
549        let cid = Cid::from_str("QmdMVqLqpba2mMB5AUjYCxubC6tLGevQFunpBkbC2UbrKS").unwrap();
550
551        let expected_pinned = Pinned {
552            source: expected_source,
553            cid,
554        };
555
556        assert_eq!(pinned, expected_pinned)
557    }
558
559    #[test]
560    fn parse_pinned_entry_with_namespace() {
561        let pinned_str =
562            "registry+core?0.0.1#QmdMVqLqpba2mMB5AUjYCxubC6tLGevQFunpBkbC2UbrKS!fuelnamespace";
563        let pinned = Pinned::from_str(pinned_str).unwrap();
564
565        let expected_source = Source {
566            name: "core".to_string(),
567            version: semver::Version::new(0, 0, 1),
568            namespace: Namespace::Domain("fuelnamespace".to_string()),
569        };
570
571        let cid = Cid::from_str("QmdMVqLqpba2mMB5AUjYCxubC6tLGevQFunpBkbC2UbrKS").unwrap();
572
573        let expected_pinned = Pinned {
574            source: expected_source,
575            cid,
576        };
577
578        assert_eq!(pinned, expected_pinned)
579    }
580
581    #[test]
582    fn test_resolve_to_cid() {
583        let mut index_file = IndexFile::default();
584
585        // Add a regular version with a valid CID
586        let valid_cid = "QmdMVqLqpba2mMB5AUjYCxubC6tLGevQFunpBkbC2UbrKS";
587        let valid_version = semver::Version::new(1, 0, 0);
588        let valid_entry = PackageEntry::new(
589            "test_package".to_string(),
590            valid_version.clone(),
591            valid_cid.to_string(),
592            None,   // no abi_cid
593            vec![], // no dependencies
594            false,  // not yanked
595        );
596        index_file.insert(valid_entry);
597
598        // Add a yanked version
599        let yanked_cid = "QmdMVqLqpba2mMB5AUjYCxubC6tLGevQFunpBkbC2UbrKR";
600        let yanked_version = semver::Version::new(0, 9, 0);
601        let yanked_entry = PackageEntry::new(
602            "test_package".to_string(),
603            yanked_version.clone(),
604            yanked_cid.to_string(),
605            None,   // no abi_cid
606            vec![], // no dependencies
607            true,   // yanked
608        );
609        index_file.insert(yanked_entry);
610
611        // Add another version just to have multiple available
612        let other_cid = "QmdMVqLqpba2mMB5AUjYCxubC6tLGevQFunpBkbC2UbrKT";
613        let other_version = semver::Version::new(1, 1, 0);
614        let other_entry = PackageEntry::new(
615            "test_package".to_string(),
616            other_version.clone(),
617            other_cid.to_string(),
618            None,   // no abi_cid
619            vec![], // no dependencies
620            false,  // not yanked
621        );
622        index_file.insert(other_entry);
623
624        // Test Case 1: Successful resolution
625        let valid_source = Source {
626            name: "test_package".to_string(),
627            version: valid_version.clone(),
628            namespace: Namespace::Flat,
629        };
630        let valid_pinned = Pinned {
631            source: valid_source,
632            cid: Cid::from_str(valid_cid).unwrap(),
633        };
634
635        let result = resolve_to_cid(&index_file, &valid_pinned);
636        assert!(result.is_ok());
637        let valid_cid = Cid::from_str(valid_cid).unwrap();
638        assert_eq!(result.unwrap(), valid_cid);
639
640        // Test Case 2: Error when version doesn't exist
641        let nonexistent_version = semver::Version::new(2, 0, 0);
642        let nonexistent_source = Source {
643            name: "test_package".to_string(),
644            version: nonexistent_version,
645            namespace: Namespace::Flat,
646        };
647        let nonexistent_pinned = Pinned {
648            source: nonexistent_source,
649            // this cid just a placeholder, as this version does not exists
650            cid: valid_cid,
651        };
652
653        let result = resolve_to_cid(&index_file, &nonexistent_pinned);
654        assert!(result.is_err());
655        let error_msg = result.unwrap_err().to_string();
656        assert!(error_msg.contains("Version 2.0.0 not found"));
657        assert!(
658            error_msg.contains("Other available versions: [1.1.0,0.9.0,1.0.0]")
659                || error_msg.contains("Other available versions: [0.9.0,1.0.0,1.1.0]")
660                || error_msg.contains("Other available versions: [1.0.0,0.9.0,1.1.0]")
661                || error_msg.contains("Other available versions: [0.9.0,1.1.0,1.0.0]")
662                || error_msg.contains("Other available versions: [1.0.0,1.1.0,0.9.0]")
663                || error_msg.contains("Other available versions: [1.1.0,1.0.0,0.9.0]")
664        );
665
666        // Test Case 3: Error when version is yanked
667        let yanked_source = Source {
668            name: "test_package".to_string(),
669            version: yanked_version.clone(),
670            namespace: Namespace::Flat,
671        };
672        let yanked_pinned = Pinned {
673            source: yanked_source,
674            cid: Cid::from_str(yanked_cid).unwrap(),
675        };
676
677        let result = resolve_to_cid(&index_file, &yanked_pinned);
678        assert!(result.is_err());
679        let error_msg = result.unwrap_err().to_string();
680        assert!(error_msg.contains("Version 0.9.0 of test_package is yanked"));
681        assert!(
682            error_msg.contains("Other avaiable versions: [1.1.0,1.0.0]")
683                || error_msg.contains("Other avaiable versions: [1.0.0,1.1.0]")
684        );
685    }
686}
forc_pkg/source/reg/mod.rs

forc_pkg/source/reg/
mod.rs
