acorn/
lib.rs

1//! # 🌱 ACORN Library
2//! > "Plant an ACORN and grow your research"
3//!
4//! `acorn-lib` is a one-stop-shop for everything related to building and maintaining research activity data (RAD)-related technology, including the Accessible Content Optimization for Research Needs (ACORN) tool.
5//! The modules, structs, enums and constants found here support the ACORN CLI, which checks, analyzes, and exports research activity data into useable formats.
6//!
7// Policy: `wasm` marks portable allocation-capable APIs. Cargo feature
8// unification means tooling such as `--all-features` may combine it with `std`;
9// wasm-specific code should use `#[cfg(all(feature = "wasm", not(feature = "std")))]`.
10#![cfg_attr(not(feature = "std"), no_std)]
11
12extern crate alloc;
13// Current schema derive macros emit `::std` paths even when host APIs are disabled.
14#[cfg(not(feature = "std"))]
15extern crate std;
16
17#[doc(hidden)]
18#[cfg(feature = "cmd")]
19pub use acorn_macros::cmd_sh_words;
20
21use core::convert::Infallible;
22use core::str::FromStr;
23use derive_more::Display;
24use fluent_uri::{Uri, UriRef};
25use serde::{Deserialize, Serialize};
26#[cfg(feature = "std")]
27use tracing::debug;
28use tracing::{error, trace, warn};
29use urlencoding::encode;
30
31#[cfg(feature = "analysis")]
32pub mod analyzer;
33#[cfg(feature = "doctor")]
34pub mod doctor;
35pub mod error;
36#[cfg(feature = "std")]
37pub mod io;
38pub mod prelude;
39pub mod schema;
40pub mod util;
41#[cfg(all(feature = "std", feature = "analysis"))]
42use crate::analyzer::{link_check, Check};
43#[cfg(feature = "std")]
44use crate::io::api::{github, gitlab, Configuration};
45#[cfg(feature = "std")]
46use crate::io::http::get;
47#[cfg(feature = "std")]
48use crate::io::uri_to_path;
49use crate::prelude::{format, String, ToString, Vec};
50#[cfg(feature = "std")]
51use crate::prelude::{Path, PathBuf};
52#[cfg(feature = "std")]
53use crate::schema::ControlledVocabulary;
54use crate::util::constants::app::DEFAULT_HUGGINGFACE_DOMAIN;
55use crate::util::Label;
56pub use error::{AcornError, AcornResult};
57use strum::EnumIs;
58
59/// **Descriptive/persistent** location reference for use in configuration (buckets, repositories, etc.).
60///
61/// This is a **data type** meant for storage and serialization — use [`io::Source`] when you
62/// actually need to **read** bytes from a local path or URL at runtime.
63///
64/// Supports both raw URI strings (`Simple`) and structured scheme+URI pairs (`Detailed`).
65///
66/// See [`io::Source`] for the complementary **operational/transient** type that handles I/O.
67#[derive(Clone, Debug, Deserialize, Display, Eq, PartialEq, Serialize)]
68#[serde(untagged)]
69pub enum Location {
70    /// Just the URI string (assumes remote location)
71    Simple(String),
72    /// Location defined by URI and scheme - intended for use with remote or local locations
73    #[display("{uri}")]
74    Detailed {
75        /// URI Scheme
76        ///
77        /// See [RFC 8089] for more information
78        ///
79        /// [RFC 8089]: https://datatracker.ietf.org/doc/rfc8089/
80        scheme: Scheme,
81        /// Full URI value
82        uri: String,
83        /// Optional branch, tag, or revision for versioned locations
84        #[serde(default, skip_serializing_if = "Option::is_none")]
85        revision: Option<String>,
86    },
87}
88/// Git hosting repository data
89#[derive(Clone, Debug, Display, EnumIs, Eq, PartialEq, Serialize, Deserialize)]
90#[serde(tag = "provider", rename_all = "lowercase")]
91pub enum Repository {
92    /// Generic Git repository
93    /// ### Note
94    /// > This repository type should be used for local and offline repositories. Having the associated data be version controlled by Git is recommended, but not required.
95    #[display("git")]
96    Git {
97        /// Repository location information
98        #[serde(alias = "uri")]
99        location: Location,
100    },
101    /// GitHub
102    ///
103    /// See <https://docs.github.com/en/rest/reference/repos>
104    #[display("github")]
105    GitHub {
106        /// Repository location information
107        #[serde(alias = "uri")]
108        location: Location,
109    },
110    /// GitLab
111    ///
112    /// See <https://docs.gitlab.com/api/repositories/#list-repository-tree>
113    #[display("gitlab")]
114    GitLab {
115        /// Integer ID of GitLab project
116        ///
117        /// See <https://docs.gitlab.com/api/projects/#get-a-single-project> for more information
118        id: Option<u64>,
119        /// Repository location information
120        #[serde(alias = "uri")]
121        location: Location,
122    },
123    /// Hugging Face
124    ///
125    /// See <https://huggingface.co/docs/hub/repositories-getting-started>
126    #[display("huggingface")]
127    HuggingFace {
128        /// Repository location information
129        #[serde(alias = "uri")]
130        location: Location,
131    },
132}
133/// URI Scheme
134///
135/// See [RFC 8089] for more information
136///
137/// [RFC 8089]: https://datatracker.ietf.org/doc/rfc8089/
138#[derive(Clone, Debug, Default, Deserialize, Display, EnumIs, Eq, PartialEq, Serialize)]
139#[serde(rename_all = "lowercase")]
140pub enum Scheme {
141    /// Secure HTTP
142    #[default]
143    #[display("https")]
144    HTTPS,
145    /// Insecure HTTP included primarily for contexts necessitating its use (ex., local development)
146    #[display("http")]
147    HTTP,
148    /// Local file or folder
149    #[display("file")]
150    File,
151    /// Unsupported scheme (e.g., insecure, not implemented, etc.)
152    Unsupported,
153}
154/// Struct for release data from GitLab or GitHub
155#[derive(Clone, Debug, Serialize, Deserialize)]
156pub struct Release {
157    /// Name of release
158    pub name: String,
159    /// Tag name of release
160    /// ### Example
161    /// > `v1.0.0`
162    pub tag_name: String,
163    /// Prose description of release
164    #[serde(alias = "body")]
165    pub description: String,
166    /// Date of release creation
167    pub created_at: String,
168    /// Date of release publication
169    #[serde(alias = "published_at")]
170    pub released_at: String,
171    /// Release response message
172    pub message: Option<String>,
173}
174impl FromStr for Location {
175    type Err = Infallible;
176
177    fn from_str(s: &str) -> Result<Self, Self::Err> {
178        Ok(Self::from(s))
179    }
180}
181impl From<&str> for Location {
182    fn from(s: &str) -> Self {
183        match UriRef::parse(s).ok().and_then(|uri| uri.scheme()) {
184            | Some(scheme) => Location::Detailed {
185                scheme: Scheme::from(scheme.as_str()),
186                uri: s.to_string(),
187                revision: None,
188            },
189            | None => Location::Simple(s.to_string()),
190        }
191    }
192}
193impl<'a> From<&'a Location> for &'a str {
194    fn from(value: &'a Location) -> Self {
195        match value {
196            | Location::Simple(value) | Location::Detailed { uri: value, .. } => value.as_str(),
197        }
198    }
199}
200impl Location {
201    /// Returns true when a source string points to a local filesystem path.
202    pub fn is_local(&self) -> bool {
203        let value = match self {
204            | Location::Simple(value) | Location::Detailed { uri: value, .. } => value.trim(),
205        };
206        let is_file_scheme = match self {
207            | Location::Detailed { scheme, .. } => scheme.is_file(),
208            | Location::Simple(_) => false,
209        };
210        let is_local_path = value.starts_with("file:") || value.starts_with("./") || value.starts_with("../") || {
211            #[cfg(feature = "std")]
212            {
213                Path::new(value).is_absolute()
214            }
215            #[cfg(not(feature = "std"))]
216            {
217                false
218            }
219        };
220        is_file_scheme || is_local_path
221    }
222    /// Convert the location URI to a normalized filesystem path.
223    #[cfg(feature = "std")]
224    pub fn uri_as_path(&self) -> PathBuf {
225        uri_to_path(self.uri().unwrap_or_default())
226    }
227    /// Return the normalized path when the location identifies an available local source.
228    #[cfg(feature = "std")]
229    pub fn local_path(&self) -> Option<PathBuf> {
230        let path = self.uri_as_path();
231        (self.is_local() || path.exists()).then_some(path)
232    }
233    /// Get associated location hash
234    /// > Useful for standardizing file path handling across local and remote contexts
235    /// ### Example
236    /// ```rust
237    /// use acorn::Location;
238    ///
239    /// let location = Location::Simple("https://code.ornl.gov/research-enablement/buckets/nssd".to_string());
240    /// assert_eq!(location.hash(), "code_ornl_gov_research-enablement_buckets_nssd");
241    /// ```
242    pub fn hash(&self) -> String {
243        let host = self.host().unwrap_or_default().replace('.', "_");
244        let segments = self
245            .path()
246            .map(|p| {
247                p.split('/')
248                    .filter(|s| !(s.is_empty() || *s == "."))
249                    .map(|s| s.to_string())
250                    .collect::<Vec<_>>()
251            })
252            .unwrap_or_default();
253        [host, segments.join("_").to_lowercase()]
254            .into_iter()
255            .filter(|x| !x.is_empty())
256            .collect::<Vec<String>>()
257            .join("_")
258    }
259    /// Get associated location value scheme (e.g., https, file, etc.)
260    /// ### Example
261    /// ```rust
262    /// use acorn::{Location, Scheme};
263    ///
264    /// let location = Location::Simple("https://code.ornl.gov/research-enablement/buckets/nssd".to_string());
265    /// assert_eq!(location.scheme(), Scheme::HTTPS);
266    /// let location = Location::Simple("file://localhost/buckets/nssd".to_string());
267    /// assert_eq!(location.scheme(), Scheme::File);
268    /// ```
269    pub fn scheme(&self) -> Scheme {
270        match self {
271            | Location::Simple(value) => Uri::parse(value.as_str())
272                .map(|uri| Scheme::from(uri.scheme().as_str()))
273                .unwrap_or(Scheme::Unsupported),
274            | Location::Detailed { scheme, .. } => scheme.clone(),
275        }
276    }
277    /// Check if a location exists (i.e., is reachable and accessible)
278    #[cfg(all(feature = "std", feature = "analysis"))]
279    pub async fn exists(self) -> bool {
280        let uri = self.uri();
281        let scheme = self.scheme();
282        if scheme == Scheme::HTTP {
283            warn!("=> {} HTTP is supported but only advised in local development scenarios", Label::skip());
284        }
285        match scheme {
286            | Scheme::HTTPS | Scheme::HTTP => match uri {
287                | Some(uri) => match link_check(Some(uri), None).await {
288                    | Check { success, .. } if success => true,
289                    | _ => false,
290                },
291                | None => false,
292            },
293            | Scheme::File => match uri {
294                | Some(_) => PathBuf::from(self.path().unwrap_or_default()).exists(),
295                | None => false,
296            },
297            | Scheme::Unsupported => false,
298        }
299    }
300    /// Extract and return URI string from a location value
301    pub fn uri(&self) -> Option<String> {
302        match self {
303            | Location::Simple(value) => Some(value.clone()),
304            | Location::Detailed { scheme, uri, .. } => match Uri::parse(uri.as_str()) {
305                | Ok(parsed) => {
306                    let authority = parsed.authority().map(|auth| auth.as_str().to_string());
307                    let path = parsed.path().to_string();
308                    let query = parsed.query().map(|q| format!("?{q}")).unwrap_or_default();
309                    let fragment = parsed.fragment().map(|f| format!("#{f}")).unwrap_or_default();
310                    Some(match authority {
311                        | Some(auth) if !auth.is_empty() => format!("{scheme}://{auth}{path}{query}{fragment}"),
312                        | _ => format!("{scheme}:{path}{query}{fragment}"),
313                    })
314                }
315                | Err(_) => {
316                    warn!("=> {} Parse URI - {uri}", Label::fail());
317                    Some(format!("{scheme}://{uri}"))
318                }
319            },
320        }
321    }
322    /// Get host from location URI
323    pub fn host(&self) -> Option<String> {
324        match self.uri() {
325            | Some(value) => Uri::parse(value.as_str())
326                .ok()
327                .and_then(|uri| uri.authority().map(|auth| auth.host().to_string())),
328            | None => None,
329        }
330    }
331    /// Get path from location URI
332    pub fn path(&self) -> Option<String> {
333        match self.uri() {
334            | Some(value) => Uri::parse(value.as_str()).ok().map(|uri| uri.path().to_string()),
335            | None => None,
336        }
337    }
338    /// Get port from location URI
339    pub fn port(&self) -> Option<u16> {
340        match self.uri() {
341            | Some(value) => Uri::parse(value.as_str())
342                .ok()
343                .and_then(|uri| uri.authority().and_then(|auth| auth.port_to_u16().ok()).flatten()),
344            | None => None,
345        }
346    }
347}
348impl Default for Repository {
349    fn default() -> Self {
350        Self::Git {
351            location: Location::Simple("file:///".to_string()),
352        }
353    }
354}
355impl Repository {
356    /// Classify a remote repository URL using its overt provider domain
357    pub fn from_remote(value: &str, domain: &str) -> Option<Self> {
358        let location = Location::from(value);
359        let host = location.host().map(|host| host.trim_start_matches("www.").to_ascii_lowercase());
360        let configured_url = if domain.contains("://") {
361            domain.to_string()
362        } else {
363            format!("https://{domain}")
364        };
365        let configured = Location::from(configured_url.as_str());
366        let is_gitlab = configured.host() == location.host() && configured.port() == location.port();
367        match (location.scheme(), host.as_deref()) {
368            | (Scheme::HTTP | Scheme::HTTPS, Some("github.com")) => Some(Self::GitHub { location }),
369            | (Scheme::HTTP | Scheme::HTTPS, Some(host)) if host == DEFAULT_HUGGINGFACE_DOMAIN => Some(Self::HuggingFace { location }),
370            | (Scheme::HTTP | Scheme::HTTPS, Some(_)) if is_gitlab => Some(Self::GitLab { id: None, location }),
371            | _ => None,
372        }
373    }
374    /// Get repository domain (e.g., "github.com" or "code.ornl.gov")
375    pub fn domain(&self) -> Option<String> {
376        self.location().host()
377    }
378    /// Return whether or not the associated URI for a repository is local (e.g., has "file" scheme)
379    pub fn is_local(&self) -> bool {
380        let local_schemes = [Scheme::File];
381        local_schemes.contains(&self.clone().location().scheme())
382    }
383    /// Get metadata for latest release of a Gitlab or GitHub repository
384    #[cfg(feature = "std")]
385    pub async fn latest_release(self) -> Option<Release> {
386        match self.releases().await {
387            | releases if releases.is_empty() => None,
388            | releases => match releases.into_iter().next() {
389                | Some(release) => {
390                    trace!("=> {} Latest {:#?}", Label::using(), release);
391                    Some(release)
392                }
393                | None => None,
394            },
395        }
396    }
397    /// Get repository location
398    pub fn location(&self) -> Location {
399        match self.clone() {
400            | Repository::Git { location, .. }
401            | Repository::GitHub { location, .. }
402            | Repository::GitLab { location, .. }
403            | Repository::HuggingFace { location, .. } => location,
404        }
405    }
406    /// Get repository ID
407    pub fn id(&self) -> Option<String> {
408        match self {
409            | Repository::Git { .. } | Repository::GitHub { .. } => None,
410            | Repository::HuggingFace { location } => location.path().map(|path| path.trim_start_matches('/').to_string()),
411            | Repository::GitLab { id, location } => match id {
412                | Some(value) => Some(value.to_string()),
413                | None => match location.path() {
414                    | Some(path) => match path.strip_prefix('/') {
415                        | Some(stripped) if !stripped.is_empty() => {
416                            let encoded = encode(stripped).to_string();
417                            trace!(encoded, "=> {} ID", Label::using());
418                            Some(encoded)
419                        }
420                        | _ => None,
421                    },
422                    | None => {
423                        warn!("=> {} Parse GitLab URI", Label::fail());
424                        None
425                    }
426                },
427            },
428        }
429    }
430    /// Return the provider-native repository path or identifier.
431    pub fn project_path(&self) -> Option<String> {
432        match self {
433            | Repository::GitHub { location } | Repository::HuggingFace { location } => location.path().and_then(|path| {
434                let mut segments = path.trim_matches('/').split('/');
435                segments
436                    .next()
437                    .zip(segments.next())
438                    .map(|(owner, repository)| format!("{owner}/{}", repository.trim_end_matches(".git")))
439            }),
440            | Repository::GitLab { id: Some(id), .. } => Some(id.to_string()),
441            | Repository::GitLab { id: None, location } => location.path().and_then(|path| {
442                let project_path = path.trim_matches('/').split("/-/").next().unwrap_or_default().trim_end_matches(".git");
443                (project_path.split('/').count() >= 2).then(|| project_path.to_string())
444            }),
445            | Repository::Git { .. } => None,
446        }
447    }
448    /// Fetch canonical programming-language names for the first supported repository.
449    #[cfg(feature = "std")]
450    pub async fn technology(repositories: &[Self], options: Option<gitlab::Options>) -> Option<Vec<String>> {
451        let options = options.unwrap_or_else(gitlab::Options::from_env);
452        match repositories.iter().find(|repository| repository.is_git_hub()) {
453            | Some(repository) => github::languages(repository.project_path()?)
454                .await
455                .ok()
456                .map(|values| ControlledVocabulary::normalize("technology", values).into_values()),
457            | None => match repositories
458                .iter()
459                .find(|repository| repository.is_git_lab())
460                .and_then(Repository::project_path)
461                .map(|path| options.with_identifier(path))
462            {
463                | Some(options) => gitlab::language_use(&options).await.ok().map(|response| {
464                    ControlledVocabulary::normalize("technology", response.languages.into_iter().map(|language| language.name)).into_values()
465                }),
466                | None => None,
467            },
468        }
469    }
470    #[cfg(feature = "std")]
471    async fn releases(self) -> Vec<Release> {
472        let maybe_url = match &self {
473            | Repository::Git { .. } | Repository::HuggingFace { .. } => None,
474            | Repository::GitHub { location } => {
475                let host = location.host();
476                let path = location.path();
477                match (host, path) {
478                    | (Some(host), Some(path)) => Some(format!("https://api.{host}/repos{path}/releases")),
479                    | (None, _) => {
480                        error!("=> {} Parse GitHub URI host", Label::fail());
481                        None
482                    }
483                    | (_, None) => {
484                        error!("=> {} Parse GitHub URI", Label::fail());
485                        None
486                    }
487                }
488            }
489            | Repository::GitLab { location, .. } => match self.id() {
490                | Some(id) => match location.host() {
491                    | Some(host) => Some(format!("https://{host}/api/v4/projects/{id}/releases")),
492                    | None => {
493                        error!("=> {} Parse GitLab URI host", Label::fail());
494                        None
495                    }
496                },
497                | None => None,
498            },
499        };
500        if let Some(url) = maybe_url {
501            debug!(url, "=> {}", Label::using());
502            match get(url).send().await {
503                | Ok(response) => {
504                    let text = response.text().await;
505                    match text {
506                        | Ok(text) => {
507                            if text.contains("API rate limit exceeded") {
508                                error!("=> {} GitHub API rate limit exceeded", Label::fail());
509                                vec![]
510                            } else {
511                                let releases: Vec<Release> = match serde_json::from_str(&text) {
512                                    | Ok(values) => values,
513                                    | Err(why) => {
514                                        error!("=> {} Parse {} API JSON response - {why}", self, Label::fail());
515                                        vec![]
516                                    }
517                                };
518                                releases
519                            }
520                        }
521                        | Err(why) => {
522                            error!("=> {} Parse {} API text response - {why}", self, Label::fail());
523                            vec![]
524                        }
525                    }
526                }
527                | Err(why) => {
528                    error!("=> {} Download {} releases - {why}", self, Label::fail());
529                    vec![]
530                }
531            }
532        } else {
533            vec![]
534        }
535    }
536    /// Get URL for raw data of a file at a given path
537    pub fn raw_url(&self, path: String) -> Option<String> {
538        match self {
539            | Repository::GitHub { location, .. } => match location.path() {
540                | Some(ref value) => Some(format!("https://raw.githubusercontent.com{value}/refs/heads/main/{path}")),
541                | None => {
542                    error!("=> {} Parse GitHub URI", Label::fail());
543                    None
544                }
545            },
546            | Repository::GitLab { location, .. } => Some(format!("{location}/-/raw/main/{path}")),
547            | Repository::Git { .. } | Repository::HuggingFace { .. } => None,
548        }
549    }
550}
551impl From<&str> for Scheme {
552    fn from(value: &str) -> Self {
553        match value.to_ascii_lowercase().as_str() {
554            | "https" => Scheme::HTTPS,
555            | "http" => Scheme::HTTP,
556            | "file" => Scheme::File,
557            | _ => Scheme::Unsupported,
558        }
559    }
560}
561
562#[cfg(test)]
563mod tests {
564    #![allow(
565        clippy::arithmetic_side_effects,
566        clippy::expect_used,
567        clippy::indexing_slicing,
568        clippy::panic,
569        clippy::unwrap_used
570    )]
571    use super::{Location, Repository, Scheme};
572    #[cfg(feature = "std")]
573    use crate::prelude::PathBuf;
574
575    #[test]
576    fn test_scheme_from_str() {
577        assert_eq!(Scheme::from("https"), Scheme::HTTPS);
578        assert_eq!(Scheme::from("HTTP"), Scheme::HTTP);
579        assert_eq!(Scheme::from("file"), Scheme::File);
580        assert_eq!(Scheme::from("ssh"), Scheme::Unsupported);
581    }
582    #[cfg(feature = "std")]
583    #[test]
584    fn test_location_uri_as_path_normalizes_file_uri() {
585        assert_eq!(
586            Location::from("file:./models/qwen.gguf").uri_as_path(),
587            PathBuf::from("./models/qwen.gguf")
588        );
589    }
590    #[cfg(feature = "std")]
591    #[test]
592    fn test_location_local_path_filters_remote_sources() {
593        assert_eq!(Location::from("./Cargo.toml").local_path(), Some(PathBuf::from("./Cargo.toml")));
594        assert_eq!(Location::from("https://example.com/input.json").local_path(), None);
595    }
596    #[test]
597    fn test_repository_default_is_local_git() {
598        let repository = Repository::default();
599        assert!(repository.is_local());
600        match repository {
601            | Repository::Git { location } => {
602                assert_eq!(location.to_string(), "file:///");
603            }
604            | _ => panic!("Repository default should be Git with local file URI"),
605        }
606    }
607    #[test]
608    fn test_repository_from_remote_requires_overt_provider_domain() {
609        let github = Repository::from_remote("https://www.github.com/openai/codex/tree/main", "gitlab.com").unwrap();
610        assert!(github.is_git_hub());
611        assert_eq!(github.project_path(), Some("openai/codex".to_string()));
612        let gitlab = Repository::from_remote("https://code.ornl.gov/group/project/-/tree/main", "code.ornl.gov").unwrap();
613        assert!(gitlab.is_git_lab());
614        assert_eq!(gitlab.project_path(), Some("group/project".to_string()));
615        assert_eq!(Repository::from_remote("https://example.org/group/project", "gitlab.com"), None);
616        assert_eq!(Repository::from_remote("git@github.com:openai/codex.git", "gitlab.com"), None);
617    }
618    #[test]
619    fn test_repository_id_prefers_explicit_gitlab_id() {
620        let repository = Repository::GitLab {
621            id: Some(16689),
622            location: Location::Simple("https://code.ornl.gov/research-enablement/acorn".to_string()),
623        };
624        assert_eq!(repository.id(), Some("16689".to_string()));
625    }
626    #[test]
627    fn test_repository_id_falls_back_to_encoded_gitlab_path() {
628        let repository = Repository::GitLab {
629            id: None,
630            location: Location::Simple("https://code.ornl.gov/research-enablement/acorn".to_string()),
631        };
632        assert_eq!(repository.id(), Some("research-enablement%2Facorn".to_string()));
633    }
634    #[test]
635    fn test_repository_id_returns_none_without_gitlab_id_or_valid_uri() {
636        let repository = Repository::GitLab {
637            id: None,
638            location: Location::Simple("not a uri".to_string()),
639        };
640        assert_eq!(repository.id(), None);
641    }
642}
643
644#[cfg(all(test, feature = "std"))]
645mod test;
acorn/lib.rs

acorn/
lib.rs
