acorn/
lib.rs

1//! # 🌱 ACORN Library
2//! > "Plant an ACORN and grow your research"
3//!
4//! `acorn-lib` is a one-stop-shop for everything related to building and maintaining research activity data (RAD)-related technology, including the Accessible Content Optimization for Research Needs (ACORN) tool.
5//! The modules, structs, enums and constants found here support the ACORN CLI, which checks, analyzes, and exports research activity data into useable formats.
6//!
7// Policy: `wasm` marks portable allocation-capable APIs. Cargo feature
8// unification means tooling such as `--all-features` may combine it with `std`;
9// wasm-specific code should use `#[cfg(all(feature = "wasm", not(feature = "std")))]`.
10#![cfg_attr(not(feature = "std"), no_std)]
11
12extern crate alloc;
13// Current schema derive macros emit `::std` paths even when host APIs are disabled.
14#[cfg(not(feature = "std"))]
15extern crate std;
16
17#[doc(hidden)]
18#[cfg(feature = "cmd")]
19pub use acorn_macros::cmd_sh_words;
20
21use core::convert::Infallible;
22use core::str::FromStr;
23use derive_more::Display;
24use fluent_uri::{Uri, UriRef};
25use serde::{Deserialize, Serialize};
26#[cfg(feature = "std")]
27use tracing::debug;
28use tracing::{error, trace, warn};
29use urlencoding::encode;
30
31#[cfg(feature = "analysis")]
32pub mod analyzer;
33#[cfg(feature = "doctor")]
34pub mod doctor;
35pub mod error;
36#[cfg(feature = "std")]
37pub mod io;
38pub mod prelude;
39pub mod schema;
40pub mod util;
41#[cfg(all(feature = "std", feature = "analysis"))]
42use crate::analyzer::{link_check, Check};
43#[cfg(feature = "std")]
44use crate::io::api::{github, gitlab, Configuration};
45#[cfg(feature = "std")]
46use crate::io::http::get;
47#[cfg(feature = "std")]
48use crate::io::uri_to_path;
49use crate::prelude::{format, String, ToString, Vec};
50#[cfg(feature = "std")]
51use crate::prelude::{Path, PathBuf};
52#[cfg(feature = "std")]
53use crate::schema::ControlledVocabulary;
54use crate::util::constants::app::DEFAULT_HUGGINGFACE_DOMAIN;
55use crate::util::Label;
56pub use error::{AcornError, AcornResult};
57use strum::EnumIs;
58
59/// **Descriptive/persistent** location reference for use in configuration (buckets, repositories, etc.).
60///
61/// This is a **data type** meant for storage and serialization — use [`io::Source`] when you
62/// actually need to **read** bytes from a local path or URL at runtime.
63///
64/// Supports both raw URI strings (`Simple`) and structured scheme+URI pairs (`Detailed`).
65///
66/// See [`io::Source`] for the complementary **operational/transient** type that handles I/O.
67#[derive(Clone, Debug, Deserialize, Display, Eq, PartialEq, Serialize)]
68#[serde(untagged)]
69pub enum Location {
70    /// Just the URI string (assumes remote location)
71    Simple(String),
72    /// Location defined by URI and scheme - intended for use with remote or local locations
73    #[display("{uri}")]
74    Detailed {
75        /// URI Scheme
76        ///
77        /// See [RFC 8089] for more information
78        ///
79        /// [RFC 8089]: https://datatracker.ietf.org/doc/rfc8089/
80        scheme: Scheme,
81        /// Full URI value
82        uri: String,
83        /// Optional branch, tag, or revision for versioned locations
84        #[serde(default, skip_serializing_if = "Option::is_none")]
85        revision: Option<String>,
86    },
87}
88/// Git hosting repository data
89#[derive(Clone, Debug, Display, EnumIs, Eq, PartialEq, Serialize, Deserialize)]
90#[serde(tag = "provider", rename_all = "lowercase")]
91pub enum Repository {
92    /// Generic Git repository
93    /// ### Note
94    /// > This repository type should be used for local and offline repositories. Having the associated data be version controlled by Git is recommended, but not required.
95    #[display("git")]
96    Git {
97        /// Repository location information
98        #[serde(alias = "uri")]
99        location: Location,
100    },
101    /// GitHub
102    ///
103    /// See <https://docs.github.com/en/rest/reference/repos>
104    #[display("github")]
105    GitHub {
106        /// Repository location information
107        #[serde(alias = "uri")]
108        location: Location,
109    },
110    /// GitLab
111    ///
112    /// See <https://docs.gitlab.com/api/repositories/#list-repository-tree>
113    #[display("gitlab")]
114    GitLab {
115        /// Integer ID of GitLab project
116        ///
117        /// See <https://docs.gitlab.com/api/projects/#get-a-single-project> for more information
118        id: Option<u64>,
119        /// Repository location information
120        #[serde(alias = "uri")]
121        location: Location,
122    },
123    /// Hugging Face
124    ///
125    /// See <https://huggingface.co/docs/hub/repositories-getting-started>
126    #[display("huggingface")]
127    HuggingFace {
128        /// Repository location information
129        #[serde(alias = "uri")]
130        location: Location,
131    },
132}
133/// URI Scheme
134///
135/// See [RFC 8089] for more information
136///
137/// [RFC 8089]: https://datatracker.ietf.org/doc/rfc8089/
138#[derive(Clone, Debug, Default, Deserialize, Display, EnumIs, Eq, PartialEq, Serialize)]
139#[serde(rename_all = "lowercase")]
140pub enum Scheme {
141    /// Secure HTTP
142    #[default]
143    #[display("https")]
144    HTTPS,
145    /// Insecure HTTP included primarily for contexts necessitating its use (ex., local development)
146    #[display("http")]
147    HTTP,
148    /// Local file or folder
149    #[display("file")]
150    File,
151    /// Unsupported scheme (e.g., insecure, not implemented, etc.)
152    Unsupported,
153}
154/// Struct for release data from GitLab or GitHub
155#[derive(Clone, Debug, Serialize, Deserialize)]
156pub struct Release {
157    /// Name of release
158    pub name: String,
159    /// Tag name of release
160    /// ### Example
161    /// > `v1.0.0`
162    pub tag_name: String,
163    /// Prose description of release
164    #[serde(alias = "body")]
165    pub description: String,
166    /// Date of release creation
167    pub created_at: String,
168    /// Date of release publication
169    #[serde(alias = "published_at")]
170    pub released_at: String,
171    /// Release response message
172    pub message: Option<String>,
173}
174impl FromStr for Location {
175    type Err = Infallible;
176
177    fn from_str(s: &str) -> Result<Self, Self::Err> {
178        Ok(Self::from(s))
179    }
180}
181impl From<&str> for Location {
182    fn from(s: &str) -> Self {
183        match UriRef::parse(s).ok().and_then(|uri| uri.scheme()) {
184            | Some(scheme) => Location::Detailed {
185                scheme: Scheme::from(scheme.as_str()),
186                uri: s.to_string(),
187                revision: None,
188            },
189            | None => Location::Simple(s.to_string()),
190        }
191    }
192}
193impl<'a> From<&'a Location> for &'a str {
194    fn from(value: &'a Location) -> Self {
195        match value {
196            | Location::Simple(value) | Location::Detailed { uri: value, .. } => value.as_str(),
197        }
198    }
199}
200impl Location {
201    /// Returns true when a source string points to a local filesystem path.
202    pub fn is_local(&self) -> bool {
203        let value = match self {
204            | Location::Simple(value) | Location::Detailed { uri: value, .. } => value.trim(),
205        };
206        let is_file_scheme = match self {
207            | Location::Detailed { scheme, .. } => scheme.is_file(),
208            | Location::Simple(_) => false,
209        };
210        let is_local_path = value.starts_with("file:") || value.starts_with("./") || value.starts_with("../") || {
211            #[cfg(feature = "std")]
212            {
213                Path::new(value).is_absolute()
214            }
215            #[cfg(not(feature = "std"))]
216            {
217                false
218            }
219        };
220        is_file_scheme || is_local_path
221    }
222    /// Convert the location URI to a normalized filesystem path.
223    #[cfg(feature = "std")]
224    pub fn uri_as_path(&self) -> PathBuf {
225        uri_to_path(self.uri().unwrap_or_default())
226    }
227    /// Return the normalized path when the location identifies an available local source.
228    #[cfg(feature = "std")]
229    pub fn local_path(&self) -> Option<PathBuf> {
230        let path = self.uri_as_path();
231        (self.is_local() || path.exists()).then_some(path)
232    }
233    /// Get associated location hash
234    /// > Useful for standardizing file path handling across local and remote contexts
235    /// ### Example
236    /// ```rust
237    /// use acorn::Location;
238    ///
239    /// let location = Location::Simple("https://code.ornl.gov/research-enablement/buckets/nssd".to_string());
240    /// assert_eq!(location.hash(), "code_ornl_gov_research-enablement_buckets_nssd");
241    /// ```
242    pub fn hash(&self) -> String {
243        let host = self.host().unwrap_or_default().replace('.', "_");
244        let segments = self
245            .path()
246            .map(|p| {
247                p.split('/')
248                    .filter(|s| !(s.is_empty() || *s == "."))
249                    .map(|s| s.to_string())
250                    .collect::<Vec<_>>()
251            })
252            .unwrap_or_default();
253        [host, segments.join("_").to_lowercase()]
254            .into_iter()
255            .filter(|x| !x.is_empty())
256            .collect::<Vec<String>>()
257            .join("_")
258    }
259    /// Get associated location value scheme (e.g., https, file, etc.)
260    /// ### Example
261    /// ```rust
262    /// use acorn::{Location, Scheme};
263    ///
264    /// let location = Location::Simple("https://code.ornl.gov/research-enablement/buckets/nssd".to_string());
265    /// assert_eq!(location.scheme(), Scheme::HTTPS);
266    /// let location = Location::Simple("file://localhost/buckets/nssd".to_string());
267    /// assert_eq!(location.scheme(), Scheme::File);
268    /// ```
269    pub fn scheme(&self) -> Scheme {
270        match self {
271            | Location::Simple(value) => Uri::parse(value.as_str())
272                .map(|uri| Scheme::from(uri.scheme().as_str()))
273                .unwrap_or(Scheme::Unsupported),
274            | Location::Detailed { scheme, .. } => scheme.clone(),
275        }
276    }
277    /// Check if a location exists (i.e., is reachable and accessible)
278    #[cfg(all(feature = "std", feature = "analysis"))]
279    pub async fn exists(self) -> bool {
280        let uri = self.uri();
281        let scheme = self.scheme();
282        if scheme == Scheme::HTTP {
283            warn!("=> {} HTTP is supported but only advised in local development scenarios", Label::skip());
284        }
285        match scheme {
286            | Scheme::HTTPS | Scheme::HTTP => match uri {
287                | Some(uri) => match link_check(Some(uri), None).await {
288                    | Check { success, .. } if success => true,
289                    | _ => false,
290                },
291                | None => false,
292            },
293            | Scheme::File => match uri {
294                | Some(_) => PathBuf::from(self.path().unwrap_or_default()).exists(),
295                | None => false,
296            },
297            | Scheme::Unsupported => false,
298        }
299    }
300    /// Extract and return URI string from a location value
301    pub fn uri(&self) -> Option<String> {
302        match self {
303            | Location::Simple(value) => Some(value.clone()),
304            | Location::Detailed { scheme, uri, .. } => match Uri::parse(uri.as_str()) {
305                | Ok(parsed) => {
306                    let authority = parsed.authority().map(|auth| auth.as_str().to_string());
307                    let path = parsed.path().to_string();
308                    let query = parsed.query().map(|q| format!("?{q}")).unwrap_or_default();
309                    let fragment = parsed.fragment().map(|f| format!("#{f}")).unwrap_or_default();
310                    Some(match authority {
311                        | Some(auth) if !auth.is_empty() => format!("{scheme}://{auth}{path}{query}{fragment}"),
312                        | _ => format!("{scheme}:{path}{query}{fragment}"),
313                    })
314                }
315                | Err(_) => {
316                    warn!("=> {} Parse URI - {uri}", Label::fail());
317                    Some(format!("{scheme}://{uri}"))
318                }
319            },
320        }
321    }
322    /// Get host from location URI
323    pub fn host(&self) -> Option<String> {
324        match self.uri() {
325            | Some(value) => Uri::parse(value.as_str())
326                .ok()
327                .and_then(|uri| uri.authority().map(|auth| auth.host().to_string())),
328            | None => None,
329        }
330    }
331    /// Get path from location URI
332    pub fn path(&self) -> Option<String> {
333        match self.uri() {
334            | Some(value) => Uri::parse(value.as_str()).ok().map(|uri| uri.path().to_string()),
335            | None => None,
336        }
337    }
338    /// Get port from location URI
339    pub fn port(&self) -> Option<u16> {
340        match self.uri() {
341            | Some(value) => Uri::parse(value.as_str())
342                .ok()
343                .and_then(|uri| uri.authority().and_then(|auth| auth.port_to_u16().ok()).flatten()),
344            | None => None,
345        }
346    }
347}
348impl Default for Repository {
349    fn default() -> Self {
350        Self::Git {
351            location: Location::Simple("file:///".to_string()),
352        }
353    }
354}
355impl Repository {
356    /// Classify a remote repository URL using its overt provider domain
357    pub fn from_remote(value: &str, domain: &str) -> Option<Self> {
358        let location = Location::from(value);
359        let host = location.host().map(|host| host.trim_start_matches("www.").to_ascii_lowercase());
360        let configured_url = if domain.contains("://") {
361            domain.to_string()
362        } else {
363            format!("https://{domain}")
364        };
365        let configured = Location::from(configured_url.as_str());
366        let is_gitlab = configured.host() == location.host() && configured.port() == location.port();
367        match (location.scheme(), host.as_deref()) {
368            | (Scheme::HTTP | Scheme::HTTPS, Some("github.com")) => Some(Self::GitHub { location }),
369            | (Scheme::HTTP | Scheme::HTTPS, Some(host)) if host == DEFAULT_HUGGINGFACE_DOMAIN => Some(Self::HuggingFace { location }),
370            | (Scheme::HTTP | Scheme::HTTPS, Some(_)) if is_gitlab => Some(Self::GitLab { id: None, location }),
371            | _ => None,
372        }
373    }
374    /// Get repository domain (e.g., "github.com" or "code.ornl.gov")
375    pub fn domain(&self) -> Option<String> {
376        self.location().host()
377    }
378    /// Return whether or not the associated URI for a repository is local (e.g., has "file" scheme)
379    pub fn is_local(&self) -> bool {
380        self.location().is_local()
381    }
382    /// Get metadata for latest release of a Gitlab or GitHub repository
383    #[cfg(feature = "std")]
384    pub async fn latest_release(self) -> Option<Release> {
385        match self.releases().await {
386            | releases if releases.is_empty() => None,
387            | releases => match releases.into_iter().next() {
388                | Some(release) => {
389                    trace!("=> {} Latest {:#?}", Label::using(), release);
390                    Some(release)
391                }
392                | None => None,
393            },
394        }
395    }
396    /// Get repository location
397    pub fn location(&self) -> Location {
398        match self.clone() {
399            | Repository::Git { location, .. }
400            | Repository::GitHub { location, .. }
401            | Repository::GitLab { location, .. }
402            | Repository::HuggingFace { location, .. } => location,
403        }
404    }
405    /// Get repository ID
406    pub fn id(&self) -> Option<String> {
407        match self {
408            | Repository::Git { .. } | Repository::GitHub { .. } => None,
409            | Repository::HuggingFace { location } => location.path().map(|path| path.trim_start_matches('/').to_string()),
410            | Repository::GitLab { id, location } => match id {
411                | Some(value) => Some(value.to_string()),
412                | None => match location.path() {
413                    | Some(path) => match path.strip_prefix('/') {
414                        | Some(stripped) if !stripped.is_empty() => {
415                            let encoded = encode(stripped).to_string();
416                            trace!(encoded, "=> {} ID", Label::using());
417                            Some(encoded)
418                        }
419                        | _ => None,
420                    },
421                    | None => {
422                        warn!("=> {} Parse GitLab URI", Label::fail());
423                        None
424                    }
425                },
426            },
427        }
428    }
429    /// Return the provider-native repository path or identifier.
430    pub fn project_path(&self) -> Option<String> {
431        match self {
432            | Repository::GitHub { location } | Repository::HuggingFace { location } => location.path().and_then(|path| {
433                let mut segments = path.trim_matches('/').split('/');
434                segments
435                    .next()
436                    .zip(segments.next())
437                    .map(|(owner, repository)| format!("{owner}/{}", repository.trim_end_matches(".git")))
438            }),
439            | Repository::GitLab { id: Some(id), .. } => Some(id.to_string()),
440            | Repository::GitLab { id: None, location } => location.path().and_then(|path| {
441                let project_path = path.trim_matches('/').split("/-/").next().unwrap_or_default().trim_end_matches(".git");
442                (project_path.split('/').count() >= 2).then(|| project_path.to_string())
443            }),
444            | Repository::Git { .. } => None,
445        }
446    }
447    /// Fetch canonical programming-language names for the first supported repository.
448    #[cfg(feature = "std")]
449    pub async fn technology(repositories: &[Self], options: Option<gitlab::Options>) -> Option<Vec<String>> {
450        let options = options.unwrap_or_else(gitlab::Options::from_env);
451        match repositories.iter().find(|repository| repository.is_git_hub()) {
452            | Some(repository) => github::languages(repository.project_path()?)
453                .await
454                .ok()
455                .map(|values| ControlledVocabulary::normalize("technology", values).into_values()),
456            | None => match repositories
457                .iter()
458                .find(|repository| repository.is_git_lab())
459                .and_then(Repository::project_path)
460                .map(|path| options.with_identifier(path))
461            {
462                | Some(options) => gitlab::language_use(&options).await.ok().map(|response| {
463                    ControlledVocabulary::normalize("technology", response.languages.into_iter().map(|language| language.name)).into_values()
464                }),
465                | None => None,
466            },
467        }
468    }
469    #[cfg(feature = "std")]
470    async fn releases(self) -> Vec<Release> {
471        let maybe_url = match &self {
472            | Repository::Git { .. } | Repository::HuggingFace { .. } => None,
473            | Repository::GitHub { location } => {
474                let host = location.host();
475                let path = location.path();
476                match (host, path) {
477                    | (Some(host), Some(path)) => Some(format!("https://api.{host}/repos{path}/releases")),
478                    | (None, _) => {
479                        error!("=> {} Parse GitHub URI host", Label::fail());
480                        None
481                    }
482                    | (_, None) => {
483                        error!("=> {} Parse GitHub URI", Label::fail());
484                        None
485                    }
486                }
487            }
488            | Repository::GitLab { location, .. } => match self.id() {
489                | Some(id) => match location.host() {
490                    | Some(host) => Some(format!("https://{host}/api/v4/projects/{id}/releases")),
491                    | None => {
492                        error!("=> {} Parse GitLab URI host", Label::fail());
493                        None
494                    }
495                },
496                | None => None,
497            },
498        };
499        if let Some(url) = maybe_url {
500            debug!(url, "=> {}", Label::using());
501            match get(url).send().await {
502                | Ok(response) => {
503                    let text = response.text().await;
504                    match text {
505                        | Ok(text) => {
506                            if text.contains("API rate limit exceeded") {
507                                error!("=> {} GitHub API rate limit exceeded", Label::fail());
508                                vec![]
509                            } else {
510                                let releases: Vec<Release> = match serde_json::from_str(&text) {
511                                    | Ok(values) => values,
512                                    | Err(why) => {
513                                        error!("=> {} Parse {} API JSON response - {why}", self, Label::fail());
514                                        vec![]
515                                    }
516                                };
517                                releases
518                            }
519                        }
520                        | Err(why) => {
521                            error!("=> {} Parse {} API text response - {why}", self, Label::fail());
522                            vec![]
523                        }
524                    }
525                }
526                | Err(why) => {
527                    error!("=> {} Download {} releases - {why}", self, Label::fail());
528                    vec![]
529                }
530            }
531        } else {
532            vec![]
533        }
534    }
535    /// Get URL for raw data of a file at a given path
536    pub fn raw_url(&self, path: String) -> Option<String> {
537        match self {
538            | Repository::GitHub { location, .. } => match location.path() {
539                | Some(ref value) => Some(format!("https://raw.githubusercontent.com{value}/refs/heads/main/{path}")),
540                | None => {
541                    error!("=> {} Parse GitHub URI", Label::fail());
542                    None
543                }
544            },
545            | Repository::GitLab { location, .. } => Some(format!("{location}/-/raw/main/{path}")),
546            | Repository::Git { .. } | Repository::HuggingFace { .. } => None,
547        }
548    }
549}
550impl From<&str> for Scheme {
551    fn from(value: &str) -> Self {
552        match value.to_ascii_lowercase().as_str() {
553            | "https" => Scheme::HTTPS,
554            | "http" => Scheme::HTTP,
555            | "file" => Scheme::File,
556            | _ => Scheme::Unsupported,
557        }
558    }
559}
560
561#[cfg(test)]
562mod tests {
563    #![allow(
564        clippy::arithmetic_side_effects,
565        clippy::expect_used,
566        clippy::indexing_slicing,
567        clippy::panic,
568        clippy::unwrap_used
569    )]
570    use super::{Location, Repository, Scheme};
571    #[cfg(feature = "std")]
572    use crate::prelude::PathBuf;
573
574    #[test]
575    fn test_scheme_from_str() {
576        assert_eq!(Scheme::from("https"), Scheme::HTTPS);
577        assert_eq!(Scheme::from("HTTP"), Scheme::HTTP);
578        assert_eq!(Scheme::from("file"), Scheme::File);
579        assert_eq!(Scheme::from("ssh"), Scheme::Unsupported);
580    }
581    #[cfg(feature = "std")]
582    #[test]
583    fn test_location_uri_as_path_normalizes_file_uri() {
584        assert_eq!(
585            Location::from("file:./models/qwen.gguf").uri_as_path(),
586            PathBuf::from("./models/qwen.gguf")
587        );
588    }
589    #[cfg(feature = "std")]
590    #[test]
591    fn test_location_local_path_filters_remote_sources() {
592        assert_eq!(Location::from("./Cargo.toml").local_path(), Some(PathBuf::from("./Cargo.toml")));
593        assert_eq!(Location::from("https://example.com/input.json").local_path(), None);
594    }
595    #[test]
596    fn test_repository_default_is_local_git() {
597        let repository = Repository::default();
598        assert!(repository.is_local());
599        match repository {
600            | Repository::Git { location } => {
601                assert_eq!(location.to_string(), "file:///");
602            }
603            | _ => panic!("Repository default should be Git with local file URI"),
604        }
605    }
606    #[test]
607    fn test_repository_from_remote_requires_overt_provider_domain() {
608        let github = Repository::from_remote("https://www.github.com/openai/codex/tree/main", "gitlab.com").unwrap();
609        assert!(github.is_git_hub());
610        assert_eq!(github.project_path(), Some("openai/codex".to_string()));
611        let gitlab = Repository::from_remote("https://code.ornl.gov/group/project/-/tree/main", "code.ornl.gov").unwrap();
612        assert!(gitlab.is_git_lab());
613        assert_eq!(gitlab.project_path(), Some("group/project".to_string()));
614        assert_eq!(Repository::from_remote("https://example.org/group/project", "gitlab.com"), None);
615        assert_eq!(Repository::from_remote("git@github.com:openai/codex.git", "gitlab.com"), None);
616    }
617    #[test]
618    fn test_repository_id_prefers_explicit_gitlab_id() {
619        let repository = Repository::GitLab {
620            id: Some(16689),
621            location: Location::Simple("https://code.ornl.gov/research-enablement/acorn".to_string()),
622        };
623        assert_eq!(repository.id(), Some("16689".to_string()));
624    }
625    #[test]
626    fn test_repository_id_falls_back_to_encoded_gitlab_path() {
627        let repository = Repository::GitLab {
628            id: None,
629            location: Location::Simple("https://code.ornl.gov/research-enablement/acorn".to_string()),
630        };
631        assert_eq!(repository.id(), Some("research-enablement%2Facorn".to_string()));
632    }
633    #[test]
634    fn test_repository_id_returns_none_without_gitlab_id_or_valid_uri() {
635        let repository = Repository::GitLab {
636            id: None,
637            location: Location::Simple("not a uri".to_string()),
638        };
639        assert_eq!(repository.id(), None);
640    }
641}
642
643#[cfg(all(test, feature = "std"))]
644mod test;
acorn/lib.rs

acorn/
lib.rs
