use crate::cmd;
use crate::prelude::CommandOutput;
use crate::prelude::{
canonicalize, consts, create_dir_all, current_dir, io, remove_file, var, var_os, write, BufReader, Cursor, File, HashSet, OpenOptions, Path,
PathBuf, Read, Write,
};
#[cfg(any(unix, target_os = "wasi", target_os = "redox"))]
use crate::prelude::{set_permissions, OpenOptionsExt, Permissions, PermissionsExt};
use crate::util::constants::app::{APPLICATION, LARGE_FILE_THRESHOLD_BYTES, ORGANIZATION, QUALIFIER};
#[cfg(windows)]
use crate::util::file_extension;
use crate::util::{generate_guid, suffix, Checksum, ChecksumAlgorithm, Label, MimeType, SemanticVersion, StringConversion, ToStrings};
use chrono::Utc;
use color_eyre::eyre::{eyre, Report, Result};
use core::fmt;
use core::pin::Pin;
use core::time::Duration;
use data_encoding::HEXUPPER;
use directories::ProjectDirs;
use fancy_regex::Regex;
use futures::stream::{self, StreamExt};
use futures::Future;
use glob::glob;
use indicatif::{ProgressBar, ProgressStyle};
use is_executable::IsExecutable;
use rand::rngs::OsRng;
use ring::digest::{Context, SHA256};
use rsa::pkcs8::{EncodePrivateKey, EncodePublicKey};
use rsa::{RsaPrivateKey, RsaPublicKey};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use strum::AsRefStr;
use tokio::runtime::{Builder, Runtime};
use tracing::{debug, error, info, trace, warn};
use which::which;
use zip::write::SimpleFileOptions;
use zip::{ZipArchive, ZipWriter};
pub mod api;
pub mod bagit;
pub mod config;
pub mod database;
pub mod docx;
pub mod http;
#[cfg(feature = "agentic")]
pub mod mcp;
#[cfg(feature = "powerpoint")]
pub mod powerpoint;
pub mod source;
pub use source::Source;
pub type ApiFuture<'a> = Pin<Box<dyn Future<Output = ApiResult<()>> + 'a>>;
pub type ApiResult<T> = Result<T, Report>;
pub type RsaKeyPair = (rsa::RsaPrivateKey, rsa::RsaPublicKey);
pub trait FromCommand {
fn from_command<S>(name: S) -> Option<Self>
where
Self: Sized,
S: Into<String> + core::marker::Copy;
}
pub trait FromPath {
fn from_path<P>(value: &P) -> Self
where
P: AsRef<Path> + ?Sized;
}
pub trait InputOutput: Sized {
fn read(path: impl Into<PathBuf>) -> ApiResult<Self>;
fn read_cff(_path: impl Into<PathBuf>) -> ApiResult<Self> {
Err(eyre!("CFF read not implemented for this type"))
}
fn read_json(path: PathBuf) -> ApiResult<Self>;
fn read_markdown(_path: PathBuf) -> Option<Self> {
None
}
fn read_yaml(path: PathBuf) -> ApiResult<Self>;
fn write(&self, path: impl Into<PathBuf>) -> ApiResult<()>;
fn write_cff(&self, _path: impl Into<PathBuf>) -> ApiResult<()> {
Err(eyre!("CFF write not implemented for this type"))
}
fn write_json(&self, path: impl Into<PathBuf>) -> ApiResult<()>;
fn write_markdown(&self, _path: impl Into<PathBuf>) -> ApiResult<()> {
Err(eyre!("Markdown write not implemented for this type"))
}
fn write_yaml(&self, path: impl Into<PathBuf>) -> ApiResult<()>;
}
#[derive(AsRefStr, Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(untagged, rename_all = "snake_case")]
pub enum Executor {
#[serde(alias = "Singularity", alias = "singularity")]
Apptainer,
Docker,
Podman,
Sandbox,
#[serde(alias = "zsh", alias = "pwsh", alias = "cmd", alias = "local")]
Shell,
#[serde(alias = "remote")]
Ssh,
#[serde(alias = "k8s")]
Kubernetes,
#[serde(alias = "vm")]
VirtualMachine,
Other(String),
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(untagged)]
pub enum License {
Multiple(Vec<String>),
Single(String),
}
#[derive(Clone, Copy, Debug, Default)]
pub enum ProgressType {
#[default]
Bar,
Spinner,
Counter,
Silent,
}
#[derive(Debug, Deserialize)]
pub struct GitlabMergeRequestDiffResponse {
new_path: String,
}
pub struct StringList<'a>(pub &'a Vec<PathBuf>);
impl fmt::Display for Executor {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let value = match self {
| Executor::Apptainer => "apptainer",
| Executor::Docker => "docker",
| Executor::Podman => "podman",
| Executor::Sandbox => "sandbox",
| Executor::Shell => "shell",
| Executor::Ssh => "ssh",
| Executor::Kubernetes => "kubernetes",
| Executor::VirtualMachine => "virtual_machine",
| Executor::Other(value) => value.as_str(),
};
write!(f, "{value}")
}
}
impl From<&str> for Executor {
fn from(value: &str) -> Self {
match value.to_lowercase().as_str() {
| "apptainer" | "singularity" => Executor::Apptainer,
| "docker" => Executor::Docker,
| "podman" => Executor::Podman,
| "sandbox" => Executor::Sandbox,
| "shell" => Executor::Shell,
| "ssh" => Executor::Ssh,
| "kubernetes" | "k8s" => Executor::Kubernetes,
| "virtual machine" | "virtual_machine" | "vm" => Executor::VirtualMachine,
| other => Executor::Other(other.to_string()),
}
}
}
impl From<Executor> for std::ffi::OsString {
fn from(value: Executor) -> Self {
Self::from(value.to_string())
}
}
impl From<Executor> for String {
fn from(value: Executor) -> Self {
value.to_string()
}
}
impl Executor {
pub fn default_gitlab_runner_config_directory() -> &'static str {
match cfg!(target_os = "macos") {
| true => "/Users/Shared/gitlab-runner/config",
| false => "/srv/gitlab-runner/config",
}
}
pub fn command(&self) -> Option<&str> {
match self {
| Executor::Docker => Some("docker"),
| Executor::Podman => Some("podman"),
| Executor::Apptainer => Some("apptainer"),
| Executor::Shell | Executor::Ssh | Executor::Kubernetes | Executor::Sandbox | Executor::VirtualMachine => None,
| Executor::Other(value) => Some(value.as_str()),
}
}
pub fn gitlab_runner_type(&self) -> &str {
match self {
| Executor::Docker | Executor::Podman => "docker",
| Executor::Shell => "shell",
| Executor::Ssh => "ssh",
| Executor::Kubernetes => "kubernetes",
| Executor::VirtualMachine => match consts::OS {
| "macos" => "parallels",
| _ => "virtualbox",
},
| Executor::Apptainer | Executor::Sandbox | Executor::Other(_) => "docker",
}
}
pub fn socket(&self) -> Option<String> {
match self {
| Executor::Docker | Executor::Apptainer => {
Some("/var/run/docker.sock".to_string())
}
| Executor::Podman => {
if let Some(value) = var_os("XDG_RUNTIME_DIR") {
let path = PathBuf::from(value).join("podman/podman.sock");
if path.exists() {
Some(path.to_absolute_string())
} else {
None
}
} else {
let path = PathBuf::from("/run/podman/podman.sock");
if path.exists() {
Some(path.to_absolute_string())
} else {
None
}
}
}
| Executor::Shell | Executor::Ssh | Executor::Kubernetes | Executor::Sandbox | Executor::VirtualMachine | Executor::Other(_) => None,
}
}
}
impl FromCommand for SemanticVersion {
#[cfg(feature = "std")]
fn from_command<S>(name: S) -> Option<SemanticVersion>
where
S: Into<String> + core::marker::Copy,
{
let command = name.into();
if command_exists(command.clone()) {
match cmd!(&command, ["--version"]) {
| Ok(output) if output.status.success() => output.stdout().lines().next().map(SemanticVersion::from),
| Ok(_) | Err(_) => None,
}
} else {
None
}
}
}
impl FromPath for MimeType {
fn from_path<P>(value: &P) -> MimeType
where
P: AsRef<Path> + ?Sized,
{
MimeType::from(value.as_ref().display().to_string())
}
}
impl<P: Into<PathBuf> + Clone> ToStrings for Vec<P> {
fn to_strings(&self) -> Vec<String> {
self.iter()
.map(|p| <P as Into<PathBuf>>::into(p.clone()).to_string_lossy().to_string())
.collect()
}
fn to_absolute_strings(&self) -> Vec<String> {
self.iter().map(|p| <P as Into<PathBuf>>::into(p.clone()).to_absolute_string()).collect()
}
}
impl ProgressType {
fn template(&self) -> Option<&'static str> {
match self {
| ProgressType::Bar => Some(Label::PROGRESS_BAR_TEMPLATE),
| ProgressType::Spinner => Some(Label::PROGRESS_SPINNER_TEMPLATE),
| ProgressType::Counter => Some(Label::PROGRESS_COUNTER_TEMPLATE),
| ProgressType::Silent => None,
}
}
fn is_indeterminate(&self) -> bool {
matches!(self, ProgressType::Spinner)
}
}
impl StringConversion for PathBuf {
fn file_name_with_parent(&self) -> String {
file_name_with_parent(self.clone())
}
fn to_absolute_string(&self) -> String {
to_absolute_string(self.clone())
}
}
pub fn archive(path: PathBuf, destination: Option<PathBuf>) -> ApiResult<PathBuf> {
let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored);
let zip_file_path = match destination {
| Some(value) => value,
| None => path.with_extension("zip"),
};
info!("=> {} Create archive at {}", Label::using(), zip_file_path.to_absolute_string());
let prepared = if zip_file_path.exists() {
match zip_file_path.symlink_metadata() {
| Ok(metadata) => {
let file_type = metadata.file_type();
if file_type.is_symlink() {
error!("=> {} Create zip archive — destination cannot be a symlink", Label::fail());
false
} else if metadata.is_file() {
match remove_file(zip_file_path.clone()) {
| Ok(_) => true,
| Err(why) => {
error!("=> {} Prepare zip destination — {why}", Label::fail());
false
}
}
} else {
error!("=> {} Create zip archive — destination exists and is not a file", Label::fail());
false
}
}
| Err(why) => {
error!("=> {} Inspect zip destination — {why}", Label::fail());
false
}
}
} else {
true
};
let zip_file = if prepared {
match OpenOptions::new().write(true).create_new(true).open(&zip_file_path) {
| Ok(zip_file) => Some(ZipWriter::new(zip_file)),
| Err(why) => {
error!("=> {} Create zip archive — {why}", Label::fail());
None
}
}
} else {
None
};
if let Some(mut zip) = zip_file {
let files = files_all(path.clone(), None).into_iter().filter(|x| x.is_file());
for file_path in files {
if let Ok(file) = File::open(file_path.clone()) {
let name = match path.canonicalize() {
| Ok(relative) => file_path.strip_prefix(relative).unwrap_or_else(|_| &file_path),
| Err(_) => &file_path,
};
trace!(
file = name.to_path_buf().to_absolute_string(),
"=> {} Add file to archive",
Label::using()
);
match zip.start_file_from_path(name, options) {
| Ok(_) => {
let mut buffer = Vec::new();
match io::copy(&mut file.take(u64::MAX), &mut buffer) {
| Ok(_) => match zip.write_all(&buffer) {
| Ok(_) => {}
| Err(why) => {
error!(file = file_path.to_absolute_string(), "=> {} Write zip archive — {why}", Label::fail())
}
},
| Err(why) => {
error!("=> {} Copy buffer - {why}", Label::fail())
}
}
}
| Err(why) => {
error!(file = file_path.to_absolute_string(), "=> {} Start zip archive - {why}", Label::fail());
}
}
}
}
match zip.finish() {
| Ok(_) => Ok(zip_file_path),
| Err(why) => {
error!(file = path.to_absolute_string(), "=> {} Finish zip archive - {why}", Label::fail());
Err(why.into())
}
}
} else {
Err(eyre!("Unable to create zip archive"))
}
}
pub fn async_runtime() -> Runtime {
debug!("=> {} Async runtime", Label::using());
#[allow(clippy::unwrap_used)]
Builder::new_current_thread().enable_all().build().unwrap()
}
pub fn command_exists<S>(name: S) -> bool
where
S: Into<String>,
{
let command = name.into();
match which(&command) {
| Ok(value) => {
let path = value.clone().to_absolute_string();
match value.try_exists() {
| Ok(true) => {
debug!(path, "=> {} Command", Label::found());
true
}
| _ => {
debug!(path, "=> {} Command", Label::not_found());
false
}
}
}
| Err(_) => {
warn!("=> {} Command {}", Label::not_found(), command);
false
}
}
}
pub fn create_rsa_keypair() -> ApiResult<RsaKeyPair> {
let bits = 2048;
let mut rng = OsRng;
match RsaPrivateKey::new(&mut rng, bits) {
| Ok(private_key) => {
let public_key = RsaPublicKey::from(&private_key);
Ok((private_key, public_key))
}
| Err(why) => {
error!("=> {} Create RSA key pair — {why}", Label::fail());
Err(eyre!("Failed to create RSA key pair — {why}"))
}
}
}
pub fn current_date() -> String {
Utc::now().format("%Y-%m-%d").to_string()
}
pub async fn download_binary<S, P>(url: S, destination: P) -> ApiResult<PathBuf>
where
S: Into<String> + Clone + core::marker::Copy,
P: Into<PathBuf> + Clone,
{
let url_string: String = url.into();
let dest: PathBuf = destination.clone().into();
let filename = PathBuf::from(url_string.clone())
.file_name()
.and_then(|f| f.to_str())
.unwrap_or("downloaded_file")
.to_string();
match http::get(url_string.clone()).send().await {
| Ok(data) => match data.bytes().await {
| Ok(content) => {
let output = dest.clone().join(filename.clone());
match write(output.clone(), content.as_slice()) {
| Ok(_) => {
debug!(filename, "=> {} Downloaded", Label::output());
Ok(output)
}
| Err(why) => Err(eyre!("Failed to write {filename} - {why}")),
}
}
| Err(_) => Err(eyre!("No content downloaded from {url_string}")),
},
| Err(_) => Err(eyre!("Failed to download {url_string}")),
}
}
pub fn env_var_is_truthy(name: impl AsRef<str>) -> Option<bool> {
var(name.as_ref())
.ok()
.map(|value| matches!(value.trim().to_ascii_lowercase().as_str(), "1" | "true" | "yes" | "on"))
}
pub fn extract_zip(path: PathBuf, destination: Option<PathBuf>) -> ApiResult<PathBuf> {
let root = match destination {
| Some(value) => value,
| None => standard_project_folder("extract", None),
};
match File::open(path.clone()) {
| Ok(zip_file) => match ZipArchive::new(zip_file) {
| Ok(mut archive) => {
let success = (0..archive.len()).all(|index| match archive.by_index(index) {
| Ok(mut file) => {
let target = root.join(file.name());
if let Some(parent) = target.parent() {
match create_dir_all(parent) {
| Ok(_) => {}
| Err(why) => error!(path = parent.to_path_buf().to_absolute_string(), "=> {} Create - {}", Label::fail(), why),
}
}
match OpenOptions::new().write(true).create_new(true).open(&target) {
| Ok(mut output_file) => match io::copy(&mut file, &mut output_file) {
| Ok(_) => true,
| Err(why) => {
error!(path = target.to_absolute_string(), "=> {} Copy file content - {why}", Label::fail());
false
}
},
| Err(_) => {
error!(path = target.to_absolute_string(), "=> {} Create file", Label::fail());
false
}
}
}
| Err(why) => {
error!(path = path.to_absolute_string(), "=> {} Extract file - {why}", Label::fail());
false
}
});
if success {
info!(path = root.to_absolute_string(), "=> {} Extract zip archive", Label::pass());
Ok(root)
} else {
error!(path = root.to_absolute_string(), "=> {} Extract zip archive", Label::fail());
Err(eyre!("Failed to extract zip archive"))
}
}
| Err(why) => {
error!(path = path.to_absolute_string(), "=> {} Read zip archive - {why}", Label::fail());
Err(eyre!("Failed to read zip archive - {why}"))
}
},
| Err(why) => {
error!(path = path.to_absolute_string(), "=> {} Read file - {why}", Label::fail());
Err(eyre!("Failed to read file - {why}"))
}
}
}
pub fn file_checksum<P>(path: P, algorithm: Option<&'static ring::digest::Algorithm>) -> Option<Checksum>
where
P: Into<PathBuf>,
{
let value = path.into();
let digest_algorithm = algorithm.unwrap_or(&SHA256);
let checksum_algorithm = ChecksumAlgorithm::from(digest_algorithm);
match File::open(value.clone()) {
| Ok(file) => {
let mut buffer = [0; 1024];
let mut context = Context::new(digest_algorithm);
let mut reader = BufReader::new(file);
loop {
let count = match reader.read(&mut buffer) {
| Ok(c) => c,
| Err(err) => {
error!(
error = err.to_string(),
path = value.to_absolute_string(),
"=> {} Read file checksum",
Label::fail()
);
return None;
}
};
if count == 0 {
break;
}
context.update(buffer.get(..count).unwrap_or(&[]));
}
let digest = context.finish();
let result = HEXUPPER.encode(digest.as_ref());
Some(Checksum {
algorithm: checksum_algorithm,
checksum_value: result.to_lowercase(),
})
}
| Err(err) => {
error!(
error = err.to_string(),
path = value.to_absolute_string(),
"=> {} Read file",
Label::fail()
);
None
}
}
}
pub fn files_all(path: PathBuf, extensions: Option<Vec<&str>>) -> Vec<PathBuf> {
let path = uri_to_path(path);
fn paths_to_vec(paths: glob::Paths) -> Vec<PathBuf> {
paths.collect::<Vec<_>>().into_iter().filter_map(|x| x.ok()).collect::<Vec<_>>()
}
fn pattern(path: PathBuf, extension: &str) -> String {
let ext = &extension.to_lowercase();
let result = format!("{}/**/*.{}", path.to_absolute_string(), ext);
debug!("=> {} {result}", Label::using());
result
}
if path.is_dir() {
match extensions {
| Some(values) => {
values
.into_iter()
.map(|extension| {
let glob_pattern = pattern(path.clone(), extension);
glob(&glob_pattern)
})
.filter_map(|x| x.ok())
.flat_map(paths_to_vec)
.fold((HashSet::new(), Vec::new()), |(mut seen, mut ordered), path| {
if seen.insert(path.clone()) {
ordered.push(path);
}
(seen, ordered)
})
.1
}
| None => match glob(&format!("{}/**/*", path.to_absolute_string())) {
| Ok(paths) => paths_to_vec(paths),
| Err(why) => {
error!("=> {} Get all files (Glob) - {why}", Label::fail());
vec![]
}
},
}
} else {
if extensions.is_some() {
warn!(
path = path.clone().to_absolute_string(),
"=> {} Extension passed with single file to files_all()...was this intended?",
Label::using()
);
}
vec![path]
}
}
pub fn files_from_git_branch(value: &str, extensions: Option<Vec<&str>>) -> Vec<PathBuf> {
if command_exists("git".to_owned()) {
let default_branch = match git_default_branch_name() {
| Some(value) => value,
| None => "main".to_string(),
};
let args = vec!["diff", "--name-only", &default_branch, "--merge-base", value];
match cmd!("git", args) {
| Ok(output) if output.status.success() => filter_git_command_result(output.stdout(), extensions),
| Ok(output) => {
let why = output.stderr();
let message = if why.is_empty() {
format!("process exited with status {}", output.status)
} else {
why
};
error!("=> {} Get files from Git branch - {}", Label::fail(), message);
vec![]
}
| Err(why) => {
error!("=> {} Get files from Git branch - {why}", Label::fail());
vec![]
}
}
} else {
vec![]
}
}
pub fn files_from_git_commit(value: &str, extensions: Option<Vec<&str>>) -> Vec<PathBuf> {
if command_exists("git".to_owned()) {
let args = vec!["diff-tree", "--no-commit-id", "--name-only", "-r", value];
let result = cmd!("git", args);
debug!("=> {} Git command response - {result:?}", Label::using());
let files = match result {
| Ok(output) if output.status.success() => filter_git_command_result(output.stdout(), extensions),
| Ok(output) => {
let why = output.stderr();
let message = if why.is_empty() {
format!("process exited with status {}", output.status)
} else {
why
};
error!("=> {} Get files from Git commit - {}", Label::fail(), message);
vec![]
}
| Err(why) => {
error!("=> {} Get files from Git commit - {why}", Label::fail());
vec![]
}
};
debug!(
"=> {} Found {} file{} from Git commit - {files:?}",
Label::using(),
files.len(),
suffix(files.len())
);
files
} else {
vec![]
}
}
pub async fn files_from_gitlab_merge_request(extensions: Option<Vec<&str>>) -> Vec<PathBuf> {
let root = var("CI_API_V4_URL").unwrap_or_default();
let project_id = var("CI_MERGE_REQUEST_PROJECT_ID").unwrap_or_default();
let merge_request_iid = var("CI_MERGE_REQUEST_IID").unwrap_or_default();
let path = format!("/projects/{project_id}/merge_requests/{merge_request_iid}/diffs");
let url = format!("{root}{path}");
match http::get(url).send().await {
| Ok(response) => {
let content: serde_json::Result<Vec<GitlabMergeRequestDiffResponse>> = response.text().await.map_or_else(
|_| Err(serde_json::Error::io(io::Error::other("Failed to read response text"))),
|body| serde_json::from_str(&body),
);
match content {
| Ok(data) => {
debug!("=> {} GitLab API merge request diff response - {data:#?}", Label::using());
let results = data.into_iter().map(|x| PathBuf::from(x.new_path)).collect::<Vec<PathBuf>>();
match extensions {
| Some(values) => results
.into_iter()
.filter(|path| values.iter().any(|ext| MimeType::from_path(path).file_type() == *ext.to_lowercase()))
.collect::<Vec<_>>(),
| None => results,
}
}
| Err(why) => {
error!("=> {} Parse GitLab API merge request diff response - {why}", Label::fail());
vec![]
}
}
}
| Err(why) => {
error!("=> {} Get GitLab API merge request diff response - {why}", Label::fail());
vec![]
}
}
}
pub fn filter_git_command_result(value: String, extensions: Option<Vec<&str>>) -> Vec<PathBuf> {
match extensions {
| Some(values) => value
.to_lowercase()
.split("\n")
.map(PathBuf::from)
.filter(|path| values.iter().any(|ext| MimeType::from_path(path).file_type() == *ext.to_lowercase()))
.collect::<Vec<_>>(),
| None => value.to_lowercase().split("\n").map(PathBuf::from).collect::<Vec<_>>(),
}
}
pub fn filter_ignored(paths: Vec<PathBuf>, ignore: Option<String>) -> ApiResult<Vec<PathBuf>> {
match ignore {
| Some(ignore_pattern) => match Regex::new(&ignore_pattern) {
| Ok(re) => Ok(paths
.into_iter()
.map(to_absolute_string)
.filter(|x| !re.is_match(x).unwrap_or(false))
.map(PathBuf::from)
.collect()),
| Err(why) => Err(eyre!("Invalid regex/filter pattern: {why}")),
},
| None => Ok(paths),
}
}
pub fn filter_ignored_with_root(paths: Vec<PathBuf>, ignore: Option<String>, root: PathBuf) -> ApiResult<Vec<PathBuf>> {
match ignore {
| Some(ignore_pattern) => match Regex::new(&ignore_pattern) {
| Ok(re) => {
let root = if root.is_file() {
root.parent().map(|value| value.to_path_buf()).unwrap_or(root)
} else {
root
};
let normalized_root = canonicalize(root.clone()).unwrap_or(root);
let mut filtered: Vec<PathBuf> = vec![];
for path in paths {
let normalized_path = canonicalize(path.clone()).unwrap_or(path.clone());
match normalized_path.strip_prefix(&normalized_root) {
| Ok(relative) => {
let value = relative.to_string_lossy().to_string().replace('\\', "/");
if !re.is_match(&value).unwrap_or(false) {
filtered.push(path);
}
}
| Err(_) => {
return Err(eyre!(
"Path '{}' is outside resolved root '{}'",
normalized_path.to_absolute_string(),
normalized_root.to_absolute_string()
));
}
}
}
Ok(filtered)
}
| Err(why) => Err(eyre!("Invalid regex/filter pattern: {why}")),
},
| None => Ok(paths),
}
}
pub fn first_env_var(names: &[&str]) -> Option<String> {
names.iter().find_map(|name| var(name).ok())
}
pub fn folder_size<P: Into<PathBuf>>(path: P) -> u64 {
files_all(path.into(), None)
.into_iter()
.filter_map(|p| p.metadata().ok())
.filter(|m| m.is_file())
.map(|m| m.len())
.sum()
}
pub fn git_branch_name() -> Option<String> {
if command_exists("git".to_owned()) {
let args = vec!["symbolic-ref", "--short", "HEAD"];
match cmd!("git", args) {
| Ok(output) if output.status.success() => output.stdout().split("/").last().map(|x| x.to_string()),
| Ok(_) => None,
| Err(_) => None,
}
} else {
None
}
}
pub fn git_default_branch_name() -> Option<String> {
if command_exists("git".to_owned()) {
let args = vec!["symbolic-ref", "refs/remotes/origin/HEAD", "--short"];
match cmd!("git", args) {
| Ok(output) if output.status.success() => output.stdout().split("/").last().map(|x| x.to_string()),
| Ok(_) => None,
| Err(_) => None,
}
} else {
None
}
}
pub fn image_paths<P>(root: P) -> Vec<PathBuf>
where
P: Into<PathBuf> + Clone,
{
let extensions = ["jpg", "jpeg", "png", "svg", "gif"];
let mut files = extensions
.iter()
.flat_map(|ext| glob(&format!("{}/**/*.{}", root.clone().into().display(), ext)))
.flat_map(|paths| paths.collect::<Vec<_>>())
.flatten()
.collect::<Vec<PathBuf>>();
files.sort();
files
}
#[cfg(any(unix, target_os = "wasi", target_os = "redox"))]
pub fn make_executable<P>(path: P) -> bool
where
P: Into<PathBuf> + Clone,
{
let path = path.into();
let create_with_mode = OpenOptions::new().write(true).create_new(true).mode(0o755).open(path.as_path());
match create_with_mode {
| Ok(_) => path.is_executable(),
| Err(why) => {
if why.kind() == io::ErrorKind::AlreadyExists {
match set_permissions(path.as_path(), Permissions::from_mode(0o755)) {
| Ok(()) => path.is_executable(),
| Err(why) => {
debug!(path = path.to_absolute_string(), "=> {} Set permissions — {why}", Label::fail());
false
}
}
} else {
debug!(path = path.to_absolute_string(), "=> {} Create executable file — {why}", Label::fail());
false
}
}
}
}
#[cfg(windows)]
pub fn make_executable<P>(path: P) -> bool
where
P: Into<PathBuf> + Clone,
{
let binary = match file_extension(path.clone().into().to_absolute_string()) {
| None => path.into().with_extension("exe"),
| _ => path.into(),
};
debug!("=> {} {binary:#?}", Label::using());
binary.is_executable()
}
pub fn file_name_with_parent(value: impl Into<PathBuf>) -> String {
let path = value.into();
let name = path.file_name().and_then(|value| value.to_str()).unwrap_or_default().to_string();
if path.is_dir() {
name
} else {
let parent_name = path
.parent()
.and_then(|value| value.file_name())
.and_then(|value| value.to_str())
.unwrap_or_default();
if parent_name.is_empty() {
name
} else {
format!("{parent_name}/{name}")
}
}
}
pub fn parent<P>(path: P) -> PathBuf
where
P: Into<PathBuf> + Clone,
{
let default = PathBuf::from(".");
match path.clone().into().canonicalize() {
| Ok(value) => match value.parent() {
| Some(value) => value.to_path_buf(),
| None => {
warn!("=> {} Resolve parent path", Label::fail());
default
}
},
| Err(why) => {
debug!("=> {} Resolve absolute path - {why}", Label::fail());
match path.into().parent() {
| Some(value) if !value.to_path_buf().to_absolute_string().is_empty() => value.to_path_buf(),
| Some(_) | None => {
warn!("=> {} Parent path was empty or could not be resolved", Label::fail());
default
}
}
}
}
}
pub fn read_file<P>(path: P) -> ApiResult<String>
where
P: Into<PathBuf> + Clone + Send,
{
let path_buf = path.into();
let filename = path_buf.file_name().unwrap_or_default().to_string_lossy().to_string();
let is_large_file = match path_buf.metadata() {
| Ok(metadata) => metadata.len() >= LARGE_FILE_THRESHOLD_BYTES,
| Err(_) => false,
};
if is_large_file {
trace!(filename, "=> {} Read file with large-file strategy", Label::using());
read_large_file(path_buf)
} else {
match File::open(&path_buf) {
| Ok(file) => {
let mut reader = BufReader::new(file);
let mut content = String::new();
match reader.read_to_string(&mut content) {
| Ok(_) => Ok(content),
| Err(why) => Err(eyre!("Failed to read file content — {why}")),
}
}
| Err(why) => {
error!(filename, "=> {} Read file", Label::fail());
Err(eyre!("Failed to read file — {why}"))
}
}
}
}
pub fn read_large_file<P>(path: P) -> ApiResult<String>
where
P: Into<PathBuf> + Clone + Send,
{
match File::open(path.into()) {
| Ok(file) => {
let capacity = file
.metadata()
.ok()
.and_then(|metadata| usize::try_from(metadata.len()).ok())
.unwrap_or(0);
let mut reader = BufReader::with_capacity(1024 * 1024, file);
let mut content = if capacity > 0 { String::with_capacity(capacity) } else { String::new() };
match reader.read_to_string(&mut content) {
| Ok(_) => Ok(content),
| Err(why) => Err(eyre!("Failed to read large file content — {why}")),
}
}
| Err(why) => Err(eyre!("Failed to read large file — {why}")),
}
}
pub fn standard_project_folder(namespace: &str, default: Option<PathBuf>) -> PathBuf {
let root = match default {
| Some(value) => value,
| None => match ProjectDirs::from(QUALIFIER, ORGANIZATION, APPLICATION) {
| Some(dirs) => dirs.cache_dir().join(namespace).to_path_buf(),
| None => PathBuf::from(format!("./{namespace}")),
},
};
match create_dir_all(root.clone()) {
| Ok(_) => {}
| Err(why) => error!(directory = root.clone().to_absolute_string(), "=> {} Create - {why}", Label::fail()),
};
root.join(generate_guid())
}
pub fn to_absolute_string<P>(path: P) -> String
where
P: Into<PathBuf> + Clone,
{
let result = match canonicalize(path.clone().into().as_path()) {
| Ok(value) => value,
| Err(_) => path.into(),
};
let s = result.display().to_string();
#[cfg(windows)]
let s = s.strip_prefix(r"\\?\").unwrap_or(&s).to_string();
s
}
pub fn unique_file_extensions(paths: &[PathBuf]) -> Vec<String> {
let mut extensions = paths
.iter()
.filter_map(|path| path.extension().map(|extension| extension.to_string_lossy().to_lowercase()))
.collect::<HashSet<_>>()
.into_iter()
.collect::<Vec<_>>();
extensions.sort_unstable();
extensions
}
pub fn uri_to_path<P>(path: P) -> PathBuf
where
P: Into<PathBuf>,
{
let value: PathBuf = path.into();
let s = value.to_string_lossy().into_owned();
s.strip_prefix("file://")
.map(|stripped| {
#[cfg(windows)]
let normalized = match stripped.get(1..3) {
| Some(drive) if drive.contains(':') => &stripped[1..],
| _ => stripped,
};
#[cfg(not(windows))]
let normalized = stripped;
PathBuf::from(normalized)
})
.unwrap_or(value)
}
pub fn create_progress_bar(count: usize, progress_type: ProgressType) -> ProgressBar {
if matches!(progress_type, ProgressType::Silent) {
ProgressBar::hidden()
} else {
let progress = if progress_type.is_indeterminate() {
let spinner = ProgressBar::new_spinner();
spinner.enable_steady_tick(Duration::from_millis(120));
spinner
} else {
ProgressBar::new(count as u64)
};
if let Some(template) = progress_type.template() {
#[allow(clippy::unwrap_used)]
progress.set_style(ProgressStyle::with_template(template).unwrap());
}
progress
}
}
pub fn apply_progress_style(progress: &ProgressBar, template: &str) {
#[allow(clippy::unwrap_used)]
progress.set_style(ProgressStyle::with_template(template).unwrap());
}
pub fn finish_progress_bar(progress: &ProgressBar, message: String) {
#[allow(clippy::unwrap_used)]
progress.set_style(ProgressStyle::with_template(" {msg}").unwrap());
progress.finish_with_message(message);
}
pub async fn with_progress<T, U, M, F, Fut>(
items: Vec<T>,
message: M,
operation: F,
finish_message: impl FnOnce(usize) -> String,
buffer_size: Option<usize>,
progress_type: ProgressType,
) -> ApiResult<Vec<U>>
where
M: for<'a> Fn(&'a T) -> String,
F: Fn(T) -> Fut,
Fut: Future<Output = ApiResult<U>>,
{
let concurrency = buffer_size.unwrap_or(10).max(1);
let count = items.len();
let progress = create_progress_bar(count, progress_type);
if matches!(progress_type, ProgressType::Spinner) {
progress.enable_steady_tick(Duration::from_millis(120));
}
let output = stream::iter(items)
.map(|item| {
let msg = message(&item);
let future = operation(item);
async move {
let result = future.await;
(msg, result)
}
})
.buffer_unordered(concurrency)
.map(|(msg, result)| {
progress.set_message(msg);
progress.inc(1);
result
})
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<ApiResult<Vec<_>>>();
if !matches!(progress_type, ProgressType::Silent) {
finish_progress_bar(&progress, finish_message(count));
}
output
}
pub fn write_file<P>(path: P, content: String) -> ApiResult<()>
where
P: Into<PathBuf>,
{
write(path.into(), content.as_bytes())
.map(|_| ())
.map_err(|why| eyre!("Failed to write file - {why}"))
}
pub async fn write_file_bytes<P, F, Fut, E>(path: P, get_bytes: F) -> ApiResult<()>
where
P: Into<PathBuf>,
F: FnOnce() -> Fut,
Fut: Future<Output = Result<Vec<u8>, E>>,
E: Into<Report>,
{
let path = path.into();
match path.parent() {
| Some(parent) => {
let folder = parent.display().to_string();
match create_dir_all(folder.clone()) {
| Ok(_) => match OpenOptions::new().write(true).create_new(true).open(&path) {
| Ok(mut file) => match get_bytes().await.map_err(Into::into) {
| Ok(bytes) => {
let mut content = Cursor::new(bytes);
match io::copy(&mut content, &mut file) {
| Ok(_) => Ok(()),
| Err(why) => Err(eyre!("Failed to write bytes — {why}")),
}
}
| Err(why) => Err(why),
},
| Err(why) => Err(eyre!("Failed to create output file — {why}")),
},
| Err(why) => Err(eyre!("Failed to create output folder — {why}")),
}
}
| None => Err(eyre!("Output path has no parent directory")),
}
}
pub fn write_rsa_keypair<P>(values: RsaKeyPair, path: Option<P>) -> ApiResult<(PathBuf, PathBuf)>
where
P: Into<PathBuf>,
{
let resolved = match path {
| Some(p) => Ok(p.into()),
| None => match current_dir() {
| Ok(cwd) => Ok(cwd.join("id_rsa")),
| Err(why) => Err(eyre!("Failed to get current directory — {why}")),
},
};
match resolved {
| Ok(path) => {
let (private_key, public_key) = values;
match private_key.to_pkcs8_pem(rsa::pkcs8::LineEnding::LF) {
| Ok(private_key_pem) => match public_key.to_public_key_pem(rsa::pkcs8::LineEnding::LF) {
| Ok(public_key_pem) => {
let public_key_path = PathBuf::from(format!("{}.pub", path.display()));
let private_key_path = path.clone();
match write_file(path, (*private_key_pem).clone()) {
| Ok(_) => match write_file(public_key_path.clone(), public_key_pem) {
| Ok(_) => Ok((private_key_path, public_key_path)),
| Err(why) => Err(why),
},
| Err(why) => Err(why),
}
}
| Err(why) => {
error!("=> {} Write RSA keypair (public key) — {why}", Label::fail());
Err(eyre!("Failed to serialize public key to PEM — {why}"))
}
},
| Err(why) => {
error!("=> {} Write RSA keypair (private key) — {why}", Label::fail());
Err(eyre!("Failed to serialize private key to PEM — {why}"))
}
}
}
| Err(why) => Err(why),
}
}
#[cfg(test)]
mod tests;