pub mod error;
mod git;
pub mod provider;
use std::fs;
use std::fs::File;
use std::path::Path;
use std::path::PathBuf;
use fs2::FileExt;
use log::{debug, error, info, trace};
use slug::slugify;
#[macro_use]
extern crate serde_derive;
use rayon::iter::{IndexedParallelIterator, IntoParallelRefIterator, ParallelIterator};
use time::OffsetDateTime;
use junit_report::{ReportBuilder, TestCase, TestCaseBuilder, TestSuite, TestSuiteBuilder};
use prometheus::register_gauge_vec;
use prometheus::{Encoder, TextEncoder};
use provider::{MirrorError, MirrorResult, Provider};
use git::{Git, GitWrapper};
use error::{GitMirrorError, Result};
pub fn mirror_repo(
origin: &str,
destination: &str,
refspec: &Option<Vec<String>>,
lfs: bool,
opts: &MirrorOptions,
) -> Result<()> {
if opts.dry_run {
return Ok(());
}
let origin_dir = Path::new(&opts.mirror_dir).join(slugify(origin));
debug!("Using origin dir: {0:?}", origin_dir);
let git = Git::new(opts.git_executable.clone(), opts.mirror_lfs);
git.git_version()?;
if opts.mirror_lfs {
git.git_lfs_version()?;
}
if origin_dir.is_dir() {
info!("Local Update for {}", origin);
git.git_update_mirror(origin, &origin_dir, lfs)?;
} else if !origin_dir.exists() {
info!("Local Checkout for {}", origin);
git.git_clone_mirror(origin, &origin_dir, lfs)?;
} else {
return Err(GitMirrorError::GenericError(format!(
"Local origin dir is a file: {origin_dir:?}"
)));
}
info!("Push to destination {}", destination);
git.git_push_mirror(destination, &origin_dir, refspec, lfs)?;
if opts.remove_workrepo {
fs::remove_dir_all(&origin_dir).map_err(|e| {
GitMirrorError::GenericError(format!(
"Unable to delete working repository: {} because of error: {}",
&origin_dir.to_string_lossy(),
e
))
})?;
}
Ok(())
}
fn run_sync_task(v: &[MirrorResult], label: &str, opts: &MirrorOptions) -> TestSuite {
rayon::ThreadPoolBuilder::new()
.num_threads(opts.worker_count)
.build_global()
.unwrap();
let proj_total =
register_gauge_vec!("git_mirror_total", "Total projects", &["mirror"]).unwrap();
let proj_skip =
register_gauge_vec!("git_mirror_skip", "Skipped projects", &["mirror"]).unwrap();
let proj_fail = register_gauge_vec!("git_mirror_fail", "Failed projects", &["mirror"]).unwrap();
let proj_ok = register_gauge_vec!("git_mirror_ok", "OK projects", &["mirror"]).unwrap();
let proj_start = register_gauge_vec!(
"git_mirror_project_start",
"Start of project mirror as unix timestamp",
&["origin", "destination", "mirror"]
)
.unwrap();
let proj_end = register_gauge_vec!(
"git_mirror_project_end",
"End of project mirror as unix timestamp",
&["origin", "destination", "mirror"]
)
.unwrap();
let total = v.len();
let results = v
.par_iter()
.enumerate()
.map(|(i, x)| {
proj_total.with_label_values(&[label]).inc();
let start = OffsetDateTime::now_utc();
match x {
Ok(x) => {
let name = format!("{} -> {}", x.origin, x.destination);
let proj_fail = proj_fail.clone();
let proj_ok = proj_ok.clone();
let proj_start = proj_start.clone();
let proj_end = proj_end.clone();
let label = label.to_string();
println!(
"START {}/{} [{}]: {}",
i,
total,
OffsetDateTime::now_utc(),
name
);
proj_start
.with_label_values(&[&x.origin, &x.destination, &label])
.set(OffsetDateTime::now_utc().unix_timestamp() as f64);
let refspec = match &x.refspec {
Some(r) => {
debug!("Using repo specific refspec: {:?}", r);
&x.refspec
}
None => {
match opts.refspec.clone() {
Some(r) => {
debug!("Using global custom refspec: {:?}", r);
}
None => {
debug!("Using no custom refspec.");
}
}
&opts.refspec
}
};
trace!("Refspec used: {:?}", refspec);
match mirror_repo(&x.origin, &x.destination, refspec, x.lfs, opts) {
Ok(_) => {
println!(
"END(OK) {}/{} [{}]: {}",
i,
total,
OffsetDateTime::now_utc(),
name
);
proj_end
.with_label_values(&[&x.origin, &x.destination, &label])
.set(OffsetDateTime::now_utc().unix_timestamp() as f64);
proj_ok.with_label_values(&[&label]).inc();
TestCaseBuilder::success(&name, OffsetDateTime::now_utc() - start)
.build()
}
Err(e) => {
println!(
"END(FAIL) {}/{} [{}]: {} ({})",
i,
total,
OffsetDateTime::now_utc(),
name,
e
);
proj_end
.with_label_values(&[&x.origin, &x.destination, &label])
.set(OffsetDateTime::now_utc().unix_timestamp() as f64);
proj_fail.with_label_values(&[&label]).inc();
error!("Unable to sync repo {} ({})", name, e);
TestCaseBuilder::error(
&name,
OffsetDateTime::now_utc() - start,
"sync error",
&format!("{e:?}"),
)
.build()
}
}
}
Err(e) => {
proj_skip.with_label_values(&[label]).inc();
let duration = OffsetDateTime::now_utc() - start;
match e {
MirrorError::Description(d, se) => {
error!("Error parsing YAML: {}, Error: {:?}", d, se);
TestCaseBuilder::error("", duration, "parse error", &format!("{e:?}"))
.build()
}
MirrorError::Skip(url) => {
println!(
"SKIP {}/{} [{}]: {}",
i,
total,
OffsetDateTime::now_utc(),
url
);
TestCaseBuilder::skipped(url).build()
}
}
}
}
})
.collect::<Vec<TestCase>>();
let success = results.iter().filter(|x| x.is_success()).count();
let ts = TestSuiteBuilder::new("Sync Job")
.add_testcases(results)
.build();
println!(
"DONE [{2}]: {0}/{1}",
success,
total,
OffsetDateTime::now_utc()
);
ts
}
pub struct MirrorOptions {
pub mirror_dir: PathBuf,
pub dry_run: bool,
pub metrics_file: Option<PathBuf>,
pub junit_file: Option<PathBuf>,
pub worker_count: usize,
pub git_executable: String,
pub refspec: Option<Vec<String>>,
pub remove_workrepo: bool,
pub fail_on_sync_error: bool,
pub mirror_lfs: bool,
}
pub fn do_mirror(provider: Box<dyn Provider>, opts: &MirrorOptions) -> Result<()> {
let start_time = register_gauge_vec!(
"git_mirror_start_time",
"Start time of the sync as unix timestamp",
&["mirror"]
)
.unwrap();
let end_time = register_gauge_vec!(
"git_mirror_end_time",
"End time of the sync as unix timestamp",
&["mirror"]
)
.unwrap();
trace!("Create mirror directory at {:?}", opts.mirror_dir);
fs::create_dir_all(&opts.mirror_dir).map_err(|e| {
GitMirrorError::GenericError(format!(
"Unable to create mirror dir: {:?} ({})",
&opts.mirror_dir, e
))
})?;
let lockfile_path = opts.mirror_dir.join("git-mirror.lock");
let lockfile = fs::File::create(&lockfile_path).map_err(|e| {
GitMirrorError::GenericError(format!(
"Unable to open lockfile: {:?} ({})",
&lockfile_path, e
))
})?;
lockfile.try_lock_exclusive().map_err(|e| {
GitMirrorError::GenericError(format!(
"Another instance is already running against the same mirror directory: {:?} ({})",
&opts.mirror_dir, e
))
})?;
trace!("Aquired lockfile: {:?}", &lockfile);
let v = provider.get_mirror_repos().map_err(|e| -> GitMirrorError {
GitMirrorError::GenericError(format!("Unable to get mirror repos ({e})"))
})?;
start_time
.with_label_values(&[&provider.get_label()])
.set(OffsetDateTime::now_utc().unix_timestamp() as f64);
let ts = run_sync_task(&v, &provider.get_label(), opts);
end_time
.with_label_values(&[&provider.get_label()])
.set(OffsetDateTime::now_utc().unix_timestamp() as f64);
match opts.metrics_file {
Some(ref f) => write_metrics(f),
None => trace!("Skipping metrics file creation"),
};
let error_count = ts.errors() + ts.failures();
match opts.junit_file {
Some(ref f) => write_junit_report(f, ts),
None => trace!("Skipping junit report"),
}
if opts.fail_on_sync_error && error_count > 0 {
Err(GitMirrorError::SyncError(error_count))
} else {
Ok(())
}
}
fn write_metrics(f: &Path) {
let mut file = File::create(f).unwrap();
let encoder = TextEncoder::new();
let metric_familys = prometheus::gather();
encoder.encode(&metric_familys, &mut file).unwrap();
}
fn write_junit_report(f: &Path, ts: TestSuite) {
let report = ReportBuilder::default().add_testsuite(ts).build();
let mut file = File::create(f).unwrap();
report.write_xml(&mut file).unwrap();
}