git_mirror/
lib.rs

1/*
2 * Copyright (c) 2017 Pascal Bach
3 *
4 * SPDX-License-Identifier:     MIT
5 */
6
7pub mod error;
8mod git;
9pub mod provider;
10
11use std::fs;
12use std::fs::File;
13use std::path::Path;
14use std::path::PathBuf;
15
16// File locking
17use fs2::FileExt;
18
19// Used for error and debug logging
20use log::{debug, error, info, trace};
21
22// Used to create sane local directory names
23use slug::slugify;
24
25// Macros for serde
26#[macro_use]
27extern crate serde_derive;
28
29// Used to allow multiple paralell sync tasks
30use rayon::iter::{IndexedParallelIterator, IntoParallelRefIterator, ParallelIterator};
31
32// Time handling
33use time::OffsetDateTime;
34
35use junit_report::{ReportBuilder, TestCase, TestCaseBuilder, TestSuite, TestSuiteBuilder};
36
37// Monitoring;
38use prometheus::register_gauge_vec;
39use prometheus::{Encoder, TextEncoder};
40
41use provider::{MirrorError, MirrorResult, Provider};
42
43use git::{Git, GitWrapper};
44
45use error::{GitMirrorError, Result};
46
47pub fn mirror_repo(
48    origin: &str,
49    destination: &str,
50    refspec: &Option<Vec<String>>,
51    lfs: bool,
52    opts: &MirrorOptions,
53) -> Result<()> {
54    if opts.dry_run {
55        return Ok(());
56    }
57
58    let origin_dir = Path::new(&opts.mirror_dir).join(slugify(origin));
59    debug!("Using origin dir: {origin_dir:?}");
60
61    let git = Git::new(opts.git_executable.clone(), opts.mirror_lfs);
62
63    git.git_version()?;
64
65    if opts.mirror_lfs {
66        git.git_lfs_version()?;
67    }
68
69    if origin_dir.is_dir() {
70        info!("Local Update for {origin}");
71
72        git.git_update_mirror(origin, &origin_dir, lfs)?;
73    } else if !origin_dir.exists() {
74        info!("Local Checkout for {origin}");
75
76        git.git_clone_mirror(origin, &origin_dir, lfs)?;
77    } else {
78        return Err(GitMirrorError::GenericError(format!(
79            "Local origin dir is a file: {origin_dir:?}"
80        )));
81    }
82
83    info!("Push to destination {destination}");
84
85    git.git_push_mirror(destination, &origin_dir, refspec, lfs)?;
86
87    if opts.remove_workrepo {
88        fs::remove_dir_all(&origin_dir).map_err(|e| {
89            GitMirrorError::GenericError(format!(
90                "Unable to delete working repository: {} because of error: {}",
91                &origin_dir.to_string_lossy(),
92                e
93            ))
94        })?;
95    }
96
97    Ok(())
98}
99
100fn run_sync_task(v: &[MirrorResult], label: &str, opts: &MirrorOptions) -> TestSuite {
101    // Give the work to the worker pool
102    rayon::ThreadPoolBuilder::new()
103        .num_threads(opts.worker_count)
104        .build_global()
105        .unwrap();
106
107    let proj_total =
108        register_gauge_vec!("git_mirror_total", "Total projects", &["mirror"]).unwrap();
109    let proj_skip =
110        register_gauge_vec!("git_mirror_skip", "Skipped projects", &["mirror"]).unwrap();
111    let proj_fail = register_gauge_vec!("git_mirror_fail", "Failed projects", &["mirror"]).unwrap();
112    let proj_ok = register_gauge_vec!("git_mirror_ok", "OK projects", &["mirror"]).unwrap();
113    let proj_start = register_gauge_vec!(
114        "git_mirror_project_start",
115        "Start of project mirror as unix timestamp",
116        &["origin", "destination", "mirror"]
117    )
118    .unwrap();
119    let proj_end = register_gauge_vec!(
120        "git_mirror_project_end",
121        "End of project mirror as unix timestamp",
122        &["origin", "destination", "mirror"]
123    )
124    .unwrap();
125
126    let total = v.len();
127    let results = v
128        .par_iter()
129        .enumerate()
130        .map(|(i, x)| {
131            proj_total.with_label_values(&[label]).inc();
132            let start = OffsetDateTime::now_utc();
133            match x {
134                Ok(x) => {
135                    let name = format!("{} -> {}", x.origin, x.destination);
136                    let proj_fail = proj_fail.clone();
137                    let proj_ok = proj_ok.clone();
138                    let proj_start = proj_start.clone();
139                    let proj_end = proj_end.clone();
140                    let label = label.to_string();
141                    println!(
142                        "START {}/{} [{}]: {}",
143                        i,
144                        total,
145                        OffsetDateTime::now_utc(),
146                        name
147                    );
148                    proj_start
149                        .with_label_values(&[&x.origin, &x.destination, &label])
150                        .set(OffsetDateTime::now_utc().unix_timestamp() as f64);
151                    let refspec = match &x.refspec {
152                        Some(r) => {
153                            debug!("Using repo specific refspec: {r:?}");
154                            &x.refspec
155                        }
156                        None => {
157                            match opts.refspec.clone() {
158                                Some(r) => {
159                                    debug!("Using global custom refspec: {r:?}");
160                                }
161                                None => {
162                                    debug!("Using no custom refspec.");
163                                }
164                            }
165                            &opts.refspec
166                        }
167                    };
168                    trace!("Refspec used: {refspec:?}");
169                    match mirror_repo(&x.origin, &x.destination, refspec, x.lfs, opts) {
170                        Ok(_) => {
171                            println!(
172                                "END(OK) {}/{} [{}]: {}",
173                                i,
174                                total,
175                                OffsetDateTime::now_utc(),
176                                name
177                            );
178                            proj_end
179                                .with_label_values(&[&x.origin, &x.destination, &label])
180                                .set(OffsetDateTime::now_utc().unix_timestamp() as f64);
181                            proj_ok.with_label_values(&[&label]).inc();
182                            TestCaseBuilder::success(&name, OffsetDateTime::now_utc() - start)
183                                .build()
184                        }
185                        Err(e) => {
186                            println!(
187                                "END(FAIL) {}/{} [{}]: {} ({})",
188                                i,
189                                total,
190                                OffsetDateTime::now_utc(),
191                                name,
192                                e
193                            );
194                            proj_end
195                                .with_label_values(&[&x.origin, &x.destination, &label])
196                                .set(OffsetDateTime::now_utc().unix_timestamp() as f64);
197                            proj_fail.with_label_values(&[&label]).inc();
198                            error!("Unable to sync repo {name} ({e})");
199                            TestCaseBuilder::error(
200                                &name,
201                                OffsetDateTime::now_utc() - start,
202                                "sync error",
203                                &format!("{e:?}"),
204                            )
205                            .build()
206                        }
207                    }
208                }
209                Err(e) => {
210                    proj_skip.with_label_values(&[label]).inc();
211                    let duration = OffsetDateTime::now_utc() - start;
212
213                    match e {
214                        MirrorError::Description(d, se) => {
215                            error!("Error parsing YAML: {d}, Error: {se:?}");
216                            TestCaseBuilder::error("", duration, "parse error", &format!("{e:?}"))
217                                .build()
218                        }
219                        MirrorError::Skip(url) => {
220                            println!(
221                                "SKIP {}/{} [{}]: {}",
222                                i,
223                                total,
224                                OffsetDateTime::now_utc(),
225                                url
226                            );
227                            TestCaseBuilder::skipped(url).build()
228                        }
229                    }
230                }
231            }
232        })
233        .collect::<Vec<TestCase>>();
234
235    let success = results.iter().filter(|x| x.is_success()).count();
236    let ts = TestSuiteBuilder::new("Sync Job")
237        .add_testcases(results)
238        .build();
239    println!(
240        "DONE [{2}]: {0}/{1}",
241        success,
242        total,
243        OffsetDateTime::now_utc()
244    );
245    ts
246}
247
248pub struct MirrorOptions {
249    pub mirror_dir: PathBuf,
250    pub dry_run: bool,
251    pub metrics_file: Option<PathBuf>,
252    pub junit_file: Option<PathBuf>,
253    pub worker_count: usize,
254    pub git_executable: String,
255    pub refspec: Option<Vec<String>>,
256    pub remove_workrepo: bool,
257    pub fail_on_sync_error: bool,
258    pub mirror_lfs: bool,
259}
260
261pub fn do_mirror(provider: Box<dyn Provider>, opts: &MirrorOptions) -> Result<()> {
262    let start_time = register_gauge_vec!(
263        "git_mirror_start_time",
264        "Start time of the sync as unix timestamp",
265        &["mirror"]
266    )
267    .unwrap();
268    let end_time = register_gauge_vec!(
269        "git_mirror_end_time",
270        "End time of the sync as unix timestamp",
271        &["mirror"]
272    )
273    .unwrap();
274
275    // Make sure the mirror directory exists
276    trace!("Create mirror directory at {:?}", opts.mirror_dir);
277    fs::create_dir_all(&opts.mirror_dir).map_err(|e| {
278        GitMirrorError::GenericError(format!(
279            "Unable to create mirror dir: {:?} ({})",
280            &opts.mirror_dir, e
281        ))
282    })?;
283
284    // Check that only one instance is running against a mirror directory
285    let lockfile_path = opts.mirror_dir.join("git-mirror.lock");
286    let lockfile = fs::File::create(&lockfile_path).map_err(|e| {
287        GitMirrorError::GenericError(format!(
288            "Unable to open lockfile: {:?} ({})",
289            &lockfile_path, e
290        ))
291    })?;
292
293    lockfile.try_lock_exclusive().map_err(|e| {
294        GitMirrorError::GenericError(format!(
295            "Another instance is already running against the same mirror directory: {:?} ({})",
296            &opts.mirror_dir, e
297        ))
298    })?;
299
300    trace!("Aquired lockfile: {:?}", &lockfile);
301
302    // Get the list of repos to sync from gitlabsss
303    let v = provider.get_mirror_repos().map_err(|e| -> GitMirrorError {
304        GitMirrorError::GenericError(format!("Unable to get mirror repos ({e})"))
305    })?;
306
307    start_time
308        .with_label_values(&[&provider.get_label()])
309        .set(OffsetDateTime::now_utc().unix_timestamp() as f64);
310
311    let ts = run_sync_task(&v, &provider.get_label(), opts);
312
313    end_time
314        .with_label_values(&[&provider.get_label()])
315        .set(OffsetDateTime::now_utc().unix_timestamp() as f64);
316
317    match opts.metrics_file {
318        Some(ref f) => write_metrics(f),
319        None => trace!("Skipping metrics file creation"),
320    };
321
322    // Check if any tasks failed
323    let error_count = ts.errors() + ts.failures();
324
325    match opts.junit_file {
326        Some(ref f) => write_junit_report(f, ts),
327        None => trace!("Skipping junit report"),
328    }
329
330    if opts.fail_on_sync_error && error_count > 0 {
331        Err(GitMirrorError::SyncError(error_count))
332    } else {
333        Ok(())
334    }
335}
336
337fn write_metrics(f: &Path) {
338    let mut file = File::create(f).unwrap();
339    let encoder = TextEncoder::new();
340    let metric_familys = prometheus::gather();
341    encoder.encode(&metric_familys, &mut file).unwrap();
342}
343
344fn write_junit_report(f: &Path, ts: TestSuite) {
345    let report = ReportBuilder::default().add_testsuite(ts).build();
346    let mut file = File::create(f).unwrap();
347    report.write_xml(&mut file).unwrap();
348}
git_mirror/lib.rs

git_mirror/
lib.rs
