zhao-cli 0.2.0

Deterministic, offline change-review and CI gate for data transformation projects.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165

//! Resolves the Baseline for `zhao check`: either the manifest explicitly
//! passed via `--state`, or -- when that's absent -- a Baseline zhao
//! compiles itself from the git-native merge-base commit, with no external
//! artifact required.

use std::path::{Path, PathBuf};

use zhao_core::adapters::dbt::DbtAdapterError;
use zhao_core::git::{self, GitError};
use zhao_core::model::ParsedProject;

use crate::adapter::ResolvedAdapter;

/// Everything that can go wrong resolving a Baseline.
#[derive(Debug, thiserror::Error)]
pub enum BaselineError {
    /// The `--state` manifest couldn't be read or parsed.
    #[error("{path}: {source}")]
    Manifest {
        /// The `--state` path that failed.
        path: String,
        /// The underlying parse error.
        #[source]
        source: DbtAdapterError,
    },
    /// Resolving or checking out the git-native merge-base commit failed.
    #[error(transparent)]
    Git(#[from] GitError),
    /// Compiling or parsing the merge-base commit failed.
    #[error(transparent)]
    Dbt(#[from] DbtAdapterError),
}

/// Resolves the Baseline `zhao check` diffs the current project against.
///
/// `adapter` is the already-resolved Transformation Tool Adapter (see
/// `crate::adapter::ResolvedAdapter`) -- every `parse`/`compile`/`deps`
/// call below goes through it, never a hardcoded concrete adapter type.
///
/// If `state_path` is given, it's parsed directly (`extra_args` is
/// ignored in this path -- there's no `dbt` invocation to pass them to).
/// Otherwise, the merge-base commit between `HEAD` and `against` is
/// resolved in the git repository containing `project_dir`, checked out
/// into a temporary worktree, compiled there with `dbt`, and parsed from
/// that worktree -- all without requiring any externally-supplied
/// Baseline artifact. `dbt deps` is run first whenever the worktree's
/// project directory has a `packages.yml` or `dependencies.yml`, since
/// `dbt compile` fails on any `ref()`/macro from an as-yet-uninstalled
/// package -- exactly the state a freshly checked-out worktree is in the
/// first time. `dbt_command` (from `--dbt-command`/`zhao.yml`'s
/// `dbt-command`, defaulting to `"dbt"`) is the executable/prefix both
/// invocations use; `extra_args` (from `--dbt-arg`/`--dbt-args`) are
/// appended verbatim to both the `dbt deps` and `dbt compile` invocations.
///
/// Before the temporary worktree is torn down (it's removed the moment
/// this function returns -- see [`git::Worktree`]'s own `Drop`), its
/// compiled `target/manifest.json` is copied to
/// `<project_dir>/target/zhao/baseline_manifest.json`, so what the
/// Baseline actually compiled to is still inspectable afterward instead
/// of vanishing along with the worktree. Best-effort: a failure to copy
/// (permissions, a read-only `target/`, ...) is reported to stderr as a
/// warning and never fails Baseline resolution itself -- the same
/// "a sidecar artifact failing to write shouldn't fail the actual
/// command" precedent `target/zhao/run-metadata.json` already follows.
/// Only meaningful for this git-native path -- `--state <path>` returns
/// above, before anything is compiled, so there's nothing to capture.
pub fn resolve(
    adapter: &ResolvedAdapter,
    state_path: Option<&Path>,
    project_dir: &Path,
    against: &str,
    dbt_command: &str,
    extra_args: &[String],
) -> Result<ParsedProject, BaselineError> {
    if let Some(path) = state_path {
        return adapter
            .parse(path)
            .map_err(|source| BaselineError::Manifest {
                path: path.display().to_string(),
                source,
            });
    }

    let repo_root = git::repo_root(project_dir)?;
    let canonical_repo_root = repo_root.canonicalize().unwrap_or(repo_root);
    let canonical_project_dir = project_dir
        .canonicalize()
        .unwrap_or_else(|_| project_dir.to_path_buf());
    let relative_project_dir = canonical_project_dir
        .strip_prefix(&canonical_repo_root)
        .unwrap_or(Path::new("."));

    let merge_base = git::resolve_merge_base(&canonical_repo_root, against)?;
    let worktree = git::create_worktree(&canonical_repo_root, &merge_base)?;
    let worktree_project_dir = worktree.path().join(relative_project_dir);

    if worktree_project_dir.join("packages.yml").exists()
        || worktree_project_dir.join("dependencies.yml").exists()
    {
        crate::log::log_dbt_result(
            "deps",
            &worktree_project_dir,
            project_dir,
            adapter.deps(&worktree_project_dir, dbt_command, extra_args),
        )?;
    }
    crate::log::log_dbt_result(
        "compile",
        &worktree_project_dir,
        project_dir,
        adapter.compile(&worktree_project_dir, dbt_command, extra_args),
    )?;

    let manifest_path = worktree_project_dir.join("target").join("manifest.json");
    capture_baseline_manifest(&manifest_path, project_dir);
    Ok(adapter.parse(&manifest_path)?)
}

/// Routes a `dbt compile`/`dbt deps` subcommand's captured stdout/stderr
/// into `real_project_dir`'s daily run log -- both a successful run's
/// output (previously discarded entirely) and a failing one's (already
/// captured on the error, for #30's terminal error message; this adds
/// the same content to the log for post-hoc inspection too) -- then
/// passes the `Result` straight through unchanged. See issue #36.
///
/// `worktree_project_dir` is where the subcommand actually ran (the
/// temporary git worktree, gone by the time anyone reads the log --
/// recorded in the log entry for context regardless);
/// `real_project_dir` is where the log itself lives, same as every
/// other `target/zhao/` artifact `resolve` produces.
/// Copies `manifest_path` (the just-compiled Baseline's manifest, still
/// inside the temporary worktree at this point) to
/// `<project_dir>/target/zhao/baseline_manifest.json`. See [`resolve`]'s
/// own doc comment for why, and for the best-effort/non-fatal contract.
fn capture_baseline_manifest(manifest_path: &Path, project_dir: &Path) {
    let dest_dir = project_dir.join("target").join("zhao");
    if let Err(err) = std::fs::create_dir_all(&dest_dir) {
        eprintln!(
            "warning: could not create {} to capture the baseline manifest: {err}",
            dest_dir.display()
        );
        return;
    }

    // Written via a temp file in the same directory, then renamed into
    // place -- the same atomic-write precedent `target/zhao/run-
    // metadata.json` (`crate::metadata::write`) already follows, so a
    // failure partway through (disk full, the process killed) can never
    // leave a truncated/corrupt `baseline_manifest.json` overwriting a
    // previously good one from an earlier run.
    let dest: PathBuf = dest_dir.join("baseline_manifest.json");
    let write_result = (|| -> std::io::Result<()> {
        let contents = std::fs::read(manifest_path)?;
        let mut temp_file = tempfile::NamedTempFile::new_in(&dest_dir)?;
        std::io::Write::write_all(&mut temp_file, &contents)?;
        temp_file.persist(&dest).map_err(|err| err.error)?;
        Ok(())
    })();
    if let Err(err) = write_result {
        eprintln!(
            "warning: could not capture the baseline manifest to {}: {err}",
            dest.display()
        );
    }
}