spack/
subprocess.rs

1/* Copyright 2022-2023 Danny McClanahan */
2/* SPDX-License-Identifier: (Apache-2.0 OR MIT) */
3
4/* use super_process::{base, exe, fs, stream, sync}; */
5
6pub mod python {
7  use super_process::{
8    base::{self, CommandBase},
9    exe, fs,
10    sync::SyncInvocable,
11  };
12
13  use async_trait::async_trait;
14  use displaydoc::Display;
15  use once_cell::sync::Lazy;
16  use regex::Regex;
17  use thiserror::Error;
18
19  use std::{env, ffi::OsString, io, path::Path, str};
20
21  /// Things that can go wrong when detecting python.
22  #[derive(Debug, Display, Error)]
23  pub enum PythonError {
24    /// unknown error: {0}
25    UnknownError(String),
26    /// error executing command: {0}
27    Command(#[from] exe::CommandErrorWrapper),
28    /// error setting up command: {0}
29    Setup(#[from] base::SetupErrorWrapper),
30    /// io error: {0}
31    Io(#[from] io::Error),
32  }
33
34  #[derive(Debug, Clone)]
35  pub struct PythonInvocation {
36    exe: exe::Exe,
37    inner: exe::Command,
38  }
39
40  #[async_trait]
41  impl CommandBase for PythonInvocation {
42    async fn setup_command(self) -> Result<exe::Command, base::SetupError> {
43      let Self { exe, mut inner } = self;
44      inner.unshift_new_exe(exe);
45      Ok(inner)
46    }
47  }
48
49  /// Refers to a particular python executable [`PYTHON_CMD`] first on the
50  /// `$PATH`.
51  #[derive(Debug, Clone)]
52  pub struct FoundPython {
53    pub exe: exe::Exe,
54    /// Version string parsed from the python executable.
55    pub version: String,
56  }
57
58  /// Pattern we match against when executing [`Python::detect`].
59  pub static PYTHON_VERSION_REGEX: Lazy<Regex> =
60    Lazy::new(|| Regex::new(r"^Python (3\.[0-9]+\.[0-9]+).*\n$").unwrap());
61
62  impl FoundPython {
63    fn determine_python_exename() -> exe::Exe {
64      let exe_name: OsString = env::var_os("SPACK_PYTHON").unwrap_or_else(|| "python3".into());
65      let exe_path = Path::new(&exe_name).to_path_buf();
66      exe::Exe(fs::File(exe_path))
67    }
68
69    /// Check for a valid python installation by parsing the output of
70    /// `--version`.
71    pub async fn detect() -> Result<Self, PythonError> {
72      let py = Self::determine_python_exename();
73      let command = PythonInvocation {
74        exe: py.clone(),
75        inner: exe::Command {
76          argv: ["--version"].as_ref().into(),
77          ..Default::default()
78        },
79      }
80      .setup_command()
81      .await
82      .map_err(|e| e.with_context(format!("in FoundPython::detect(py = {:?})", &py)))?;
83      let output = command.invoke().await?;
84      let stdout = str::from_utf8(&output.stdout).map_err(|e| {
85        PythonError::UnknownError(format!(
86          "could not parse utf8 from '{} --version' stdout ({}); received:\n{:?}",
87          &py, &e, &output.stdout
88        ))
89      })?;
90      match PYTHON_VERSION_REGEX.captures(stdout) {
91        Some(m) => Ok(Self {
92          exe: py,
93          version: m.get(1).unwrap().as_str().to_string(),
94        }),
95        None => Err(PythonError::UnknownError(format!(
96          "could not parse '{} --version'; received:\n(stdout):\n{}",
97          py, &stdout,
98        ))),
99      }
100    }
101
102    pub(crate) fn with_python_exe(self, inner: exe::Command) -> PythonInvocation {
103      let Self { exe, .. } = self;
104      PythonInvocation { exe, inner }
105    }
106  }
107
108  #[cfg(test)]
109  mod test {
110    use super::*;
111
112    use tokio;
113
114    #[tokio::test]
115    async fn test_detect_python() -> Result<(), crate::Error> {
116      let _python = FoundPython::detect().await.unwrap();
117      Ok(())
118    }
119  }
120}
121
122pub mod spack {
123  use super::python;
124  use crate::{commands, summoning};
125  use super_process::{
126    base::{self, CommandBase},
127    exe, fs,
128    sync::SyncInvocable,
129  };
130
131  use async_trait::async_trait;
132  use displaydoc::Display;
133  use thiserror::Error;
134
135  use std::{
136    io,
137    path::{Path, PathBuf},
138    str,
139  };
140
141  #[derive(Debug, Display, Error)]
142  pub enum InvocationSummoningError {
143    /// error validating arguments: {0}
144    Validation(#[from] base::SetupErrorWrapper),
145    /// error executing command: {0}
146    Command(#[from] exe::CommandErrorWrapper),
147    /// error summoning: {0}
148    Summon(#[from] summoning::SummoningError),
149    /// error finding compilers: {0}
150    CompilerFind(#[from] commands::compiler_find::CompilerFindError),
151    /// error bootstrapping: {0}
152    Bootstrap(#[from] commands::install::InstallError),
153    /// error location python: {0}
154    Python(#[from] python::PythonError),
155    /// io error: {0}
156    Io(#[from] io::Error),
157  }
158
159  /// Builder for spack subprocesss.
160  #[derive(Debug, Clone)]
161  pub struct SpackInvocation {
162    /// Information about the python executable used to execute spack with.
163    python: python::FoundPython,
164    /// Information about the spack checkout.
165    repo: summoning::SpackRepo,
166    /// Version parsed from executing with '--version'.
167    #[allow(dead_code)]
168    pub version: String,
169  }
170
171  pub(crate) static SUMMON_CUR_PROCESS_LOCK: once_cell::sync::Lazy<tokio::sync::Mutex<()>> =
172    once_cell::sync::Lazy::new(|| tokio::sync::Mutex::new(()));
173
174  impl SpackInvocation {
175    pub(crate) fn cache_location(&self) -> &Path { self.repo.cache_location() }
176
177    /// Create an instance.
178    ///
179    /// You should prefer to call [`Self::clone`] on the first instance you
180    /// construct instead of repeatedly calling this method when executing
181    /// multiple spack subprocesss in a row.
182    pub async fn create(
183      python: python::FoundPython,
184      repo: summoning::SpackRepo,
185    ) -> Result<Self, InvocationSummoningError> {
186      let script_path = format!("{}", repo.script_path.display());
187      let command = python
188        .clone()
189        .with_python_exe(exe::Command {
190          argv: [&script_path, "--version"].as_ref().into(),
191          ..Default::default()
192        })
193        .setup_command()
194        .await
195        .map_err(|e| e.with_context(format!("with py {:?} and repo {:?}", &python, &repo)))?;
196      let output = command.clone().invoke().await?;
197      let version = str::from_utf8(&output.stdout)
198        .map_err(|e| format!("utf8 decoding error {}: from {:?}", e, &output.stdout))
199        .and_then(|s| {
200          s.strip_suffix('\n')
201            .ok_or_else(|| format!("failed to strip final newline from output: '{}'", s))
202        })
203        .map_err(|e: String| {
204          python::PythonError::UnknownError(format!(
205            "error parsing '{} {} --version' output: {}",
206            &python.exe, &script_path, e
207          ))
208        })?
209        .to_string();
210      Ok(Self {
211        python,
212        repo,
213        version,
214      })
215    }
216
217    async fn ensure_compilers_found(&self) -> Result<(), InvocationSummoningError> {
218      let find_site_compilers = commands::compiler_find::CompilerFind {
219        spack: self.clone(),
220        paths: vec![PathBuf::from("/usr/bin")],
221        scope: Some("site".to_string()),
222      };
223      find_site_compilers.compiler_find().await?;
224      Ok(())
225    }
226
227    async fn bootstrap(
228      &self,
229      cache_dir: summoning::CacheDir,
230    ) -> Result<(), InvocationSummoningError> {
231      let bootstrap_proof_name: PathBuf = format!("{}.bootstrap_proof", cache_dir.dirname()).into();
232      let bootstrap_proof_path = cache_dir.location().join(bootstrap_proof_name);
233
234      match tokio::fs::File::open(&bootstrap_proof_path).await {
235        Ok(_) => return Ok(()),
236        /* If not found, continue. */
237        Err(e) if e.kind() == io::ErrorKind::NotFound => (),
238        Err(e) => return Err(e.into()),
239      }
240
241      let bootstrap_lock_name: PathBuf = format!("{}.bootstrap_lock", cache_dir.dirname()).into();
242      let bootstrap_lock_path = cache_dir.location().join(bootstrap_lock_name);
243      let mut lockfile =
244        tokio::task::spawn_blocking(move || fslock::LockFile::open(&bootstrap_lock_path))
245          .await
246          .unwrap()?;
247      /* This unlocks the lockfile upon drop! */
248      let _lockfile = tokio::task::spawn_blocking(move || {
249        lockfile.lock_with_pid()?;
250        Ok::<_, io::Error>(lockfile)
251      })
252      .await
253      .unwrap()?;
254
255      /* See if the target file was created since we locked the lockfile. */
256      if tokio::fs::File::open(&bootstrap_proof_path).await.is_ok() {
257        /* If so, return early! */
258        return Ok(());
259      }
260
261      eprintln!(
262        "bootstrapping spack {}",
263        crate::versions::patches::PATCHES_TOPLEVEL_COMPONENT,
264      );
265
266      self.ensure_compilers_found().await?;
267
268      let bootstrap_install = commands::install::Install {
269        spack: self.clone(),
270        spec: commands::CLISpec::new("zlib"),
271        verbosity: Default::default(),
272        env: None,
273        repos: None,
274      };
275      let installed_spec = bootstrap_install.install_find().await?;
276
277      use tokio::io::AsyncWriteExt;
278      let mut proof = tokio::fs::File::create(bootstrap_proof_path).await?;
279      proof
280        .write_all(format!("{}", installed_spec.hashed_spec()).as_bytes())
281        .await?;
282
283      Ok(())
284    }
285
286    /// Create an instance via [`Self::create`], with good defaults.
287    pub async fn summon() -> Result<Self, InvocationSummoningError> {
288      /* Our use of file locking within the summoning process does not
289       * differentiate between different threads within the same process, so
290       * we additionally lock in-process here. */
291      let _lock = SUMMON_CUR_PROCESS_LOCK.lock().await;
292
293      let python = python::FoundPython::detect().await?;
294      let cache_dir = summoning::CacheDir::get_or_create().await?;
295      let spack_repo = summoning::SpackRepo::summon(cache_dir.clone()).await?;
296      let spack = Self::create(python, spack_repo).await?;
297      spack.bootstrap(cache_dir).await?;
298      Ok(spack)
299    }
300
301    /// Get a [`CommandBase`] instance to execute spack with the given `argv`.
302    pub(crate) fn with_spack_exe(self, inner: exe::Command) -> ReadiedSpackInvocation {
303      let Self { python, repo, .. } = self;
304      ReadiedSpackInvocation {
305        python,
306        repo,
307        inner,
308      }
309    }
310  }
311
312  #[cfg(test)]
313  mod test {
314    use super::*;
315
316    use crate::{subprocess::python::*, summoning::*};
317
318    use tokio;
319
320    #[tokio::test]
321    async fn test_summon() -> Result<(), crate::Error> {
322      let spack = SpackInvocation::summon().await?;
323      // This is the current version number for the spack installation.
324      assert_eq!(spack.version, "1.0.0.dev0");
325      Ok(())
326    }
327
328    #[tokio::test]
329    async fn test_create_invocation() -> Result<(), crate::Error> {
330      let _lock = SUMMON_CUR_PROCESS_LOCK.lock().await;
331
332      // Access a few of the relevant files and directories.
333      let python = FoundPython::detect().await.unwrap();
334      let cache_dir = CacheDir::get_or_create().await.unwrap();
335      let spack_exe = SpackRepo::summon(cache_dir).await.unwrap();
336      let spack = SpackInvocation::create(python, spack_exe).await?;
337
338      // This is the current version number for the spack installation.
339      assert_eq!(spack.version, "1.0.0.dev0");
340      Ok(())
341    }
342  }
343
344  pub(crate) struct ReadiedSpackInvocation {
345    pub python: python::FoundPython,
346    pub repo: summoning::SpackRepo,
347    pub inner: exe::Command,
348  }
349
350  #[async_trait]
351  impl base::CommandBase for ReadiedSpackInvocation {
352    async fn setup_command(self) -> Result<exe::Command, base::SetupError> {
353      let Self {
354        python,
355        repo:
356          summoning::SpackRepo {
357            script_path,
358            repo_path,
359            ..
360          },
361        mut inner,
362      } = self;
363
364      assert!(inner.wd.is_none(), "assuming working dir was not yet set");
365      inner.wd = Some(fs::Directory(repo_path));
366
367      assert!(inner.exe.is_empty());
368      inner.unshift_new_exe(exe::Exe(fs::File(script_path)));
369      let py = python.with_python_exe(inner);
370
371      Ok(py.setup_command().await?)
372    }
373  }
374}
spack/subprocess.rs

spack/
subprocess.rs
