shep 0.24.0

The shep binary: a process manager that keeps a flock of long-running processes alive on macOS, Linux and Windows, with logs, watch and cron restarts, and webhook alerts
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
//! Turns a CLI target token into `AppConfig`s.
//!
//! [`resolve_target`] walks `start`'s precedence: `-`, an explicit
//! `--flockfile`, a recognised extension, or a bare script path. A `.js`
//! Flockfile is evaluated through node via [`evaluate_js_flockfile`].

use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use std::time::Duration;

use shep_core::config::{AppConfig, DeclaredApp, FlockFormat, Flockfile, FlockfileError};

use crate::commands::bounded::{Bounded, run_bounded};
use crate::commands::lifecycle::default_cwd_to_flockfile_dir;
use crate::exit::ExitCode;

/// What [`resolve_target`] can fail with
///
/// Mapped to exit codes by [`target_exit_code`], not by a `From` impl.
/// `start`'s daemon-side failures are `shep_client`'s errors instead.
#[derive(Debug)]
pub enum TargetError {
    /// `target` was `-` and stdin was not valid UTF-8, or the read failed.
    Stdin(std::io::Error),
    /// The extension named a recognised Flockfile format, but the file
    /// could not be read.
    Read {
        /// The path that failed to read.
        path: PathBuf,
        /// The underlying IO failure.
        source: std::io::Error,
    },
    /// The source read fine but failed Flockfile validation.
    Flockfile(FlockfileError),
    /// `target` named nothing at any tier of `start`'s precedence: no sheep
    /// by id or name, no fold, no Flockfile, and no path on disk.
    Unresolvable {
        /// The raw target string.
        target: String,
    },
    /// `--flockfile` was given for a path whose extension names no format
    /// this can read.
    UnknownFlockfileFormat {
        /// The path as the operator wrote it.
        path: PathBuf,
    },
    /// A `.js` Flockfile could not be evaluated. `node_missing` separates
    /// "install node" from "your config threw", which exit differently.
    /// No `path` field: every `detail` already names it.
    Js {
        /// What went wrong, already phrased for the operator.
        detail: String,
        /// `true` when node itself was not found on `PATH`.
        node_missing: bool,
    },
}

impl std::fmt::Display for TargetError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            Self::Stdin(err) => write!(f, "failed to read stdin: {err}"),
            Self::Read { path, source } => {
                write!(f, "failed to read {}: {source}", path.display())
            }
            Self::Flockfile(err) => write!(f, "{err}"),
            Self::Unresolvable { target } => {
                write!(
                    f,
                    "{target} is not a sheep, a fold, `-`, a recognised Flockfile, or an \
                     existing path"
                )?;
                // Only for a word that could have been an assignment and was
                // not: a name holding a separator was always a path.
                match target.split_once('=') {
                    Some((name, _)) if !name.is_empty() && !name.contains(['/', '\\']) => write!(
                        f,
                        "; `{name}` is not a name an assignment can use, which takes a letter \
                         or `_` and then letters, digits or `_`"
                    ),
                    _ => Ok(()),
                }
            }
            Self::UnknownFlockfileFormat { path } => write!(
                f,
                "--flockfile needs a .toml, .yaml, .yml, .json, .json5 or .js file; {} is none of those",
                path.display()
            ),
            Self::Js { detail, .. } => f.write_str(detail),
        }
    }
}

impl core::error::Error for TargetError {
    fn source(&self) -> Option<&(dyn core::error::Error + 'static)> {
        match self {
            Self::Stdin(err) | Self::Read { source: err, .. } => Some(err),
            Self::Flockfile(err) => Some(err),
            Self::Unresolvable { .. } | Self::UnknownFlockfileFormat { .. } | Self::Js { .. } => {
                None
            }
        }
    }
}

impl From<FlockfileError> for TargetError {
    fn from(source: FlockfileError) -> Self {
        Self::Flockfile(source)
    }
}

/// `start`'s mapping from a resolution failure to the exit code that reports it
pub(crate) fn target_exit_code(err: &TargetError) -> ExitCode {
    match err {
        TargetError::Stdin(_) => ExitCode::Failure,
        TargetError::Read { .. } | TargetError::Unresolvable { .. } => ExitCode::Usage,
        TargetError::Flockfile(_) => ExitCode::InvalidConfig,
        TargetError::UnknownFlockfileFormat { .. } => ExitCode::Usage,
        TargetError::Js {
            node_missing: true, ..
        } => ExitCode::Failure,
        TargetError::Js {
            node_missing: false,
            ..
        } => ExitCode::InvalidConfig,
    }
}

/// How long node gets to hand back a `.js` Flockfile's JSON before shep
/// kills it
///
/// 30s, against ~60ms to require a small module and the couple of seconds a
/// large dependency tree costs on a cold filesystem.
const JS_EVAL_BUDGET: Duration = Duration::from_secs(30);

/// The filename [`evaluate_js_flockfile`] writes [`JS_BRIDGE_SCRIPT`] to
///
/// Plain ASCII with no spaces: it reaches node as a bare relative argument,
/// so nothing needing quoting reaches a command line.
const JS_BRIDGE_FILE: &str = "shep-flockfile-bridge.js";

/// The bridge run by [`evaluate_js_flockfile`], written to a file rather
/// than passed to `node -e`
///
/// A `node` resolving to a `.cmd` shim has `cmd.exe` re-parse a `-e`
/// argument, and this script contains `&&`. A file's contents cross no
/// parser. The path comes from the environment, so a Flockfile path
/// containing `'`, `\` or a newline has no string literal to escape.
const JS_BRIDGE_SCRIPT: &str = "try { \
     process.stdout.write(JSON.stringify(require(process.env.SHEP_FLOCKFILE_PATH))); \
 } catch (err) { \
     process.stderr.write('[bridge saw ' + String(process.env.SHEP_FLOCKFILE_PATH) + '] ' + (err && err.message ? String(err.message) : String(err))); \
     process.exitCode = 1; \
 }";

/// Evaluates a `.js` Flockfile through node and returns its JSON
///
/// `SHEP_FLOCKFILE_PATH` carries the path, absolute: `require("x.js")`
/// without `./` resolves against `node_modules`. `budget` bounds node
/// exiting, not `require` returning. `docs/migration.md` quotes the
/// `node_missing` sentence by hand.
///
/// # Errors
/// - [`TargetError::Read`] if the path could not be canonicalized.
/// - [`TargetError::Js`] with `node_missing` if node is not on `PATH`.
/// - [`TargetError::Js`] if node failed, could not be spawned, ran past
///   `budget`, or left a process holding the output.
pub(crate) fn evaluate_js_flockfile(path: &Path, budget: Duration) -> Result<String, TargetError> {
    let absolute = std::fs::canonicalize(path).map_err(|source| TargetError::Read {
        path: path.to_path_buf(),
        source,
    })?;
    // Bound until this function returns: dropping a `TempDir` deletes the
    // loader node is still reading.
    let scratch = tempfile::Builder::new()
        .prefix("shep-js-bridge")
        .tempdir()
        .map_err(|source| TargetError::Read {
            path: path.to_path_buf(),
            source,
        })?;
    let loader = scratch.path().join(JS_BRIDGE_FILE);
    std::fs::write(&loader, JS_BRIDGE_SCRIPT).map_err(|source| TargetError::Read {
        path: loader.clone(),
        source,
    })?;
    let mut command = std::process::Command::new("node");
    command
        .arg(JS_BRIDGE_FILE)
        .current_dir(scratch.path())
        .env(
            "SHEP_FLOCKFILE_PATH",
            shep_core::paths::strip_verbatim_prefix(&absolute).as_os_str(),
        )
        .stdin(std::process::Stdio::null());
    let output = match run_bounded(&mut command, budget) {
        Ok(Bounded::Exited(output)) => output,
        Ok(Bounded::Killed) => {
            return Err(TargetError::Js {
                detail: format!(
                    "node was still running {} after {}s, so shep killed it; a Flockfile \
                     module has to export its config and let node exit, and one that leaves a \
                     server listening or a timer armed does not",
                    path.display(),
                    budget.as_secs_f32()
                ),
                node_missing: false,
            });
        }
        Ok(Bounded::OutputHeldOpen) => {
            return Err(TargetError::Js {
                detail: format!(
                    "node finished with {} within {}s, but a process it left behind still \
                     holds the output shep was reading, so shep gave up on it; a Flockfile \
                     module must not leave a child of its own on node's stdout or stderr",
                    path.display(),
                    budget.as_secs_f32()
                ),
                node_missing: false,
            });
        }
        Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
            return Err(TargetError::Js {
                detail: format!(
                    "reading a .js Flockfile runs it through node, and node was not found on PATH; \
                     install node, or convert {} to a .toml Flockfile",
                    path.display()
                ),
                node_missing: true,
            });
        }
        Err(err) => {
            return Err(TargetError::Js {
                detail: format!("could not run node for {}: {err}", path.display()),
                node_missing: false,
            });
        }
    };
    if !output.status.success() {
        let stderr = String::from_utf8_lossy(&output.stderr);
        let reason = stderr
            .lines()
            .find(|line| !line.trim().is_empty())
            .unwrap_or("node exited non-zero and said nothing");
        return Err(TargetError::Js {
            detail: format!("node could not evaluate {}: {reason}", path.display()),
            node_missing: false,
        });
    }
    String::from_utf8(output.stdout).map_err(|_utf8_error| TargetError::Js {
        detail: format!("node printed non-UTF-8 output for {}", path.display()),
        node_missing: false,
    })
}

/// `word` read as a `NAME=VALUE` assignment, or `None` when the name is one
/// no shell would accept
///
/// The name is a letter or `_` followed by letters, digits or `_`. A value
/// may hold anything, `=` included, since only the first `=` separates the
/// two. `execve` itself validates no name, but one a shell cannot expand is
/// a variable the sheep's own wrapper script could never read.
fn assignment(word: &str) -> Option<(&str, &str)> {
    let (name, value) = word.split_once('=')?;
    let mut chars = name.chars();
    let first = chars.next()?;
    if !first.is_ascii_alphabetic() && first != '_' {
        return None;
    }
    if !chars.all(|c| c.is_ascii_alphanumeric() || c == '_') {
        return None;
    }
    Some((name, value))
}

/// Splits the leading assignments off `targets`, returning them and what is
/// left
///
/// The shell's own rule: leading words that parse as assignments are the
/// environment, and the first word that does not ends the run. So `A=1
/// ./koji B=2` leaves `B=2` a target, exactly as a shell leaves it an
/// argument. A repeated name takes its last value.
pub(crate) fn split_assignments(targets: &[String]) -> (BTreeMap<String, String>, &[String]) {
    let mut env = BTreeMap::new();
    let mut rest = targets;
    while let Some((word, tail)) = rest.split_first() {
        let Some((name, value)) = assignment(word) else {
            break;
        };
        env.insert(name.to_string(), value.to_string());
        rest = tail;
    }
    (env, rest)
}

/// Reads `path` as a Flockfile in `format`, and gives every app that set no
/// `cwd` the file's own directory
///
/// The one body the two tiers of [`resolve_target_declared`] that name a
/// recognised extension share: the `--flockfile` tier, and the tier that
/// recognises the same extension on its own. They read, parse and default
/// the cwd identically, and only the tier above them differs.
///
/// # Errors
/// - [`TargetError::Read`] if `path` could not be read.
/// - [`TargetError::Flockfile`] if the source failed Flockfile validation.
fn load_flockfile(path: &Path, format: FlockFormat) -> Result<Vec<DeclaredApp>, TargetError> {
    let source = std::fs::read_to_string(path).map_err(|source| TargetError::Read {
        path: path.to_path_buf(),
        source,
    })?;
    let apps = Flockfile::parse_declared(&source, format)?;
    Ok(default_cwd_to_flockfile_dir(apps, path))
}

/// Resolves `target` into the [`AppConfig`]s `start` should register
///
/// Fixed precedence, do not widen: `-` is Flockfile JSON on `stdin`, then
/// `as_flockfile` or a recognised extension, then any path that exists.
///
/// # Errors
/// - [`TargetError::Stdin`] if `target` is `-` and `stdin` is not UTF-8.
/// - [`TargetError::Read`] if the file could not be read.
/// - [`TargetError::Flockfile`] if the source failed Flockfile validation.
/// - [`TargetError::UnknownFlockfileFormat`] under `as_flockfile`, bad extension.
/// - [`TargetError::Js`] if node failed or ran past [`JS_EVAL_BUDGET`].
/// - [`TargetError::Unresolvable`] if `target` matched none of the above.
pub fn resolve_target(
    target: &str,
    name: Option<&str>,
    stdin: &[u8],
    as_flockfile: bool,
) -> Result<Vec<AppConfig>, TargetError> {
    Ok(resolve_target_declared(target, name, stdin, as_flockfile)?
        .into_iter()
        .map(|declared| declared.config)
        .collect())
}

/// [`resolve_target`], keeping the keys each app's document literally wrote
///
/// A key set answers what a template claims, and only a document makes a
/// claim: the Flockfile tiers go through [`Flockfile::parse_declared`], and
/// a bare script path reports an empty set. A defaulted `cwd` is not
/// declared, nor are `--cwd`, `--fold` and `--interpreter`: they change what
/// a fresh app is registered with and claim nothing about an app the flock
/// already has.
///
/// # Errors
/// Every error [`resolve_target`] returns, for the same inputs.
pub(crate) fn resolve_target_declared(
    target: &str,
    name: Option<&str>,
    stdin: &[u8],
    as_flockfile: bool,
) -> Result<Vec<DeclaredApp>, TargetError> {
    let path = Path::new(target);
    match (target, FlockFormat::from_path(path)) {
        ("-", _) => {
            let source = String::from_utf8(stdin.to_vec()).map_err(|_utf8_error| {
                TargetError::Stdin(std::io::Error::new(
                    std::io::ErrorKind::InvalidData,
                    "stdin is not UTF-8",
                ))
            })?;
            Ok(Flockfile::parse_declared(&source, FlockFormat::Json)?)
        }
        (_, format) if as_flockfile => match format {
            Some(format) => load_flockfile(path, format),
            None if path.extension().and_then(|e| e.to_str()) == Some("js") => {
                let json = evaluate_js_flockfile(path, JS_EVAL_BUDGET)?;
                let apps = Flockfile::parse_declared(&json, FlockFormat::Json)?;
                Ok(default_cwd_to_flockfile_dir(apps, path))
            }
            None => Err(TargetError::UnknownFlockfileFormat {
                path: path.to_path_buf(),
            }),
        },
        (_, Some(format)) => load_flockfile(path, format),
        // Absolutised against the CLI's cwd, which is where `exists` is
        // answered. The daemon would otherwise resolve it against its own.
        _ if path.exists() => {
            let stem = path.file_stem().and_then(|s| s.to_str()).unwrap_or(target);
            // Relative only: `canonicalize` resolves symlinks, so an
            // absolute `/var/...` comes back as `/private/var/...` on macOS.
            let script = if path.is_absolute() {
                target.to_string()
            } else {
                std::fs::canonicalize(path)
                    .map(|abs| {
                        shep_core::paths::strip_verbatim_prefix(&abs)
                            .to_string_lossy()
                            .into_owned()
                    })
                    .unwrap_or_else(|_| target.to_string())
            };
            let mut app = AppConfig::minimal(name.unwrap_or(stem), &script);
            // Where the operator ran `shep start`: an unset `cwd` leaves
            // the child inheriting whatever the shepherd was spawned from.
            app.cwd = std::env::current_dir()
                .ok()
                .map(|dir| dir.to_string_lossy().into_owned());
            // Empty key sets: the command line is not a template, so a load
            // has nothing to apply to a sheep the flock already has.
            Ok(vec![DeclaredApp {
                config: app,
                declared: BTreeSet::new(),
                declared_env: BTreeSet::new(),
            }])
        }
        _ => Err(TargetError::Unresolvable {
            target: target.to_string(),
        }),
    }
}