sofie_cli/bench/
mod.rs

1mod baseline;
2mod journeys;
3mod script;
4
5use std::collections::BTreeSet;
6use std::ffi::OsStr;
7use std::fs::{self, File};
8use std::path::{Path, PathBuf};
9use std::process::{Child, Command, Stdio};
10use std::thread;
11use std::time::{Duration, Instant};
12
13use serde_json::json;
14use sofie_model::App;
15
16pub use baseline::{Comparison, Measurement, RouteResult, Verdict};
17
18use baseline::compare;
19
20use crate::build_command::{BuildOptions, build_binary};
21use crate::dev_command::app_command;
22use crate::gen_command::{GenError, io_error, read_existing};
23use crate::local_app::{
24    MEMBER_PASSWORD, bench_member, free_address, private_directory, refuse_remote_database,
25    wait_answering,
26};
27use crate::source_edits::{read, write};
28use script::bench_script;
29
30const BENCHABLE: [&str; 3] = ["local", "testing", "staging"];
31const STOP_WAIT: Duration = Duration::from_secs(30);
32const LOG_TAIL: usize = 40;
33
34pub struct BenchReport {
35    pub skipped: Vec<String>,
36    pub comparison: Comparison,
37    pub baseline: PathBuf,
38}
39
40/// Without a target it builds the app, starts it on a free local port with
41/// the shield off and its database on this machine, and loads that; a target
42/// must run with `SOFIE_SHIELD=off`, and one not on this machine is signed in
43/// to only as accounts named in `SOFIE_BENCH_<AREA>_EMAIL`/`_PASSWORD`.
44pub fn bench(
45    root: &Path,
46    target: Option<&str>,
47    environment: Option<&str>,
48) -> Result<BenchReport, GenError> {
49    let variable = std::env::var("SOFIE_ENVIRONMENT").ok();
50    let environment = bench_environment(target, environment, variable.as_deref())?;
51    if target.is_none() {
52        refuse_remote_database(root, "bench")?;
53    }
54    let app = sofie_model::load(root).map_err(GenError::Model)?;
55    let scripts = Scripts::write(root, &app)?;
56    let measured = if let Some(target) = target {
57        scripts.run(target)?
58    } else {
59        let built = build_binary(root, &app, BuildOptions::default())?;
60        let running = serve(root, &built.path, &environment, &[])?;
61        scripts.run(&running.target)?
62    };
63    let baseline_file = root.join("bench").join("baseline.json");
64    let baseline = match read_existing(&baseline_file)? {
65        Some(text) => baseline::parse(&text)
66            .map_err(|error| GenError::App(format!("{}: {error}", baseline_file.display())))?,
67        None => baseline::Measurements::new(),
68    };
69    let comparison = compare(&baseline, &measured);
70    if !comparison.regressed() {
71        write(&baseline_file, baseline::render(&comparison.next_baseline))?;
72    }
73    Ok(BenchReport {
74        skipped: scripts.skipped,
75        comparison,
76        baseline: baseline_file,
77    })
78}
79
80/// The generated k6 scripts under `bench/gen/`.
81pub(crate) struct Scripts {
82    endpoints: PathBuf,
83    journeys: Option<PathBuf>,
84    /// Each route left out, with its reason.
85    skipped: Vec<String>,
86    member_areas: BTreeSet<String>,
87}
88
89impl Scripts {
90    pub(crate) fn write(root: &Path, app: &App) -> Result<Self, GenError> {
91        let folder = root.join("bench").join("gen");
92        let script = bench_script(app)?;
93        let endpoints = folder.join("endpoints.js");
94        write(&endpoints, &script.text)?;
95        let client = journeys::client(app)?;
96        write(&folder.join("client.ts"), &client.text)?;
97        let mut member_areas: BTreeSet<String> = script.member_areas.into_iter().collect();
98        let found = journeys::journeys(app)?;
99        let runner = folder.join("journeys.ts");
100        let journeys = if found.is_empty() {
101            remove_stale(&runner)?;
102            None
103        } else {
104            write(&runner, journeys::runner(&found))?;
105            member_areas.extend(client.member_areas);
106            Some(runner)
107        };
108        Ok(Self {
109            endpoints,
110            journeys,
111            skipped: script.skipped,
112            member_areas,
113        })
114    }
115
116    pub(crate) fn run(&self, target: &str) -> Result<baseline::Measurements, GenError> {
117        let accounts = accounts(target, &self.member_areas, |name| std::env::var(name).ok())?;
118        let mut measured = run_k6(&self.endpoints, target, &accounts)?;
119        if let Some(journeys) = &self.journeys {
120            measured.extend(run_k6(journeys, target, &accounts)?);
121        }
122        Ok(measured)
123    }
124}
125
126/// The account per member area as JSON for k6's environment, so no password
127/// lands in a script or an argument list. A server not on this machine is
128/// never signed up on: a known password would leave an account anyone can use.
129fn accounts(
130    target: &str,
131    areas: &BTreeSet<String>,
132    variable: impl Fn(&str) -> Option<String>,
133) -> Result<String, GenError> {
134    let local = is_local_target(target);
135    let mut accounts = serde_json::Map::new();
136    let mut missing = Vec::new();
137    for area in areas {
138        let account = if local {
139            json!({ "email": bench_member(area), "password": MEMBER_PASSWORD, "signUp": true })
140        } else {
141            let prefix = format!("SOFIE_BENCH_{}", area.to_ascii_uppercase());
142            let (email, password) = (format!("{prefix}_EMAIL"), format!("{prefix}_PASSWORD"));
143            let (Some(email_value), Some(password_value)) = (variable(&email), variable(&password))
144            else {
145                missing.push(format!("{email} and {password} for area `{area}`"));
146                continue;
147            };
148            json!({ "email": email_value, "password": password_value, "signUp": false })
149        };
150        accounts.insert(area.clone(), account);
151    }
152    if !missing.is_empty() {
153        return Err(GenError::Usage(format!(
154            "sofie bench signs up nobody on {target}, a server not on this machine; \
155             name an existing account with {}",
156            missing.join("; ")
157        )));
158    }
159    Ok(serde_json::Value::Object(accounts).to_string())
160}
161
162fn is_local_target(target: &str) -> bool {
163    let rest = target.split_once("://").map_or(target, |(_, rest)| rest);
164    let authority = rest.split(['/', '?', '#']).next().unwrap_or_default();
165    let host_and_port = authority
166        .rsplit_once('@')
167        .map_or(authority, |(_, host)| host);
168    let host = match host_and_port.strip_prefix('[') {
169        Some(bracketed) => bracketed.split(']').next().unwrap_or_default(),
170        None => host_and_port.split(':').next().unwrap_or_default(),
171    };
172    matches!(
173        host.to_ascii_lowercase().as_str(),
174        "localhost" | "127.0.0.1" | "::1"
175    )
176}
177
178fn remove_stale(path: &Path) -> Result<(), GenError> {
179    match fs::remove_file(path) {
180        Err(error) if error.kind() != std::io::ErrorKind::NotFound => Err(io_error(path, error)),
181        _ => Ok(()),
182    }
183}
184
185/// Load against production would be an outage of our own making.
186pub fn bench_environment(
187    target: Option<&str>,
188    flag: Option<&str>,
189    variable: Option<&str>,
190) -> Result<String, GenError> {
191    let environment = match (flag.or(variable), target) {
192        (Some(named), _) => named,
193        (None, None) => "local",
194        (None, Some(target)) => {
195            return Err(GenError::Usage(format!(
196                "name the environment {target} runs in with `--environment` or SOFIE_ENVIRONMENT; \
197                 sofie bench loads only {}",
198                BENCHABLE.join(", ")
199            )));
200        }
201    };
202    if let Some(target) = target.filter(|target| environment == "local" && !is_local_target(target))
203    {
204        return Err(GenError::Usage(format!(
205            "{target} is not local, yet its environment is named `local`; name the environment \
206             it runs in with `--environment`"
207        )));
208    }
209    if !BENCHABLE.contains(&environment) {
210        return Err(GenError::Usage(format!(
211            "sofie bench refuses the `{environment}` environment; it loads only {}",
212            BENCHABLE.join(", ")
213        )));
214    }
215    Ok(environment.to_owned())
216}
217
218pub(crate) struct Running {
219    child: Child,
220    pub(crate) target: String,
221    /// Holds the app's log, removed with it.
222    directory: PathBuf,
223}
224
225impl Running {
226    /// Ends it as an orchestrator would, so what the process writes at exit is written.
227    pub(crate) fn stop(mut self) -> Result<(), GenError> {
228        let pid = self.child.id().to_string();
229        let sent = Command::new("kill")
230            .args(["-TERM", &pid])
231            .status()
232            .map_err(|source| GenError::App(format!("sending SIGTERM to {pid}: {source}")))?;
233        if !sent.success() {
234            return Err(GenError::App(format!(
235                "`kill -TERM {pid}` failed with {sent}"
236            )));
237        }
238        let deadline = Instant::now() + STOP_WAIT;
239        while Instant::now() < deadline {
240            let exited = self
241                .child
242                .try_wait()
243                .map_err(|source| GenError::App(format!("waiting for {pid}: {source}")))?;
244            match exited {
245                Some(status) if status.success() => return Ok(()),
246                Some(status) => {
247                    return Err(GenError::App(format!(
248                        "the app (pid {pid}) ended with {status} after SIGTERM"
249                    )));
250                }
251                None => {}
252            }
253            thread::sleep(Duration::from_millis(100));
254        }
255        Err(GenError::App(format!(
256            "the app (pid {pid}) did not stop within {} s of SIGTERM",
257            STOP_WAIT.as_secs()
258        )))
259    }
260}
261
262/// Killed rather than stopped: the measurement is over, and nothing it holds
263/// needs a graceful end.
264impl Drop for Running {
265    fn drop(&mut self) {
266        if let Err(error) = self.child.kill().and_then(|()| self.child.wait().map(drop)) {
267            eprintln!(
268                "sofie bench: stopping the app (pid {}): {error}",
269                self.child.id()
270            );
271        }
272        if let Err(error) = fs::remove_dir_all(&self.directory) {
273            eprintln!(
274                "sofie bench: removing {}: {error}",
275                self.directory.display()
276            );
277        }
278    }
279}
280
281pub(crate) fn serve(
282    root: &Path,
283    binary: &Path,
284    environment: &str,
285    variables: &[(&str, &OsStr)],
286) -> Result<Running, GenError> {
287    let address = free_address()?;
288    let directory = private_directory("sofie-bench")?;
289    let log = directory.join("app.log");
290    let output = File::create(&log).map_err(|source| io_error(&log, source))?;
291    let errors = output
292        .try_clone()
293        .map_err(|source| io_error(&log, source))?;
294    let child = app_command(root, binary)?
295        .arg("serve")
296        .envs(variables.iter().copied())
297        .env("SOFIE_ADDRESS", address.to_string())
298        .env("SOFIE_ENVIRONMENT", environment)
299        .env("SOFIE_MIGRATE", "on")
300        .env("SOFIE_SHIELD", "off")
301        .stdout(output)
302        .stderr(errors)
303        .spawn()
304        .map_err(|source| io_error(binary, source))?;
305    let mut running = Running {
306        child,
307        target: format!("http://{address}"),
308        directory,
309    };
310    let child = &mut running.child;
311    if let Err(error) = wait_answering(address, "/ready", || {
312        child.try_wait().map_err(|source| io_error(binary, source))
313    }) {
314        let text = read(&log)?;
315        let lines: Vec<&str> = text.lines().collect();
316        let last = lines[lines.len().saturating_sub(LOG_TAIL)..].join("\n");
317        return Err(GenError::App(format!("{error}; its last lines:\n{last}")));
318    }
319    Ok(running)
320}
321
322fn run_k6(script: &Path, target: &str, accounts: &str) -> Result<baseline::Measurements, GenError> {
323    let directory = private_directory("sofie-bench-summary")?;
324    let measured = run_k6_into(script, target, accounts, &directory.join("summary.json"));
325    fs::remove_dir_all(&directory).map_err(|source| io_error(&directory, source))?;
326    measured
327}
328
329fn run_k6_into(
330    script: &Path,
331    target: &str,
332    accounts: &str,
333    summary: &Path,
334) -> Result<baseline::Measurements, GenError> {
335    let status = Command::new("k6")
336        .arg("run")
337        .arg("--env")
338        .arg(format!(
339            "SOFIE_BENCH_TARGET={}",
340            target.trim_end_matches('/')
341        ))
342        .arg("--env")
343        .arg(format!("SOFIE_BENCH_SUMMARY={}", summary.display()))
344        .arg(script)
345        .env("SOFIE_BENCH_ACCOUNTS", accounts)
346        .stdin(Stdio::null())
347        .status()
348        .map_err(|source| {
349            GenError::Usage(format!("running k6: {source}; install k6 (https://k6.io)"))
350        })?;
351    if !status.success() {
352        return Err(GenError::App(format!("k6 failed with {status}")));
353    }
354    let text = read(summary)?;
355    baseline::parse(&text)
356        .map_err(|error| GenError::App(format!("k6's summary {}: {error}", summary.display())))
357}
358
359#[cfg(test)]
360mod tests {
361    use std::collections::BTreeSet;
362
363    use super::{accounts, bench_environment};
364
365    #[test]
366    fn signs_up_only_on_this_machine_and_takes_named_accounts_elsewhere() {
367        let areas = BTreeSet::from(["customer".to_owned()]);
368        let none = |_: &str| None;
369        let named = |name: &str| Some(format!("{name}-value"));
370
371        let local = accounts("http://127.0.0.1:4000", &areas, none).unwrap();
372        let refused = accounts("https://staging.example.com", &areas, none).unwrap_err();
373        let remote = accounts("https://staging.example.com", &areas, named).unwrap();
374
375        assert!(local.contains("\"signUp\":true"), "{local}");
376        assert!(local.contains("bench-customer@example.com"), "{local}");
377        assert!(
378            refused
379                .to_string()
380                .contains("SOFIE_BENCH_CUSTOMER_EMAIL and SOFIE_BENCH_CUSTOMER_PASSWORD"),
381            "{refused}"
382        );
383        assert!(remote.contains("\"signUp\":false"), "{remote}");
384        assert!(
385            remote.contains("SOFIE_BENCH_CUSTOMER_PASSWORD-value"),
386            "{remote}"
387        );
388        assert!(
389            !remote.contains(crate::local_app::MEMBER_PASSWORD),
390            "{remote}"
391        );
392    }
393
394    #[test]
395    fn a_target_is_local_only_on_a_loopback_host() {
396        assert!(super::is_local_target("http://localhost:3000/"));
397        assert!(super::is_local_target("http://[::1]:3000"));
398        assert!(!super::is_local_target("https://localhost.example.com"));
399        assert!(!super::is_local_target(
400            "https://127.0.0.1@shop.example.com"
401        ));
402    }
403
404    #[test]
405    fn refuses_production_and_a_target_whose_environment_is_unnamed() {
406        let production = bench_environment(None, Some("production"), None).unwrap_err();
407        let unnamed = bench_environment(Some("https://shop.example.com"), None, None).unwrap_err();
408        let from_variable = bench_environment(Some("https://shop.example.com"), None, Some("prod"));
409
410        assert!(
411            production
412                .to_string()
413                .contains("refuses the `production` environment")
414        );
415        assert!(
416            unnamed
417                .to_string()
418                .contains("name the environment https://shop.example.com")
419        );
420        assert!(from_variable.is_err());
421    }
422
423    /// A developer's shell often exports `SOFIE_ENVIRONMENT=local` for local work.
424    #[test]
425    fn refuses_calling_a_remote_target_local() {
426        let remote = "https://shop.example.com";
427
428        let from_variable = bench_environment(Some(remote), None, Some("local")).unwrap_err();
429        let from_flag = bench_environment(Some(remote), Some("local"), None);
430        let here = bench_environment(Some("http://127.0.0.1:8080"), None, Some("local"));
431
432        assert!(
433            from_variable.to_string().contains("is not local"),
434            "{from_variable}"
435        );
436        assert!(from_flag.is_err());
437        assert_eq!(here.unwrap(), "local");
438    }
439
440    #[test]
441    fn accepts_local_testing_and_staging_with_the_flag_over_the_variable() {
442        assert_eq!(bench_environment(None, None, None).unwrap(), "local");
443        assert_eq!(
444            bench_environment(
445                Some("https://staging.example.com"),
446                Some("staging"),
447                Some("production")
448            )
449            .unwrap(),
450            "staging"
451        );
452        assert_eq!(
453            bench_environment(None, None, Some("testing")).unwrap(),
454            "testing"
455        );
456    }
457}
sofie_cli/bench/mod.rs

sofie_cli/bench/
mod.rs
