ryra-test 0.8.7

E2E test runner for ryra using ephemeral QEMU VMs
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
use std::collections::BTreeMap;
use std::path::Path;

use anyhow::{Context, Result};
use serde::Deserialize;

/// Parsed test.toml — the unified test definition format.
#[derive(Debug, Deserialize)]
pub struct TestToml {
    #[serde(default)]
    pub test: Option<TestMeta>,
    #[serde(default)]
    pub setup: Option<SetupSection>,
    #[serde(default)]
    pub tests: Vec<TestDef>,
    #[serde(default)]
    pub steps: Vec<StepDef>,
}

#[derive(Debug, Deserialize)]
pub struct TestMeta {
    pub name: Option<String>,
    #[serde(default)]
    pub browser: bool,
    /// Optional RAM override (MB). When set, bypasses auto-calculation from
    /// service requirements. Use for tests that run many services and need
    /// more headroom than the sum of individual recommendations.
    pub ram: Option<u32>,
    /// Declares that this test performs privileged operations (shells out to
    /// `sudo`). When true, the bare runner acquires sudo credentials once up
    /// front so captured, non-TTY steps don't fail trying to prompt mid-run.
    /// The runner already auto-detects the common case — writing `*.internal`
    /// hostnames to `/etc/hosts` for OIDC/HTTPS URLs — so most tests never set
    /// this; it's the escape hatch for any *other* sudo a test needs.
    #[serde(default)]
    pub requires_sudo: bool,
}

#[derive(Debug, Deserialize)]
pub struct SetupSection {
    #[serde(default)]
    pub services: Vec<String>,
    #[serde(default)]
    pub quadlets: Vec<String>,
}

/// A single named test within a test.toml file.
///
/// Two shapes are accepted for backwards compatibility during the
/// [[tests]]-array migration:
///
/// - **Multi-step (new)**: `steps` non-empty; `run` unset. Produces a
///   lifecycle-style execution reading the given steps directly.
/// - **Shell (legacy)**: `run` set; `steps` empty. Relies on `[setup]`
///   at the file level to deploy services before running `run`.
///
/// Exactly one of `run` / `steps` must be present — validated at parse time.
#[derive(Debug, Clone, Deserialize)]
pub struct TestDef {
    pub name: String,
    /// Legacy: a single shell command run after `[setup]` services deploy.
    #[serde(default)]
    pub run: Option<String>,
    /// New: a sequence of lifecycle steps (add / wait / http / shell / …).
    #[serde(default)]
    pub steps: Vec<StepDef>,
    #[serde(default = "default_timeout")]
    pub timeout: u64,
    #[serde(default)]
    pub env: BTreeMap<String, String>,
    /// Needs a browser VM image (for Playwright steps). Can also be set
    /// at the file level via `[test] browser = true`.
    #[serde(default)]
    pub browser: bool,
    /// Per-test RAM override (MB). File-level `[test] ram` still works.
    pub ram: Option<u32>,
    /// Per-test `requires_sudo`. File-level `[test] requires_sudo` still works;
    /// either being true marks the test as needing elevated privileges.
    #[serde(default)]
    pub requires_sudo: bool,
}

fn default_timeout() -> u64 {
    30
}

fn default_add_timeout() -> u64 {
    300
}

fn default_http_status() -> u16 {
    200
}

fn default_content_type() -> String {
    "application/json".into()
}

/// HTTP method for the `http` test step. Kept as a typed enum so parsing
/// rejects typos at the boundary (per CLAUDE.md: enums over strings).
#[derive(Debug, Clone, Copy, Default, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum HttpMethod {
    #[default]
    Get,
    Post,
    Put,
    Delete,
}

impl HttpMethod {
    /// Upper-case verb for curl's `-X` flag.
    pub fn as_curl_arg(self) -> &'static str {
        match self {
            HttpMethod::Get => "GET",
            HttpMethod::Post => "POST",
            HttpMethod::Put => "PUT",
            HttpMethod::Delete => "DELETE",
        }
    }
}

/// Retry configuration for run steps. The runner re-executes the command
/// up to `attempts` times, sleeping `interval` seconds between tries.
#[derive(Debug, Clone, Deserialize)]
pub struct PollConfig {
    /// Seconds between retries.
    pub interval: u64,
    /// Maximum number of attempts before giving up.
    pub attempts: u64,
}

/// A lifecycle test step — serde deserializes directly into the correct
/// variant based on the `action` field. Invalid field combinations are
/// rejected at parse time rather than runtime.
#[derive(Debug, Clone, Deserialize)]
#[serde(tag = "action", rename_all = "lowercase")]
pub enum StepDef {
    Add {
        service: String,
        #[serde(default)]
        args: Option<String>,
        #[serde(default)]
        env: BTreeMap<String, String>,
        #[serde(default = "default_add_timeout")]
        timeout: u64,
        /// Set by local-project discovery (`--project`): add by this path
        /// instead of by registry name, so the project's own
        /// `service.toml` resolves. Never set from test.toml; `service`
        /// remains the real registered name for every other use.
        #[serde(skip)]
        project_path: Option<std::path::PathBuf>,
    },
    Remove {
        service: String,
    },
    Wait {
        service: String,
        #[serde(default = "default_timeout")]
        timeout: u64,
    },
    /// Shell command step. Fails the test on non-zero exit code.
    Shell {
        name: String,
        run: String,
        #[serde(default = "default_timeout")]
        timeout: u64,
        /// Optional retry configuration. When set, the runner re-executes
        /// the command on failure, up to `attempts` times.
        #[serde(default)]
        poll: Option<PollConfig>,
    },
    /// HTTP request step. Sends a request and checks the response status code.
    /// The URL supports shell variable expansion (e.g., `$SERVICE_PORT_HTTP`)
    /// after sourcing service `.env` files. Follows redirects automatically.
    Http {
        #[serde(default)]
        name: Option<String>,
        url: String,
        #[serde(default)]
        method: HttpMethod,
        /// Request body for POST/PUT. Shell heredoc-safe: arbitrary bytes
        /// are supported including quotes and newlines.
        #[serde(default)]
        body: Option<String>,
        /// Content-Type header for requests with a body. Defaults to
        /// application/json since most API triggers we use ship JSON.
        #[serde(default = "default_content_type")]
        content_type: String,
        /// Extra request headers (e.g., `apikey`, `Authorization`). Values
        /// support shell variable expansion after `.env` sourcing.
        #[serde(default)]
        headers: BTreeMap<String, String>,
        #[serde(default = "default_http_status")]
        status: u16,
        /// When set, only source this service's `.env` file (needed when
        /// multiple services define the same port variable).
        #[serde(default)]
        service: Option<String>,
        #[serde(default)]
        poll: Option<PollConfig>,
        #[serde(default = "default_timeout")]
        timeout: u64,
    },
    /// Playwright browser test step.
    Playwright {
        #[serde(default)]
        name: Option<String>,
        spec: String,
        #[serde(default)]
        env: BTreeMap<String, String>,
        #[serde(default = "default_browser_timeout")]
        timeout: u64,
    },
    /// Inbucket mail-delivery assertion. Polls inbucket's `/api/v1/mailbox/
    /// <mailbox>` endpoint until a non-empty response arrives; when
    /// `contains` is set, additionally requires that substring in the raw
    /// JSON body. Collapses the 8-line port-discovery + curl-poll pattern
    /// that previously lived in every SMTP test into one step.
    Mail {
        #[serde(default)]
        name: Option<String>,
        /// Local-part of the recipient address (`smtptest` for `smtptest@example.com`).
        mailbox: String,
        /// Optional substring required in the response body. Matches
        /// against the raw inbucket JSON, which includes subject + body.
        #[serde(default)]
        contains: Option<String>,
        /// Retry config. Defaults favour short SMTP mail delivery; apps
        /// with async mail queues (twenty, supabase) should widen these.
        #[serde(default = "default_mail_poll")]
        poll: PollConfig,
        #[serde(default = "default_timeout")]
        timeout: u64,
    },
}

fn default_mail_poll() -> PollConfig {
    PollConfig {
        interval: 2,
        attempts: 30,
    }
}

fn default_browser_timeout() -> u64 {
    120
}

impl std::fmt::Display for StepDef {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            StepDef::Add { service, .. } => write!(f, "add {service}"),
            StepDef::Remove { service } => write!(f, "remove {service}"),
            StepDef::Wait { service, .. } => write!(f, "wait {service}"),
            StepDef::Shell { name, .. } => write!(f, "shell: {name}"),
            StepDef::Http { name, url, .. } => {
                write!(f, "http: {}", name.as_deref().unwrap_or(url))
            }
            StepDef::Playwright { name, spec, .. } => {
                write!(f, "browser: {}", name.as_deref().unwrap_or(spec))
            }
            StepDef::Mail { name, mailbox, .. } => {
                write!(f, "mail: {}", name.as_deref().unwrap_or(mailbox))
            }
        }
    }
}

impl StepDef {
    /// The service name referenced by this step, if any.
    pub fn service(&self) -> Option<&str> {
        match self {
            StepDef::Add { service, .. }
            | StepDef::Remove { service }
            | StepDef::Wait { service, .. } => Some(service),
            _ => None,
        }
    }

    /// Whether this step is a setup step (vs. a test/assertion step).
    /// Used by `--retest` to skip setup and only re-run test steps.
    pub fn is_setup(&self) -> bool {
        matches!(
            self,
            StepDef::Add { .. } | StepDef::Remove { .. } | StepDef::Wait { .. }
        )
    }

    /// Human-readable name for this step (used in output).
    pub fn step_name(&self) -> String {
        format!("{self}")
    }

    /// Multi-line description for `--list -v`. Shows every field that
    /// meaningfully changes behaviour (args, env, headers, body, …).
    /// The caller indents each returned line.
    pub fn describe(&self) -> Vec<String> {
        let mut lines = Vec::new();
        match self {
            StepDef::Add {
                service,
                args,
                env,
                timeout,
                ..
            } => {
                let args_s = args
                    .as_deref()
                    .filter(|s| !s.is_empty())
                    .map(|a| format!(" {a}"))
                    .unwrap_or_default();
                lines.push(format!("ryra add {service}{args_s}  (timeout={timeout}s)"));
                for (k, v) in env {
                    lines.push(format!("  env {k}={v}"));
                }
            }
            StepDef::Remove { service } => lines.push(format!("ryra remove --purge {service}")),
            StepDef::Wait { service, timeout } => {
                lines.push(format!("wait for {service}.service  (timeout={timeout}s)"));
            }
            StepDef::Shell {
                name,
                run,
                timeout,
                poll,
            } => {
                let poll_s = match poll {
                    Some(p) => {
                        format!(
                            "  poll={{interval={}s, attempts={}}}",
                            p.interval, p.attempts
                        )
                    }
                    None => String::new(),
                };
                lines.push(format!("shell '{name}'  (timeout={timeout}s{poll_s})"));
                for l in run.trim().lines() {
                    lines.push(format!("  | {l}"));
                }
            }
            StepDef::Http {
                name,
                url,
                method,
                body,
                content_type,
                headers,
                status,
                service,
                poll,
                timeout,
            } => {
                let label = name.as_deref().unwrap_or("(anon)");
                let verb = method.as_curl_arg();
                lines.push(format!(
                    "http '{label}': {verb} {url}  (expect {status}, timeout={timeout}s)"
                ));
                if let Some(svc) = service {
                    lines.push(format!("  env-source: {svc}/.env"));
                }
                for (k, v) in headers {
                    lines.push(format!("  header {k}: {v}"));
                }
                if let Some(b) = body {
                    lines.push(format!("  content-type: {content_type}"));
                    for l in b.trim().lines() {
                        lines.push(format!("  body> {l}"));
                    }
                }
                if let Some(p) = poll {
                    lines.push(format!(
                        "  poll: every {}s, up to {} attempts",
                        p.interval, p.attempts
                    ));
                }
            }
            StepDef::Playwright {
                name,
                spec,
                env,
                timeout,
            } => {
                let label = name.as_deref().unwrap_or(spec);
                lines.push(format!(
                    "playwright '{label}': spec={spec}  (timeout={timeout}s)"
                ));
                for (k, v) in env {
                    lines.push(format!("  env {k}={v}"));
                }
            }
            StepDef::Mail {
                name,
                mailbox,
                contains,
                poll,
                timeout,
            } => {
                let label = name.as_deref().unwrap_or(mailbox);
                lines.push(format!(
                    "mail '{label}': mailbox={mailbox}  (timeout={timeout}s)"
                ));
                if let Some(c) = contains {
                    lines.push(format!("  contains: {c}"));
                }
                lines.push(format!(
                    "  poll: every {}s, up to {} attempts",
                    poll.interval, poll.attempts
                ));
            }
        }
        lines
    }
}

impl TestToml {
    /// Read and deserialize a test.toml file, then validate it.
    pub fn parse(path: &Path) -> Result<Self> {
        let content = std::fs::read_to_string(path)
            .with_context(|| format!("failed to read test.toml at {}", path.display()))?;
        let parsed: Self = toml::from_str(&content)
            .with_context(|| format!("failed to parse test.toml at {}", path.display()))?;
        parsed.validate(path)?;
        Ok(parsed)
    }

    /// Validate structural invariants after deserialization.
    ///
    /// Most field-level validation is handled by serde (the tagged enum
    /// rejects missing required fields at parse time). This only checks
    /// cross-field invariants that serde can't express.
    pub fn validate(&self, path: &Path) -> Result<()> {
        let ctx = path.display();

        // Top-level [[tests]] coexists with [[steps]] ONLY if all [[tests]]
        // are new-format (each brings its own `steps`). The legacy shape
        // (shell-style `run`-based tests with a shared [setup]) remains
        // mutually exclusive with top-level [[steps]].
        let has_legacy_run_tests = self
            .tests
            .iter()
            .any(|t| t.run.is_some() && t.steps.is_empty());
        if has_legacy_run_tests && !self.steps.is_empty() {
            anyhow::bail!(
                "{ctx}: test.toml cannot mix [setup]+[[tests]] (legacy shell) with top-level [[steps]] — \
                 migrate to the new [[tests]] + [[tests.steps]] format instead",
            );
        }

        for t in &self.tests {
            let has_run = t.run.is_some();
            let has_steps = !t.steps.is_empty();
            if has_run == has_steps {
                anyhow::bail!(
                    "{ctx}: test '{}' must set exactly one of `run` or `steps` \
                     (got run={}, steps={})",
                    t.name,
                    has_run,
                    has_steps,
                );
            }
        }

        Ok(())
    }

    /// True if this is a lifecycle test (uses [[steps]] instead of [[tests]]).
    pub fn is_lifecycle(&self) -> bool {
        !self.steps.is_empty()
    }

    /// True if this test requires a browser VM image.
    pub fn needs_browser(&self) -> bool {
        self.test.as_ref().is_some_and(|t| t.browser)
    }

    /// Explicit RAM override (MB) from [test] metadata, if set.
    pub fn ram_override(&self) -> Option<u32> {
        self.test.as_ref().and_then(|t| t.ram)
    }

    /// File-level `[test] requires_sudo` flag.
    pub fn requires_sudo(&self) -> bool {
        self.test.as_ref().is_some_and(|t| t.requires_sudo)
    }

    /// The test name from [test] metadata, or the file stem as a fallback.
    pub fn name_or_default(&self, path: &Path) -> String {
        if let Some(ref meta) = self.test
            && let Some(ref name) = meta.name
        {
            return name.clone();
        }
        path.file_stem()
            .and_then(|s| s.to_str())
            .unwrap_or("unknown")
            .to_string()
    }

    /// All services referenced: from setup.services + any `add` steps.
    pub fn referenced_services(&self) -> Vec<String> {
        let mut services: Vec<String> = self
            .setup
            .as_ref()
            .map_or_else(Vec::new, |s| s.services.clone());

        for step in &self.steps {
            if let StepDef::Add { service, .. } = step
                && !services.contains(service)
            {
                services.push(service.clone());
            }
        }

        services
    }

    /// Quadlet files declared in [setup].
    pub fn quadlet_files(&self) -> Vec<String> {
        self.setup
            .as_ref()
            .map_or_else(Vec::new, |s| s.quadlets.clone())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::io::Write as _;

    fn write_temp(content: &str) -> (tempfile::TempDir, std::path::PathBuf) {
        let dir = tempfile::tempdir().expect("tempdir");
        let path = dir.path().join("test.toml");
        let mut f = std::fs::File::create(&path).expect("create");
        f.write_all(content.as_bytes()).expect("write");
        (dir, path)
    }

    #[test]
    fn reject_mixed_tests_and_steps() {
        let toml = r#"
[[tests]]
name = "foo"
run = "true"

[[steps]]
action = "add"
service = "bar"
"#;
        let (_dir, path) = write_temp(toml);
        let result = TestToml::parse(&path);
        assert!(result.is_err(), "expected error for mixed tests+steps");
        let msg = format!("{:#}", result.unwrap_err());
        assert!(msg.contains("[[tests]]") || msg.contains("[[steps]]"));
    }

    #[test]
    fn name_from_metadata() {
        let toml = r#"
[test]
name = "my explicit name"

[[tests]]
name = "check"
run = "true"
"#;
        let (_dir, path) = write_temp(toml);
        let parsed = TestToml::parse(&path).expect("parse");
        assert_eq!(parsed.name_or_default(&path), "my explicit name");
    }

    #[test]
    fn name_from_filename() {
        let toml = r#"
[[tests]]
name = "check"
run = "true"
"#;
        let dir = tempfile::tempdir().expect("tempdir");
        let path = dir.path().join("immich-sso.toml");
        std::fs::write(&path, toml).expect("write");
        let parsed = TestToml::parse(&path).expect("parse");
        assert_eq!(parsed.name_or_default(&path), "immich-sso");
    }

    #[test]
    fn browser_step_requires_spec() {
        let toml = r#"
[[steps]]
action = "playwright"
"#;
        let (_dir, path) = write_temp(toml);
        let result = TestToml::parse(&path);
        assert!(result.is_err());
        let msg = format!("{:#}", result.unwrap_err());
        assert!(msg.contains("spec") || msg.contains("missing field"));
    }

    #[test]
    fn run_step_rejects_missing_name() {
        let toml = r#"
[[steps]]
action = "shell"
run = "true"
"#;
        let (_dir, path) = write_temp(toml);
        let result = TestToml::parse(&path);
        assert!(result.is_err(), "run step without 'name' should fail");
    }

    #[test]
    fn add_step_default_timeout() {
        let toml = r#"
[[steps]]
action = "add"
service = "whoami"
"#;
        let (_dir, path) = write_temp(toml);
        let parsed = TestToml::parse(&path).expect("parse");
        if let StepDef::Add { timeout, .. } = parsed.steps[0] {
            assert_eq!(timeout, 300);
        } else {
            panic!("expected Add step");
        }
    }

    #[test]
    fn http_step_defaults() {
        let toml = r#"
[[steps]]
action = "http"
url = "http://localhost:8080"
"#;
        let (_dir, path) = write_temp(toml);
        let parsed = TestToml::parse(&path).expect("parse");
        if let StepDef::Http {
            status, timeout, ..
        } = parsed.steps[0]
        {
            assert_eq!(status, 200);
            assert_eq!(timeout, 30);
        } else {
            panic!("expected Http step");
        }
    }

    #[test]
    fn mail_step_defaults() {
        let toml = r#"
[[steps]]
action = "mail"
mailbox = "smtptest"
"#;
        let (_dir, path) = write_temp(toml);
        let parsed = TestToml::parse(&path).expect("parse");
        if let StepDef::Mail {
            ref contains,
            ref poll,
            timeout,
            ..
        } = parsed.steps[0]
        {
            assert!(contains.is_none(), "contains defaults to None");
            assert_eq!(poll.interval, 2, "default poll interval");
            assert_eq!(poll.attempts, 30, "default poll attempts");
            assert_eq!(timeout, 30);
        } else {
            panic!("expected Mail step");
        }
    }

    #[test]
    fn is_setup_classification() {
        let toml = r#"
[[steps]]
action = "add"
service = "whoami"

[[steps]]
action = "remove"
service = "whoami"

[[steps]]
action = "wait"
service = "whoami"

[[steps]]
action = "shell"
name = "check"
run = "true"

[[steps]]
action = "http"
url = "http://localhost:8080"

[[steps]]
action = "playwright"
spec = "test.spec.ts"
"#;
        let (_dir, path) = write_temp(toml);
        let parsed = TestToml::parse(&path).expect("parse");
        assert!(parsed.steps[0].is_setup(), "add should be setup");
        assert!(parsed.steps[1].is_setup(), "remove should be setup");
        assert!(parsed.steps[2].is_setup(), "wait should be setup");
        assert!(!parsed.steps[3].is_setup(), "shell should not be setup");
        assert!(!parsed.steps[4].is_setup(), "http should not be setup");
        assert!(
            !parsed.steps[5].is_setup(),
            "playwright should not be setup"
        );
    }
}