1use std::path::{Path, PathBuf};
2use std::time::Instant;
3
4use anyhow::Context;
5
6use crate::Result;
7use crate::cli_util;
8use crate::suite::auth::{AuthInit, SuiteAuthManager};
9use crate::suite::cleanup::{CleanupContext, run_case_cleanup};
10use crate::suite::filter::selection_skip_reason;
11use crate::suite::resolve::write_file;
12use crate::suite::results::{SuiteCaseResult, SuiteRunResults, SuiteRunSummary};
13use crate::suite::runtime::{
14 path_relative_to_repo_or_abs, resolve_effective_env, resolve_effective_no_history,
15 resolve_rest_base_url, sanitize_id, time_iso_now, time_run_id_now,
16};
17use crate::suite::schema::LoadedSuite;
18
19mod context;
20mod graphql;
21mod grpc;
22mod progress;
23mod rest;
24mod websocket;
25
26pub use context::{SuiteRunOptions, SuiteRunOutput};
27
28fn mask_args_for_command_snippet(args: &[String]) -> String {
29 if args.is_empty() {
30 return String::new();
31 }
32
33 let mut out: Vec<String> = Vec::new();
34 let mut mask_next = false;
35 for a in args {
36 if mask_next {
37 out.push("REDACTED".to_string());
38 mask_next = false;
39 continue;
40 }
41 if a == "--token" || a == "--jwt" {
42 out.push(a.clone());
43 mask_next = true;
44 continue;
45 }
46 if let Some((k, _v)) = a.split_once('=')
47 && (k == "--token" || k == "--jwt")
48 {
49 out.push(format!("{k}=REDACTED"));
50 continue;
51 }
52 out.push(a.clone());
53 }
54
55 out.into_iter()
56 .map(|a| cli_util::shell_quote(&a))
57 .collect::<Vec<_>>()
58 .join(" ")
59}
60
61pub fn run_suite(
62 repo_root: &Path,
63 loaded: LoadedSuite,
64 mut options: SuiteRunOptions,
65) -> Result<SuiteRunOutput> {
66 let mut progress = progress::SuiteProgress::new(options.progress.take());
67
68 let run_id = time_run_id_now()?;
69 let started_at = time_iso_now()?;
70
71 let run_dir_abs = options.output_dir_base.join(&run_id);
72 std::fs::create_dir_all(&run_dir_abs)
73 .with_context(|| format!("create output dir: {}", run_dir_abs.display()))?;
74
75 let suite_file_rel = path_relative_to_repo_or_abs(repo_root, &loaded.suite_path);
76 let output_dir_rel = path_relative_to_repo_or_abs(repo_root, &run_dir_abs);
77 let suite_name_value = context::suite_display_name(&loaded);
78
79 let defaults = &loaded.manifest.defaults;
80
81 let mut auth_init_message: Option<String> = None;
82 let mut auth_manager: Option<SuiteAuthManager> = match loaded.manifest.auth.clone() {
83 None => None,
84 Some(auth) => match SuiteAuthManager::init_from_suite(auth, defaults)? {
85 AuthInit::Disabled { message } => {
86 auth_init_message = message;
87 None
88 }
89 AuthInit::Enabled(mgr) => Some(*mgr),
90 },
91 };
92
93 let mut total: u32 = 0;
94 let mut passed: u32 = 0;
95 let mut failed: u32 = 0;
96 let mut skipped: u32 = 0;
97
98 let mut cases_out: Vec<SuiteCaseResult> = Vec::new();
99
100 let mut case_index: u64 = 0;
101
102 for c in &loaded.manifest.cases {
103 total += 1;
104
105 let id = c.id.trim().to_string();
106 let safe_id = sanitize_id(&id);
107
108 case_index = case_index.saturating_add(1);
109 progress.on_case_start(case_index, if id.is_empty() { &safe_id } else { &id });
110
111 let tags = c.tags.clone();
112 let ty = context::case_type_normalized(&c.case_type);
113
114 let effective_env = resolve_effective_env(&c.env, defaults);
115 let effective_no_history = resolve_effective_no_history(c.no_history, defaults);
116
117 if let Some(reason) = selection_skip_reason(
118 &id,
119 &tags,
120 &options.required_tags,
121 &options.only_ids,
122 &options.skip_ids,
123 ) {
124 skipped += 1;
125 cases_out.push(SuiteCaseResult {
126 id,
127 case_type: ty,
128 status: "skipped".to_string(),
129 duration_ms: 0,
130 tags,
131 command: None,
132 message: Some(reason.as_str().to_string()),
133 assertions: None,
134 stdout_file: None,
135 stderr_file: None,
136 });
137 continue;
138 }
139
140 let start = Instant::now();
141
142 let mut status: String;
143 let mut message: Option<String> = None;
144 let mut assertions: Option<serde_json::Value> = None;
145 let mut command_snippet: Option<String> = None;
146 let mut stdout_file_abs: Option<PathBuf> = None;
147 let mut stderr_file_abs: Option<PathBuf> = None;
148
149 let mut rest_config_dir = String::new();
150 let mut rest_url = String::new();
151 let mut rest_token = String::new();
152 let mut gql_config_dir = String::new();
153 let mut gql_url = String::new();
154 let mut gql_jwt = String::new();
155 let mut access_token_for_case = String::new();
156
157 match ty.as_str() {
158 "rest" => {
159 match rest::prepare_rest_case(
160 repo_root,
161 c,
162 &id,
163 defaults,
164 &options.env_rest_url,
165 &options.env_gql_url,
166 options.allow_writes_flag,
167 &effective_env,
168 auth_manager.as_mut(),
169 )? {
170 rest::PrepareOutcome::Ready(plan) => {
171 let rest::RestCasePlan {
172 request_abs,
173 request_file,
174 config_dir,
175 url,
176 token,
177 access_token_for_case: prepared_access_token,
178 } = plan;
179
180 rest_config_dir = config_dir;
181 rest_url = url;
182 rest_token = token;
183 access_token_for_case = prepared_access_token;
184
185 let out = rest::run_rest_case(
186 repo_root,
187 &run_dir_abs,
188 &safe_id,
189 effective_no_history,
190 &effective_env,
191 defaults,
192 &options.env_rest_url,
193 &rest_config_dir,
194 &rest_url,
195 &rest_token,
196 &access_token_for_case,
197 &request_abs,
198 &request_file,
199 )?;
200
201 status = out.status;
202 message = out.message;
203 command_snippet = out.command_snippet;
204 stdout_file_abs = Some(out.stdout_path);
205 stderr_file_abs = Some(out.stderr_path);
206
207 match status.as_str() {
208 "passed" => passed += 1,
209 "failed" => failed += 1,
210 _ => {}
211 }
212 }
213 rest::PrepareOutcome::Skipped { message: msg } => {
214 status = "skipped".to_string();
215 message = Some(msg);
216 skipped += 1;
217 }
218 rest::PrepareOutcome::Failed { message: msg } => {
219 status = "failed".to_string();
220 message = Some(msg);
221 failed += 1;
222 }
223 }
224 }
225 "rest-flow" | "rest_flow" => {
226 match rest::prepare_rest_flow_case(
227 repo_root,
228 c,
229 &id,
230 defaults,
231 &options.env_rest_url,
232 options.allow_writes_flag,
233 &effective_env,
234 )? {
235 rest::PrepareOutcome::Ready(plan) => {
236 let login_abs = plan.login_abs;
237 let request_abs = plan.request_abs;
238 let login_request_file = plan.login_request_file;
239 let main_request_file = plan.main_request_file;
240 rest_config_dir = plan.config_dir;
241 rest_url = plan.url;
242 let token_jq = plan.token_jq;
243
244 let stderr_path = run_dir_abs.join(format!("{safe_id}.stderr.log"));
245 write_file(&stderr_path, b"")?;
246 stderr_file_abs = Some(stderr_path.clone());
247
248 let base_url = match resolve_rest_base_url(
249 repo_root,
250 &rest_config_dir,
251 &rest_url,
252 &effective_env,
253 defaults,
254 &options.env_rest_url,
255 ) {
256 Ok(v) => v,
257 Err(err) => {
258 write_file(&stderr_path, format!("{err:#}\n").as_bytes())?;
259 status = "failed".to_string();
260 message = Some("rest_flow_login_failed".to_string());
261 failed += 1;
262 cases_out.push(case_result(
263 &id,
264 &ty,
265 &tags,
266 &status,
267 start.elapsed(),
268 command_snippet.clone(),
269 message.clone(),
270 assertions.clone(),
271 None,
272 Some(&stderr_path),
273 repo_root,
274 ));
275 if options.fail_fast && status == "failed" {
276 break;
277 }
278 continue;
279 }
280 };
281
282 let mut login_args: Vec<String> = Vec::new();
284 login_args.push("call".to_string());
285 login_args.push("--config-dir".to_string());
286 login_args.push(rest_config_dir.clone());
287 if effective_no_history {
288 login_args.push("--no-history".to_string());
289 }
290 if !rest_url.trim().is_empty() {
291 login_args.push("--url".to_string());
292 login_args.push(rest_url.clone());
293 } else if !effective_env.trim().is_empty() {
294 login_args.push("--env".to_string());
295 login_args.push(effective_env.clone());
296 }
297 login_args.push(path_relative_to_repo_or_abs(repo_root, &login_abs));
298
299 let mut main_args: Vec<String> = Vec::new();
300 main_args.push("call".to_string());
301 main_args.push("--config-dir".to_string());
302 main_args.push(rest_config_dir.clone());
303 if effective_no_history {
304 main_args.push("--no-history".to_string());
305 }
306 if !rest_url.trim().is_empty() {
307 main_args.push("--url".to_string());
308 main_args.push(rest_url.clone());
309 } else if !effective_env.trim().is_empty() {
310 main_args.push("--env".to_string());
311 main_args.push(effective_env.clone());
312 }
313 main_args.push(path_relative_to_repo_or_abs(repo_root, &request_abs));
314
315 let login_args_snip = mask_args_for_command_snippet(&login_args);
316 let main_args_snip = mask_args_for_command_snippet(&main_args);
317 let token_expr_q = cli_util::shell_quote(&token_jq);
318 command_snippet = Some(format!(
319 "ACCESS_TOKEN=\"$(REST_TOKEN_NAME= ACCESS_TOKEN= {} {} | jq -r {token_expr_q})\" REST_TOKEN_NAME= {} {}",
320 cli_util::shell_quote("api-rest"),
321 login_args_snip,
322 cli_util::shell_quote("api-rest"),
323 main_args_snip
324 ));
325
326 let login_executed = match crate::rest::runner::execute_rest_request(
328 &login_request_file,
329 &base_url,
330 None,
331 ) {
332 Ok(v) => v,
333 Err(err) => {
334 write_file(&stderr_path, format!("{err:#}\n").as_bytes())?;
335 status = "failed".to_string();
336 message = Some("rest_flow_login_failed".to_string());
337 failed += 1;
338 cases_out.push(case_result(
339 &id,
340 &ty,
341 &tags,
342 &status,
343 start.elapsed(),
344 command_snippet,
345 message,
346 assertions,
347 None,
348 Some(&stderr_path),
349 repo_root,
350 ));
351 if options.fail_fast && status == "failed" {
352 break;
353 }
354 continue;
355 }
356 };
357
358 if let Err(err) = crate::rest::expect::evaluate_main_response(
359 &login_request_file.request,
360 &login_executed,
361 ) {
362 write_file(&stderr_path, format!("{err:#}\n").as_bytes())?;
363 status = "failed".to_string();
364 message = Some("rest_flow_login_failed".to_string());
365 failed += 1;
366 cases_out.push(case_result(
367 &id,
368 &ty,
369 &tags,
370 &status,
371 start.elapsed(),
372 command_snippet,
373 message,
374 assertions,
375 None,
376 Some(&stderr_path),
377 repo_root,
378 ));
379 if options.fail_fast && status == "failed" {
380 break;
381 }
382 continue;
383 }
384
385 let login_json: serde_json::Value =
386 match serde_json::from_slice(&login_executed.response.body) {
387 Ok(v) => v,
388 Err(_) => serde_json::Value::Null,
389 };
390 let token = crate::jq::query_raw(&login_json, &token_jq)
391 .ok()
392 .and_then(|lines| lines.into_iter().next())
393 .unwrap_or_default();
394 let token = token.trim().to_string();
395 if token.is_empty() || token == "null" {
396 let hint = "Failed to extract token from login response.\nHint: set cases[i].tokenJq to the token field (e.g. .accessToken).\n";
397 write_file(&stderr_path, hint.as_bytes())?;
398 status = "failed".to_string();
399 message = Some("rest_flow_token_extract_failed".to_string());
400 failed += 1;
401 cases_out.push(case_result(
402 &id,
403 &ty,
404 &tags,
405 &status,
406 start.elapsed(),
407 command_snippet,
408 message,
409 assertions,
410 None,
411 Some(&stderr_path),
412 repo_root,
413 ));
414 if options.fail_fast && status == "failed" {
415 break;
416 }
417 continue;
418 }
419
420 let stdout_path = run_dir_abs.join(format!("{safe_id}.response.json"));
422 write_file(&stdout_path, b"")?;
423 stdout_file_abs = Some(stdout_path.clone());
424
425 match crate::rest::runner::execute_rest_request(
426 &main_request_file,
427 &base_url,
428 Some(&token),
429 ) {
430 Ok(executed) => {
431 write_file(&stdout_path, &executed.response.body)?;
432 if let Err(err) = crate::rest::expect::evaluate_main_response(
433 &main_request_file.request,
434 &executed,
435 ) {
436 write_file(&stderr_path, format!("{err:#}\n").as_bytes())?;
437 status = "failed".to_string();
438 message = Some("rest_flow_request_failed".to_string());
439 failed += 1;
440 } else {
441 status = "passed".to_string();
442 passed += 1;
443 }
444 }
445 Err(err) => {
446 write_file(&stderr_path, format!("{err:#}\n").as_bytes())?;
447 status = "failed".to_string();
448 message = Some("rest_flow_request_failed".to_string());
449 failed += 1;
450 }
451 }
452 }
453 rest::PrepareOutcome::Skipped { message: msg } => {
454 status = "skipped".to_string();
455 message = Some(msg);
456 skipped += 1;
457 }
458 rest::PrepareOutcome::Failed { message: msg } => {
459 status = "failed".to_string();
460 message = Some(msg);
461 failed += 1;
462 }
463 }
464 }
465 "graphql" => {
466 match graphql::prepare_graphql_case(
467 repo_root,
468 c,
469 &id,
470 defaults,
471 &options.env_rest_url,
472 &options.env_gql_url,
473 options.allow_writes_flag,
474 &effective_env,
475 auth_manager.as_mut(),
476 )? {
477 graphql::PrepareOutcome::Ready(plan) => {
478 let graphql::GraphqlCasePlan {
479 op_abs,
480 vars_abs,
481 config_dir,
482 url,
483 jwt,
484 access_token_for_case: prepared_access_token,
485 } = plan;
486
487 gql_config_dir = config_dir;
488 gql_url = url;
489 gql_jwt = jwt;
490 access_token_for_case = prepared_access_token;
491
492 let expect_jq_raw = c.expect.as_ref().map(|e| e.jq.as_str()).unwrap_or("");
493 let graphql::GraphqlCaseRunOutput {
494 status: graphql_status,
495 message: graphql_message,
496 assertions: graphql_assertions,
497 command_snippet: graphql_command_snippet,
498 stdout_path,
499 stderr_path,
500 skip_cleanup,
501 } = graphql::run_graphql_case(
502 repo_root,
503 &run_dir_abs,
504 &safe_id,
505 effective_no_history,
506 &effective_env,
507 defaults,
508 &options.env_gql_url,
509 &gql_config_dir,
510 &gql_url,
511 &gql_jwt,
512 &access_token_for_case,
513 &op_abs,
514 vars_abs.as_deref(),
515 c.allow_errors,
516 expect_jq_raw,
517 )?;
518
519 status = graphql_status;
520 message = graphql_message;
521 assertions = graphql_assertions;
522 command_snippet = graphql_command_snippet;
523 stdout_file_abs = Some(stdout_path);
524 stderr_file_abs = Some(stderr_path);
525
526 match status.as_str() {
527 "passed" => passed += 1,
528 "failed" => failed += 1,
529 _ => {}
530 }
531
532 if skip_cleanup {
533 cases_out.push(case_result(
534 &id,
535 &ty,
536 &tags,
537 &status,
538 start.elapsed(),
539 command_snippet.clone(),
540 message.clone(),
541 assertions.clone(),
542 stdout_file_abs.as_deref(),
543 stderr_file_abs.as_deref(),
544 repo_root,
545 ));
546
547 if options.fail_fast && status == "failed" {
548 break;
549 }
550 continue;
551 }
552 }
553 graphql::PrepareOutcome::Skipped { message: msg } => {
554 status = "skipped".to_string();
555 message = Some(msg);
556 skipped += 1;
557 }
558 graphql::PrepareOutcome::Failed { message: msg } => {
559 status = "failed".to_string();
560 message = Some(msg);
561 failed += 1;
562 }
563 }
564 }
565 "grpc" => match grpc::prepare_grpc_case(repo_root, c, &id, defaults)? {
566 grpc::PrepareOutcome::Ready(plan) => {
567 let grpc::GrpcCasePlan {
568 request_abs,
569 request_file,
570 config_dir,
571 url,
572 token,
573 } = plan;
574 let grpc_config_dir = config_dir;
575 let grpc_url = url;
576 let grpc_token = token;
577
578 let grpc::GrpcCaseRunOutput {
579 status: grpc_status,
580 message: grpc_message,
581 assertions: grpc_assertions,
582 command_snippet: grpc_command_snippet,
583 stdout_path,
584 stderr_path,
585 } = grpc::run_grpc_case(
586 repo_root,
587 &run_dir_abs,
588 &safe_id,
589 effective_no_history,
590 &effective_env,
591 defaults,
592 &options.env_grpc_url,
593 &grpc_config_dir,
594 &grpc_url,
595 &grpc_token,
596 &request_abs,
597 &request_file,
598 )?;
599
600 status = grpc_status;
601 message = grpc_message;
602 assertions = grpc_assertions;
603 command_snippet = grpc_command_snippet;
604 stdout_file_abs = Some(stdout_path);
605 stderr_file_abs = Some(stderr_path);
606
607 match status.as_str() {
608 "passed" => passed += 1,
609 "failed" => failed += 1,
610 _ => {}
611 }
612 }
613 },
614 "websocket" => match websocket::prepare_websocket_case(repo_root, c, &id, defaults)? {
615 websocket::PrepareOutcome::Ready(plan) => {
616 let websocket::WebsocketCasePlan {
617 request_abs,
618 request_file,
619 config_dir,
620 url,
621 token,
622 } = plan;
623 let ws_config_dir = config_dir;
624 let ws_url = url;
625 let ws_token = token;
626
627 let websocket::WebsocketCaseRunOutput {
628 status: ws_status,
629 message: ws_message,
630 assertions: ws_assertions,
631 command_snippet: ws_command_snippet,
632 stdout_path,
633 stderr_path,
634 } = websocket::run_websocket_case(
635 repo_root,
636 &run_dir_abs,
637 &safe_id,
638 effective_no_history,
639 &effective_env,
640 defaults,
641 &options.env_ws_url,
642 &ws_config_dir,
643 &ws_url,
644 &ws_token,
645 &request_abs,
646 &request_file,
647 )?;
648
649 status = ws_status;
650 message = ws_message;
651 assertions = ws_assertions;
652 command_snippet = ws_command_snippet;
653 stdout_file_abs = Some(stdout_path);
654 stderr_file_abs = Some(stderr_path);
655
656 match status.as_str() {
657 "passed" => passed += 1,
658 "failed" => failed += 1,
659 _ => {}
660 }
661 }
662 },
663 other => {
664 anyhow::bail!("Unknown case type '{other}' for case '{id}'");
665 }
666 }
667
668 let duration_ms = start.elapsed().as_millis().try_into().unwrap_or(u64::MAX);
669
670 if status == "passed" || status == "failed" {
672 let cleanup_ok = if let (Some(stderr_path), Some(cleanup)) =
673 (stderr_file_abs.as_deref(), c.cleanup.as_ref())
674 {
675 let mut ctx = CleanupContext {
676 repo_root,
677 run_dir: &run_dir_abs,
678 case_id: &id,
679 safe_id: &safe_id,
680 main_response_file: stdout_file_abs.as_deref(),
681 main_stderr_file: stderr_path,
682 allow_writes_flag: options.allow_writes_flag,
683 effective_env: &effective_env,
684 effective_no_history,
685 suite_defaults: defaults,
686 env_rest_url: &options.env_rest_url,
687 env_gql_url: &options.env_gql_url,
688 rest_config_dir: &rest_config_dir,
689 rest_url: &rest_url,
690 rest_token: &rest_token,
691 gql_config_dir: &gql_config_dir,
692 gql_url: &gql_url,
693 gql_jwt: &gql_jwt,
694 access_token_for_case: &access_token_for_case,
695 auth_manager: auth_manager.as_mut(),
696 cleanup: Some(cleanup),
697 };
698 run_case_cleanup(&mut ctx)?
699 } else {
700 true
701 };
702
703 if !cleanup_ok && status == "passed" {
704 status = "failed".to_string();
705 message = Some("cleanup_failed".to_string());
706 passed = passed.saturating_sub(1);
707 failed += 1;
708 }
709 }
710
711 cases_out.push(SuiteCaseResult {
712 id,
713 case_type: ty,
714 status: status.clone(),
715 duration_ms,
716 tags,
717 command: command_snippet,
718 message,
719 assertions,
720 stdout_file: stdout_file_abs
721 .as_deref()
722 .map(|p| path_relative_to_repo_or_abs(repo_root, p)),
723 stderr_file: stderr_file_abs
724 .as_deref()
725 .map(|p| path_relative_to_repo_or_abs(repo_root, p)),
726 });
727
728 if options.fail_fast && status == "failed" {
729 break;
730 }
731 }
732
733 let finished_at = time_iso_now()?;
734
735 let results = SuiteRunResults {
736 version: 1,
737 suite: suite_name_value,
738 suite_file: suite_file_rel,
739 run_id,
740 started_at,
741 finished_at,
742 output_dir: output_dir_rel,
743 summary: SuiteRunSummary {
744 total,
745 passed,
746 failed,
747 skipped,
748 },
749 cases: cases_out,
750 };
751
752 if let Some(msg) = auth_init_message {
753 let path = run_dir_abs.join("auth.disabled.log");
755 let _ = write_file(&path, format!("{msg}\n").as_bytes());
756 }
757
758 Ok(SuiteRunOutput {
759 run_dir_abs,
760 results,
761 })
762}
763
764#[allow(clippy::too_many_arguments)]
765fn case_result(
766 id: &str,
767 case_type: &str,
768 tags: &[String],
769 status: &str,
770 duration: std::time::Duration,
771 command: Option<String>,
772 message: Option<String>,
773 assertions: Option<serde_json::Value>,
774 stdout_file: Option<&Path>,
775 stderr_file: Option<&Path>,
776 repo_root: &Path,
777) -> SuiteCaseResult {
778 SuiteCaseResult {
779 id: id.to_string(),
780 case_type: case_type.to_string(),
781 status: status.to_string(),
782 duration_ms: duration.as_millis().try_into().unwrap_or(u64::MAX),
783 tags: tags.to_vec(),
784 command,
785 message,
786 assertions,
787 stdout_file: stdout_file.map(|p| path_relative_to_repo_or_abs(repo_root, p)),
788 stderr_file: stderr_file.map(|p| path_relative_to_repo_or_abs(repo_root, p)),
789 }
790}
791
792#[cfg(test)]
793mod tests {
794 use super::*;
795 use crate::suite::runtime;
796 use crate::suite::safety::MSG_WRITE_CASES_DISABLED;
797 use crate::suite::schema::{SuiteCase, SuiteDefaults, SuiteManifest};
798 use nils_term::progress::Progress;
799 use nils_term::progress::{ProgressDrawTarget, ProgressEnabled, ProgressOptions};
800 use nils_test_support::fixtures::{GraphqlSetupFixture, RestSetupFixture};
801 use pretty_assertions::assert_eq;
802 use std::collections::HashSet;
803 use std::io::{Read, Write};
804 use std::net::{TcpListener, TcpStream};
805 use std::sync::mpsc;
806 use std::sync::{Arc, Mutex};
807 use std::thread;
808 use std::time::Duration;
809
810 use tempfile::TempDir;
811
812 fn base_case(id: &str, case_type: &str) -> SuiteCase {
813 SuiteCase {
814 id: id.to_string(),
815 case_type: case_type.to_string(),
816 tags: Vec::new(),
817 env: String::new(),
818 no_history: None,
819 allow_write: false,
820 config_dir: String::new(),
821 url: String::new(),
822 token: String::new(),
823 request: String::new(),
824 grpc_proto: None,
825 grpc_import_paths: None,
826 grpc_plaintext: None,
827 login_request: String::new(),
828 token_jq: String::new(),
829 jwt: String::new(),
830 op: String::new(),
831 vars: None,
832 allow_errors: false,
833 expect: None,
834 cleanup: None,
835 }
836 }
837
838 struct TestServer {
839 base_url: String,
840 shutdown: mpsc::Sender<()>,
841 join: Option<thread::JoinHandle<()>>,
842 }
843
844 impl Drop for TestServer {
845 fn drop(&mut self) {
846 let _ = self.shutdown.send(());
847 if let Some(j) = self.join.take() {
848 let _ = j.join();
849 }
850 }
851 }
852
853 fn read_until_headers_end(stream: &mut TcpStream) {
854 let _ = stream.set_read_timeout(Some(Duration::from_secs(2)));
855 let mut buf = Vec::new();
856 let mut tmp = [0u8; 2048];
857 loop {
858 match stream.read(&mut tmp) {
859 Ok(0) => break,
860 Ok(n) => {
861 buf.extend_from_slice(&tmp[..n]);
862 if buf.windows(4).any(|w| w == b"\r\n\r\n") {
863 break;
864 }
865 if buf.len() > 64 * 1024 {
866 break;
867 }
868 }
869 Err(_) => break,
870 }
871 }
872 }
873
874 fn write_json_response(stream: &mut TcpStream, body: &[u8]) {
875 let mut resp = Vec::new();
876 resp.extend_from_slice(b"HTTP/1.1 200 OK\r\n");
877 resp.extend_from_slice(format!("Content-Length: {}\r\n", body.len()).as_bytes());
878 resp.extend_from_slice(b"Content-Type: application/json\r\n");
879 resp.extend_from_slice(b"\r\n");
880 resp.extend_from_slice(body);
881 let _ = stream.write_all(&resp);
882 let _ = stream.flush();
883 }
884
885 fn start_server() -> TestServer {
886 let listener = TcpListener::bind("127.0.0.1:0").expect("bind");
887 listener.set_nonblocking(true).expect("nonblocking");
888 let addr = listener.local_addr().expect("addr");
889 let base_url = format!("http://{addr}");
890
891 let (tx, rx) = mpsc::channel::<()>();
892 let join = thread::spawn(move || {
893 loop {
894 if rx.try_recv().is_ok() {
895 break;
896 }
897 match listener.accept() {
898 Ok((mut stream, _peer)) => {
899 read_until_headers_end(&mut stream);
900 write_json_response(&mut stream, br#"{"ok":true}"#);
901 }
902 Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {
903 thread::sleep(Duration::from_millis(5));
904 }
905 Err(_) => break,
906 }
907 }
908 });
909
910 TestServer {
911 base_url,
912 shutdown: tx,
913 join: Some(join),
914 }
915 }
916
917 fn read_output(buffer: &Arc<Mutex<Vec<u8>>>) -> String {
918 String::from_utf8_lossy(&buffer.lock().expect("buffer lock")).to_string()
919 }
920
921 fn normalize_progress_output(s: &str) -> String {
922 let mut out = String::with_capacity(s.len());
923 let bytes = s.as_bytes();
924 let mut i = 0;
925 while i < bytes.len() {
926 if bytes[i] == b'\r' {
927 i += 1;
928 continue;
929 }
930
931 if bytes[i] == 0x1b && i + 1 < bytes.len() && bytes[i + 1] == b'[' {
932 i += 2;
933 while i < bytes.len() {
934 let b = bytes[i];
935 i += 1;
936 if b.is_ascii_alphabetic() {
937 break;
938 }
939 }
940 continue;
941 }
942
943 out.push(bytes[i] as char);
944 i += 1;
945 }
946 out
947 }
948
949 #[test]
950 fn suite_runner_sanitize_id_matches_expected_replacements() {
951 assert_eq!(runtime::sanitize_id("rest.health"), "rest.health");
952 assert_eq!(runtime::sanitize_id("a b c"), "a-b-c");
953 assert_eq!(runtime::sanitize_id(""), "case");
954 }
955
956 #[test]
957 fn suite_runner_sanitize_id_handles_punctuation_and_unicode() {
958 assert_eq!(runtime::sanitize_id("foo,bar"), "foo-bar");
959 assert_eq!(runtime::sanitize_id("foo✅bar"), "foo-bar");
960 assert_eq!(runtime::sanitize_id("✅foo"), "foo");
961 assert_eq!(runtime::sanitize_id("foo!!!"), "foo");
962 assert_eq!(runtime::sanitize_id("!!!"), "case");
963 }
964
965 #[test]
966 fn suite_runner_masks_token_args_in_command_snippet() {
967 let args = vec![
968 "--token".to_string(),
969 "secret".to_string(),
970 "--jwt=jwt-value".to_string(),
971 "--other".to_string(),
972 "keep".to_string(),
973 "--token=inline".to_string(),
974 "--jwt".to_string(),
975 "another".to_string(),
976 ];
977
978 let masked = mask_args_for_command_snippet(&args);
979
980 assert_eq!(
981 masked,
982 "'--token' 'REDACTED' '--jwt=REDACTED' '--other' 'keep' '--token=REDACTED' '--jwt' 'REDACTED'"
983 );
984 }
985
986 #[test]
987 fn suite_runner_resolve_rest_base_url_precedence() {
988 let fixture = RestSetupFixture::new();
989 fixture.write_endpoints_env("REST_URL_STAGE=http://env-file\n");
990
991 let defaults = SuiteDefaults::default();
992 assert_eq!(
993 runtime::resolve_rest_base_url(&fixture.root, "setup/rest", "", "", &defaults, "")
994 .unwrap(),
995 "http://localhost:6700"
996 );
997 assert_eq!(
998 runtime::resolve_rest_base_url(&fixture.root, "setup/rest", "", "stage", &defaults, "")
999 .unwrap(),
1000 "http://env-file"
1001 );
1002 assert_eq!(
1003 runtime::resolve_rest_base_url(
1004 &fixture.root,
1005 "setup/rest",
1006 "",
1007 "stage",
1008 &defaults,
1009 "http://env-var"
1010 )
1011 .unwrap(),
1012 "http://env-var"
1013 );
1014
1015 let mut defaults_with_url = SuiteDefaults::default();
1016 defaults_with_url.rest.url = "http://default".to_string();
1017 assert_eq!(
1018 runtime::resolve_rest_base_url(
1019 &fixture.root,
1020 "setup/rest",
1021 "",
1022 "stage",
1023 &defaults_with_url,
1024 "http://env-var"
1025 )
1026 .unwrap(),
1027 "http://default"
1028 );
1029 assert_eq!(
1030 runtime::resolve_rest_base_url(
1031 &fixture.root,
1032 "setup/rest",
1033 "http://override",
1034 "stage",
1035 &defaults_with_url,
1036 "http://env-var"
1037 )
1038 .unwrap(),
1039 "http://override"
1040 );
1041 }
1042
1043 #[test]
1044 fn suite_runner_resolve_gql_url_precedence() {
1045 let fixture = GraphqlSetupFixture::new();
1046 fixture.write_endpoints_env("GQL_URL_STAGE=http://env-file/graphql\n");
1047
1048 let defaults = SuiteDefaults::default();
1049 assert_eq!(
1050 runtime::resolve_gql_url(&fixture.root, "setup/graphql", "", "", &defaults, "")
1051 .unwrap(),
1052 "http://localhost:6700/graphql"
1053 );
1054 assert_eq!(
1055 runtime::resolve_gql_url(&fixture.root, "setup/graphql", "", "stage", &defaults, "")
1056 .unwrap(),
1057 "http://env-file/graphql"
1058 );
1059 assert_eq!(
1060 runtime::resolve_gql_url(
1061 &fixture.root,
1062 "setup/graphql",
1063 "",
1064 "stage",
1065 &defaults,
1066 "http://env-var/graphql"
1067 )
1068 .unwrap(),
1069 "http://env-var/graphql"
1070 );
1071
1072 let mut defaults_with_url = SuiteDefaults::default();
1073 defaults_with_url.graphql.url = "http://default/graphql".to_string();
1074 assert_eq!(
1075 runtime::resolve_gql_url(
1076 &fixture.root,
1077 "setup/graphql",
1078 "",
1079 "stage",
1080 &defaults_with_url,
1081 "http://env-var/graphql"
1082 )
1083 .unwrap(),
1084 "http://default/graphql"
1085 );
1086 assert_eq!(
1087 runtime::resolve_gql_url(
1088 &fixture.root,
1089 "setup/graphql",
1090 "http://override/graphql",
1091 "stage",
1092 &defaults_with_url,
1093 "http://env-var/graphql"
1094 )
1095 .unwrap(),
1096 "http://override/graphql"
1097 );
1098 }
1099
1100 #[test]
1101 fn suite_runner_resolve_rest_token_profile_from_tokens_files() {
1102 let fixture = RestSetupFixture::new();
1103 fixture.write_tokens_env("REST_TOKEN_TEAM_ALPHA=base\n");
1104 fixture.write_tokens_local_env("REST_TOKEN_TEAM_ALPHA=local\n");
1105
1106 let token = runtime::resolve_rest_token_profile(&fixture.setup_dir, "team alpha").unwrap();
1107 assert_eq!(token, "local");
1108 }
1109
1110 #[test]
1111 fn suite_runner_token_profile_ignores_leading_separators() {
1112 let fixture = RestSetupFixture::new();
1113 fixture.write_tokens_env("REST_TOKEN_TEAM_ALPHA=base\n");
1114
1115 let token = runtime::resolve_rest_token_profile(&fixture.setup_dir, "-team alpha").unwrap();
1116 assert_eq!(token, "base");
1117 }
1118
1119 #[test]
1120 fn suite_runner_empty_suite_has_zero_counts() {
1121 let tmp = TempDir::new().unwrap();
1122 let suite_path = tmp.path().join("suite.json");
1123 std::fs::write(&suite_path, br#"{"version":1,"cases":[]}"#).unwrap();
1124
1125 let manifest = SuiteManifest {
1126 version: 1,
1127 name: String::new(),
1128 defaults: SuiteDefaults::default(),
1129 auth: None,
1130 cases: Vec::new(),
1131 };
1132 let loaded = LoadedSuite {
1133 suite_path: suite_path.clone(),
1134 manifest,
1135 };
1136
1137 let options = SuiteRunOptions {
1138 required_tags: Vec::new(),
1139 only_ids: HashSet::new(),
1140 skip_ids: HashSet::new(),
1141 allow_writes_flag: true,
1142 fail_fast: false,
1143 output_dir_base: tmp.path().join("out"),
1144 env_rest_url: String::new(),
1145 env_gql_url: String::new(),
1146 env_grpc_url: String::new(),
1147 env_ws_url: String::new(),
1148 progress: None,
1149 };
1150
1151 let out = run_suite(tmp.path(), loaded, options).unwrap();
1152 assert_eq!(out.results.summary.total, 0);
1153 assert_eq!(out.results.summary.passed, 0);
1154 assert_eq!(out.results.summary.failed, 0);
1155 assert_eq!(out.results.summary.skipped, 0);
1156 assert!(out.run_dir_abs.is_dir());
1157 }
1158
1159 #[test]
1160 fn suite_runner_skips_case_when_tag_mismatch() {
1161 let tmp = TempDir::new().unwrap();
1162 let suite_path = tmp.path().join("suite.json");
1163 std::fs::write(&suite_path, br#"{"version":1,"cases":[]}"#).unwrap();
1164
1165 let mut case = base_case("case-1", "rest");
1166 case.tags = vec!["smoke".to_string()];
1167
1168 let manifest = SuiteManifest {
1169 version: 1,
1170 name: String::new(),
1171 defaults: SuiteDefaults::default(),
1172 auth: None,
1173 cases: vec![case],
1174 };
1175 let loaded = LoadedSuite {
1176 suite_path: suite_path.clone(),
1177 manifest,
1178 };
1179
1180 let options = SuiteRunOptions {
1181 required_tags: vec!["fast".to_string()],
1182 only_ids: HashSet::new(),
1183 skip_ids: HashSet::new(),
1184 allow_writes_flag: true,
1185 fail_fast: false,
1186 output_dir_base: tmp.path().join("out"),
1187 env_rest_url: String::new(),
1188 env_gql_url: String::new(),
1189 env_grpc_url: String::new(),
1190 env_ws_url: String::new(),
1191 progress: None,
1192 };
1193
1194 let out = run_suite(tmp.path(), loaded, options).unwrap();
1195 assert_eq!(out.results.summary.total, 1);
1196 assert_eq!(out.results.summary.skipped, 1);
1197 assert_eq!(out.results.cases[0].status, "skipped");
1198 assert_eq!(
1199 out.results.cases[0].message.as_deref(),
1200 Some("tag_mismatch")
1201 );
1202 }
1203
1204 #[test]
1205 fn suite_runner_skips_write_case_when_writes_disabled() {
1206 let tmp = TempDir::new().unwrap();
1207 let suite_path = tmp.path().join("suite.json");
1208 std::fs::write(&suite_path, br#"{"version":1,"cases":[]}"#).unwrap();
1209
1210 let request_path = tmp.path().join("requests/post.request.json");
1211 std::fs::create_dir_all(request_path.parent().unwrap()).unwrap();
1212 std::fs::write(
1213 &request_path,
1214 br#"{"method":"POST","path":"/do","expect":{"status":200}}"#,
1215 )
1216 .unwrap();
1217
1218 let mut case = base_case("write-case", "rest");
1219 case.allow_write = true;
1220 case.request = "requests/post.request.json".to_string();
1221
1222 let manifest = SuiteManifest {
1223 version: 1,
1224 name: String::new(),
1225 defaults: SuiteDefaults::default(),
1226 auth: None,
1227 cases: vec![case],
1228 };
1229 let loaded = LoadedSuite {
1230 suite_path: suite_path.clone(),
1231 manifest,
1232 };
1233
1234 let options = SuiteRunOptions {
1235 required_tags: Vec::new(),
1236 only_ids: HashSet::new(),
1237 skip_ids: HashSet::new(),
1238 allow_writes_flag: false,
1239 fail_fast: false,
1240 output_dir_base: tmp.path().join("out"),
1241 env_rest_url: String::new(),
1242 env_gql_url: String::new(),
1243 env_grpc_url: String::new(),
1244 env_ws_url: String::new(),
1245 progress: None,
1246 };
1247
1248 let out = run_suite(tmp.path(), loaded, options).unwrap();
1249 assert_eq!(out.results.summary.total, 1);
1250 assert_eq!(out.results.summary.skipped, 1);
1251 assert_eq!(out.results.cases[0].status, "skipped");
1252 assert_eq!(
1253 out.results.cases[0].message.as_deref(),
1254 Some(MSG_WRITE_CASES_DISABLED)
1255 );
1256 }
1257
1258 #[test]
1259 fn suite_runner_unknown_case_type_is_error() {
1260 let tmp = TempDir::new().unwrap();
1261 let suite_path = tmp.path().join("suite.json");
1262 std::fs::write(&suite_path, br#"{"version":1,"cases":[]}"#).unwrap();
1263
1264 let case = base_case("case-1", "weird");
1265 let manifest = SuiteManifest {
1266 version: 1,
1267 name: String::new(),
1268 defaults: SuiteDefaults::default(),
1269 auth: None,
1270 cases: vec![case],
1271 };
1272 let loaded = LoadedSuite {
1273 suite_path: suite_path.clone(),
1274 manifest,
1275 };
1276
1277 let options = SuiteRunOptions {
1278 required_tags: Vec::new(),
1279 only_ids: HashSet::new(),
1280 skip_ids: HashSet::new(),
1281 allow_writes_flag: true,
1282 fail_fast: false,
1283 output_dir_base: tmp.path().join("out"),
1284 env_rest_url: String::new(),
1285 env_gql_url: String::new(),
1286 env_grpc_url: String::new(),
1287 env_ws_url: String::new(),
1288 progress: None,
1289 };
1290
1291 let err = run_suite(tmp.path(), loaded, options).unwrap_err();
1292 assert!(format!("{err:#}").contains("Unknown case type 'weird'"));
1293 }
1294
1295 #[test]
1296 fn suite_runner_no_history_flag_in_command_snippet() {
1297 let tmp = TempDir::new().unwrap();
1298 let suite_path = tmp.path().join("suite.json");
1299 std::fs::write(&suite_path, br#"{"version":1,"cases":[]}"#).unwrap();
1300
1301 let request_path = tmp.path().join("requests/health.request.json");
1302 std::fs::create_dir_all(request_path.parent().unwrap()).unwrap();
1303 std::fs::write(
1304 &request_path,
1305 br#"{"method":"GET","path":"/health","expect":{"status":200}}"#,
1306 )
1307 .unwrap();
1308
1309 let server = start_server();
1310
1311 let defaults = SuiteDefaults {
1312 no_history: true,
1313 ..Default::default()
1314 };
1315
1316 let mut case = base_case("case-1", "rest");
1317 case.request = "requests/health.request.json".to_string();
1318
1319 let manifest = SuiteManifest {
1320 version: 1,
1321 name: String::new(),
1322 defaults,
1323 auth: None,
1324 cases: vec![case],
1325 };
1326 let loaded = LoadedSuite {
1327 suite_path: suite_path.clone(),
1328 manifest,
1329 };
1330
1331 let options = SuiteRunOptions {
1332 required_tags: Vec::new(),
1333 only_ids: HashSet::new(),
1334 skip_ids: HashSet::new(),
1335 allow_writes_flag: true,
1336 fail_fast: false,
1337 output_dir_base: tmp.path().join("out"),
1338 env_rest_url: server.base_url.clone(),
1339 env_gql_url: String::new(),
1340 env_grpc_url: String::new(),
1341 env_ws_url: String::new(),
1342 progress: None,
1343 };
1344
1345 let out = run_suite(tmp.path(), loaded, options).unwrap();
1346 let cmd = out.results.cases[0].command.as_deref().unwrap_or("");
1347 assert!(cmd.contains("--no-history"));
1348 assert!(cmd.contains("--url"));
1349 }
1350
1351 #[test]
1352 fn suite_runner_progress_disabled_is_noop() {
1353 let tmp = TempDir::new().unwrap();
1354 let suite_path = tmp.path().join("suite.json");
1355 std::fs::write(&suite_path, br#"{"version":1,"cases":[]}"#).unwrap();
1356
1357 let mut case = base_case("case-1", "rest");
1358 case.tags = vec!["smoke".to_string()];
1359
1360 let manifest = SuiteManifest {
1361 version: 1,
1362 name: String::new(),
1363 defaults: SuiteDefaults::default(),
1364 auth: None,
1365 cases: vec![case],
1366 };
1367 let loaded = LoadedSuite {
1368 suite_path: suite_path.clone(),
1369 manifest,
1370 };
1371
1372 let buffer = Arc::new(Mutex::new(Vec::new()));
1373 let progress = Progress::new(
1374 1,
1375 ProgressOptions::default()
1376 .with_enabled(ProgressEnabled::Off)
1377 .with_draw_target(ProgressDrawTarget::to_writer(buffer.clone()))
1378 .with_width(Some(60)),
1379 );
1380
1381 let options = SuiteRunOptions {
1382 required_tags: vec!["fast".to_string()],
1383 only_ids: HashSet::new(),
1384 skip_ids: HashSet::new(),
1385 allow_writes_flag: true,
1386 fail_fast: false,
1387 output_dir_base: tmp.path().join("out"),
1388 env_rest_url: String::new(),
1389 env_gql_url: String::new(),
1390 env_grpc_url: String::new(),
1391 env_ws_url: String::new(),
1392 progress: Some(progress),
1393 };
1394
1395 let out = run_suite(tmp.path(), loaded, options).unwrap();
1396 assert_eq!(out.results.summary.total, 1);
1397 assert_eq!(out.results.summary.skipped, 1);
1398 assert!(read_output(&buffer).is_empty());
1399 }
1400
1401 #[test]
1402 fn suite_runner_progress_updates_position_and_message() {
1403 let tmp = TempDir::new().unwrap();
1404 let suite_path = tmp.path().join("suite.json");
1405 std::fs::write(&suite_path, br#"{"version":1,"cases":[]}"#).unwrap();
1406
1407 let mut case = base_case("case-1", "rest");
1408 case.tags = vec!["smoke".to_string()];
1409
1410 let manifest = SuiteManifest {
1411 version: 1,
1412 name: String::new(),
1413 defaults: SuiteDefaults::default(),
1414 auth: None,
1415 cases: vec![case],
1416 };
1417 let loaded = LoadedSuite {
1418 suite_path: suite_path.clone(),
1419 manifest,
1420 };
1421
1422 let buffer = Arc::new(Mutex::new(Vec::new()));
1423 let progress = Progress::new(
1424 1,
1425 ProgressOptions::default()
1426 .with_enabled(ProgressEnabled::Auto)
1427 .with_draw_target(ProgressDrawTarget::to_writer(buffer.clone()))
1428 .with_width(Some(60)),
1429 );
1430
1431 let options = SuiteRunOptions {
1432 required_tags: vec!["fast".to_string()],
1433 only_ids: HashSet::new(),
1434 skip_ids: HashSet::new(),
1435 allow_writes_flag: true,
1436 fail_fast: false,
1437 output_dir_base: tmp.path().join("out"),
1438 env_rest_url: String::new(),
1439 env_gql_url: String::new(),
1440 env_grpc_url: String::new(),
1441 env_ws_url: String::new(),
1442 progress: Some(progress),
1443 };
1444
1445 let out = run_suite(tmp.path(), loaded, options).unwrap();
1446 assert_eq!(out.results.summary.total, 1);
1447 assert_eq!(out.results.summary.skipped, 1);
1448
1449 let normalized = normalize_progress_output(&read_output(&buffer));
1450 assert!(normalized.contains("1/1"), "output was: {normalized:?}");
1451 assert!(normalized.contains("case-1"), "output was: {normalized:?}");
1452 }
1453}