mod common;
use assert_cmd::cargo::cargo_bin;
use predicates::prelude::*;
use serde_json::Value;
use std::path::{Path, PathBuf};
use std::process::Output;
use std::sync::OnceLock;
use std::time::{Duration, Instant};
const TIMEOUT_SECS: u64 = 2;
const COUNT_TIMEOUT_SECS: u64 = 1;
const SIGNAL_START_TIMEOUT_SECS: u64 = 30;
const SLACK_MS: u64 = 1000;
const STARTUP_ALLOWANCE_CAP_MS: u64 = 500;
const CEILING_NUMERATOR: u64 = 3;
const CEILING_DENOMINATOR: u64 = 2;
const FIXTURE_BUDGET_FACTOR: u64 = 3;
const FIXTURE_FLOOR: u64 = 8 * 1024 * 1024;
const FIXTURE_CAP: u64 = 64 * 1024 * 1024 * 1024;
const FILE_MAX: u64 = 128 * 1024 * 1024;
const MAX_ATTEMPTS: usize = 3;
const GROWTH_FACTOR: u64 = 4;
const PROBE_SAMPLES: usize = 2;
const PROBE_TIMEOUT_SECS: u64 = 120;
const MAX_FILESIZE: &str = "4294967296";
const SIGNAL_FILE_BYTES: u64 = 4 * 1024 * 1024;
const TOKEN: &str = "tok=1";
const MISS_TOKEN: &str = "XXNOPE";
const FIXTURE_NAME: &str = "oversized.txt";
const CALIBRATION_SLOTS: usize = 4;
const LINE: &str = "tok=1 tok=1 tok=1 tok=1 tok=1 tok=1 tok=1 tok=1\n";
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
enum Verb {
Replace,
Search,
Count,
ReplaceFuzzyMiss,
EditFuzzyMiss,
}
impl Verb {
fn slot(self) -> usize {
match self {
Verb::Replace => 0,
Verb::Search => 1,
Verb::Count => 2,
Verb::ReplaceFuzzyMiss | Verb::EditFuzzyMiss => 3,
}
}
fn probe_bytes(self) -> u64 {
match self {
Verb::Count => 256 * 1024 * 1024,
Verb::Replace | Verb::Search | Verb::ReplaceFuzzyMiss | Verb::EditFuzzyMiss => {
16 * 1024 * 1024
}
}
}
fn budget_secs(self) -> u64 {
match self {
Verb::Count | Verb::EditFuzzyMiss => COUNT_TIMEOUT_SECS,
Verb::Replace | Verb::Search | Verb::ReplaceFuzzyMiss => TIMEOUT_SECS,
}
}
fn command(self, ws: &str) -> Vec<String> {
match self {
Verb::Replace => ["replace", "--dry-run", TOKEN, "tok=2", ws]
.map(String::from)
.to_vec(),
Verb::Search => ["search", TOKEN, ws].map(String::from).to_vec(),
Verb::Count => ["count", ws].map(String::from).to_vec(),
Verb::ReplaceFuzzyMiss => [
"replace",
"--dry-run",
"--fuzzy",
"auto",
MISS_TOKEN,
"x",
ws,
]
.map(String::from)
.to_vec(),
Verb::EditFuzzyMiss => vec![
"edit".into(),
format!("{ws}/{FIXTURE_NAME}"),
"--old".into(),
MISS_TOKEN.into(),
"--new".into(),
"x".into(),
],
}
}
}
impl std::fmt::Display for Verb {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let name = match self {
Verb::Replace => "replace",
Verb::Search => "search",
Verb::Count => "count",
Verb::ReplaceFuzzyMiss => "replace-fuzzy-miss",
Verb::EditFuzzyMiss => "edit-fuzzy-miss",
};
f.write_str(name)
}
}
#[derive(Clone, Copy)]
struct Calibration {
bytes_per_ms: u64,
base_ms: u64,
}
struct Run {
_dir: tempfile::TempDir,
output: Output,
elapsed_ms: u64,
budget_ms: u64,
base_ms: u64,
fixture_bytes: u64,
}
impl Run {
fn ceiling_ms(&self) -> u64 {
let startup = self.base_ms.min(STARTUP_ALLOWANCE_CAP_MS);
self.budget_ms * CEILING_NUMERATOR / CEILING_DENOMINATOR + SLACK_MS + 2 * startup
}
}
fn write_fixture_of(dir: &Path, target: u64) -> PathBuf {
use std::io::Write as _;
let path = dir.join(FIXTURE_NAME);
let links = target.div_ceil(FILE_MAX).max(1) - 1;
let target = target.min(FILE_MAX);
let chunk = LINE.repeat((1024 * 1024) / LINE.len());
let mut writer =
std::io::BufWriter::new(std::fs::File::create(&path).expect("create oversized fixture"));
let mut written = 0;
while written < target {
writer
.write_all(chunk.as_bytes())
.expect("write oversized fixture");
written += chunk.len() as u64;
}
writer.flush().expect("flush oversized fixture");
drop(writer);
for n in 1..=links {
let link = dir.join(format!("oversized-{n}.txt"));
let _ = std::fs::remove_file(&link);
std::fs::hard_link(&path, &link).expect("hard-link oversized fixture");
}
path
}
fn parse_strict(stdout: &[u8]) -> (Vec<Value>, Vec<String>) {
let mut events = Vec::new();
let mut bad = Vec::new();
let text = String::from_utf8_lossy(stdout);
for line in text.lines().filter(|l| !l.trim().is_empty()) {
match serde_json::from_str::<Value>(line) {
Ok(v) => events.push(v),
Err(_) => bad.push(line.to_string()),
}
}
(events, bad)
}
fn summary_event(events: &[Value]) -> Option<&Value> {
events.iter().find(|e| e["type"] == "summary")
}
fn run_verb(verb: Verb, dir: &Path, timeout_secs: u64) -> (Output, u64) {
let ws = dir.to_str().expect("utf8 workspace");
let started = Instant::now();
let output = common::hermetic(std::process::Command::new(cargo_bin("atomwrite")))
.args([
"--timeout-secs",
&timeout_secs.to_string(),
"--no-auto-heal",
"--no-progress",
"--max-filesize",
MAX_FILESIZE,
"--workspace",
ws,
])
.args(verb.command(ws))
.output()
.expect("run verb");
let elapsed_us = started.elapsed().as_micros() as u64;
(output, elapsed_us)
}
fn fastest_run_us(verb: Verb, dir: &Path) -> u64 {
(0..PROBE_SAMPLES)
.map(|_| run_verb(verb, dir, PROBE_TIMEOUT_SECS).1)
.min()
.expect("at least one probe sample")
}
fn measure(verb: Verb) -> Calibration {
let dir = tempfile::tempdir().expect("tempdir");
write_fixture_of(dir.path(), LINE.len() as u64);
let base_us = fastest_run_us(verb, dir.path());
write_fixture_of(dir.path(), verb.probe_bytes());
let probe_us = fastest_run_us(verb, dir.path());
let scan_us = probe_us.saturating_sub(base_us).max(1);
Calibration {
bytes_per_ms: (verb.probe_bytes() * 1000 / scan_us).max(1),
base_ms: base_us / 1000,
}
}
fn calibration(verb: Verb) -> Calibration {
static CACHE: [OnceLock<Calibration>; CALIBRATION_SLOTS] = [
OnceLock::new(),
OnceLock::new(),
OnceLock::new(),
OnceLock::new(),
];
*CACHE[verb.slot()].get_or_init(|| measure(verb))
}
fn timed_run(verb: Verb) -> Run {
let cal = calibration(verb);
let budget_ms = verb.budget_secs() * 1000;
let mut bytes = cal
.bytes_per_ms
.saturating_mul(budget_ms * FIXTURE_BUDGET_FACTOR)
.clamp(FIXTURE_FLOOR, FIXTURE_CAP);
for _ in 0..MAX_ATTEMPTS {
let dir = tempfile::tempdir().expect("tempdir");
write_fixture_of(dir.path(), bytes);
let (output, elapsed_us) = run_verb(verb, dir.path(), verb.budget_secs());
let elapsed_ms = elapsed_us / 1000;
eprintln!(
"{verb}: fixture {bytes} bytes, {} bytes/ms probed, startup {} ms, elapsed {elapsed_ms} ms of {budget_ms} ms",
cal.bytes_per_ms, cal.base_ms
);
let (events, _) = parse_strict(&output.stdout);
let observed_cancel = events.iter().any(|e| e["type"] == "cancelled");
let finished_on_its_own = matches!(output.status.code(), Some(0 | 1 | 65))
&& !observed_cancel
&& elapsed_ms < budget_ms;
if !finished_on_its_own {
return Run {
_dir: dir,
output,
elapsed_ms,
budget_ms,
base_ms: cal.base_ms,
fixture_bytes: bytes,
};
}
assert!(
bytes < FIXTURE_CAP,
"{verb}: a {bytes}-byte fixture finished in {elapsed_ms} ms, inside the \
{budget_ms} ms budget; this host outruns FIXTURE_CAP, so no cancel can be \
observed. Raise FIXTURE_CAP"
);
bytes = bytes.saturating_mul(GROWTH_FACTOR).min(FIXTURE_CAP);
}
panic!(
"{verb}: {MAX_ATTEMPTS} fixtures finished inside the {budget_ms} ms budget, so no \
cancel was observed"
);
}
#[test]
fn replace_honours_global_deadline_inside_file() {
let run = timed_run(Verb::Replace);
let ceiling = run.ceiling_ms();
assert!(
run.elapsed_ms < ceiling,
"process took {} ms, above the {ceiling} ms ceiling (budget {} ms scaled \
by 1.5 + {SLACK_MS} ms slack + 2 x {} ms startup, fixture {} bytes); the \
deadline did not interrupt the scan inside the file",
run.elapsed_ms,
run.budget_ms,
run.base_ms,
run.fixture_bytes
);
let code = run.output.status.code();
assert_eq!(
code,
Some(124),
"expected exit 124 for the elapsed deadline, got {code:?}; stderr: {}",
String::from_utf8_lossy(&run.output.stderr)
);
}
#[test]
fn replace_data_channel_is_pure_ndjson_on_timeout() {
let run = timed_run(Verb::Replace);
let (events, bad) = parse_strict(&run.output.stdout);
assert!(
bad.is_empty(),
"data channel carried {} unparseable line(s): {:?}",
bad.len(),
bad
);
assert!(
!events.is_empty(),
"expected at least the terminal envelope on stdout, got none"
);
let stderr = String::from_utf8_lossy(&run.output.stderr);
assert!(
stderr.contains("global timeout elapsed"),
"the deadline notice must be emitted on the error channel, stderr was: {stderr}"
);
}
#[test]
fn cancelled_run_emits_cancelled_envelope_and_no_summary() {
let run = timed_run(Verb::Replace);
let (events, _) = parse_strict(&run.output.stdout);
assert!(
summary_event(&events).is_none(),
"a cancelled run must not emit a partial summary, stdout was: {events:?}"
);
let cancelled = events
.iter()
.find(|e| e["type"] == "cancelled")
.unwrap_or_else(|| {
panic!("a cancelled run must emit a `cancelled` envelope, stdout was: {events:?}")
});
assert_eq!(
cancelled["reason"].as_str(),
Some("global --timeout-secs deadline elapsed"),
"the reason must name the real cause, envelope was: {cancelled}"
);
assert!(
cancelled.get("signal").is_none_or(Value::is_null),
"an elapsed deadline is not a signal, envelope was: {cancelled}"
);
assert_eq!(
cancelled["deadline_exceeded"], true,
"an elapsed deadline must say so in the field, envelope was: {cancelled}"
);
let last = events.last().expect("at least one event");
assert_eq!(
last["type"], "cancelled",
"the last data-channel line must be the cancel, stdout was: {events:?}"
);
}
#[test]
fn completed_run_still_emits_its_summary() {
let dir = tempfile::tempdir().expect("tempdir");
std::fs::write(dir.path().join("small.txt"), "tok=1 tok=1 tok=1\n").expect("write fixture");
let (output, _) = run_verb(Verb::Replace, dir.path(), 60);
assert!(output.status.success(), "small run must succeed");
let (events, bad) = parse_strict(&output.stdout);
assert!(
bad.is_empty(),
"unparseable lines on the data channel: {bad:?}"
);
let summary = summary_event(&events).expect("a completed run must emit its summary");
assert_eq!(summary["files_matched"], 1, "summary was: {summary}");
assert!(
events.iter().all(|e| e["type"] != "cancelled"),
"a completed run must not emit a cancel envelope, stdout was: {events:?}"
);
}
#[test]
fn timeout_notice_predicate_matches_stderr() {
let run = timed_run(Verb::Replace);
let stderr = String::from_utf8_lossy(&run.output.stderr).into_owned();
assert_eq!(run.output.status.code(), Some(124), "stderr: {stderr}");
assert!(
predicate::str::contains("global timeout elapsed").eval(stderr.as_str()),
"the deadline notice must reach the operator on stderr, stderr was: {stderr}"
);
}
#[test]
fn every_content_scanning_verb_reports_the_interruption() {
for verb in [Verb::Search, Verb::Count] {
let run = timed_run(verb);
let output = &run.output;
let (events, bad) = parse_strict(&output.stdout);
assert!(
bad.is_empty(),
"{verb} put unparseable lines on the data channel: {bad:?}"
);
let cancelled = events
.iter()
.find(|e| e["type"] == "cancelled")
.unwrap_or_else(|| {
panic!(
"{verb} must report the interruption, status {:?}, elapsed {} ms, \
fixture {} bytes, stdout tail: {:?}",
output.status,
run.elapsed_ms,
run.fixture_bytes,
events.last()
)
});
assert_eq!(
cancelled["deadline_exceeded"], true,
"{verb} must name the deadline as the cause, envelope was: {cancelled}"
);
assert!(
cancelled.get("signal").is_none_or(Value::is_null),
"{verb} reported a signal that never arrived, envelope was: {cancelled}"
);
let last = events.last().expect("at least one event");
assert_eq!(
last["type"], "cancelled",
"{verb} must end on the cancel record, stdout was: {events:?}"
);
}
}
fn assert_fuzzy_miss_is_cancelled(verb: Verb) {
let run = timed_run(verb);
let ceiling = run.ceiling_ms();
assert!(
run.elapsed_ms < ceiling,
"{verb} took {} ms, above the {ceiling} ms ceiling (budget {} ms, startup {} ms, \
fixture {} bytes); the cascade did not honour the deadline",
run.elapsed_ms,
run.budget_ms,
run.base_ms,
run.fixture_bytes
);
let stderr = String::from_utf8_lossy(&run.output.stderr);
assert_eq!(
run.output.status.code(),
Some(124),
"{verb}: an elapsed deadline is exit 124, never MATCH_FAILED (65) or NO_MATCHES (1); \
stderr: {stderr}"
);
let (events, bad) = parse_strict(&run.output.stdout);
assert!(
bad.is_empty(),
"{verb} put unparseable lines on the data channel: {bad:?}"
);
let last = events.last().unwrap_or_else(|| {
panic!("{verb} must end on an NDJSON record, stdout was empty; stderr: {stderr}")
});
assert!(
last["type"] == "cancelled" || last["ok"] == false,
"{verb} must end on a cancel or error record, last record was: {last}"
);
assert!(
events
.iter()
.all(|e| e["code"] != "MATCH_FAILED" && e["code"] != "NO_MATCHES"),
"{verb} reported a miss instead of the cancel, stdout was: {events:?}"
);
assert!(
summary_event(&events).is_none(),
"{verb}: a cancelled run must not emit a summary, stdout was: {events:?}"
);
}
#[test]
fn fuzzy_replace_miss_honours_global_deadline() {
assert_fuzzy_miss_is_cancelled(Verb::ReplaceFuzzyMiss);
}
#[test]
fn fuzzy_edit_miss_honours_global_deadline() {
assert_fuzzy_miss_is_cancelled(Verb::EditFuzzyMiss);
}
#[test]
fn fuzzy_cascade_reports_cancel_not_a_miss_when_shutdown_is_armed() {
use atomwrite::error::AtomwriteError;
use atomwrite::fuzzy::{MatchOpts, apply_fuzzy_one_pass, match_pair_with};
use std::sync::atomic::Ordering;
let signal = atomwrite::signal::get_or_install_handlers().expect("install handlers");
signal.flag().store(true, Ordering::Release);
let content = "alpha beta gamma\ndelta epsilon zeta\n".repeat(64);
let err = match_pair_with(&content, MISS_TOKEN, "x", MatchOpts::default())
.expect_err("a cancelled cascade is an error");
assert!(
matches!(err, AtomwriteError::Cancelled { .. }),
"match_pair_with must return Cancelled, got {err:?}"
);
let err = apply_fuzzy_one_pass(&content, MISS_TOKEN, "x", MatchOpts::default(), None)
.expect_err("a cancelled one-pass is an error, not zero matches");
assert!(
matches!(err, AtomwriteError::Cancelled { .. }),
"apply_fuzzy_one_pass must pass Cancelled through, got {err:?}"
);
let err = apply_fuzzy_one_pass(&content, MISS_TOKEN, "x", MatchOpts::default(), Some(3))
.expect_err("the multi-hit path must pass Cancelled through too");
assert!(
matches!(err, AtomwriteError::Cancelled { .. }),
"multi-hit apply_fuzzy_one_pass must pass Cancelled through, got {err:?}"
);
}
#[cfg(unix)]
#[test]
fn a_signal_reports_itself_and_not_a_deadline() {
use std::io::Read as _;
use std::process::Stdio;
use std::sync::mpsc;
let dir = tempfile::tempdir().expect("tempdir");
write_fixture_of(dir.path(), SIGNAL_FILE_BYTES);
let ws = dir.path().to_str().expect("utf8 workspace").to_string();
let mut child = common::hermetic(std::process::Command::new(assert_cmd::cargo::cargo_bin(
"atomwrite",
)))
.args([
"--timeout-secs",
"600",
"--no-auto-heal",
"--no-progress",
"--max-filesize",
"200000000",
"--workspace",
&ws,
"search",
TOKEN,
&ws,
])
.stdout(Stdio::piped())
.stderr(Stdio::null())
.spawn()
.expect("spawn search");
let mut stdout = child.stdout.take().expect("stdout pipe");
let (started_tx, started_rx) = mpsc::channel::<()>();
let reader = std::thread::spawn(move || {
let mut buf = Vec::new();
let mut chunk = [0u8; 8192];
let mut notified = false;
loop {
match stdout.read(&mut chunk) {
Ok(0) | Err(_) => break,
Ok(n) => {
buf.extend_from_slice(&chunk[..n]);
if !notified {
notified = true;
let _ = started_tx.send(());
}
}
}
}
buf
});
let observed_output = started_rx
.recv_timeout(Duration::from_secs(SIGNAL_START_TIMEOUT_SECS))
.is_ok();
if !observed_output {
let _ = child.kill();
let _ = child.wait();
panic!("the search produced no output within the start timeout to interrupt");
}
unsafe {
libc::kill(child.id() as i32, libc::SIGTERM);
}
let status = child.wait().expect("wait");
let all = reader.join().expect("reader thread");
let (events, _) = parse_strict(&all);
let cancelled = events.iter().find(|e| e["type"] == "cancelled");
let Some(cancelled) = cancelled else {
panic!(
"a signalled run must report the interruption, status {status:?}, stdout tail: {:?}",
events.last()
);
};
assert_eq!(
cancelled["deadline_exceeded"], false,
"a signal is not an elapsed deadline, envelope was: {cancelled}"
);
assert_eq!(
cancelled["signal"].as_str(),
Some("SIGTERM"),
"the envelope must name the signal that arrived, envelope was: {cancelled}"
);
assert_eq!(
cancelled["reason"].as_str(),
Some("cancelled by signal"),
"the reason must name the signal, not a deadline that never fired, envelope was: {cancelled}"
);
}