1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree.
*/
use std::cell::RefCell;
use std::ffi::OsString;
use std::path::Path;
use std::path::PathBuf;
use std::rc::Rc;
use clap::Parser;
use hermit::Error;
use hermit::HermitData;
use hermit::Id;
use hermit::Shebang;
use reverie::process::ExitStatus;
use super::container::deterministic_container;
use super::gdb_client::CLIENT_EXITED_BEFORE_CONNECTING;
use super::gdb_client::GdbClientWatch;
use super::global_opts::GlobalOpts;
use super::run::NetworkingMode;
/// Command-line options for the "replay" subcommand.
#[derive(Debug, Parser, Clone)]
pub struct ReplayOpts {
/// The recording to replay, by default `last`: the newest one `hermit
/// record` or `hermit run --record-networking` made, as a bare `rr
/// replay` replays the newest rr recording. It may be `last`, `last-run`
/// (the newest of any kind, including the manifest a plain `hermit run`
/// leaves when it draws an input from the host), a recording ID or a unique prefix
/// of at least 6 hex digits, or the path of a manifest.json or of its
/// directory; `hermit list` lists them. A syscall recording (`hermit
/// record`) replays under gdb, or to the end with `--autopilot`. Any
/// other recording replays by rerunning its program from its manifest,
/// as `hermit run --from RECORDING` does, to the end: a network
/// recording offline from its trace, a run that drew an input from the
/// host with the values it drew. To serve such a rerun to gdb, run
/// `hermit run --from RECORDING --gdbserver`.
#[clap(value_name = "RECORDING")]
recording: Option<OsString>,
/// The global options this invocation gave, which a rerun passes on.
#[clap(skip)]
global_argv: Vec<OsString>,
/// Directory where recorded syscall data is stored.
#[clap(long, value_name = "DIR", env = "HERMIT_DATA_DIR")]
data_dir: Option<PathBuf>,
/// The port to use for the gdb server.
#[clap(long, default_value = "1234")]
// FIXME: This shouldn't exist. It'd be better to use a Unix domain socket.
gdbserver_port: u16,
/// Replay to the end without an attached debugger.
#[clap(long, short)]
autopilot: bool,
/// Serve the replay through the gdb remote protocol without launching gdb.
#[clap(long, conflicts_with = "autopilot")]
serve_only: bool,
/// Additional gdb command passed by `-ex`
#[clap(long, value_delimiter = ';', conflicts_with = "serve_only")]
gdbex: Vec<String>,
/// Serve the replay to one Debug Adapter Protocol client, such as an
/// editor, on the TCP address [HOST:]PORT (HOST defaults to 127.0.0.1).
/// This runs `hermit-dap --replay` from beside this executable, which
/// supports stepBack and reverseContinue. See DEBUGGING.md.
#[clap(
long,
value_name = "[HOST:]PORT",
conflicts_with_all = ["autopilot", "serve_only", "gdbex"]
)]
dap_listen: Option<String>,
}
impl ReplayOpts {
pub fn main(&self, global: &GlobalOpts) -> Result<ExitStatus, Error> {
use crate::manifest::Kind;
use crate::manifest::Target;
if self.dap_listen.is_some() {
// Before resolving the recording, so a missing adapter is named
// first, whatever the recording turns out to be.
super::dap_adapter::hermit_dap_beside_hermit()?;
}
let store = HermitData::from(self.data_dir.as_ref());
let spec = self
.recording
.clone()
.unwrap_or_else(|| hermit::LAST.into());
// Every kind of recording has one name and one verb
// (https://github.com/rrnewton/hermit/issues/4271): a syscall
// recording replays here; any other reruns from its manifest.
let (hermit, id) = match crate::manifest::resolve(store.data_dir(), &spec)? {
Target::Recording { data_dir, id } => (HermitData::with_dir(data_dir), id),
Target::Manifest(path) => {
let manifest = crate::manifest::read(&path)?;
if manifest.kind != Kind::FullSyscall {
return Err(self.rerun(global, &path, &manifest, store.data_dir()));
}
// The directory named, as the replayer finds it: its parent
// and its name.
let directory = std::path::absolute(crate::manifest::directory_of(&path))?;
let (data_dir, id) =
crate::manifest::recording_location(&directory, Some(&manifest.id))?;
(HermitData::with_dir(data_dir), id)
}
};
if let Some(address) = &self.dap_listen {
return Err(self.exec_dap_adapter(&hermit, id, address));
}
let metadata = hermit.recording_metadata(id)?;
if !metadata.cpu_profile.is_default() {
// A v3 recording replays only on a qualifying host; qualify before
// the container, as `record start` does. The replayer compares the
// host facts (https://github.com/rrnewton/hermit/issues/4243).
let policy = |refusal: hermit::cpu_profile_admission::CpuProfileRefusal| {
Error::new(super::container::PolicyRefusal).context(refusal.to_string())
};
hermit::cpu_profile_admission::precheck_replay(
metadata.cpu_profile,
metadata.cpu_profile_host.as_ref(),
metadata.cpu_table_digest.as_deref(),
)
.map_err(policy)?;
let gdb_served = !self.autopilot;
let request = hermit::cpu_profile_admission::record_or_replay_request(gdb_served);
hermit::cpu_profile_admission::admit(&request).map_err(|refusal| {
Error::new(super::container::PolicyRefusal)
.context(format!("--cpu-profile x86-64-v3 refused: {refusal}"))
})?;
}
let recorded_local = metadata.local_networking == Some(true);
let network = replay_network(self.autopilot, recorded_local);
if recorded_local && network == NetworkingMode::Host {
// Same override, and same warning, as `hermit run --gdbserver`.
eprintln!(
"hermit replay: WARNING: recording {id} used --network local, but a \
gdb-served replay runs on the host network; syscalls replayed by \
re-execution may see different interfaces and ports"
);
}
if self.autopilot || self.serve_only {
let (mut container, identity) = deterministic_container()?;
network.configure(&mut container, false);
let options = self.clone();
let global = global.clone();
let resources = format!("replay {} and identity mounts", hermit.data_dir().display());
super::owned_container::run(
&mut container,
(identity, hermit),
resources,
true,
"with_container",
None,
move |(_, hermit)| options.container_main(&global, options.autopilot, hermit, id),
)
.map(|(value, _guards)| value)
} else {
// Find the path to the executable so that GDB can use it to resolve
// symbols.
let exe = hermit.data_dir().join(format!("{}/exe", id));
let real_exe = Shebang::new(&exe).map_or(exe, |s| s.interpreter().into());
// Run the gdb client outside of the PID namespace. This cannot be done
// inside of the PID namespace because it would perturb the
// deterministic PID allocation that is needed for the replay.
let mut gdb_command = std::process::Command::new("gdb");
gdb_command
.arg(real_exe)
.arg("-quiet")
.arg("-iex")
// don't prompt (dialog) when breakpoint symbol doesn't exist.
.arg("set breakpoint pending on")
.arg("-ex")
.arg(format!("target remote :{}", self.gdbserver_port));
for ex in &self.gdbex {
gdb_command.arg("-ex").arg(ex);
}
// TODO: For replay, we ought to construct the container from
// `metadata.json`. That logic belongs in `hermit::replay`, but we have
// to initialize logging inside the container because it may spawn a
// thread. If we can guarantee that tracing won't spawn a thread, then
// that restriction be lifted.
// Same unbounded accept as the record path: the client is spawned
// before the container that binds the port, so a client that dies
// early leaves the gdbserver waiting for a peer that cannot arrive.
// ⚠️ CORRECTION TO WHAT LANDED HERE. This comment used to claim this
// file's `wait()` "WAS reached on every path ... so it never leaked a
// zombie", and that only the record path leaked. That is wrong:
// `deterministic_container()?` on the next line precedes the wait, so
// an error building the container skipped it here too. BOTH files
// leaked, and the merged description of hermit#2654 says otherwise.
// Recorded rather than quietly deleted, because the claim is in a
// landed commit message where it cannot be edited.
let gdb_watch = GdbClientWatch::spawn(gdb_command, self.gdbserver_port)?;
let (mut container, identity) = deterministic_container()?;
let guards = Rc::new(RefCell::new((identity, hermit, gdb_watch)));
let resources = format!(
"replay {}, identity mounts and GDB watcher",
guards.borrow().1.data_dir().display()
);
let options = self.clone();
let global = global.clone();
let result = super::owned_container::run(
&mut container,
Rc::clone(&guards),
resources,
true,
"with_container",
None,
move |guards| {
options.container_main(&global, options.autopilot, &guards.borrow().1, id)
},
);
// On unresolved cleanup the factory retains the SAME guard scope.
// Do not signal watcher completion while its container still owns it.
if result.as_ref().is_err_and(|e| {
e.downcast_ref::<super::owned_container::ParentCleanupUnconfirmed>()
.is_some()
}) {
return result.map(|(status, _)| status);
}
let finished = guards.borrow_mut().2.finish();
match (result, finished) {
(Ok((status, _)), Ok(_)) => Ok(status),
(Ok(_), Err(watcher)) => Err(watcher),
(Err(primary), Err(watcher)) => {
Err(primary.context(format!("GDB watcher also failed: {watcher:#}")))
}
(Err(primary), Ok(true)) => Err(primary.context(CLIENT_EXITED_BEFORE_CONNECTING)),
(Err(primary), Ok(false)) => Err(primary),
}
}
}
/// Sets the global options this invocation gave, which a rerun passes on.
pub(crate) fn set_global_argv(&mut self, global_argv: Vec<OsString>) {
self.global_argv = global_argv;
}
/// Replaces this process with `hermit [GLOBAL] run --from MANIFEST`, the
/// replay of a recording that is not a syscall recording, run in the
/// data directory it came from. Returns only the error that kept it
/// from doing so.
fn rerun(
&self,
global: &GlobalOpts,
path: &Path,
manifest: &crate::manifest::Manifest,
data_dir: &Path,
) -> Error {
use std::os::unix::process::CommandExt;
let id = &manifest.id;
let kind = manifest.kind.name();
if let Some(address) = &self.dap_listen {
return anyhow::anyhow!(
"recording {id} is of kind {kind}, which replays by rerunning its program \
rather than under gdb, so it has no reverse execution; serve the rerun to a \
DAP client with: hermit run --from {} --dap-listen={address}",
path.display()
);
}
if self.serve_only || !self.gdbex.is_empty() {
return anyhow::anyhow!(
"recording {id} is of kind {kind}, which replays by rerunning its program \
rather than under gdb; serve the rerun to gdb with: hermit run --from {} \
--gdbserver",
path.display()
);
}
let exe = match std::env::current_exe() {
Ok(exe) => exe,
Err(error) => return Error::from(error).context("Failed to locate hermit"),
};
// A WARN: the program runs again from its saved inputs, which is not a
// replay of its system calls. On stderr even under `--log-file`: the
// rerun opens that log afresh and would erase it.
super::tracing::emit_before_tracing(
global.log,
super::tracing::EarlyLog::Stderr(global.log_budget()),
|| {
::tracing::warn!(
target: super::global_opts::CONTROLLER_TARGET,
"recording {id} ({kind}) replays by rerunning its program: hermit run --from {}",
super::run::home_relative(path)
)
},
);
let error = std::process::Command::new(&exe)
.args(&self.global_argv)
.arg("run")
.arg("--from")
.arg(path)
.env("HERMIT_DATA_DIR", data_dir)
.exec();
Error::from(error).context(format!("Failed to execute {}", exe.display()))
}
/// Replaces this process with `hermit-dap --listen ADDRESS --replay ID`,
/// after telling the user what a DAP client's attach request must name.
/// Returns only the error that kept it from doing so.
fn exec_dap_adapter(&self, hermit: &HermitData, id: Id, address: &str) -> Error {
use std::os::unix::process::CommandExt;
let adapter = match super::dap_adapter::hermit_dap_beside_hermit() {
Ok(adapter) => adapter,
Err(error) => return error,
};
let exe = hermit.data_dir().join(format!("{}/exe", id));
let program = Shebang::new(&exe).map_or(exe, |s| s.interpreter().into());
eprintln!(
"hermit replay: starting hermit-dap to serve recording {id} to one DAP client; {}",
super::dap_adapter::attach_hint(&program, self.gdbserver_port)
);
let error = std::process::Command::new(&adapter)
.arg("--listen")
.arg(address)
.arg("--replay")
.arg(id.to_string())
.arg("--data-dir")
.arg(hermit.data_dir())
.arg("--gdbserver-port")
.arg(self.gdbserver_port.to_string())
.exec();
Error::from(error).context(format!("Failed to execute {}", adapter.display()))
}
fn container_main(
&self,
global: &GlobalOpts,
autopilot: bool,
hermit: &HermitData,
id: Id,
) -> Result<ExitStatus, Error> {
let _guard = global.init_tracing();
if autopilot {
hermit.replay(id)
} else {
hermit.replay_with_gdbserver(id, self.gdbserver_port)
}
}
}
/// The network a replay runs in. Syscalls the replayer does not serve from
/// the recording are executed for real, so an autopilot replay uses the
/// network the recording used; recordings that predate the stored choice ran
/// on the host network. A replay served to gdb uses the host network so the
/// gdb client can reach its gdbserver port, as `hermit run --gdbserver` does.
fn replay_network(autopilot: bool, recorded_local: bool) -> NetworkingMode {
if autopilot && recorded_local {
NetworkingMode::Local
} else {
NetworkingMode::Host
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn only_an_autopilot_replay_of_a_local_recording_is_local() {
assert_eq!(replay_network(true, true), NetworkingMode::Local);
assert_eq!(replay_network(true, false), NetworkingMode::Host);
assert_eq!(replay_network(false, true), NetworkingMode::Host);
assert_eq!(replay_network(false, false), NetworkingMode::Host);
}
}