1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree.
*/
use std::cell::RefCell;
use std::path::PathBuf;
use std::rc::Rc;
use clap::Parser;
use hermit::Context;
use hermit::Error;
use hermit::HermitData;
use hermit::Id;
use hermit::Shebang;
use reverie::process::ExitStatus;
use super::container::deterministic_container;
use super::gdb_client::CLIENT_EXITED_BEFORE_CONNECTING;
use super::gdb_client::GdbClientWatch;
use super::global_opts::GlobalOpts;
use super::run::NetworkingMode;
/// Command-line options for the "replay" subcommand.
#[derive(Debug, Parser, Clone)]
pub struct ReplayOpts {
/// The ID of the recording to replay. This is obtained by first running
/// `hermit record`. This is optional. Defaults to the last successful
/// recording.
#[clap(value_name = "ID")]
id: Option<Id>,
/// Directory where recorded syscall data is stored.
#[clap(long, value_name = "DIR", env = "HERMIT_DATA_DIR")]
data_dir: Option<PathBuf>,
/// The port to use for the gdb server.
#[clap(long, default_value = "1234")]
// FIXME: This shouldn't exist. It'd be better to use a Unix domain socket.
gdbserver_port: u16,
/// Replay to the end without an attached debugger.
#[clap(long, short)]
autopilot: bool,
/// Serve the replay through the gdb remote protocol without launching gdb.
#[clap(long, conflicts_with = "autopilot")]
serve_only: bool,
/// Additional gdb command passed by `-ex`
#[clap(long, value_delimiter = ';', conflicts_with = "serve_only")]
gdbex: Vec<String>,
}
impl ReplayOpts {
pub fn main(&self, global: &GlobalOpts) -> Result<ExitStatus, Error> {
let hermit = HermitData::from(self.data_dir.as_ref());
let id = match self.id {
Some(id) => id,
None => hermit
.last_id()
.context("Failed to find last recording ID")?,
};
let recorded_local = hermit.recording_metadata(id)?.local_networking == Some(true);
let network = replay_network(self.autopilot, recorded_local);
if recorded_local && network == NetworkingMode::Host {
// Same override, and same warning, as `hermit run --gdbserver`.
eprintln!(
"hermit replay: WARNING: recording {id} used --network local, but a \
gdb-served replay runs on the host network; syscalls replayed by \
re-execution may see different interfaces and ports"
);
}
if self.autopilot || self.serve_only {
let (mut container, identity) = deterministic_container()?;
network.configure(&mut container, false);
let options = self.clone();
let global = global.clone();
let resources = format!("replay {} and identity mounts", hermit.data_dir().display());
super::owned_container::run(
&mut container,
(identity, hermit),
resources,
true,
"with_container",
None,
move |(_, hermit)| options.container_main(&global, options.autopilot, hermit, id),
)
.map(|(value, _guards)| value)
} else {
// Find the path to the executable so that GDB can use it to resolve
// symbols.
let exe = hermit.data_dir().join(format!("{}/exe", id));
let real_exe = Shebang::new(&exe).map_or(exe, |s| s.interpreter().into());
// Run the gdb client outside of the PID namespace. This cannot be done
// inside of the PID namespace because it would perturb the
// deterministic PID allocation that is needed for the replay.
let mut gdb_command = std::process::Command::new("gdb");
gdb_command
.arg(real_exe)
.arg("-quiet")
.arg("-iex")
// don't prompt (dialog) when breakpoint symbol doesn't exist.
.arg("set breakpoint pending on")
.arg("-ex")
.arg(format!("target remote :{}", self.gdbserver_port));
for ex in &self.gdbex {
gdb_command.arg("-ex").arg(ex);
}
// TODO: For replay, we ought to construct the container from
// `metadata.json`. That logic belongs in `hermit::replay`, but we have
// to initialize logging inside the container because it may spawn a
// thread. If we can guarantee that tracing won't spawn a thread, then
// that restriction be lifted.
// Same unbounded accept as the record path: the client is spawned
// before the container that binds the port, so a client that dies
// early leaves the gdbserver waiting for a peer that cannot arrive.
// ⚠️ CORRECTION TO WHAT LANDED HERE. This comment used to claim this
// file's `wait()` "WAS reached on every path ... so it never leaked a
// zombie", and that only the record path leaked. That is wrong:
// `deterministic_container()?` on the next line precedes the wait, so
// an error building the container skipped it here too. BOTH files
// leaked, and the merged description of hermit#2654 says otherwise.
// Recorded rather than quietly deleted, because the claim is in a
// landed commit message where it cannot be edited.
let gdb_watch = GdbClientWatch::spawn(gdb_command, self.gdbserver_port)?;
let (mut container, identity) = deterministic_container()?;
let guards = Rc::new(RefCell::new((identity, hermit, gdb_watch)));
let resources = format!(
"replay {}, identity mounts and GDB watcher",
guards.borrow().1.data_dir().display()
);
let options = self.clone();
let global = global.clone();
let result = super::owned_container::run(
&mut container,
Rc::clone(&guards),
resources,
true,
"with_container",
None,
move |guards| {
options.container_main(&global, options.autopilot, &guards.borrow().1, id)
},
);
// On unresolved cleanup the factory retains the SAME guard scope.
// Do not signal watcher completion while its container still owns it.
if result.as_ref().is_err_and(|e| {
e.downcast_ref::<super::owned_container::ParentCleanupUnconfirmed>()
.is_some()
}) {
return result.map(|(status, _)| status);
}
let finished = guards.borrow_mut().2.finish();
match (result, finished) {
(Ok((status, _)), Ok(_)) => Ok(status),
(Ok(_), Err(watcher)) => Err(watcher),
(Err(primary), Err(watcher)) => {
Err(primary.context(format!("GDB watcher also failed: {watcher:#}")))
}
(Err(primary), Ok(true)) => Err(primary.context(CLIENT_EXITED_BEFORE_CONNECTING)),
(Err(primary), Ok(false)) => Err(primary),
}
}
}
fn container_main(
&self,
global: &GlobalOpts,
autopilot: bool,
hermit: &HermitData,
id: Id,
) -> Result<ExitStatus, Error> {
let _guard = global.init_tracing();
if autopilot {
hermit.replay(id)
} else {
hermit.replay_with_gdbserver(id, self.gdbserver_port)
}
}
}
/// The network a replay runs in. Syscalls the replayer does not serve from
/// the recording are executed for real, so an autopilot replay uses the
/// network the recording used; recordings that predate the stored choice ran
/// on the host network. A replay served to gdb uses the host network so the
/// gdb client can reach its gdbserver port, as `hermit run --gdbserver` does.
fn replay_network(autopilot: bool, recorded_local: bool) -> NetworkingMode {
if autopilot && recorded_local {
NetworkingMode::Local
} else {
NetworkingMode::Host
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn only_an_autopilot_replay_of_a_local_recording_is_local() {
assert_eq!(replay_network(true, true), NetworkingMode::Local);
assert_eq!(replay_network(true, false), NetworkingMode::Host);
assert_eq!(replay_network(false, true), NetworkingMode::Host);
assert_eq!(replay_network(false, false), NetworkingMode::Host);
}
}