use std::ffi::OsString;
use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener};
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};
use tokio::io::{AsyncBufReadExt, BufReader};
use tokio::process::{Child, Command};
use tokio::task::JoinHandle;
use tracing::{debug, warn};
use super::browser::BrowserKind;
use super::error::ManagerError;
use super::status::{
DriverId, DriverLogLine, DriverLogSubscription, DriverStream, Emitter, LogSubscribers, Status,
};
#[derive(Debug, Clone, Copy, Default)]
pub enum StdioMode {
#[default]
Tracing,
Inherit,
Null,
}
impl StdioMode {
fn to_stdio(self) -> Stdio {
match self {
StdioMode::Tracing => Stdio::piped(),
StdioMode::Inherit => Stdio::inherit(),
StdioMode::Null => Stdio::null(),
}
}
}
pub(crate) struct SpawnConfig {
pub host: IpAddr,
pub ready_timeout: Duration,
pub shutdown_timeout: Duration,
pub stdio: StdioMode,
pub args: Vec<OsString>,
#[cfg_attr(not(windows), allow(dead_code))]
pub show_console_window: bool,
}
impl Default for SpawnConfig {
fn default() -> Self {
Self {
host: IpAddr::V4(Ipv4Addr::LOCALHOST),
ready_timeout: Duration::from_secs(30),
shutdown_timeout: Duration::from_secs(10),
stdio: StdioMode::default(),
args: Vec::new(),
show_console_window: false,
}
}
}
pub(crate) struct ManagedDriverProcess {
pub host: IpAddr,
pub port: u16,
pub browser: BrowserKind,
pub version: String,
pub driver_id: DriverId,
pub log_subscribers: LogSubscribers,
child: Option<Child>,
shutdown_timeout: Duration,
shutdown: Arc<AtomicBool>,
pump_handles: Vec<JoinHandle<()>>,
emitter: Emitter,
}
impl std::fmt::Debug for ManagedDriverProcess {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ManagedDriverProcess")
.field("host", &self.host)
.field("port", &self.port)
.field("browser", &self.browser)
.finish()
}
}
pub(crate) struct SpawnContext<'a> {
pub driver_id: DriverId,
pub version: &'a str,
pub emitter: &'a Emitter,
pub manager_log_subscribers: LogSubscribers,
}
impl ManagedDriverProcess {
pub(crate) async fn spawn(
binary: &Path,
browser: BrowserKind,
cfg: &SpawnConfig,
ctx: SpawnContext<'_>,
) -> Result<Self, ManagerError> {
const MAX_PORT_ATTEMPTS: u8 = 3;
let mut last_err: Option<ManagerError> = None;
for attempt in 0..MAX_PORT_ATTEMPTS {
let port = pick_port(cfg.host)?;
match spawn_at_port(binary, browser, cfg, port, &ctx).await {
Ok(p) => return Ok(p),
Err(e) if is_port_in_use(&e) => {
debug!("driver port {port} already in use (attempt {attempt}): {e}");
last_err = Some(e);
}
Err(e) => return Err(e),
}
}
Err(last_err.unwrap_or_else(|| ManagerError::Spawn("port allocation exhausted".into())))
}
pub(crate) fn url(&self) -> String {
format!("http://{}:{}", self.host, self.port)
}
pub(crate) fn subscribe_log<F>(&self, f: F) -> DriverLogSubscription
where
F: Fn(&DriverLogLine) + Send + Sync + 'static,
{
self.log_subscribers.add(f)
}
#[cfg(test)]
pub(crate) async fn shutdown(self) -> std::io::Result<()> {
let budget = self.shutdown_timeout;
self.shutdown_with_budget(budget).await
}
pub(crate) async fn shutdown_with_budget(mut self, budget: Duration) -> std::io::Result<()> {
let budget = budget.min(self.shutdown_timeout);
let url = format!("{}/shutdown", self.url());
if let Some(child) = self.child.as_mut() {
if matches!(self.browser, BrowserKind::Chrome | BrowserKind::Edge) && !budget.is_zero()
{
let _ = tokio::time::timeout(budget, async {
if let Ok(client) = reqwest::Client::builder()
.no_proxy()
.redirect(reqwest::redirect::Policy::none())
.build()
{
let _ = client.get(url).send().await;
let _ = child.wait().await;
}
})
.await;
}
if child.try_wait()?.is_none() {
child.kill().await?;
}
}
self.child.take();
Ok(())
}
}
fn is_port_in_use(err: &ManagerError) -> bool {
let msg = match err {
ManagerError::Spawn(s) => s.as_str(),
_ => return false,
};
let lower = msg.to_ascii_lowercase();
lower.contains("address already in use")
|| lower.contains("only one usage of each socket address")
|| lower.contains("addrinuse")
}
async fn spawn_at_port(
binary: &Path,
browser: BrowserKind,
cfg: &SpawnConfig,
port: u16,
ctx: &SpawnContext<'_>,
) -> Result<ManagedDriverProcess, ManagerError> {
let mut cmd = Command::new(binary);
cmd.args(&cfg.args);
cmd.arg(format!("--port={port}"));
cmd.stdout(cfg.stdio.to_stdio());
cmd.stderr(cfg.stdio.to_stdio());
cmd.stdin(Stdio::null());
cmd.kill_on_drop(true);
#[cfg(windows)]
if !cfg.show_console_window {
const CREATE_NO_WINDOW: u32 = 0x0800_0000;
cmd.creation_flags(CREATE_NO_WINDOW);
}
if matches!(browser, BrowserKind::Chrome | BrowserKind::Edge)
&& cfg.host != IpAddr::V4(Ipv4Addr::LOCALHOST)
{
cmd.arg(format!("--allowed-ips={}", cfg.host));
}
if matches!(browser, BrowserKind::Firefox) {
cmd.arg("--websocket-port=0");
}
let mut child = cmd
.spawn()
.map_err(|e| ManagerError::Spawn(format!("spawn {}: {}", binary.display(), e)))?;
let pid = child.id().unwrap_or(0);
ctx.emitter.emit(Status::DriverProcessSpawned {
browser,
version: ctx.version.to_string(),
pid,
port,
binary: PathBuf::from(binary),
});
let shutdown = Arc::new(AtomicBool::new(false));
let log_subscribers = LogSubscribers::new();
let mut pump_handles = Vec::new();
if matches!(cfg.stdio, StdioMode::Tracing) {
let line_ctx = LogLineContext {
driver_id: ctx.driver_id,
browser,
version: ctx.version.to_string(),
port,
};
if let Some(stdout) = child.stdout.take() {
pump_handles.push(spawn_pump(
DriverStream::Stdout,
stdout,
Arc::clone(&shutdown),
line_ctx.clone(),
log_subscribers.clone(),
ctx.manager_log_subscribers.clone(),
));
}
if let Some(stderr) = child.stderr.take() {
pump_handles.push(spawn_pump(
DriverStream::Stderr,
stderr,
Arc::clone(&shutdown),
line_ctx,
log_subscribers.clone(),
ctx.manager_log_subscribers.clone(),
));
}
}
let ready_started = Instant::now();
if let Err(e) = wait_until_ready(cfg.host, port, cfg.ready_timeout).await {
let _ = child.kill().await;
for h in &pump_handles {
h.abort();
}
return Err(e);
}
let url = format!("http://{}:{}", cfg.host, port);
ctx.emitter.emit(Status::DriverReady {
browser,
version: ctx.version.to_string(),
url,
elapsed: ready_started.elapsed(),
});
Ok(ManagedDriverProcess {
host: cfg.host,
port,
browser,
version: ctx.version.to_string(),
driver_id: ctx.driver_id,
log_subscribers,
child: Some(child),
shutdown_timeout: cfg.shutdown_timeout,
shutdown,
pump_handles,
emitter: ctx.emitter.clone(),
})
}
fn pick_port(host: IpAddr) -> Result<u16, ManagerError> {
let listener = TcpListener::bind(SocketAddr::new(host, 0))
.map_err(|e| ManagerError::Spawn(format!("bind ephemeral port: {e}")))?;
let port =
listener.local_addr().map_err(|e| ManagerError::Spawn(format!("local_addr: {e}")))?.port();
drop(listener);
Ok(port)
}
#[derive(Clone)]
struct LogLineContext {
driver_id: DriverId,
browser: BrowserKind,
version: String,
port: u16,
}
fn spawn_pump<R>(
stream: DriverStream,
reader: R,
shutdown: Arc<AtomicBool>,
ctx: LogLineContext,
process_subs: LogSubscribers,
manager_subs: LogSubscribers,
) -> JoinHandle<()>
where
R: tokio::io::AsyncRead + Unpin + Send + 'static,
{
let stream_label = match stream {
DriverStream::Stdout => "stdout",
DriverStream::Stderr => "stderr",
};
tokio::spawn(async move {
let mut lines = BufReader::new(reader).lines();
loop {
if shutdown.load(Ordering::Relaxed) {
break;
}
match lines.next_line().await {
Ok(Some(line)) => {
debug!(target: "thirtyfour::manager::driver", stream = stream_label, line = %line);
let log = DriverLogLine {
driver_id: ctx.driver_id,
browser: ctx.browser,
version: ctx.version.clone(),
port: ctx.port,
stream,
line,
};
process_subs.dispatch(&log);
manager_subs.dispatch(&log);
}
Ok(None) => break,
Err(e) => {
warn!(target: "thirtyfour::manager::driver", stream = stream_label, error = %e);
break;
}
}
}
})
}
async fn wait_until_ready(host: IpAddr, port: u16, timeout: Duration) -> Result<(), ManagerError> {
let url = format!("http://{host}:{port}/status");
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(2))
.build()
.map_err(|e| ManagerError::Spawn(e.to_string()))?;
let deadline = Instant::now() + timeout;
while Instant::now() < deadline {
if let Ok(resp) = client.get(&url).send().await
&& resp.status().is_success()
&& let Ok(body) = resp.json::<serde_json::Value>().await
&& body
.get("value")
.and_then(|v| v.get("ready"))
.and_then(|v| v.as_bool())
.unwrap_or(false)
{
return Ok(());
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
Err(ManagerError::DriverNotReady(timeout))
}
impl Drop for ManagedDriverProcess {
fn drop(&mut self) {
self.shutdown.store(true, Ordering::Relaxed);
if let Some(mut child) = self.child.take()
&& let Err(e) = child.start_kill()
{
warn!(target: "thirtyfour::manager", error = %e, "failed to kill driver");
}
for h in self.pump_handles.drain(..) {
h.abort();
}
self.emitter.emit(Status::DriverShutdown {
browser: self.browser,
version: self.version.clone(),
port: self.port,
});
}
}
#[cfg(all(test, unix))]
mod shutdown_tests {
use super::super::status::Subscription;
use super::*;
use std::sync::atomic::AtomicUsize;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::process::{ChildStdin, ChildStdout};
struct Fixture {
process: ManagedDriverProcess,
stdin: ChildStdin,
stdout: ChildStdout,
events: Arc<AtomicUsize>,
_subscription: Subscription,
}
fn fixture(port: u16, timeout: Duration) -> Fixture {
let mut child = Command::new("sh")
.args(["-c", "read line; printf cleaned"])
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::null())
.kill_on_drop(true)
.spawn()
.unwrap();
let stdin = child.stdin.take().unwrap();
let stdout = child.stdout.take().unwrap();
let emitter = Emitter::new();
let events = Arc::new(AtomicUsize::new(0));
let counter = Arc::clone(&events);
let subscription = emitter.add(move |event| {
if matches!(event, Status::DriverShutdown { .. }) {
counter.fetch_add(1, Ordering::SeqCst);
}
});
Fixture {
process: ManagedDriverProcess {
host: IpAddr::V4(Ipv4Addr::LOCALHOST),
port,
browser: BrowserKind::Chrome,
version: "fixture".into(),
driver_id: DriverId::from_raw(0),
log_subscribers: LogSubscribers::new(),
child: Some(child),
shutdown_timeout: timeout,
shutdown: Arc::new(AtomicBool::new(false)),
pump_handles: Vec::new(),
emitter,
},
stdin,
stdout,
events,
_subscription: subscription,
}
}
async fn read_output(mut stdout: ChildStdout) -> String {
let mut output = String::new();
tokio::time::timeout(Duration::from_secs(3), stdout.read_to_string(&mut output))
.await
.expect("child must exit and close stdout")
.unwrap();
output
}
#[tokio::test]
async fn shutdown_waits_for_cleanup_after_response_or_eof() {
for respond in [true, false] {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let Fixture {
process,
mut stdin,
stdout,
events,
_subscription,
} = fixture(listener.local_addr().unwrap().port(), Duration::from_secs(3));
let shutdown_flag = Arc::clone(&process.shutdown);
let server = tokio::spawn(async move {
let (stream, _) = listener.accept().await.unwrap();
let mut reader = BufReader::new(stream);
let mut request = String::new();
reader.read_line(&mut request).await.unwrap();
assert_eq!(request, "GET /shutdown HTTP/1.1\r\n");
let mut stream = reader.into_inner();
if respond {
stream
.write_all(
b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
)
.await
.unwrap();
}
drop(stream);
tokio::time::sleep(Duration::from_millis(30)).await;
assert!(!shutdown_flag.load(Ordering::Relaxed));
stdin.write_all(b"exit\n").await.unwrap();
});
process.shutdown().await.unwrap();
server.await.unwrap();
assert_eq!(read_output(stdout).await, "cleaned");
assert_eq!(events.load(Ordering::SeqCst), 1);
}
}
#[tokio::test]
async fn shutdown_bounds_hanging_request_or_child_and_reaps() {
for respond in [false, true] {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let Fixture {
process,
stdin,
stdout,
events,
_subscription,
} = fixture(listener.local_addr().unwrap().port(), Duration::from_millis(100));
let server = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.unwrap();
if respond {
stream
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
.await
.unwrap();
}
std::future::pending::<()>().await;
});
tokio::time::timeout(Duration::from_secs(3), process.shutdown())
.await
.expect("shutdown must be bounded")
.unwrap();
assert_eq!(read_output(stdout).await, "");
assert_eq!(events.load(Ordering::SeqCst), 1);
drop(stdin);
server.abort();
}
}
#[tokio::test]
async fn cancelled_shutdown_kills_child_and_emits_once() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let Fixture {
process,
stdin,
stdout,
events,
_subscription,
} = fixture(listener.local_addr().unwrap().port(), Duration::from_secs(30));
let (accepted_tx, accepted_rx) = tokio::sync::oneshot::channel();
let server = tokio::spawn(async move {
let (_stream, _) = listener.accept().await.unwrap();
accepted_tx.send(()).unwrap();
std::future::pending::<()>().await;
});
let task = tokio::spawn(process.shutdown());
tokio::time::timeout(Duration::from_secs(3), accepted_rx).await.unwrap().unwrap();
task.abort();
assert!(task.await.unwrap_err().is_cancelled());
assert_eq!(read_output(stdout).await, "");
assert_eq!(events.load(Ordering::SeqCst), 1);
drop(stdin);
server.abort();
}
}