#[cfg(any(unix, windows))]
mod unix;
#[cfg(all(unix, any(feature = "unixexec", feature = "ibus", target_os = "macos")))]
mod pipe;
mod reader;
pub(crate) use reader::Reader;
mod tcp;
#[cfg(any(test, all(windows, feature = "default-rt")))]
mod connect;
#[cfg(any(test, all(windows, feature = "default-rt")))]
pub(crate) use connect::connect_blocking;
#[cfg(all(test, any(unix, windows)))]
pub(crate) mod tests;
#[cfg(unix)]
use std::os::fd::{AsFd, BorrowedFd};
use std::{
fmt,
future::{Future, poll_fn},
io::{self, IoSlice},
net::Shutdown,
slice,
sync::Arc,
task::{Context, Poll},
};
#[cfg(not(target_os = "wasi"))]
use socket2::SockRef;
#[cfg(all(unix, any(feature = "unixexec", feature = "ibus", target_os = "macos")))]
pub(crate) use pipe::PipeOps;
pub(crate) use tcp::TcpOps;
#[cfg(any(unix, windows))]
pub(crate) use unix::UnixOps;
#[cfg(all(unix, feature = "bus-impl"))]
pub(crate) use unix::peer_credentials;
#[cfg(all(unix, any(feature = "default-rt", feature = "tokio")))]
pub(crate) use unix::unix_socket_address;
use super::{Interest, IoSource, Runtime, erased::ErasedRegistration, traits};
use crate::{
Message,
conn::AuthMechanism,
connection::socket::{RecvmsgResult, Socket, Split, WriteHalf, Writes},
fdo::ConnectionCredentials,
log::trace,
};
#[derive(Debug)]
pub(crate) struct RegisteredIo<O> {
registration: Registration,
source: IoSource,
runtime: Runtime,
ops: O,
}
impl<O> RegisteredIo<O>
where
O: SocketOps,
{
pub(crate) fn new(runtime: &Runtime, source: IoSource, ops: O) -> io::Result<Self> {
Ok(Self {
registration: runtime.register_io_source(source.clone())?,
source,
runtime: runtime.clone(),
ops,
})
}
#[cfg(all(target_os = "linux", feature = "handover"))]
pub(crate) fn source(&self) -> &IoSource {
&self.source
}
pub(crate) async fn recv(&self, buffer: &mut [u8]) -> RecvmsgResult {
self.read_with(|| self.ops.recv(&self.source, buffer)).await
}
pub(crate) async fn read_with<T>(
&self,
operation: impl FnMut() -> io::Result<T>,
) -> io::Result<T> {
self.io(Interest::Readable, operation).await
}
pub(crate) async fn write_with<T>(
&self,
operation: impl FnMut() -> io::Result<T>,
) -> io::Result<T> {
self.io(Interest::Writable, operation).await
}
async fn io<T>(
&self,
interest: Interest,
mut operation: impl FnMut() -> io::Result<T>,
) -> io::Result<T> {
poll_fn(|cx| self.poll_io(cx, interest, &mut operation)).await
}
fn poll_io<T>(
&self,
cx: &mut Context<'_>,
interest: Interest,
mut operation: impl FnMut() -> io::Result<T>,
) -> Poll<io::Result<T>> {
self.registration.poll_io(cx, interest, || {
loop {
match operation() {
Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
result => return result,
}
}
})
}
}
pub(crate) trait PollSend: fmt::Debug + Send + Sync {
fn poll_send(
&self,
cx: &mut Context<'_>,
buffers: &[IoSlice<'_>],
#[cfg(unix)] fds: &[BorrowedFd<'_>],
) -> Poll<io::Result<usize>>;
fn try_send(
&self,
buffers: &[IoSlice<'_>],
#[cfg(unix)] fds: &[BorrowedFd<'_>],
) -> io::Result<usize>;
}
impl<O> PollSend for RegisteredIo<O>
where
O: SocketOps,
{
fn poll_send(
&self,
cx: &mut Context<'_>,
buffers: &[IoSlice<'_>],
#[cfg(unix)] fds: &[BorrowedFd<'_>],
) -> Poll<io::Result<usize>> {
self.poll_io(cx, Interest::Writable, || {
self.ops.send(
&self.source,
buffers,
#[cfg(unix)]
fds,
)
})
}
fn try_send(
&self,
buffers: &[IoSlice<'_>],
#[cfg(unix)] fds: &[BorrowedFd<'_>],
) -> io::Result<usize> {
loop {
match self.ops.send(
&self.source,
buffers,
#[cfg(unix)]
fds,
) {
Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
sent => return sent,
}
}
}
}
pub(crate) trait Progress {
fn wrote(&mut self, done: usize, cut: Option<(&Message, usize)>);
fn stops(&mut self, cx: &mut Context<'_>) -> bool;
}
impl Progress for () {
fn wrote(&mut self, _done: usize, _cut: Option<(&Message, usize)>) {}
fn stops(&mut self, _cx: &mut Context<'_>) -> bool {
false
}
}
pub(crate) async fn write_messages<'m, S, P>(
socket: &S,
mut msgs: &'m [Message],
progress: &mut P,
) -> io::Result<&'m [Message]>
where
S: PollSend + ?Sized,
P: Progress,
{
let mut done = 0;
while !msgs.is_empty() {
let mut bytes = msgs[0].data().len();
let len = if carries_fds(&msgs[0]) {
1
} else {
1 + msgs[1..]
.iter()
.take(MESSAGES_PER_WRITE - 1)
.take_while(|msg| {
let joins = bytes < BYTES_PER_WRITE && !carries_fds(msg);
bytes += msg.data().len();
joins
})
.count()
};
let (write, rest) = msgs.split_at(len);
let mut buffers = [IoSlice::new(&[]); MESSAGES_PER_WRITE];
for (buffer, msg) in buffers.iter_mut().zip(write) {
trace!("Sending message: {:?}", msg);
*buffer = IoSlice::new(msg.data());
}
let mut buffers = &mut buffers[..len];
#[cfg(unix)]
let fds: Vec<_> = write[0].data().fds().iter().map(|fd| fd.as_fd()).collect();
#[cfg(unix)]
let mut fds = &fds[..];
while !buffers.is_empty() {
let polled = poll_fn(|cx| {
poll_write(
socket,
cx,
buffers,
#[cfg(unix)]
fds,
progress,
)
});
let written = match polled.await {
Ok(written) => written,
Err(e) if e.kind() == io::ErrorKind::WouldBlock => {
return Ok(unwritten(msgs, len - buffers.len(), buffers[0].len()));
}
Err(e) => return Err(e),
};
#[cfg(unix)]
{
fds = &[];
}
IoSlice::advance_slices(&mut buffers, written);
let whole = len - buffers.len();
let ended_in = buffers.first().and_then(|left| {
let msg = &write[whole];
let written = msg.data().len() - left.len();
(written > 0).then_some((msg, written))
});
progress.wrote(done + whole, ended_in);
}
for msg in write {
trace!(
"Sent message with serial: {}",
msg.primary_header().serial_num()
);
}
done += len;
msgs = rest;
}
Ok(&[])
}
pub(crate) async fn write_message<'m, S, P>(
socket: &S,
msg: &'m Message,
progress: &mut P,
) -> io::Result<&'m [Message]>
where
S: PollSend + ?Sized,
P: Progress,
{
trace!("Sending message: {:?}", msg);
let data = msg.data();
#[cfg(unix)]
let fds: Vec<_> = data.fds().iter().map(|fd| fd.as_fd()).collect();
#[cfg(unix)]
let mut fds = &fds[..];
let mut written = 0;
while written < data.len() {
let buffers = [IoSlice::new(&data[written..])];
let polled = poll_fn(|cx| {
poll_write(
socket,
cx,
&buffers,
#[cfg(unix)]
fds,
progress,
)
});
let sent = match polled.await {
Ok(sent) => sent,
Err(e) if e.kind() == io::ErrorKind::WouldBlock => {
return Ok(if written > 0 {
&[]
} else {
slice::from_ref(msg)
});
}
Err(e) => return Err(e),
};
#[cfg(unix)]
{
fds = &[];
}
written += sent;
let cut = (written < data.len()).then_some((msg, written));
progress.wrote(usize::from(cut.is_none()), cut);
}
trace!(
"Sent message with serial: {}",
msg.primary_header().serial_num()
);
Ok(&[])
}
pub(crate) fn poll_write<S, P>(
socket: &S,
cx: &mut Context<'_>,
buffers: &[IoSlice<'_>],
#[cfg(unix)] fds: &[BorrowedFd<'_>],
progress: &mut P,
) -> Poll<io::Result<usize>>
where
S: PollSend + ?Sized,
P: Progress,
{
match socket.poll_send(
cx,
buffers,
#[cfg(unix)]
fds,
) {
Poll::Pending if progress.stops(cx) => Poll::Ready(socket.try_send(
buffers,
#[cfg(unix)]
fds,
)),
polled => polled,
}
}
pub(crate) fn registered<S, O>(runtime: &Runtime, stream: S, ops: O) -> io::Result<RegisteredIo<O>>
where
S: Into<Owned>,
O: SocketOps,
{
let source = IoSource::from(stream.into());
#[cfg(not(target_os = "wasi"))]
SockRef::from(&source).set_nonblocking(true)?;
#[cfg(target_os = "wasi")]
rustix::io::ioctl_fionbio(&source, true)?;
RegisteredIo::new(runtime, source, ops)
}
#[cfg(any(unix, target_os = "wasi"))]
type Owned = std::os::fd::OwnedFd;
#[cfg(windows)]
type Owned = std::os::windows::io::OwnedSocket;
pub(crate) trait SocketOps: fmt::Debug + Send + Sync + 'static {
fn recv(&self, source: &IoSource, buffer: &mut [u8]) -> RecvmsgResult;
fn send(
&self,
source: &IoSource,
buffers: &[IoSlice<'_>],
#[cfg(unix)] fds: &[BorrowedFd<'_>],
) -> io::Result<usize>;
fn can_pass_unix_fd(&self) -> bool {
false
}
fn auth_mechanism(&self) -> AuthMechanism {
AuthMechanism::External
}
fn peer_credentials(
&self,
source: &IoSource,
runtime: &Runtime,
) -> impl Future<Output = io::Result<ConnectionCredentials>> + Send {
let _ = (source, runtime);
std::future::ready(Ok(ConnectionCredentials::default()))
}
#[cfg(any(target_os = "freebsd", target_os = "dragonfly"))]
fn sends_credentials_byte(&self) -> bool {
false
}
fn shutdown(&self, source: &IoSource) -> io::Result<()> {
shutdown(source)
}
}
#[derive(Debug)]
pub(crate) enum Registration {
#[cfg(feature = "default-rt")]
ZRuntime(super::zruntime::Registration),
#[cfg(feature = "tokio")]
Tokio(super::tokio_rt::Registration),
External(Box<dyn ErasedRegistration>),
}
impl Registration {
pub(crate) fn poll_io<T>(
&self,
cx: &mut Context<'_>,
interest: Interest,
operation: impl FnMut() -> io::Result<T>,
) -> Poll<io::Result<T>> {
match self {
#[cfg(feature = "default-rt")]
Self::ZRuntime(registration) => {
traits::PollIo::poll_io(registration, cx, interest, operation)
}
#[cfg(feature = "tokio")]
Self::Tokio(registration) => {
traits::PollIo::poll_io(registration, cx, interest, operation)
}
Self::External(registration) => {
traits::PollIo::poll_io(registration, cx, interest, operation)
}
}
}
}
#[async_trait::async_trait]
impl<O> WriteHalf for Arc<RegisteredIo<O>>
where
O: SocketOps,
{
async fn send_messages(&mut self, msgs: &[Message]) -> crate::Result<()> {
write_messages(&**self, msgs, &mut ()).await?;
Ok(())
}
async fn sendmsg(
&mut self,
buffer: &[u8],
#[cfg(unix)] fds: &[BorrowedFd<'_>],
) -> io::Result<usize> {
let socket = &**self;
socket
.write_with(|| {
socket.ops.send(
&socket.source,
&[IoSlice::new(buffer)],
#[cfg(unix)]
fds,
)
})
.await
}
#[cfg(any(target_os = "freebsd", target_os = "dragonfly"))]
async fn send_zero_byte(&mut self) -> io::Result<Option<usize>> {
if !self.ops.sends_credentials_byte() {
return Ok(None);
}
let socket = &**self;
socket
.write_with(|| {
unix::send_credentials_byte(std::os::fd::AsRawFd::as_raw_fd(&socket.source))
})
.await
.map(Some)
}
async fn close(&mut self) -> io::Result<()> {
self.ops.shutdown(&self.source)
}
fn can_pass_unix_fd(&self) -> bool {
self.ops.can_pass_unix_fd()
}
async fn peer_credentials(&mut self) -> io::Result<ConnectionCredentials> {
self.ops.peer_credentials(&self.source, &self.runtime).await
}
fn can_resume_writes(&self) -> bool {
true
}
}
fn unwritten(msgs: &[Message], next: usize, left: usize) -> &[Message] {
let started = left < msgs[next].data().len();
&msgs[next + usize::from(started)..]
}
#[cfg(unix)]
fn carries_fds(msg: &Message) -> bool {
!msg.data().fds().is_empty()
}
#[cfg(not(unix))]
fn carries_fds(_msg: &Message) -> bool {
false
}
pub(crate) const MESSAGES_PER_WRITE: usize = 16;
const BYTES_PER_WRITE: usize = 16 * 1024;
impl<O> Socket for RegisteredIo<O>
where
O: SocketOps,
{
type ReadHalf = Reader<O>;
type WriteHalf = Arc<Self>;
fn split(self) -> Split<Reader<O>, Arc<Self>> {
let socket = Arc::new(self);
Split::with_writes(
Reader::new(socket.clone()),
socket.clone(),
Some(Writes::Driven(socket)),
)
}
}
#[cfg(not(target_os = "wasi"))]
fn shutdown(source: &IoSource) -> io::Result<()> {
SockRef::from(source).shutdown(Shutdown::Both)
}
#[cfg(target_os = "wasi")]
fn shutdown(source: &IoSource) -> io::Result<()> {
use std::{
mem::ManuallyDrop,
net::TcpStream,
os::fd::{AsRawFd, FromRawFd},
};
let stream = ManuallyDrop::new(unsafe { TcpStream::from_raw_fd(source.as_raw_fd()) });
stream.shutdown(Shutdown::Both)
}
fn recv(source: &IoSource, buffer: &mut [u8]) -> RecvmsgResult {
#[cfg(unix)]
{
let (read, _) = rustix::net::recv(source, buffer, rustix::net::RecvFlags::empty())?;
Ok((read, vec![]))
}
#[cfg(windows)]
{
let socket = SockRef::from(source);
std::io::Read::read(&mut &*socket, buffer)
}
#[cfg(target_os = "wasi")]
{
Ok(rustix::io::read(source, buffer)?)
}
}
fn send(source: &IoSource, buffers: &[IoSlice<'_>]) -> io::Result<usize> {
#[cfg(unix)]
{
let mut control = rustix::net::SendAncillaryBuffer::default();
Ok(rustix::net::sendmsg(
source,
buffers,
&mut control,
SEND_FLAGS,
)?)
}
#[cfg(windows)]
{
let socket = SockRef::from(source);
std::io::Write::write_vectored(&mut &*socket, buffers)
}
#[cfg(target_os = "wasi")]
{
Ok(rustix::io::writev(source, buffers)?)
}
}
#[cfg(all(
unix,
not(any(
target_os = "macos",
target_os = "ios",
target_os = "tvos",
target_os = "visionos",
target_os = "watchos",
target_os = "redox"
))
))]
const SEND_FLAGS: rustix::net::SendFlags = rustix::net::SendFlags::NOSIGNAL;
#[cfg(all(
unix,
any(
target_os = "macos",
target_os = "ios",
target_os = "tvos",
target_os = "visionos",
target_os = "watchos",
target_os = "redox"
)
))]
const SEND_FLAGS: rustix::net::SendFlags = rustix::net::SendFlags::empty();