use std::{
collections::{BTreeMap, BTreeSet},
io, mem,
panic::{self, AssertUnwindSafe},
sync::{
Arc, Condvar, Mutex, MutexGuard, PoisonError, Weak,
atomic::{AtomicBool, AtomicU64, Ordering},
},
thread::{self, ThreadId},
time::{Duration, Instant},
};
use crate::{
config::{Cancellation, Config},
error::{CallError, Error},
error_object::{ErrorCode, ErrorObject},
handles::CancelToken,
id::Id,
message::{write_call, write_numbered_call, write_response},
pending::Pending,
raw::Json,
sink::{Drain, Sink},
slot::{self, Outcome, Slot, Waiter},
stats::{Stats, Tally, bump},
text,
trace::event,
};
const SCRATCH_KEEP: usize = 64 * 1024;
#[derive(Clone)]
pub(crate) enum Scope {
Free,
Turn(Arc<Turn>),
Deferred { id: Id<'static>, serial: u64 },
}
#[derive(Debug, Default)]
pub(crate) struct Turn {
done: AtomicBool,
progress: AtomicU64,
}
const NO_PROGRESS: u64 = u64::MAX;
impl Turn {
pub(crate) fn new() -> Self {
Self {
done: AtomicBool::new(false),
progress: AtomicU64::new(NO_PROGRESS),
}
}
pub(crate) fn finish(&self) {
self.done.store(true, Ordering::Release);
}
}
pub(crate) trait Link: Send + Sync {
fn notify(&self, method: &str, params: Option<&Json<'_>>) -> Result<(), Error>;
fn notify_in(
&self,
scope: &Scope,
progress: Option<f64>,
method: &str,
params: Option<&Json<'_>>,
) -> Result<(), Error>;
fn request(
&self,
method: &str,
params: Option<&Json<'_>>,
timeout: Option<Duration>,
) -> Result<Issued, Error>;
fn cancel(&self, id: u64) -> bool;
fn expire(&self);
fn respond(
&self,
id: &Id<'static>,
serial: u64,
result: Result<&Json<'_>, &ErrorObject>,
) -> Result<(), Error>;
fn on_endpoint_thread(&self) -> bool;
fn is_closed(&self) -> bool;
fn stats(&self) -> Stats;
}
pub(crate) struct Issued {
pub(crate) id: u64,
pub(crate) outcome: Arc<Slot>,
pub(crate) deadline: Option<Instant>,
}
pub(crate) struct Shared<K> {
inner: Mutex<Inner<K>>,
timer: Arc<Timer>,
closed: Arc<AtomicBool>,
}
impl<K: Sink> Shared<K> {
pub(crate) fn new(sink: K, config: &Config) -> Arc<Self> {
let timer = Arc::new(Timer::default());
let closed = Arc::new(AtomicBool::new(false));
Arc::new_cyclic(|me| Self {
timer: Arc::clone(&timer),
closed: Arc::clone(&closed),
inner: Mutex::new(Inner {
me: Weak::clone(me),
timer,
closed_flag: closed,
sink,
sink_closed: false,
closed: false,
broken: None,
corked: false,
scratch: String::new(),
pending: Pending::new(config.max_pending),
in_flight: IdMap::default(),
batches: BTreeMap::new(),
next_batch: 0,
next_serial: 0,
gate: if config.gate.is_some() {
GateState::Closed
} else {
GateState::Open
},
held: Vec::new(),
held_bytes: 0,
held_requests: BTreeSet::new(),
stats: Stats::default(),
settings: Settings::from(config),
epoch: Instant::now(),
endpoint_thread: None,
}),
})
}
pub(crate) fn lock(&self) -> MutexGuard<'_, Inner<K>> {
self.inner.lock().unwrap_or_else(PoisonError::into_inner)
}
pub(crate) fn is_closed(&self) -> bool {
self.closed.load(Ordering::Acquire)
}
}
impl<K> Drop for Shared<K> {
fn drop(&mut self) {
self.timer.stop();
}
}
impl<K: Sink> Link for Shared<K> {
fn notify(&self, method: &str, params: Option<&Json<'_>>) -> Result<(), Error> {
self.lock().notify(method, params)
}
fn notify_in(
&self,
scope: &Scope,
progress: Option<f64>,
method: &str,
params: Option<&Json<'_>>,
) -> Result<(), Error> {
let mut inner = self.lock();
if inner.closed {
return Err(Error::Closed);
}
let last = match scope {
Scope::Free => None,
Scope::Turn(turn) => {
if turn.done.load(Ordering::Acquire) {
return Err(Error::AlreadyAnswered);
}
let bits = turn.progress.load(Ordering::Acquire);
(bits != NO_PROGRESS).then(|| f64::from_bits(bits))
}
Scope::Deferred { id, serial } => match inner.in_flight.get(id) {
Some(request) if request.serial == *serial => request.progress,
_ => return Err(Error::AlreadyAnswered),
},
};
if let (Some(next), Some(last)) = (progress, last) {
if next <= last {
return Err(Error::ProgressNotIncreasing { last, next });
}
}
inner.notify(method, params)?;
if let Some(next) = progress {
match scope {
Scope::Free => {}
Scope::Turn(turn) => turn.progress.store(next.to_bits(), Ordering::Release),
Scope::Deferred { id, .. } => {
if let Some(request) = inner.in_flight.get_mut(id) {
request.progress = Some(next);
}
}
}
}
Ok(())
}
fn request(
&self,
method: &str,
params: Option<&Json<'_>>,
timeout: Option<Duration>,
) -> Result<Issued, Error> {
self.lock().request(method, params, timeout)
}
fn cancel(&self, id: u64) -> bool {
self.lock().cancel_outgoing(id)
}
fn expire(&self) {
self.lock().expire();
}
fn respond(
&self,
id: &Id<'static>,
serial: u64,
result: Result<&Json<'_>, &ErrorObject>,
) -> Result<(), Error> {
self.lock().respond_deferred(id, serial, result)
}
fn on_endpoint_thread(&self) -> bool {
self.lock().endpoint_thread == Some(thread::current().id())
}
fn is_closed(&self) -> bool {
Shared::is_closed(self)
}
fn stats(&self) -> Stats {
self.lock().stats()
}
}
pub(crate) struct Settings {
pub(crate) cancellation: Option<Cancellation>,
pub(crate) request_timeout: Option<Duration>,
pub(crate) max_in_flight: usize,
pub(crate) max_held_bytes: usize,
pub(crate) timer: bool,
pub(crate) omit_null_id: bool,
pub(crate) overloaded_code: ErrorCode,
}
impl From<&Config> for Settings {
fn from(config: &Config) -> Self {
Self {
cancellation: config.cancellation.clone(),
request_timeout: config.request_timeout,
max_in_flight: config.max_in_flight,
max_held_bytes: config.max_held_bytes,
timer: config.timer,
omit_null_id: config.omit_null_id,
overloaded_code: config.overloaded_code,
}
}
}
#[derive(Default)]
struct Timer {
state: Mutex<TimerState>,
wake: Condvar,
}
#[derive(Default)]
struct TimerState {
due: Option<Instant>,
running: bool,
stopped: bool,
}
impl Timer {
fn lock(&self) -> MutexGuard<'_, TimerState> {
self.state.lock().unwrap_or_else(PoisonError::into_inner)
}
fn stop(&self) {
self.lock().stopped = true;
self.wake.notify_all();
}
}
fn run_timer<K: Sink>(timer: &Timer, shared: &Weak<Shared<K>>) {
let mut state = timer.lock();
while !state.stopped {
let Some(due) = state.due else {
state = timer
.wake
.wait(state)
.unwrap_or_else(PoisonError::into_inner);
continue;
};
let now = Instant::now();
if due > now {
state = timer
.wake
.wait_timeout(state, due - now)
.unwrap_or_else(PoisonError::into_inner)
.0;
continue;
}
state.due = None;
drop(state);
let Some(shared) = shared.upgrade() else {
return;
};
{
let mut inner = shared.lock();
inner.expire();
inner.arm();
}
drop(shared);
state = timer.lock();
}
state.running = false;
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) enum GateState {
Open,
Closed,
Opening(Id<'static>),
}
struct InFlight {
token: CancelToken,
batch: Option<u64>,
serial: u64,
progress: Option<f64>,
}
struct IdMap<V> {
numbers: BTreeMap<i64, V>,
strings: BTreeMap<Box<str>, V>,
raw_numbers: BTreeMap<Box<str>, V>,
null: Option<V>,
}
impl<V> Default for IdMap<V> {
fn default() -> Self {
Self {
numbers: BTreeMap::new(),
strings: BTreeMap::new(),
raw_numbers: BTreeMap::new(),
null: None,
}
}
}
impl<V> IdMap<V> {
fn len(&self) -> usize {
self.numbers.len()
+ self.strings.len()
+ self.raw_numbers.len()
+ usize::from(self.null.is_some())
}
fn contains(&self, id: &Id<'_>) -> bool {
match id {
Id::Number(n) => self.numbers.contains_key(n),
Id::String(s) => self.strings.contains_key(&**s),
Id::RawNumber(s) => self.raw_numbers.contains_key(&**s),
Id::Null => self.null.is_some(),
}
}
fn insert(&mut self, id: Id<'_>, value: V) -> Option<V> {
match id {
Id::Number(n) => self.numbers.insert(n, value),
Id::String(s) => self.strings.insert(s.into_owned().into_boxed_str(), value),
Id::RawNumber(s) => self
.raw_numbers
.insert(s.into_owned().into_boxed_str(), value),
Id::Null => self.null.replace(value),
}
}
fn get(&self, id: &Id<'_>) -> Option<&V> {
match id {
Id::Number(n) => self.numbers.get(n),
Id::String(s) => self.strings.get(&**s),
Id::RawNumber(s) => self.raw_numbers.get(&**s),
Id::Null => self.null.as_ref(),
}
}
fn get_mut(&mut self, id: &Id<'_>) -> Option<&mut V> {
match id {
Id::Number(n) => self.numbers.get_mut(n),
Id::String(s) => self.strings.get_mut(&**s),
Id::RawNumber(s) => self.raw_numbers.get_mut(&**s),
Id::Null => self.null.as_mut(),
}
}
fn remove(&mut self, id: &Id<'_>) -> Option<V> {
match id {
Id::Number(n) => self.numbers.remove(n),
Id::String(s) => self.strings.remove(&**s),
Id::RawNumber(s) => self.raw_numbers.remove(&**s),
Id::Null => self.null.take(),
}
}
fn into_values(self) -> impl Iterator<Item = V> {
self.numbers
.into_values()
.chain(self.strings.into_values())
.chain(self.raw_numbers.into_values())
.chain(self.null)
}
}
struct Batch {
remaining: usize,
body: String,
}
struct Held {
request: Option<u64>,
body: String,
}
#[derive(Clone, Copy)]
enum Sending {
Response,
Notification,
Request(u64),
}
pub(crate) struct Inner<K> {
me: Weak<Shared<K>>,
timer: Arc<Timer>,
closed_flag: Arc<AtomicBool>,
sink: K,
sink_closed: bool,
closed: bool,
broken: Option<io::Error>,
corked: bool,
scratch: String,
pending: Pending<Waiter>,
in_flight: IdMap<InFlight>,
batches: BTreeMap<u64, Batch>,
next_batch: u64,
next_serial: u64,
pub(crate) gate: GateState,
held: Vec<Held>,
held_bytes: usize,
held_requests: BTreeSet<u64>,
stats: Stats,
pub(crate) settings: Settings,
epoch: Instant,
pub(crate) endpoint_thread: Option<ThreadId>,
}
impl<K: Sink> Inner<K> {
pub(crate) fn sink_mut(&mut self) -> &mut K {
&mut self.sink
}
pub(crate) fn is_closed(&self) -> bool {
self.closed
}
pub(crate) fn stats(&self) -> Stats {
Stats {
pending: self.pending.len(),
in_flight: self.in_flight.len(),
held: self.held.len(),
..self.stats
}
}
pub(crate) fn take_in(&mut self, tally: &Tally) {
self.stats.take_in(tally);
}
pub(crate) fn count_panic(&mut self) {
bump(&mut self.stats.panics);
}
pub(crate) fn configure(&mut self, config: &Config) {
self.settings = Settings::from(config);
if self.pending.is_empty() {
self.pending = Pending::new(config.max_pending);
}
if config.gate.is_none() {
self.gate = GateState::Open;
} else if self.gate == GateState::Open && self.held.is_empty() {
self.gate = GateState::Closed;
}
}
fn now(&self) -> Duration {
self.epoch.elapsed()
}
#[cfg(feature = "net")]
pub(crate) fn open_ended(&self) -> usize {
self.pending.without_deadline()
}
pub(crate) fn next_deadline(&self) -> Option<Instant> {
self.pending
.next_deadline()
.and_then(|at| self.epoch.checked_add(at))
}
fn arm(&mut self) {
if !self.settings.timer || self.closed {
return;
}
let Some(due) = self.next_deadline() else {
return;
};
let mut timer = self.timer.lock();
if timer.stopped {
return;
}
if timer.due.is_none_or(|at| due < at) {
timer.due = Some(due);
self.timer.wake.notify_all();
}
if !timer.running {
let shared = Weak::clone(&self.me);
let thread_timer = Arc::clone(&self.timer);
let started = thread::Builder::new()
.name("rpc-io-timer".into())
.spawn(move || run_timer(&thread_timer, &shared));
timer.running = started.is_ok();
}
}
pub(crate) fn cork(&mut self) {
self.corked = true;
}
fn on_endpoint_thread(&self) -> bool {
self.endpoint_thread == Some(thread::current().id())
}
pub(crate) fn uncork(&mut self) {
self.corked = false;
if !self.closed {
if let Err(error) = guard(|| self.sink.flush()) {
let _closed = self.fail(error);
}
}
}
pub(crate) fn take_broken(&mut self) -> Option<io::Error> {
self.broken.take()
}
pub(crate) fn close(&mut self) -> Option<Drain> {
if self.closed {
return None;
}
if let Err(error) = self.close_sink() {
let _closed = self.fail(error);
}
self.shut();
let drain = panic::catch_unwind(AssertUnwindSafe(|| self.sink.drain()));
drain.unwrap_or(None)
}
fn close_sink(&mut self) -> io::Result<()> {
if mem::replace(&mut self.sink_closed, true) {
return Ok(());
}
guard(|| self.sink.close())
}
fn shut(&mut self) {
event!(debug, "the connection is closed");
self.closed = true;
self.closed_flag.store(true, Ordering::Release);
self.timer.stop();
for (_, waiter) in self.pending.drain() {
waiter.send(Err(CallError::Closed));
}
for request in mem::take(&mut self.in_flight).into_values() {
request.token.cancel();
}
self.batches.clear();
self.held.clear();
self.held_requests.clear();
self.held_bytes = 0;
}
fn fail(&mut self, error: io::Error) -> Error {
event!(warn, error = %error, "the transport failed; closing the connection");
let copy = io::Error::new(error.kind(), error.to_string());
if self.broken.is_none() {
self.broken = Some(error);
}
let _released = self.close_sink();
self.shut();
Error::Io(copy)
}
fn deliver(&mut self, sending: Sending) -> Result<(), Error> {
if self.closed {
return Err(Error::Closed);
}
if self.gate != GateState::Open && !matches!(sending, Sending::Response) {
let bytes = self.held_bytes.saturating_add(self.scratch.len());
if bytes > self.settings.max_held_bytes {
event!(
warn,
held = self.held.len(),
"too much is held behind the gate; a message was refused"
);
return Err(Error::Backlog);
}
self.held_bytes = bytes;
let request = match sending {
Sending::Request(id) => Some(id),
_ => None,
};
if let Some(id) = request {
let _fresh = self.held_requests.insert(id);
}
self.held.push(Held {
request,
body: self.scratch.clone(),
});
return Ok(());
}
bump(&mut self.stats.bodies_out);
self.stats.bytes_out = self.stats.bytes_out.wrapping_add(self.scratch.len() as u64);
match sending {
Sending::Request(_) => bump(&mut self.stats.requests_out),
Sending::Notification => bump(&mut self.stats.notifications_out),
Sending::Response => {}
}
let mut sent = guard(|| self.sink.send(&self.scratch));
if sent.is_ok() && (!self.corked || !self.on_endpoint_thread()) {
sent = guard(|| self.sink.flush());
}
if self.scratch.capacity() > SCRATCH_KEEP {
self.scratch = String::new();
}
sent.map_err(|error| self.fail(error))
}
pub(crate) fn notify(&mut self, method: &str, params: Option<&Json<'_>>) -> Result<(), Error> {
if self.closed {
return Err(Error::Closed);
}
self.scratch.clear();
write_call(&mut self.scratch, None, method, params);
self.deliver(Sending::Notification)
}
pub(crate) fn request(
&mut self,
method: &str,
params: Option<&Json<'_>>,
timeout: Option<Duration>,
) -> Result<Issued, Error> {
if self.closed {
return Err(Error::Closed);
}
self.expire();
let (waiter, outcome) = slot::slot();
let deadline = timeout
.or(self.settings.request_timeout)
.and_then(|t| self.now().checked_add(t));
let (id, evicted) = self.pending.issue(waiter, deadline);
if let Some((oldest, waiter)) = evicted {
self.abandon(oldest, waiter, CallError::Evicted);
}
self.scratch.clear();
write_numbered_call(&mut self.scratch, id, method, params);
if let Err(error) = self.deliver(Sending::Request(id)) {
let _unsent = self.pending.settle(id);
return Err(error);
}
if deadline.is_some() {
self.arm();
}
Ok(Issued {
id,
outcome,
deadline: deadline.and_then(|at| self.epoch.checked_add(at)),
})
}
pub(crate) fn expire(&mut self) {
let now = self.now();
while let Some((id, waiter)) = self.pending.pop_expired(now) {
self.abandon(id, waiter, CallError::TimedOut);
}
}
pub(crate) fn cancel_outgoing(&mut self, id: u64) -> bool {
match self.pending.settle(id) {
Some(waiter) => {
self.abandon(id, waiter, CallError::Cancelled);
true
}
None => false,
}
}
pub(crate) fn settle(&mut self, id: u64, outcome: Outcome) -> bool {
if self.held_requests.contains(&id) {
return false;
}
match self.pending.settle(id) {
Some(waiter) => {
waiter.send(outcome);
true
}
None => false,
}
}
fn abandon(&mut self, id: u64, waiter: Waiter, reason: CallError) {
match reason {
CallError::TimedOut => {
bump(&mut self.stats.timed_out);
event!(debug, id, "our request timed out");
}
CallError::Evicted => {
bump(&mut self.stats.evicted);
event!(
warn,
id,
"our oldest request was evicted: too many are outstanding"
);
}
_ => bump(&mut self.stats.cancelled_out),
}
if !self.held_requests.remove(&id) && !self.closed {
self.send_cancel(id);
}
waiter.send(Err(reason));
}
fn send_cancel(&mut self, id: u64) {
let Some(cancellation) = &self.settings.cancellation else {
return;
};
self.scratch.clear();
self.scratch.push_str("{\"jsonrpc\":\"2.0\",\"method\":");
jsonx_lib::write_string(&mut self.scratch, &cancellation.method);
self.scratch.push_str(",\"params\":{");
jsonx_lib::write_string(&mut self.scratch, &cancellation.id_member);
self.scratch.push(':');
text::push_u64(&mut self.scratch, id);
self.scratch.push_str("}}");
let _best_effort = self.deliver(Sending::Notification);
}
pub(crate) fn respond(
&mut self,
id: Option<&Id<'_>>,
batch: Option<u64>,
result: Result<&Json<'_>, &ErrorObject>,
) -> Result<(), Error> {
if self.closed {
return Err(Error::Closed);
}
bump(&mut self.stats.responses_out);
if result.is_err() {
bump(&mut self.stats.errors_out);
}
self.scratch.clear();
write_response(&mut self.scratch, id, result, self.settings.omit_null_id);
let opening = match (&self.gate, id) {
(GateState::Opening(opening), Some(id)) => opening == id,
_ => false,
};
let sent = match batch {
Some(batch) => self.add_to_batch(batch, true),
None => self.deliver(Sending::Response),
};
if opening {
self.settle_gate(result.is_ok());
}
sent
}
fn add_to_batch(&mut self, batch: u64, with_response: bool) -> Result<(), Error> {
let Some(open) = self.batches.get_mut(&batch) else {
return Ok(());
};
if with_response {
if open.body.len() > 1 {
open.body.push(',');
}
open.body.push_str(&self.scratch);
}
open.remaining = open.remaining.saturating_sub(1);
if open.remaining > 0 {
return Ok(());
}
let Some(mut done) = self.batches.remove(&batch) else {
return Ok(());
};
if done.body.len() == 1 {
return Ok(());
}
done.body.push(']');
mem::swap(&mut self.scratch, &mut done.body);
self.deliver(Sending::Response)
}
fn settle_gate(&mut self, succeeded: bool) {
let held = mem::take(&mut self.held);
let live = mem::take(&mut self.held_requests);
self.held_bytes = 0;
if succeeded {
self.gate = GateState::Open;
for message in held {
if message.request.is_some_and(|id| !live.contains(&id)) {
continue;
}
self.scratch = message.body;
let sending = message
.request
.map_or(Sending::Notification, Sending::Request);
if self.deliver(sending).is_err() {
break;
}
}
} else {
self.gate = GateState::Closed;
for message in held {
if let Some(id) = message.request.filter(|id| live.contains(id)) {
if let Some(waiter) = self.pending.settle(id) {
waiter.send(Err(CallError::Cancelled));
}
}
}
}
}
pub(crate) fn open_batch(&mut self, answers: usize) -> u64 {
let id = self.next_batch;
self.next_batch = self.next_batch.wrapping_add(1);
let _fresh = self.batches.insert(
id,
Batch {
remaining: answers,
body: String::from("["),
},
);
id
}
pub(crate) fn in_flight(&self, id: &Id<'_>) -> bool {
self.in_flight.contains(id)
}
pub(crate) fn begin_deferred(
&mut self,
id: Id<'static>,
batch: Option<u64>,
) -> Option<(CancelToken, u64)> {
if self.in_flight.len() >= self.settings.max_in_flight {
return None;
}
let token = CancelToken::new();
let serial = self.next_serial;
self.next_serial = self.next_serial.wrapping_add(1);
let _fresh = self.in_flight.insert(
id,
InFlight {
token: token.clone(),
batch,
serial,
progress: None,
},
);
Some((token, serial))
}
pub(crate) fn respond_deferred(
&mut self,
id: &Id<'static>,
serial: u64,
result: Result<&Json<'_>, &ErrorObject>,
) -> Result<(), Error> {
let current = self
.in_flight
.get(id)
.is_some_and(|request| request.serial == serial);
match current.then(|| self.in_flight.remove(id)).flatten() {
Some(request) => self.respond(Some(id), request.batch, result),
None if self.closed => Err(Error::Closed),
None => Err(Error::AlreadyAnswered),
}
}
pub(crate) fn cancel_incoming(&mut self, id: &Id<'_>) -> bool {
let Some(request) = self.in_flight.remove(id) else {
return false;
};
bump(&mut self.stats.cancelled_in);
request.token.cancel();
let reply = self.settings.cancellation.as_ref().and_then(|c| c.reply);
let _sent = match reply {
Some(code) => {
let error = ErrorObject::new(code, "the request was cancelled");
self.respond(Some(id), request.batch, Err(&error))
}
None => {
if matches!(&self.gate, GateState::Opening(opening) if opening == id) {
self.settle_gate(false);
}
match request.batch {
Some(batch) => self.add_to_batch(batch, false),
None => Ok(()),
}
}
};
true
}
}
fn guard(send: impl FnOnce() -> io::Result<()>) -> io::Result<()> {
panic::catch_unwind(AssertUnwindSafe(send))
.unwrap_or_else(|_| Err(io::Error::other("the sink panicked")))
}