Skip to main content

plexus_core/plexus/
wire.rs

1//! PERF-7: the item path — an item encoded once, by its method's own event
2//! type, straight into a buffer its socket writes from.
3//!
4//! | when | then |
5//! | -- | -- |
6//! | a method yields an event under the WebSocket pipe ([`pipe`]) | [`wrap_stream`](super::streaming::wrap_stream) serializes it once, by its own `Serialize`, into the pipe's buffer (a block from its connection's pool); the `Data` item carries it as [`Content`] — JSON bytes, no `Value` |
7//! | the pipe's buffer takes an event's JSON (`json`, PERF-34) | written by a real scan, not a byte at a time, the bytes `serde_json` writes |
8//! | the same event anywhere else (in process, a nested route, a test) | it becomes a `Value` as before: in-process items stay what they cost |
9//! | a transformer reads a `Data` item's content | [`Content`] decodes its bytes once, on demand (`Deref` to `Value`); one that rewrites it replaces the whole [`Content`] — an encoded item is a `Data` item, so no rewrite can miss one (PERF-17's finding: the gated view's rewrite did) |
10//! | the pipe writes an item | the notification's prefix (once a call), the metadata (once a stream), the content type and the content's bytes, and the suffix, into the same buffer: one frame, split off as `Bytes` (no allocation while the block has room) |
11//! | the transport offers a [`FrameSink`] (plexus-transport's socket loop) | the frame goes to the socket's writer as it is; the call's first frame is fenced, so it follows the subscription's answer that jsonrpsee queued |
12//! | it does not (jsonrpsee's own sink: stdio, an in-memory module) | the item's bytes go to the sink as one raw value, as before |
13//! | one change fans out to N watchers ([`Shared`]) | the first socket that sends it encodes it, once; every other writes those bytes |
14//! | a call has an input half ([`super::input`]) | [`pipe_duplex`] takes it instead: the same frames and buffer, and beside the method's items the frames of the input half (`credit`, `ack`, `input_closed`), made here since they are this transport's and no method's. [`pipe`] is unchanged by them |
15//! | the client offered `batch` at its upgrade ([`FrameSink::batching`], PERF-8) | the items a stream has ready together, up to the batch and about [`BATCH_BYTES`], go in one notification whose `results` are their array, the metadata said once; a lone item goes at once, in today's form |
16//!
17//! PERF-8 batches frames here (`results: […]`), PERF-9 adds an encoding
18//! ([`WireEncoding`]), PERF-13 relays a child's content as bytes.
19
20use std::borrow::Cow;
21use std::cell::{Cell, RefCell};
22use std::sync::{Arc, Mutex, OnceLock};
23
24use bytes::{BufMut, Bytes, BytesMut};
25use serde::{Deserialize, Serialize, Serializer};
26use serde_json::value::RawValue;
27use serde_json::Value;
28
29use super::types::{PlexusStreamItem, StreamMetadata};
30
31mod json;
32
33// ---------------------------------------------------------------------------
34// Content
35// ---------------------------------------------------------------------------
36
37/// A `data` item's content: the method's event, encoded once (JSON bytes),
38/// or a `Value` — whichever the item was made with. Reads as a `Value`
39/// either way ([`Content::value`], `Deref`), the bytes decoded once on
40/// demand; serializes as its JSON. Equal as JSON is equal.
41#[derive(Clone, Default)]
42pub struct Content(Repr);
43
44#[derive(Clone)]
45enum Repr {
46    Value(Value),
47    /// JSON text the pipe's writer wrote (valid by construction: the bytes
48    /// serde_json writes), and its `Value` once someone asked for it.
49    Json { text: Bytes, value: OnceLock<Value> },
50}
51
52impl Default for Repr {
53    fn default() -> Self {
54        Repr::Value(Value::Null)
55    }
56}
57
58impl Content {
59    /// Content that is a `Value`.
60    pub fn from_value(value: Value) -> Self {
61        Content(Repr::Value(value))
62    }
63
64    /// Content that is JSON text, checked (a parse, no `Value` built).
65    pub fn from_json(text: Bytes) -> Result<Self, serde_json::Error> {
66        serde_json::from_slice::<serde::de::IgnoredAny>(&text)?;
67        Ok(Self::json(text))
68    }
69
70    /// JSON text the pipe's writer wrote, taken as it is.
71    pub(crate) fn json(text: Bytes) -> Self {
72        Content(Repr::Json { text, value: OnceLock::new() })
73    }
74
75    /// The content as a `Value` (encoded bytes decoded once, kept).
76    pub fn value(&self) -> &Value {
77        match &self.0 {
78            Repr::Value(v) => v,
79            Repr::Json { text, value } => value.get_or_init(|| serde_json::from_slice(text).unwrap_or(Value::Null)),
80        }
81    }
82
83    /// The content as an owned `Value`.
84    pub fn into_value(self) -> Value {
85        match self.0 {
86            Repr::Value(v) => v,
87            Repr::Json { text, value } => value.into_inner().unwrap_or_else(|| serde_json::from_slice(&text).unwrap_or(Value::Null)),
88        }
89    }
90
91    /// A copy of the content as a `Value`.
92    pub fn to_value(&self) -> Value {
93        match &self.0 {
94            Repr::Value(v) => v.clone(),
95            Repr::Json { text, value } => value.get().cloned().unwrap_or_else(|| serde_json::from_slice(text).unwrap_or(Value::Null)),
96        }
97    }
98
99    /// The content as `T`, decoded straight from its bytes when it is
100    /// encoded (no `Value` in between).
101    pub fn decode<T: serde::de::DeserializeOwned>(&self) -> Result<T, serde_json::Error> {
102        match &self.0 {
103            Repr::Value(v) => T::deserialize(v),
104            Repr::Json { text, value } => match value.get() {
105                Some(v) => T::deserialize(v),
106                None => serde_json::from_slice(text),
107            },
108        }
109    }
110
111    /// The JSON text the content was encoded as, when it was.
112    pub fn json_bytes(&self) -> Option<&Bytes> {
113        match &self.0 {
114            Repr::Json { text, .. } => Some(text),
115            Repr::Value(_) => None,
116        }
117    }
118
119    /// Whether the content was encoded once from its event (PERF-7).
120    pub fn is_encoded(&self) -> bool {
121        matches!(self.0, Repr::Json { .. })
122    }
123
124    /// Its JSON text, appended to `out`: the bytes, or the `Value` written.
125    fn write_json(&self, out: &mut BytesMut) {
126        match &self.0 {
127            Repr::Json { text, .. } => out.put_slice(text),
128            Repr::Value(v) => {
129                let _ = serde_json::to_writer((&mut *out).writer(), v);
130            }
131        }
132    }
133}
134
135impl std::ops::Deref for Content {
136    type Target = Value;
137    fn deref(&self) -> &Value {
138        self.value()
139    }
140}
141
142impl From<Value> for Content {
143    fn from(value: Value) -> Self {
144        Content::from_value(value)
145    }
146}
147
148impl From<Content> for Value {
149    fn from(content: Content) -> Self {
150        content.into_value()
151    }
152}
153
154/// Whatever a `Value` compares with — another content, a `Value`, a string,
155/// a number, a bool — a content compares with as its `Value`.
156impl<T: ?Sized> PartialEq<T> for Content
157where
158    Value: PartialEq<T>,
159{
160    fn eq(&self, other: &T) -> bool {
161        self.value() == other
162    }
163}
164
165impl PartialEq<Content> for Value {
166    fn eq(&self, other: &Content) -> bool {
167        self == other.value()
168    }
169}
170
171impl std::fmt::Debug for Content {
172    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
173        match &self.0 {
174            Repr::Value(v) => v.fmt(f),
175            Repr::Json { text, .. } => write!(f, "{}", String::from_utf8_lossy(text)),
176        }
177    }
178}
179
180impl std::fmt::Display for Content {
181    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
182        match &self.0 {
183            Repr::Value(v) => v.fmt(f),
184            Repr::Json { text, .. } => f.write_str(&String::from_utf8_lossy(text)),
185        }
186    }
187}
188
189impl Serialize for Content {
190    fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
191        if let Repr::Json { text, value } = &self.0 {
192            // serde_json writes a raw value as it is; any other serializer
193            // gets the `Value`.
194            if writing_json() {
195                if let Ok(raw) = serde_json::from_slice::<&RawValue>(text) {
196                    return raw.serialize(s);
197                }
198            }
199            if let Some(v) = value.get() {
200                return v.serialize(s);
201            }
202        }
203        self.value().serialize(s)
204    }
205}
206
207impl<'de> Deserialize<'de> for Content {
208    fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
209        Value::deserialize(d).map(Content::from_value)
210    }
211}
212
213impl schemars::JsonSchema for Content {
214    fn inline_schema() -> bool {
215        <Value as schemars::JsonSchema>::inline_schema()
216    }
217    fn schema_name() -> Cow<'static, str> {
218        <Value as schemars::JsonSchema>::schema_name()
219    }
220    fn schema_id() -> Cow<'static, str> {
221        <Value as schemars::JsonSchema>::schema_id()
222    }
223    fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema {
224        <Value as schemars::JsonSchema>::json_schema(generator)
225    }
226}
227
228// ---------------------------------------------------------------------------
229// The connection's encoding and its frames
230// ---------------------------------------------------------------------------
231
232/// The encoding of every frame on a connection, fixed when it opens. JSON
233/// text until PERF-9 adds msgpack.
234#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
235#[non_exhaustive]
236pub enum WireEncoding {
237    /// JSON-RPC 2.0 objects, one text message each.
238    #[default]
239    Json,
240}
241
242/// A frame ready for a socket, in its connection's encoding.
243#[derive(Debug, Clone)]
244pub struct WireFrame {
245    bytes: Bytes,
246    fence: bool,
247}
248
249impl WireFrame {
250    /// The frame's bytes (JSON text under [`WireEncoding::Json`]).
251    pub fn bytes(&self) -> &Bytes {
252        &self.bytes
253    }
254
255    /// The frame's bytes, owned.
256    pub fn into_bytes(self) -> Bytes {
257        self.bytes
258    }
259
260    /// A call's first frame: written only after every frame the connection's
261    /// method table queued before it (its subscription's answer among them,
262    /// which jsonrpsee queues when the call is accepted).
263    pub fn fenced(&self) -> bool {
264        self.fence
265    }
266}
267
268/// The frames a transport takes from a connection's calls (plexus-transport's
269/// socket loop puts one in each call's extensions): each item a frame, in
270/// order, written as it is. Cheap to clone.
271#[derive(Clone)]
272pub struct FrameSink {
273    tx: tokio::sync::mpsc::Sender<WireFrame>,
274    encoding: WireEncoding,
275    /// The connection's pool: one spare block a call starts in, so a short
276    /// call allocates none.
277    spare: Arc<Mutex<Option<BytesMut>>>,
278    /// The most items one notification carries (PERF-8): 0 or 1, one an item.
279    batch: usize,
280    /// BIDIR-6: the connection's input halves. Carried here, not as an entry
281    /// of its own in the extensions, which every call clones: an entry is an
282    /// allocation a call (call.one_item's +2 allocations, +192 bytes).
283    inputs: Option<Arc<super::input::InputTable>>,
284}
285
286/// The most items one notification carries when the client offered `batch`
287/// at its upgrade (PERF-8, spec/02): a client may ask for fewer.
288pub const MAX_BATCH: usize = 64;
289
290/// Where a batch stops taking items: once what it holds reaches this many
291/// bytes, so a batch of large items stays small. A queue's 1,024 frames are
292/// bounded by frames, not bytes (PERF.115-37), and each item takes its size
293/// twice from a block: 64 items of 4 KB to a frame held 280 KB a frame and
294/// allocated a 512 KiB block every few frames, 16 KiB still 4,096 B an item
295/// in one run of two, and a ticket body's batch would hold a megabyte. At
296/// 8 KiB a full queue of 4 KB items holds about 18 MB, which the call's 80
297/// spare blocks keep. PERF-17 found a count alone cost a 4 KB stream 27 %, and
298/// said a batch should stop at bytes.
299pub const BATCH_BYTES: usize = 8 * 1024;
300
301/// What the transport's writer reads a [`FrameSink`]'s frames from.
302pub type FrameReceiver = tokio::sync::mpsc::Receiver<WireFrame>;
303
304impl FrameSink {
305    /// A connection's frame queue, bounded at `capacity` frames.
306    pub fn channel(capacity: usize, encoding: WireEncoding) -> (FrameSink, FrameReceiver) {
307        let (tx, rx) = tokio::sync::mpsc::channel(capacity.max(1));
308        (FrameSink { tx, encoding, spare: Arc::new(Mutex::new(None)), batch: 0, inputs: None }, rx)
309    }
310
311    /// The same sink for a client that takes batches (it offered `batch` at
312    /// its upgrade): up to `max` items a stream has ready go in one
313    /// notification. 0 or 1: one an item, as before.
314    pub fn batching(mut self, max: usize) -> Self {
315        self.batch = max;
316        self
317    }
318
319    /// BIDIR-6: the same sink carrying the connection's input halves, where a
320    /// method with an `Input` finds its table.
321    pub fn with_inputs(mut self, table: Arc<super::input::InputTable>) -> Self {
322        self.inputs = Some(table);
323        self
324    }
325
326    /// BIDIR-6: the connection's input halves, when its transport carries
327    /// them.
328    pub fn inputs(&self) -> Option<&Arc<super::input::InputTable>> {
329        self.inputs.as_ref()
330    }
331
332    /// The most items one notification carries: 0 or 1, one an item.
333    pub fn batch(&self) -> usize {
334        self.batch
335    }
336
337    /// The connection's encoding.
338    pub fn encoding(&self) -> WireEncoding {
339        self.encoding
340    }
341
342    /// Whether the connection is gone.
343    pub fn is_closed(&self) -> bool {
344        self.tx.is_closed()
345    }
346
347    fn take_block(&self) -> Option<BytesMut> {
348        self.spare.lock().ok().and_then(|mut s| s.take())
349    }
350
351    fn give_block(&self, block: BytesMut) {
352        if block.capacity() <= Buffer::POOLED {
353            if let Ok(mut s) = self.spare.lock() {
354                if s.is_none() {
355                    *s = Some(block);
356                }
357            }
358        }
359    }
360}
361
362impl std::fmt::Debug for FrameSink {
363    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
364        f.debug_struct("FrameSink").field("encoding", &self.encoding).field("batch", &self.batch).field("closed", &self.tx.is_closed()).finish()
365    }
366}
367
368// ---------------------------------------------------------------------------
369// The buffer: blocks a call's items and frames are written into
370// ---------------------------------------------------------------------------
371
372/// A call's buffer. Each item and each frame is written at the end of the
373/// current block and split off as `Bytes` sharing it; a block is reused once
374/// every frame cut from it has been written and dropped, and a new one is
375/// allocated only when none is free.
376struct Buffer {
377    cur: BytesMut,
378    /// Blocks still holding frames in flight, kept to be reused once free.
379    spare: Vec<BytesMut>,
380    /// The size a new block is made at: grown with the frames it holds.
381    block: usize,
382    /// Whether a block has been made (or taken from the pool) yet.
383    started: bool,
384    /// Blocks made, for the tests.
385    #[cfg(test)]
386    made: usize,
387}
388
389impl Default for Buffer {
390    fn default() -> Self {
391        Buffer {
392            cur: BytesMut::new(),
393            spare: Vec::new(),
394            block: Self::FIRST,
395            started: false,
396            #[cfg(test)]
397            made: 0,
398        }
399    }
400}
401
402impl Buffer {
403    /// A first block: a short call's frames fit.
404    const FIRST: usize = 8 * 1024;
405    /// The largest block made for a stream's frames (one larger item gets a
406    /// block of its own size).
407    const MOST: usize = 512 * 1024;
408    /// The largest block a connection keeps for its next call.
409    const POOLED: usize = 64 * 1024;
410    /// Blocks kept to be reused. A stream that outruns its socket has the
411    /// connection's whole queue in flight (1,024 frames), and each item has
412    /// taken its size twice from a block, content and frame: 8.8 MB at 4 KB an
413    /// item, 36 MB at a ticket body. Blocks that hold fewer than that are
414    /// dropped before they come free, and every block filled is a new one, so
415    /// a faster encode allocates more: three held 8 KB an item at 4 KB and 37 KB
416    /// at a body (PERF-34; PERF-9 found it with msgpack, at b80cb2e). They are
417    /// bytes those frames hold anyway.
418    const SPARE: usize = 80;
419    /// Blocks kept once the queue has drained: what a long watch holds.
420    const DRAINED: usize = 3;
421
422    fn from_block(block: Option<BytesMut>) -> Self {
423        match block {
424            Some(cur) => Buffer { block: cur.capacity().max(Self::FIRST), cur, started: true, ..Buffer::default() },
425            None => Buffer::default(),
426        }
427    }
428
429    /// Room for `n` more bytes in the current block: it, or a free block, or
430    /// a new one.
431    fn room(&mut self, n: usize) {
432        if self.cur.capacity() - self.cur.len() >= n {
433            return;
434        }
435        if self.cur.try_reclaim(n) {
436            // None of this block's frames is left, and older ones went first.
437            self.spare.truncate(Self::DRAINED);
438            return;
439        }
440        // The current block holds unsent frames, or is too small: keep it
441        // for later and take a free one, if any is free.
442        let free = self.spare.iter_mut().position(|b| b.try_reclaim(n));
443        let next = match free {
444            Some(i) => self.spare.swap_remove(i),
445            None => {
446                // Every block is busy: the frames outrun the socket, so the
447                // next block holds more of them.
448                if self.started {
449                    self.block = (self.block * 2).clamp(Self::FIRST, Self::MOST);
450                }
451                self.started = true;
452                #[cfg(test)]
453                {
454                    self.made += 1;
455                }
456                BytesMut::with_capacity(self.block.max(n))
457            }
458        };
459        let old = std::mem::replace(&mut self.cur, next);
460        if old.capacity() > 0 {
461            if self.spare.len() >= Self::SPARE {
462                self.spare.remove(0);
463            }
464            self.spare.push(old);
465        }
466    }
467
468    /// What was written since the last split, as `Bytes`.
469    fn split(&mut self) -> Bytes {
470        self.cur.split().freeze()
471    }
472
473    /// The block a call with no connection leaves for its thread's next one:
474    /// all of it, with no frame of a call in flight, and not one a stream grew.
475    fn into_idle(mut self) -> Option<BytesMut> {
476        let _ = self.cur.try_reclaim(Self::FIRST);
477        (Self::FIRST..=Self::POOLED).contains(&self.cur.capacity()).then_some(self.cur)
478    }
479
480    /// The block to give back to the connection's pool: the current one,
481    /// whatever is left of it (frames still in flight keep what they hold).
482    fn into_block(self) -> Option<BytesMut> {
483        (self.cur.capacity() >= 1024).then_some(self.cur)
484    }
485}
486
487// ---------------------------------------------------------------------------
488// The pipe's encoder: the events of the stream the pipe polls
489// ---------------------------------------------------------------------------
490
491/// The pipe's encoder, for the task that polls a call's stream.
492struct Encoder {
493    /// The connection's encoding: JSON until PERF-9 reads it here.
494    #[allow(dead_code)]
495    encoding: WireEncoding,
496    buffer: RefCell<Buffer>,
497    /// How deep the poll is in streams [`wrap_stream`] wrapped: only the
498    /// outermost encodes — a stream polled inside a method's own (a call
499    /// it routes and reads in process) keeps its items as `Value`s.
500    ///
501    /// [`wrap_stream`]: super::streaming::wrap_stream
502    depth: Cell<u32>,
503    /// The last content's size, to make room for the next.
504    last: Cell<usize>,
505}
506
507tokio::task_local! {
508    static WIRE: Encoder;
509}
510
511thread_local! {
512    /// The pipe's serde_json writer is serializing an event on this thread.
513    static WRITING_JSON: Cell<u32> = const { Cell::new(0) };
514}
515
516/// Whether the serializer at work on this thread is the pipe's JSON writer
517/// (so a [`Shared`] event or an encoded [`Content`] may write its bytes as
518/// they are).
519fn writing_json() -> bool {
520    WRITING_JSON.with(|w| w.get() > 0)
521}
522
523struct WritingJson;
524
525impl WritingJson {
526    fn on() -> Self {
527        WRITING_JSON.with(|w| w.set(w.get() + 1));
528        WritingJson
529    }
530}
531
532impl Drop for WritingJson {
533    fn drop(&mut self) {
534        WRITING_JSON.with(|w| w.set(w.get().saturating_sub(1)));
535    }
536}
537
538/// Run `poll` (a wrapped stream's poll of its method's stream) one stream
539/// deeper: what it yields is not the pipe's to encode.
540pub(crate) fn nested<R>(poll: impl FnOnce() -> R) -> R {
541    if WIRE.try_with(|e| e.depth.set(e.depth.get() + 1)).is_err() {
542        return poll();
543    }
544    // Put back after the poll, a panic included.
545    struct Back;
546    impl Drop for Back {
547        fn drop(&mut self) {
548            let _ = WIRE.try_with(|e| e.depth.set(e.depth.get().saturating_sub(1)));
549        }
550    }
551    let _back = Back;
552    poll()
553}
554
555/// `event` encoded once into the pipe's buffer — when the pipe is polling
556/// this stream directly, the event is not a credential's (whose envelope is
557/// assembled as before) and it serializes. `None`: make a `Value` as before.
558pub(crate) fn encode<T: Serialize + ?Sized>(event: &T) -> Option<Content> {
559    WIRE.try_with(|e| {
560        if e.depth.get() > 0 {
561            return None;
562        }
563        let mut buffer = e.buffer.borrow_mut();
564        buffer.room(e.last.get().saturating_mul(2).max(256));
565        debug_assert!(buffer.cur.is_empty());
566        let (written, captured) = plexus_auth_core::credential::run_with_credential_capture(|| {
567            let _json = WritingJson::on();
568            json::write(&mut buffer.cur, event).is_ok()
569        });
570        if !written || !captured.is_empty() {
571            buffer.cur.clear();
572            return None;
573        }
574        e.last.set(buffer.cur.len());
575        Some(Content::json(buffer.split()))
576    })
577    .ok()
578    .flatten()
579}
580
581thread_local! {
582    /// The block of a call with no frame sink, for this thread's next one.
583    /// Such a call has no connection to keep a block, and it was half of what
584    /// a one-item call allocated (PERF-27).
585    static IDLE: Cell<Option<BytesMut>> = const { Cell::new(None) };
586}
587
588/// One call's frames: the notification around each item. With no frame sink
589/// there is none to write, so no prefix: jsonrpsee's sink writes its own.
590#[derive(Default)]
591struct Frames {
592    /// `{"jsonrpc":"2.0","method":M,"params":{"subscription":S,"result":`
593    prefix: Vec<u8>,
594    /// A `data` item's opening, `{"type":"data","metadata":…,"content_type":…,"content":`,
595    /// for the stream's metadata (shared by its items) and content type.
596    head: Option<(Arc<StreamMetadata>, Cow<'static, str>, Vec<u8>)>,
597    /// A batch's (PERF-8): `,"metadata":…,"results":[` for the stream's
598    /// metadata, and a `data` item's opening without it,
599    /// `{"type":"data","content_type":…,"content":`, for its content type.
600    batch: Option<(Arc<StreamMetadata>, Vec<u8>)>,
601    bare: Option<(Cow<'static, str>, Vec<u8>)>,
602}
603
604/// Where a notification's `params` stop and its `result` begins, in [`Frames::prefix`].
605const RESULT: &[u8] = br#","result":"#;
606
607impl Frames {
608    fn new(method: &str, subscription: &jsonrpsee::types::SubscriptionId<'_>) -> Self {
609        // Sized once: each grew from its first literal, and three reallocations a call.
610        let mut prefix = Vec::with_capacity(64);
611        prefix.extend_from_slice(br#"{"jsonrpc":"2.0","method":"#);
612        let _ = serde_json::to_writer(&mut prefix, method);
613        prefix.extend_from_slice(br#","params":{"subscription":"#);
614        let _ = serde_json::to_writer(&mut prefix, subscription);
615        prefix.extend_from_slice(RESULT);
616        Frames { prefix, head: None, batch: None, bare: None }
617    }
618
619    /// `item`'s JSON, written at the end of `out`.
620    fn item(&mut self, out: &mut BytesMut, item: &PlexusStreamItem) {
621        match item {
622            PlexusStreamItem::Data { metadata, content_type, content } => {
623                let fresh = match &self.head {
624                    Some((m, ct, _)) => !Arc::ptr_eq(m, metadata) || ct != content_type,
625                    None => true,
626                };
627                if fresh {
628                    let mut head = Vec::with_capacity(128);
629                    head.extend_from_slice(br#"{"type":"data","metadata":"#);
630                    let _ = serde_json::to_writer(&mut head, &**metadata);
631                    head.extend_from_slice(br#","content_type":"#);
632                    let _ = serde_json::to_writer(&mut head, content_type.as_ref());
633                    head.extend_from_slice(br#","content":"#);
634                    self.head = Some((metadata.clone(), content_type.clone(), head));
635                }
636                let head = &self.head.as_ref().expect("made above").2;
637                out.reserve(head.len() + content.json_bytes().map_or(64, Bytes::len) + 1);
638                out.put_slice(head);
639                content.write_json(out);
640                out.put_u8(b'}');
641            }
642            other => {
643                let _ = serde_json::to_writer((&mut *out).writer(), other);
644            }
645        }
646    }
647
648    /// `items`, ready together, in one notification (PERF-8, spec/02):
649    /// `{"jsonrpc":"2.0","method":M,"params":{"subscription":S,"metadata":…,"results":[…]}}`.
650    /// The metadata is the first `data` item's, said once: a `data` item whose
651    /// metadata writes the same leaves its own out; any other item is written
652    /// whole, as alone.
653    /// One item alone is today's frame.
654    fn batch(&mut self, buffer: &mut Buffer, items: &[PlexusStreamItem]) -> Bytes {
655        if let [item] = items {
656            return self.frame(buffer, item);
657        }
658        let shared = items.iter().find_map(|item| match item {
659            PlexusStreamItem::Data { metadata, .. } => Some(metadata),
660            _ => None,
661        });
662        let open = match shared {
663            Some(m) => {
664                if !matches!(&self.batch, Some((had, _)) if Arc::ptr_eq(had, m)) {
665                    let mut open = Vec::with_capacity(128);
666                    open.extend_from_slice(br#","metadata":"#);
667                    let _ = serde_json::to_writer(&mut open, &**m);
668                    open.extend_from_slice(br#","results":["#);
669                    self.batch = Some((m.clone(), open));
670                }
671                &self.batch.as_ref().expect("made above").1[..]
672            }
673            None => br#","results":["#,
674        };
675        let content: usize = items
676            .iter()
677            .map(|item| match item {
678                PlexusStreamItem::Data { content, .. } => content.json_bytes().map_or(64, Bytes::len) + 64,
679                _ => 256,
680            })
681            .sum();
682        let start = &self.prefix[..self.prefix.len() - RESULT.len()];
683        buffer.room(start.len() + open.len() + content + 4);
684        buffer.cur.put_slice(start);
685        buffer.cur.put_slice(open);
686        for (n, item) in items.iter().enumerate() {
687            if n > 0 {
688                buffer.cur.put_u8(b',');
689            }
690            match item {
691                PlexusStreamItem::Data { metadata, content_type, content } if shared.is_some_and(|m| same_metadata(m, metadata)) => {
692                    if !matches!(&self.bare, Some((ct, _)) if ct == content_type) {
693                        let mut bare = Vec::with_capacity(64);
694                        bare.extend_from_slice(br#"{"type":"data","content_type":"#);
695                        let _ = serde_json::to_writer(&mut bare, content_type.as_ref());
696                        bare.extend_from_slice(br#","content":"#);
697                        self.bare = Some((content_type.clone(), bare));
698                    }
699                    let bare = &self.bare.as_ref().expect("made above").1;
700                    buffer.cur.reserve(bare.len() + content.json_bytes().map_or(64, Bytes::len) + 1);
701                    buffer.cur.put_slice(bare);
702                    content.write_json(&mut buffer.cur);
703                    buffer.cur.put_u8(b'}');
704                }
705                other => self.item(&mut buffer.cur, other),
706            }
707        }
708        buffer.cur.put_slice(b"]}}");
709        buffer.split()
710    }
711
712    /// The notification carrying `item`: the frame a socket writes.
713    fn frame(&mut self, buffer: &mut Buffer, item: &PlexusStreamItem) -> Bytes {
714        let content = match item {
715            PlexusStreamItem::Data { content, .. } => content.json_bytes().map_or(256, Bytes::len),
716            _ => 256,
717        };
718        buffer.room(self.prefix.len() + content + 256);
719        buffer.cur.put_slice(&self.prefix);
720        self.item(&mut buffer.cur, item);
721        buffer.cur.put_slice(b"}}");
722        buffer.split()
723    }
724
725    /// The notification carrying `body`, JSON text this side wrote.
726    fn body(&mut self, buffer: &mut Buffer, body: &[u8]) -> Bytes {
727        buffer.room(self.prefix.len() + body.len() + 2);
728        buffer.cur.put_slice(&self.prefix);
729        buffer.cur.put_slice(body);
730        buffer.cur.put_slice(b"}}");
731        buffer.split()
732    }
733
734    /// `item` alone, for jsonrpsee's sink, which wraps it in the notification:
735    /// text of its own, its exact size, written once and not read again. The
736    /// copy and the check this took were most of the 6,644 instructions PERF-27
737    /// took off a one-item call's 50,842.
738    fn raw_item(&mut self, item: &PlexusStreamItem) -> Result<Box<RawValue>, serde_json::Error> {
739        if !matches!(item, PlexusStreamItem::Data { .. }) {
740            return serde_json::value::to_raw_value(item);
741        }
742        let mut text = BytesMut::new();
743        self.item(&mut text, item);
744        // This one stays: `Content::from_json` holds text to JSON's grammar, not to UTF-8.
745        let text = String::from_utf8(Vec::from(text)).map_err(serde::ser::Error::custom)?;
746        // SAFETY: one JSON value with nothing around it. `Frames::item` writes
747        // a `data` item's opening, then its content, then `}`. The opening is
748        // literal keys around values serde_json wrote into a `Vec`, which
749        // cannot fail. The content is one value: `encode`'s text, whole (a
750        // failed write leaves a `Value` instead); text `Content::from_json`
751        // parsed, by the same parse `RawValue::from_string` would run here; or
752        // a `Value`, which serde_json cannot fail to write.
753        // A debug build parses it here all the same (tests/no_frame_sink.rs).
754        Ok(unsafe { RawValue::from_string_unchecked(text) })
755    }
756}
757
758/// Pipe a call's stream to its accepted subscription: each item encoded
759/// once and written as one frame.
760///
761/// With a [`FrameSink`] in the call's extensions (plexus-transport's socket
762/// loop) each frame goes to the socket's writer as it is, the first fenced
763/// behind the subscription's answer; without one, each item goes to
764/// jsonrpsee's sink as one raw value. A `Request` item (an in-call
765/// question) is bound to this connection first (BIDIR-13). The pipe ends
766/// when the stream does, or when the caller lets go (an unsubscribe, a lost
767/// socket), dropping the stream at once.
768pub async fn pipe(sink: jsonrpsee::SubscriptionSink, stream: super::streaming::PlexusStream, ext: &http::Extensions) -> jsonrpsee::core::SubscriptionResult {
769    let frames = ext.get::<FrameSink>().cloned();
770    let encoding = frames.as_ref().map(FrameSink::encoding).unwrap_or_default();
771    let block = match &frames {
772        Some(to) => to.take_block(),
773        None => IDLE.try_with(Cell::take).ok().flatten(),
774    };
775    let buffer = Buffer::from_block(block);
776    let encoder = Encoder { encoding, buffer: RefCell::new(buffer), depth: Cell::new(0), last: Cell::new(0) };
777    WIRE.scope(encoder, run(sink, stream, frames)).await
778}
779
780async fn run(sink: jsonrpsee::SubscriptionSink, mut stream: super::streaming::PlexusStream, frames: Option<FrameSink>) -> jsonrpsee::core::SubscriptionResult {
781    use futures::StreamExt;
782    let connection = sink.connection_id().0;
783    let mut notification = match &frames {
784        Some(_) => Frames::new(sink.method_name(), &sink.subscription_id()),
785        None => Frames::default(),
786    };
787    let mut first = true;
788    // A batch's items, kept for the call: no allocation until two are ready together.
789    let mut ready: Vec<PlexusStreamItem> = Vec::new();
790    let out: jsonrpsee::core::SubscriptionResult = async {
791        loop {
792            let item = tokio::select! {
793                biased;
794                _ = sink.closed() => return Ok(()),
795                item = stream.next() => item,
796            };
797            let Some(item) = item else { return Ok(()) };
798            if let PlexusStreamItem::Request { request_id, .. } = &item {
799                super::bidirectional::bind_asker(request_id, connection);
800            }
801            match &frames {
802                Some(to) => {
803                    // PERF-8: what else the stream has ready now goes with it,
804                    // for a client that takes batches. Never what is not ready
805                    // yet: a lone item goes at once, in today's form.
806                    let mut ended = false;
807                    let mut held = if to.batch > 1 { size(&item) } else { BATCH_BYTES };
808                    let more = held < BATCH_BYTES;
809                    let bytes = match more.then(|| ready_now(&mut stream, &mut ended, connection)).flatten() {
810                        None => WIRE.with(|e| notification.frame(&mut e.buffer.borrow_mut(), &item)),
811                        Some(next) => {
812                            held += size(&next);
813                            ready.push(item);
814                            ready.push(next);
815                            while ready.len() < to.batch && held < BATCH_BYTES && !ended {
816                                match ready_now(&mut stream, &mut ended, connection) {
817                                    Some(more) => {
818                                        held += size(&more);
819                                        ready.push(more);
820                                    }
821                                    None => break,
822                                }
823                            }
824                            let bytes = WIRE.with(|e| notification.batch(&mut e.buffer.borrow_mut(), &ready));
825                            ready.clear();
826                            bytes
827                        }
828                    };
829                    if to.tx.send(WireFrame { bytes, fence: first }).await.is_err() || ended {
830                        return Ok(());
831                    }
832                }
833                None => {
834                    // jsonrpsee's sink takes a raw value, which it wraps in
835                    // the notification: a copy, the price of a transport
836                    // without frames of its own (stdio, a module called in memory).
837                    let raw = notification.raw_item(&item)?;
838                    drop(item);
839                    sink.send(raw).await?;
840                }
841            }
842            first = false;
843        }
844    }
845    .await;
846    drop(stream);
847    let buffer = WIRE.with(|e| std::mem::take(&mut *e.buffer.borrow_mut()));
848    match &frames {
849        Some(to) => {
850            if let Some(block) = buffer.into_block() {
851                to.give_block(block);
852            }
853        }
854        None => {
855            let _ = IDLE.try_with(|idle| idle.set(buffer.into_idle()));
856        }
857    }
858    out
859}
860
861/// About what an item writes into a batch: its encoded content, or a guess.
862fn size(item: &PlexusStreamItem) -> usize {
863    match item {
864        PlexusStreamItem::Data { content, .. } => content.json_bytes().map_or(256, Bytes::len),
865        _ => 128,
866    }
867}
868
869/// Whether two items' metadata write the same JSON: the same one, or equal
870/// (a child's items, relayed, each bring their own copy).
871fn same_metadata(a: &Arc<StreamMetadata>, b: &Arc<StreamMetadata>) -> bool {
872    Arc::ptr_eq(a, b) || (a.timestamp == b.timestamp && a.plexus_hash == b.plexus_hash && a.provenance == b.provenance)
873}
874
875/// The stream's next item if it is ready now: no waiting, so a batch never
876/// holds an item back for one that has not come. `ended` once the stream is.
877fn ready_now(stream: &mut super::streaming::PlexusStream, ended: &mut bool, connection: usize) -> Option<PlexusStreamItem> {
878    use futures::StreamExt;
879    match futures::FutureExt::now_or_never(stream.next()) {
880        Some(Some(item)) => {
881            if let PlexusStreamItem::Request { request_id, .. } = &item {
882                super::bidirectional::bind_asker(request_id, connection);
883            }
884            Some(item)
885        }
886        Some(None) => {
887            *ended = true;
888            None
889        }
890        None => None,
891    }
892}
893
894// ---------------------------------------------------------------------------
895// The pipe of a call with an input half
896// ---------------------------------------------------------------------------
897
898/// What a call with an input half answers: its stream, or (BIDIR-17's
899/// review) the route that will make it, still resolving because its handler
900/// reads its input first. An error is the item to send.
901pub(crate) enum Output<'a> {
902    Ready(super::streaming::PlexusStream),
903    Routing(std::pin::Pin<&'a mut (dyn std::future::Future<Output = Result<super::streaming::PlexusStream, Value>> + Send + 'a)>),
904}
905
906impl Output<'_> {
907    /// The next turn of the output half: an item, or the route's refusal.
908    async fn next(&mut self) -> Turn {
909        std::future::poll_fn(|cx| loop {
910            match self {
911                Output::Ready(stream) => return stream.as_mut().poll_next(cx).map(Turn::Item),
912                Output::Routing(route) => match route.as_mut().poll(cx) {
913                    std::task::Poll::Ready(Ok(stream)) => *self = Output::Ready(stream),
914                    std::task::Poll::Ready(Err(item)) => return std::task::Poll::Ready(Turn::Failed(item)),
915                    std::task::Poll::Pending => return std::task::Poll::Pending,
916                },
917            }
918        })
919        .await
920    }
921}
922
923/// [`pipe`] for a call with an input half ([`super::input`]): the subscription
924/// is already accepted, and its first frame is the first `credit`.
925pub(crate) async fn pipe_duplex(sink: jsonrpsee::SubscriptionSink, out: Output<'_>, duplex: super::input::Duplex) -> jsonrpsee::core::SubscriptionResult {
926    let encoding = duplex.to.encoding();
927    let buffer = Buffer::from_block(duplex.to.take_block());
928    let encoder = Encoder { encoding, buffer: RefCell::new(buffer), depth: Cell::new(0), last: Cell::new(0) };
929    WIRE.scope(encoder, run_duplex(sink, out, duplex)).await
930}
931
932/// One frame of the input half, as `body` says.
933async fn control(to: &FrameSink, notification: &mut Frames, body: &str, fence: bool) -> Result<(), ()> {
934    let bytes = WIRE.with(|e| notification.body(&mut e.buffer.borrow_mut(), body.as_bytes()));
935    to.tx.send(WireFrame { bytes, fence }).await.map_err(drop)
936}
937
938/// `input_closed`: the half is over, said once, before anything final.
939fn input_closed(reason: &str, ack: Option<u64>) -> String {
940    match ack {
941        Some(seq) => format!(r#"{{"type":"input_closed","reason":{},"ack":{seq}}}"#, Value::from(reason)),
942        None => format!(r#"{{"type":"input_closed","reason":{}}}"#, Value::from(reason)),
943    }
944}
945
946/// What the loop below learned this turn.
947enum Turn {
948    /// The method's next item.
949    Item(Option<PlexusStreamItem>),
950    /// Nothing to write but what the ledger owes.
951    Moved,
952    /// The call is refused.
953    Refused(super::duplex_gate::DuplexRefusal),
954    /// The route answered an error: this item.
955    Failed(Value),
956}
957
958async fn run_duplex(sink: jsonrpsee::SubscriptionSink, mut output: Output<'_>, duplex: super::input::Duplex) -> jsonrpsee::core::SubscriptionResult {
959    use futures::StreamExt;
960    let super::input::Duplex { ledger, mut gate, inbox, credit, ns, to, slot } = duplex;
961    let mut inbox = Some(inbox);
962    let mut slot = Some(slot);
963    let mut notification = Frames::new(sink.method_name(), &sink.subscription_id());
964    let connection = sink.connection_id().0;
965    let mut acked = 0u64;
966    let out: Result<(), ()> = async {
967        // Fenced behind the subscription's answer (the client holds nothing
968        // yet: it may read this before it has the id).
969        control(&to, &mut notification, &format!(r#"{{"type":"credit","n":{}}}"#, ledger.window()), true).await?;
970        loop {
971            let turn = tokio::select! {
972                biased;
973                _ = sink.closed() => return Ok(()),
974                frame = gate.next(), if inbox.is_some() => match frame {
975                    Some(Ok(batch)) => {
976                        // The handler let go of its input: the next turn says so.
977                        let _ = inbox.as_ref().expect("guarded").send(batch);
978                        Turn::Moved
979                    }
980                    Some(Err(why)) => Turn::Refused(why),
981                    None => {
982                        inbox = None;
983                        Turn::Moved
984                    }
985                },
986                _ = ledger.woken() => Turn::Moved,
987                turn = output.next() => turn,
988            };
989            if let Turn::Failed(item) = turn {
990                if slot.take().is_some() && !ledger.client_ended() {
991                    control(&to, &mut notification, &input_closed("method_ended", ledger.last_done()), false).await?;
992                }
993                return control(&to, &mut notification, &serde_json::to_string(&item).unwrap_or_default(), false).await;
994            }
995            let (item, refused) = match turn {
996                Turn::Failed(_) => unreachable!("answered above"),
997                Turn::Refused(why) => (None, Some(why)),
998                Turn::Item(item) => (Some(item), ledger.take_refusal()),
999                Turn::Moved => (None, ledger.take_refusal()),
1000            };
1001            if let Some(owed) = ledger.owed(acked) {
1002                if let Some(seq) = owed.ack {
1003                    control(&to, &mut notification, &format!(r#"{{"type":"ack","seq":{seq}}}"#), false).await?;
1004                }
1005                acked = owed.done;
1006                if owed.credit > 0 {
1007                    credit.grant(owed.credit);
1008                    control(&to, &mut notification, &format!(r#"{{"type":"credit","n":{}}}"#, owed.credit), false).await?;
1009                }
1010            }
1011            if let Some(why) = refused {
1012                // The handler's input ends with the refusal, and the call
1013                // with it: nothing the handler says after it goes out.
1014                if slot.take().is_some() && !ledger.client_ended() {
1015                    control(&to, &mut notification, &input_closed(why.kind(), ledger.last_done()), false).await?;
1016                }
1017                let item = serde_json::to_string(&why.to_item(&ns)).unwrap_or_default();
1018                return control(&to, &mut notification, &item, false).await;
1019            }
1020            if ledger.abandoned() && slot.is_some() && !ledger.client_ended() {
1021                slot = None;
1022                inbox = None;
1023                control(&to, &mut notification, &input_closed("method_ended", ledger.last_done()), false).await?;
1024            }
1025            let Some(item) = item else { continue };
1026            let Some(item) = item else { return Ok(()) };
1027            match &item {
1028                PlexusStreamItem::Request { request_id, .. } => {
1029                    super::bidirectional::bind_asker(request_id, connection);
1030                }
1031                // After `done` there is no channel to tell anything on.
1032                PlexusStreamItem::Done { .. } | PlexusStreamItem::Error { .. } => {
1033                    inbox = None;
1034                    if slot.take().is_some() && !ledger.client_ended() {
1035                        control(&to, &mut notification, &input_closed("method_ended", ledger.last_done()), false).await?;
1036                    }
1037                }
1038                _ => {}
1039            }
1040            let bytes = WIRE.with(|e| notification.frame(&mut e.buffer.borrow_mut(), &item));
1041            drop(item);
1042            to.tx.send(WireFrame { bytes, fence: false }).await.map_err(drop)?;
1043        }
1044    }
1045    .await;
1046    drop(output);
1047    let _ = out; // Err: the socket is gone, as the call's end says.
1048    if let Some(block) = WIRE.with(|e| std::mem::take(&mut *e.buffer.borrow_mut())).into_block() {
1049        to.give_block(block);
1050    }
1051    Ok(())
1052}
1053
1054// ---------------------------------------------------------------------------
1055// Shared: one event, many streams
1056// ---------------------------------------------------------------------------
1057
1058/// An event many streams send — one change fanned out to every watcher.
1059/// The first socket that sends it encodes it, once; every other writes
1060/// those bytes as they are (PERF-7's fan-out). Anywhere else (in process, a
1061/// `Value`, another encoding) it serializes as `T` does. Reads as `T`.
1062///
1063/// An event that carries a credential is not kept encoded: each stream
1064/// encodes it, so each assembles its own `_credentials` envelope.
1065pub struct Shared<T>(Arc<SharedInner<T>>);
1066
1067struct SharedInner<T> {
1068    value: T,
1069    /// The JSON the first socket wrote (`None`: never kept — a credential).
1070    json: OnceLock<Option<Box<RawValue>>>,
1071}
1072
1073impl<T> Shared<T> {
1074    /// `value`, to be sent by many streams.
1075    pub fn new(value: T) -> Self {
1076        Shared(Arc::new(SharedInner { value, json: OnceLock::new() }))
1077    }
1078
1079    /// Whether this event has been encoded (a socket has sent it).
1080    pub fn is_encoded(&self) -> bool {
1081        matches!(self.0.json.get(), Some(Some(_)))
1082    }
1083}
1084
1085impl<T: Serialize> Shared<T> {
1086    fn encoded(&self) -> Option<&RawValue> {
1087        self.0
1088            .json
1089            .get_or_init(|| {
1090                let (text, captured) = plexus_auth_core::credential::run_with_credential_capture(|| serde_json::to_string(&self.0.value));
1091                match text {
1092                    Ok(text) if captured.is_empty() => RawValue::from_string(text).ok(),
1093                    _ => None,
1094                }
1095            })
1096            .as_deref()
1097    }
1098}
1099
1100impl<T> Clone for Shared<T> {
1101    fn clone(&self) -> Self {
1102        Shared(self.0.clone())
1103    }
1104}
1105
1106impl<T> std::ops::Deref for Shared<T> {
1107    type Target = T;
1108    fn deref(&self) -> &T {
1109        &self.0.value
1110    }
1111}
1112
1113impl<T: std::fmt::Debug> std::fmt::Debug for Shared<T> {
1114    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1115        self.0.value.fmt(f)
1116    }
1117}
1118
1119impl<T: Default> Default for Shared<T> {
1120    fn default() -> Self {
1121        Shared::new(T::default())
1122    }
1123}
1124
1125impl<T: Serialize> Serialize for Shared<T> {
1126    fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
1127        if writing_json() {
1128            if let Some(raw) = self.encoded() {
1129                return raw.serialize(s);
1130            }
1131        }
1132        self.0.value.serialize(s)
1133    }
1134}
1135
1136impl<'de, T: Deserialize<'de>> Deserialize<'de> for Shared<T> {
1137    fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
1138        T::deserialize(d).map(Shared::new)
1139    }
1140}
1141
1142impl<T: schemars::JsonSchema> schemars::JsonSchema for Shared<T> {
1143    fn inline_schema() -> bool {
1144        T::inline_schema()
1145    }
1146    fn schema_name() -> Cow<'static, str> {
1147        T::schema_name()
1148    }
1149    fn schema_id() -> Cow<'static, str> {
1150        T::schema_id()
1151    }
1152    fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema {
1153        T::json_schema(generator)
1154    }
1155}
1156
1157#[cfg(test)]
1158mod tests {
1159    use std::collections::{BTreeMap, HashMap};
1160
1161    use super::*;
1162
1163    #[test]
1164    fn content_reads_as_its_value_either_way() {
1165        let v = serde_json::json!({"b": [1, 2], "a": "x"});
1166        let encoded = Content::from_json(Bytes::from_static(br#"{"b":[1,2],"a":"x"}"#)).unwrap();
1167        let valued = Content::from(v.clone());
1168        assert!(encoded.is_encoded() && !valued.is_encoded());
1169        assert_eq!(encoded, v);
1170        assert_eq!(valued, v);
1171        assert_eq!(encoded, valued);
1172        assert_eq!(encoded["a"], "x");
1173        assert_eq!(encoded.decode::<serde_json::Map<String, Value>>().unwrap().len(), 2);
1174        assert_eq!(serde_json::to_value(&encoded).unwrap(), v);
1175        assert_eq!(serde_json::to_string(&encoded).unwrap(), serde_json::to_string(&v).unwrap(), "outside the pipe it writes its Value");
1176        assert_eq!(encoded.into_value(), v);
1177        assert!(Content::from_json(Bytes::from_static(b"{nope")).is_err());
1178    }
1179
1180    /// PERF-27: a block a call was half of what a one-item call allocated on
1181    /// a transport with no frame sink, which has no connection to keep one.
1182    #[test]
1183    fn a_call_with_no_connection_leaves_its_block_whole_for_the_next() {
1184        let mut b = Buffer::default();
1185        b.room(300);
1186        b.cur.put_slice(&[1u8; 300]);
1187        drop(b.split());
1188        let block = b.into_idle().expect("a block this size is kept");
1189        assert!(block.capacity() >= Buffer::FIRST, "all of it again, not what was left: {}", block.capacity());
1190        let mut next = Buffer::from_block(Some(block));
1191        let kept = next.cur.as_ptr();
1192        next.room(300);
1193        assert_eq!(next.cur.as_ptr(), kept, "the next call makes none");
1194    }
1195
1196    /// One a stream grew is not kept, or a thread holds it for good.
1197    #[test]
1198    fn a_block_too_large_to_keep_goes() {
1199        let mut b = Buffer::default();
1200        b.room(Buffer::POOLED + 1);
1201        assert!(b.into_idle().is_none());
1202    }
1203
1204    /// A stream that outruns its socket holds the queue's 1,024 frames in
1205    /// flight. The blocks in rotation must hold them, or every block filled is
1206    /// a new one: at 4 KB an item (8.6 KB of block each, content and frame) a
1207    /// fresh block every 60 items, at a ticket body one every 15, for the
1208    /// stream's life. Red with 3 spares.
1209    #[test]
1210    fn a_queue_full_of_frames_comes_back_to_blocks_already_made() {
1211        for size in [8_600, 35_500] {
1212            let mut b = Buffer::default();
1213            let mut in_flight = std::collections::VecDeque::new();
1214            let item = vec![7u8; size];
1215            let mut ramp = 0;
1216            for n in 0..40_000 {
1217                b.room(item.len());
1218                b.cur.put_slice(&item);
1219                in_flight.push_back(b.split());
1220                if in_flight.len() > 1024 {
1221                    in_flight.pop_front();
1222                }
1223                if n == 20_000 {
1224                    ramp = b.made;
1225                }
1226            }
1227            assert_eq!(b.made, ramp, "at {size} B: past the ramp ({ramp} blocks), every block is one already made");
1228        }
1229    }
1230
1231    /// Those blocks are for the burst: a watch that lives on after one holds
1232    /// what it held before it.
1233    #[test]
1234    fn a_drained_queue_lets_its_blocks_go() {
1235        let mut b = Buffer::default();
1236        let mut in_flight = Vec::new();
1237        for _ in 0..4_000 {
1238            b.room(35_500);
1239            b.cur.put_slice(&[7u8; 35_500]);
1240            in_flight.push(b.split());
1241        }
1242        assert_eq!(b.spare.len(), Buffer::SPARE, "the burst keeps them");
1243        drop(in_flight);
1244        for _ in 0..200 {
1245            b.room(35_500);
1246            b.cur.put_slice(&[7u8; 35_500]);
1247            drop(b.split());
1248        }
1249        assert!(b.spare.len() <= Buffer::DRAINED, "drained, it keeps {}", b.spare.len());
1250    }
1251
1252    #[test]
1253    fn a_buffer_reuses_a_block_once_its_frames_are_gone() {
1254        let mut b = Buffer::default();
1255        b.room(100);
1256        b.cur.put_slice(&[1u8; 100]);
1257        let first = b.split();
1258        let block = b.cur.capacity() + 100;
1259        for _ in 0..1000 {
1260            b.room(100);
1261            b.cur.put_slice(&[2u8; 100]);
1262            drop(b.split());
1263        }
1264        assert_eq!(first.len(), 100, "a frame in flight stays whole");
1265        assert!(b.block <= Buffer::FIRST * 2, "blocks are reused, not grown, when frames are written: {} (first {block})", b.block);
1266    }
1267
1268    // -----------------------------------------------------------------------
1269    // PERF-34: the writer, held to serde_json
1270    // -----------------------------------------------------------------------
1271
1272    /// What the pipe's writer writes for `value` against `serde_json::to_vec`:
1273    /// the same bytes, or a refusal where it refuses.
1274    #[track_caller]
1275    fn same_as_serde_json<T: Serialize + ?Sized>(value: &T) {
1276        let _raw = WritingJson::on();
1277        let want = serde_json::to_vec(value);
1278        let mut got = BytesMut::new();
1279        let wrote = json::write(&mut got, value);
1280        match want {
1281            Ok(want) => {
1282                assert!(wrote.is_ok(), "serde_json wrote {}, the writer refused: {:?}", String::from_utf8_lossy(&want), wrote.err());
1283                assert_eq!(String::from_utf8_lossy(&got), String::from_utf8_lossy(&want));
1284                assert_eq!(&got[..], &want[..]);
1285            }
1286            Err(e) => assert!(wrote.is_err(), "serde_json refused ({e}), the writer wrote {}", String::from_utf8_lossy(&got)),
1287        }
1288    }
1289
1290    /// A seeded xorshift: a property test with no dependency, the same cases
1291    /// every run, and a failure that names what it failed on.
1292    struct Rng(u64);
1293
1294    impl Rng {
1295        fn next(&mut self) -> u64 {
1296            let mut x = self.0;
1297            x ^= x << 13;
1298            x ^= x >> 7;
1299            x ^= x << 17;
1300            self.0 = x;
1301            x
1302        }
1303        fn below(&mut self, n: usize) -> usize {
1304            (self.next() % n as u64) as usize
1305        }
1306    }
1307
1308    /// Everything a string can be, weighted to what breaks a scan: runs of
1309    /// plain text around each block size, every control character, quotes and
1310    /// backslashes alone and together, text that looks escaped and is not, and
1311    /// everything past ASCII, each width.
1312    fn a_string(r: &mut Rng) -> String {
1313        const PIECES: &[&str] = &[
1314            "\"", "\\", "\\\\", "\\\"", "\"\"", "\n", "\r", "\t", "\u{8}", "\u{c}", "\u{0}", "\u{1}", "\u{1f}", "\u{7f}", "\u{80}", "\u{ff}", "/", "é", "→", "—", "\u{2028}", "\u{2029}", "\u{feff}", "𝄞", "😀", "\u{10ffff}",
1315            "\\n", "\\u00e9", "\\ud83d", "\\x41", "\\", "\\u", "\\u12", "u0041", "<m-field label=\"x\">", "`code`", "{\"json\":\"in a string\"}", "null", " ",
1316        ];
1317        let mut s = String::new();
1318        for _ in 0..r.below(12) {
1319            match r.below(4) {
1320                0 => s.push_str(PIECES[r.below(PIECES.len())]),
1321                1 => s.extend(std::iter::repeat_n('x', r.below(70))),
1322                2 => s.push(char::from(r.below(128) as u8)),
1323                _ => s.push(char::from_u32(r.next() as u32 % 0x11_0000).unwrap_or('\u{fffd}')),
1324            }
1325        }
1326        s
1327    }
1328
1329    #[derive(Serialize)]
1330    #[serde(rename_all = "snake_case")]
1331    enum External {
1332        Unit,
1333        Newtype(String),
1334        Tuple(u8, String),
1335        Struct { a: u8, b: String },
1336    }
1337
1338    #[derive(Serialize)]
1339    #[serde(tag = "type", rename_all = "snake_case")]
1340    enum Internal {
1341        Unit,
1342        Newtype(Inner),
1343        Struct { n: u64, text: String, maybe: Option<String>, list: Vec<String> },
1344    }
1345
1346    #[derive(Serialize)]
1347    #[serde(tag = "t", content = "c")]
1348    enum Adjacent {
1349        Unit,
1350        Newtype(String),
1351        Tuple(u8, u8),
1352        Struct { s: String },
1353    }
1354
1355    #[derive(Serialize)]
1356    #[serde(untagged)]
1357    enum Untagged {
1358        Number(u8),
1359        Text(String),
1360        Fields { s: String },
1361    }
1362
1363    #[derive(Serialize, Clone)]
1364    struct Inner {
1365        s: String,
1366        #[serde(rename = "a key with a \"quote\" and a \n")]
1367        renamed: u8,
1368        #[serde(skip_serializing_if = "Option::is_none")]
1369        skipped: Option<u8>,
1370    }
1371
1372    /// A field's name that wants a look: the first, the next, and one past a block.
1373    #[derive(Serialize)]
1374    struct Names {
1375        #[serde(rename = "a \"first\" name\t")]
1376        first: u8,
1377        #[serde(rename = "a \\next\\ name")]
1378        next: u8,
1379        a_name_that_is_longer_than_a_block_of_sixteen: u8,
1380        #[serde(rename = "")]
1381        empty: u8,
1382        #[serde(rename = "é")]
1383        past_ascii: u8,
1384    }
1385
1386    #[derive(Serialize)]
1387    struct Flat {
1388        a: u8,
1389        #[serde(flatten)]
1390        inner: Inner,
1391        #[serde(flatten)]
1392        rest: BTreeMap<String, Value>,
1393    }
1394
1395    #[derive(Serialize)]
1396    struct Nothing;
1397
1398    #[derive(Serialize)]
1399    struct Wrapped(String);
1400
1401    #[derive(Serialize)]
1402    struct Pair(u8, String);
1403
1404    #[derive(Serialize, PartialEq, Eq, PartialOrd, Ord)]
1405    #[serde(rename_all = "snake_case")]
1406    enum KeyName {
1407        First,
1408    }
1409
1410    #[derive(Serialize, PartialEq, Eq, PartialOrd, Ord)]
1411    struct WrappedKey(String);
1412
1413    /// `serialize_bytes`, as `serde_bytes` calls it.
1414    struct Blob(&'static [u8]);
1415
1416    impl Serialize for Blob {
1417        fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
1418            s.serialize_bytes(self.0)
1419        }
1420    }
1421
1422    /// `collect_str`, in pieces.
1423    struct Pieces(String);
1424
1425    impl std::fmt::Display for Pieces {
1426        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1427            for (i, piece) in self.0.split('x').enumerate() {
1428                if i > 0 {
1429                    f.write_str("x\"")?;
1430                }
1431                f.write_str(piece)?;
1432            }
1433            Ok(())
1434        }
1435    }
1436
1437    impl Serialize for Pieces {
1438        fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
1439            s.collect_str(self)
1440        }
1441    }
1442
1443    /// A length that lies: serde_json writes what it is told, so the writer does.
1444    struct Lies(Option<usize>, u8);
1445
1446    impl Serialize for Lies {
1447        fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
1448            use serde::ser::{SerializeMap, SerializeSeq};
1449            if self.1 == 0 {
1450                let mut seq = s.serialize_seq(self.0)?;
1451                seq.serialize_element(&1)?;
1452                seq.serialize_element(&2)?;
1453                seq.end()
1454            } else {
1455                let mut map = s.serialize_map(self.0)?;
1456                map.serialize_entry("a", &1)?;
1457                map.serialize_entry("b", &2)?;
1458                map.end()
1459            }
1460        }
1461    }
1462
1463    /// A `Serialize` that fails partway, as a credential's or a user's can.
1464    struct Fails;
1465
1466    impl Serialize for Fails {
1467        fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
1468            use serde::ser::{Error, SerializeStruct};
1469            let mut st = s.serialize_struct("Fails", 2)?;
1470            st.serialize_field("written", "so far")?;
1471            st.end()?;
1472            Err(S::Error::custom("no"))
1473        }
1474    }
1475
1476    #[test]
1477    fn same_bytes_as_serde_json_on_every_shape() {
1478        same_as_serde_json(&());
1479        same_as_serde_json(&Nothing);
1480        same_as_serde_json(&Wrapped("x\"".into()));
1481        same_as_serde_json(&Pair(1, "a\n".into()));
1482        same_as_serde_json(&(1, "two", 3.5, true, None::<u8>, Some('c')));
1483        same_as_serde_json(&Vec::<u8>::new());
1484        same_as_serde_json(&Vec::<Vec<u8>>::from([vec![], vec![1], vec![]]));
1485        same_as_serde_json(&[(); 2]);
1486        same_as_serde_json(&BTreeMap::<String, u8>::new());
1487        same_as_serde_json(&BTreeMap::from([("a\"b".to_string(), BTreeMap::from([(1, 2)]))]));
1488        same_as_serde_json(&Blob(b""));
1489        same_as_serde_json(&Blob(b"\x00\xffabc"));
1490        same_as_serde_json(&'x');
1491        same_as_serde_json(&'"');
1492        same_as_serde_json(&'\u{1}');
1493        same_as_serde_json(&'𝄞');
1494        same_as_serde_json(&std::net::IpAddr::from([127, 0, 0, 1]));
1495        same_as_serde_json(&std::time::Duration::from_millis(1500));
1496        same_as_serde_json(&std::marker::PhantomData::<u8>);
1497        same_as_serde_json(&Ok::<u8, String>(1));
1498        same_as_serde_json(&Err::<u8, String>("no".into()));
1499        same_as_serde_json(&std::borrow::Cow::Borrowed("a\tb"));
1500        same_as_serde_json(&Arc::<str>::from("shared\n"));
1501        for lie in [None, Some(0), Some(1), Some(5)] {
1502            same_as_serde_json(&Lies(lie, 0));
1503            same_as_serde_json(&Lies(lie, 1));
1504        }
1505        same_as_serde_json(&Fails);
1506        same_as_serde_json(&vec![Some(Fails)]);
1507        same_as_serde_json(&External::Unit);
1508        same_as_serde_json(&External::Newtype("n\"".into()));
1509        same_as_serde_json(&External::Tuple(1, "t".into()));
1510        same_as_serde_json(&External::Struct { a: 1, b: "b".into() });
1511        same_as_serde_json(&Internal::Unit);
1512        same_as_serde_json(&Adjacent::Unit);
1513        same_as_serde_json(&Adjacent::Newtype("n".into()));
1514        same_as_serde_json(&Adjacent::Tuple(1, 2));
1515        same_as_serde_json(&Adjacent::Struct { s: "s".into() });
1516        same_as_serde_json(&Untagged::Number(1));
1517        same_as_serde_json(&Untagged::Text("t".into()));
1518        same_as_serde_json(&Untagged::Fields { s: "s".into() });
1519        same_as_serde_json(&Names { first: 1, next: 2, a_name_that_is_longer_than_a_block_of_sixteen: 3, empty: 4, past_ascii: 5 });
1520    }
1521
1522    #[test]
1523    fn same_bytes_as_serde_json_on_every_number() {
1524        same_as_serde_json(&[u8::MIN, u8::MAX]);
1525        same_as_serde_json(&[u16::MAX]);
1526        same_as_serde_json(&[u32::MAX]);
1527        same_as_serde_json(&[u64::MIN, 1, 9, 10, 99, 100, u64::MAX]);
1528        same_as_serde_json(&[u128::MAX]);
1529        same_as_serde_json(&[i8::MIN, i8::MAX]);
1530        same_as_serde_json(&[i16::MIN]);
1531        same_as_serde_json(&[i32::MIN]);
1532        same_as_serde_json(&[i64::MIN, -1, 0, i64::MAX]);
1533        same_as_serde_json(&[i128::MIN, i128::MAX]);
1534        for f in [0.0, -0.0, 0.1, 1.0, 1.5, -1.5, 1e7, 1e15, 1e16, 1e20, 1e21, 1e22, 1e100, 1e-5, 1e-7, 1e-100, 5e-324, 2.2250738585072014e-308, f64::MAX, f64::MIN, f64::EPSILON, f64::NAN, f64::INFINITY, f64::NEG_INFINITY] {
1535            same_as_serde_json(&f);
1536            same_as_serde_json(&(f as f32));
1537        }
1538        let mut r = Rng(0x9E37_79B9_7F4A_7C15);
1539        for _ in 0..20_000 {
1540            let bits = r.next();
1541            same_as_serde_json(&f64::from_bits(bits));
1542            same_as_serde_json(&f32::from_bits(bits as u32));
1543            same_as_serde_json(&(bits as i64));
1544        }
1545    }
1546
1547    /// serde_json's map keys: a string, or what reads as one; anything else
1548    /// refused by both.
1549    #[test]
1550    fn same_bytes_as_serde_json_on_every_key() {
1551        same_as_serde_json(&BTreeMap::from([(1u8, 0)]));
1552        same_as_serde_json(&BTreeMap::from([(-1i64, 0)]));
1553        same_as_serde_json(&BTreeMap::from([(u128::MAX, 0)]));
1554        same_as_serde_json(&BTreeMap::from([(i128::MIN, 0)]));
1555        same_as_serde_json(&BTreeMap::from([(true, 0)]));
1556        same_as_serde_json(&BTreeMap::from([('"', 0)]));
1557        same_as_serde_json(&BTreeMap::from([("a key\n", 0)]));
1558        same_as_serde_json(&BTreeMap::from([(KeyName::First, 0)]));
1559        same_as_serde_json(&BTreeMap::from([(WrappedKey("a\"".into()), 0)]));
1560        same_as_serde_json(&BTreeMap::from([(Some(1), 0)]));
1561        same_as_serde_json(&BTreeMap::from([(None::<u8>, 0)]));
1562        same_as_serde_json(&BTreeMap::from([(vec![1], 0)]));
1563        same_as_serde_json(&BTreeMap::from([((), 0)]));
1564        same_as_serde_json(&BTreeMap::from([((1, 2), 0)]));
1565        same_as_serde_json(&HashMap::from([(1.5f64.to_string(), 0)]));
1566        for key in [1.5f64, -0.0, 1e21, f64::NAN, f64::INFINITY] {
1567            same_as_serde_json(&FloatKey(key));
1568            same_as_serde_json(&FloatKey32(key as f32));
1569        }
1570        same_as_serde_json(&PiecesKey("axb\n".into()));
1571        same_as_serde_json(&BlobKey);
1572    }
1573
1574    struct FloatKey(f64);
1575
1576    impl Serialize for FloatKey {
1577        fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
1578            use serde::ser::SerializeMap;
1579            let mut map = s.serialize_map(Some(1))?;
1580            map.serialize_entry(&self.0, &0)?;
1581            map.end()
1582        }
1583    }
1584
1585    struct FloatKey32(f32);
1586
1587    impl Serialize for FloatKey32 {
1588        fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
1589            use serde::ser::SerializeMap;
1590            let mut map = s.serialize_map(Some(1))?;
1591            map.serialize_entry(&self.0, &0)?;
1592            map.end()
1593        }
1594    }
1595
1596    struct PiecesKey(String);
1597
1598    impl Serialize for PiecesKey {
1599        fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
1600            use serde::ser::SerializeMap;
1601            let mut map = s.serialize_map(Some(1))?;
1602            map.serialize_entry(&Pieces(self.0.clone()), &0)?;
1603            map.end()
1604        }
1605    }
1606
1607    struct BlobKey;
1608
1609    impl Serialize for BlobKey {
1610        fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
1611            use serde::ser::SerializeMap;
1612            let mut map = s.serialize_map(None)?;
1613            map.serialize_entry(&Blob(b"no"), &0)?;
1614            map.end()
1615        }
1616    }
1617
1618    /// A raw value's text goes as it is, whatever is in it; a `Content` and a
1619    /// `Shared`, encoded, are raw values to the writer.
1620    #[test]
1621    fn same_bytes_as_serde_json_on_raw_values() {
1622        #[derive(Serialize)]
1623        struct Holds {
1624            raw: Box<RawValue>,
1625            content: Content,
1626            shared: Shared<Inner>,
1627        }
1628        let raw = |text: &str| RawValue::from_string(text.to_string()).unwrap();
1629        same_as_serde_json(&raw(r#"{ "spaced" : [1, 2.50, "é"] }"#));
1630        same_as_serde_json(&raw("null"));
1631        same_as_serde_json(&Some(raw("[]")));
1632        same_as_serde_json(&vec![raw("1"), raw("\"two\"")]);
1633        same_as_serde_json(&BTreeMap::from([("k", raw("{}"))]));
1634        same_as_serde_json(&BTreeMap::from([(raw("\"a raw key\""), 0)].map(|(k, v)| (k.get().to_string(), v))));
1635        let inner = Inner { s: "s\"".into(), renamed: 1, skipped: None };
1636        let held = Holds {
1637            raw: raw("[ 1 ]"),
1638            content: Content::from_json(Bytes::from_static(br#"{"b":[1,2], "a":"x"}"#)).unwrap(),
1639            shared: Shared::new(inner.clone()),
1640        };
1641        same_as_serde_json(&held);
1642        same_as_serde_json(&Content::from(serde_json::json!({"a": 1})));
1643        same_as_serde_json(&Shared::new(inner));
1644        // a tag around a raw value, which serde_json refuses
1645        #[derive(Serialize)]
1646        #[serde(tag = "type")]
1647        enum Tagged {
1648            Raw(Box<RawValue>),
1649        }
1650        same_as_serde_json(&Tagged::Raw(raw("{}")));
1651    }
1652
1653    #[test]
1654    fn same_bytes_as_serde_json_on_every_string() {
1655        let mut r = Rng(0x2545_F491_4F6C_DD1D);
1656        for _ in 0..200_000 {
1657            let s = a_string(&mut r);
1658            same_as_serde_json(&s);
1659            same_as_serde_json(&Pieces(s));
1660        }
1661        for len in 0..300 {
1662            same_as_serde_json(&"x".repeat(len));
1663            same_as_serde_json(&"é".repeat(len));
1664            same_as_serde_json(&"\"".repeat(len));
1665            same_as_serde_json(&"\u{1}".repeat(len));
1666        }
1667        // a string that grows sixfold, past any room made for its length
1668        same_as_serde_json(&"\u{1f}".repeat(1 << 20));
1669    }
1670
1671    /// Every shape with the strings of [`a_string`] in it: keys included.
1672    #[test]
1673    fn same_bytes_as_serde_json_on_random_events() {
1674        let mut r = Rng(0x9E37_79B9_7F4A_7C15);
1675        for _ in 0..20_000 {
1676            let (a, b, c) = (a_string(&mut r), a_string(&mut r), a_string(&mut r));
1677            same_as_serde_json(&Internal::Struct { n: r.next(), text: a.clone(), maybe: (r.below(2) == 0).then(|| b.clone()), list: vec![c.clone(), a.clone()] });
1678            same_as_serde_json(&Internal::Newtype(Inner { s: b.clone(), renamed: r.next() as u8, skipped: (r.below(2) == 0).then_some(1) }));
1679            same_as_serde_json(&Flat { a: 1, inner: Inner { s: c.clone(), renamed: 2, skipped: None }, rest: BTreeMap::from([(a.clone(), Value::String(b.clone()))]) });
1680            let value = serde_json::json!({ (a.clone()): [b.clone(), null, 1, -1, 1.5, true, { (c.clone()): {} }, []] });
1681            same_as_serde_json(&value);
1682            same_as_serde_json(&Content::from(value));
1683            same_as_serde_json(&BTreeMap::from([(a, b), (c, String::new())]));
1684        }
1685    }
1686
1687    /// PERF-34: a hit at every position, around each block size, by each scan.
1688    #[test]
1689    fn scan_at_every_offset() {
1690        const ESCAPED: &[u8] = &[b'"', b'\\', 0x00, 0x01, 0x08, b'\t', b'\n', 0x0c, b'\r', 0x1f];
1691        const PLAIN: &[u8] = &[b'x', b' ', b'!', b'#', b'/', b'[', b']', b'^', 0x7f, 0x80, 0xc3, 0xff];
1692        let mut scans: Vec<(&str, unsafe fn(&[u8], *mut u8) -> usize)> = vec![("a byte at a time", json::scan::one_at_a_time), ("the best this CPU has", json::scan::copy_clean)];
1693        #[cfg(target_arch = "x86_64")]
1694        scans.push(("a CPU without AVX2", json::scan::x86::sse2));
1695        for (name, scan) in scans {
1696            for len in 0..=130usize {
1697                for fill in PLAIN {
1698                    let mut text = vec![*fill; len];
1699                    let mut copy = vec![0u8; len];
1700                    // SAFETY: `copy` is as long as `text`.
1701                    assert_eq!(unsafe { scan(&text, copy.as_mut_ptr()) }, len, "{name}: {len} of {fill:#x}");
1702                    assert_eq!(copy, text, "{name}: {len} of {fill:#x}, copied");
1703                    for at in 0..len {
1704                        for escaped in ESCAPED {
1705                            text[at] = *escaped;
1706                            // SAFETY: as above.
1707                            assert_eq!(unsafe { scan(&text, copy.as_mut_ptr()) }, at, "{name}: {escaped:#x} at {at} of {len}");
1708                            assert_eq!(&copy[..at], &text[..at], "{name}: the run before {escaped:#x} at {at} of {len}");
1709                            // the first of two
1710                            if at + 1 < len {
1711                                text[at + 1] = b'"';
1712                                // SAFETY: as above.
1713                                assert_eq!(unsafe { scan(&text, copy.as_mut_ptr()) }, at, "{name}: the first of two at {at} of {len}");
1714                                text[at + 1] = *fill;
1715                            }
1716                            text[at] = *fill;
1717                        }
1718                    }
1719                }
1720            }
1721        }
1722    }
1723
1724    // -----------------------------------------------------------------------
1725    // PERF-8: batches, for a client that offered them
1726    // -----------------------------------------------------------------------
1727
1728    /// A content type every test item carries.
1729    const CT: &str = "test.count";
1730
1731    fn meta(stamp: i64) -> Arc<StreamMetadata> {
1732        Arc::new(StreamMetadata { provenance: vec!["test".into()], plexus_hash: "ab".into(), timestamp: stamp })
1733    }
1734
1735    /// A data item as the pipe's encode makes one: its content already JSON.
1736    fn data(m: &Arc<StreamMetadata>, n: u64) -> PlexusStreamItem {
1737        let content = Content::json(Bytes::from(serde_json::to_vec(&serde_json::json!({"type": "item", "n": n})).unwrap()));
1738        PlexusStreamItem::Data { metadata: m.clone(), content_type: Cow::Borrowed(CT), content }
1739    }
1740
1741    fn done(stamp: i64) -> PlexusStreamItem {
1742        PlexusStreamItem::Done { metadata: StreamMetadata { provenance: vec!["test".into()], plexus_hash: "ab".into(), timestamp: stamp } }
1743    }
1744
1745    /// Every frame `stream` makes down the pipe, to a sink batching `batch`,
1746    /// as JSON, until its terminal item; `wait` once the stream stops for good.
1747    fn frames_of(stream: super::super::streaming::PlexusStream, batch: usize) -> Vec<Value> {
1748        let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap();
1749        rt.block_on(async move {
1750            let (to, mut frames) = FrameSink::channel(1024, WireEncoding::Json);
1751            let to = to.batching(batch);
1752            let stream = Arc::new(Mutex::new(Some(stream)));
1753            let mut module = jsonrpsee::RpcModule::new(());
1754            module
1755                .register_subscription("open", "note", "close", move |_, pending, _, mut ext| {
1756                    let (to, stream) = (to.clone(), stream.lock().unwrap().take());
1757                    async move {
1758                        let sink = pending.accept().await?;
1759                        ext.insert(to);
1760                        pipe(sink, stream.expect("one call"), &ext).await
1761                    }
1762                })
1763                .unwrap();
1764            let (_answer, _notes) = module.raw_json_request(r#"{"jsonrpc":"2.0","id":1,"method":"open","params":[]}"#, 1).await.unwrap();
1765            let mut out = Vec::new();
1766            loop {
1767                let frame = tokio::time::timeout(std::time::Duration::from_secs(5), frames.recv()).await.expect("a frame in time").expect("the pipe runs");
1768                let v: Value = serde_json::from_slice(frame.bytes()).expect("a frame is one JSON value");
1769                let last = items_of(&v).last().map(|i| i["type"] == "done" || i["type"] == "error").unwrap_or(false);
1770                out.push(v);
1771                if last {
1772                    return out;
1773                }
1774            }
1775        })
1776    }
1777
1778    /// A notification's items, as a client that offered `batch` reads them: a
1779    /// `result` alone, or each of `results`, a `data` item without `metadata`
1780    /// taking the batch's.
1781    fn items_of(note: &Value) -> Vec<Value> {
1782        let params = &note["params"];
1783        if let Some(one) = params.get("result") {
1784            return vec![one.clone()];
1785        }
1786        let shared = params.get("metadata").cloned();
1787        params["results"]
1788            .as_array()
1789            .expect("`result` or `results`")
1790            .iter()
1791            .map(|item| {
1792                let mut item = item.clone();
1793                if item["type"] == "data" && item.get("metadata").is_none() {
1794                    item["metadata"] = shared.clone().expect("a data item without metadata takes the batch's");
1795                }
1796                item
1797            })
1798            .collect()
1799    }
1800
1801    fn ready(items: Vec<PlexusStreamItem>) -> super::super::streaming::PlexusStream {
1802        Box::pin(futures::stream::iter(items))
1803    }
1804
1805    /// 200 items ready at once, then `done`: batches of 64, in order, each item
1806    /// equal as JSON to the one a client that offered nothing is sent.
1807    #[test]
1808    fn a_batch_is_the_items_ready_together_in_one_notification() {
1809        let m = meta(7);
1810        let make = || ready((0..200).map(|n| data(&m, n)).chain([done(7)]).collect());
1811        let one_each = frames_of(make(), 0);
1812        assert_eq!(one_each.len(), 201, "one an item");
1813        assert!(one_each.iter().all(|f| f["params"].get("result").is_some() && f["params"].get("results").is_none()), "today's frames");
1814        let batched = frames_of(make(), 64);
1815        let sizes: Vec<usize> = batched.iter().map(|f| items_of(f).len()).collect();
1816        assert_eq!(sizes, vec![64, 64, 64, 9], "what was ready, up to 64, the last with `done`");
1817        assert!(batched.iter().all(|f| f["params"]["results"].is_array()), "a batch is `results`");
1818        let unbatched: Vec<Value> = one_each.iter().flat_map(items_of).collect();
1819        assert_eq!(batched.iter().flat_map(items_of).collect::<Vec<_>>(), unbatched, "the same items, in order, equal as JSON");
1820        assert!(batched.iter().all(|f| f["params"]["subscription"] == batched[0]["params"]["subscription"] && f["method"] == "note"), "one call's, on its own method");
1821    }
1822
1823    /// The metadata goes once a batch where it is the same: a `data` item of
1824    /// the batch's leaves its own out; any other keeps it.
1825    #[test]
1826    fn metadata_goes_once_a_batch_where_it_is_the_same() {
1827        // a child's items, relayed, each bring their own copy of the same
1828        let (a, b, copy) = (meta(7), meta(8), meta(7));
1829        let frames = frames_of(ready(vec![data(&a, 0), data(&a, 1), data(&b, 2), data(&copy, 3), done(9)]), 64);
1830        assert_eq!(frames.len(), 1);
1831        let p = &frames[0]["params"];
1832        assert_eq!(p["metadata"], serde_json::to_value(&*a).unwrap(), "the batch's, its first data item's");
1833        let raw = p["results"].as_array().unwrap();
1834        assert!(raw[0].get("metadata").is_none() && raw[1].get("metadata").is_none() && raw[3].get("metadata").is_none(), "the same, or an equal copy: {raw:?}");
1835        assert_eq!(raw[2]["metadata"], serde_json::to_value(&*b).unwrap(), "another stream's metadata stays with its item");
1836        assert_eq!(raw[4]["metadata"]["timestamp"], 9, "`done` keeps its own");
1837    }
1838
1839    /// A stream that yields one item and then waits: that item goes at once,
1840    /// alone, in today's form. A batch is what is ready, never what a timer
1841    /// waits for.
1842    #[test]
1843    fn a_lone_item_goes_at_once_alone() {
1844        let m = meta(7);
1845        let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap();
1846        rt.block_on(async {
1847            let (to, mut frames) = FrameSink::channel(16, WireEncoding::Json);
1848            let to = to.batching(64);
1849            let stream: super::super::streaming::PlexusStream = Box::pin(futures::StreamExt::chain(futures::stream::iter([data(&m, 0)]), futures::stream::pending()));
1850            let stream = Arc::new(Mutex::new(Some(stream)));
1851            let mut module = jsonrpsee::RpcModule::new(());
1852            module
1853                .register_subscription("open", "note", "close", move |_, pending, _, mut ext| {
1854                    let (to, stream) = (to.clone(), stream.lock().unwrap().take());
1855                    async move {
1856                        let sink = pending.accept().await?;
1857                        ext.insert(to);
1858                        pipe(sink, stream.expect("one call"), &ext).await
1859                    }
1860                })
1861                .unwrap();
1862            let (_answer, _notes) = module.raw_json_request(r#"{"jsonrpc":"2.0","id":1,"method":"open","params":[]}"#, 1).await.unwrap();
1863            let frame = tokio::time::timeout(std::time::Duration::from_millis(500), frames.recv()).await.expect("at once, not when more come").unwrap();
1864            let v: Value = serde_json::from_slice(frame.bytes()).unwrap();
1865            assert_eq!(v["params"]["result"]["content"]["n"], 0, "alone, in today's form: {v}");
1866        });
1867    }
1868
1869    /// A batch of large items stops at its bytes: 4 KB items go two to a
1870    /// notification, not 64, and a ticket body goes alone.
1871    #[test]
1872    fn a_batch_stops_at_its_bytes() {
1873        let m = meta(7);
1874        let big = |n: u64, size: usize| PlexusStreamItem::Data { metadata: m.clone(), content_type: Cow::Borrowed(CT), content: Content::json(Bytes::from(format!(r#"{{"type":"item","n":{n},"pad":"{}"}}"#, "x".repeat(size)))) };
1875        let frames = frames_of(ready((0..20).map(|n| big(n, 4096)).chain([done(7)]).collect()), 64);
1876        let sizes: Vec<usize> = frames.iter().map(|f| items_of(f).len()).collect();
1877        assert!(sizes.iter().all(|&n| n <= 2), "about 8 KiB a batch: {sizes:?}");
1878        assert_eq!(sizes.iter().sum::<usize>(), 21);
1879        let frames = frames_of(ready((0..3).map(|n| big(n, 17_000)).chain([done(7)]).collect()), 64);
1880        assert!(frames.iter().take(2).all(|f| f["params"].get("result").is_some()), "a body alone, as today: {:?}", frames.iter().map(|f| items_of(f).len()).collect::<Vec<_>>());
1881    }
1882
1883    /// A client may ask for fewer than 64: no notification carries more.
1884    #[test]
1885    fn a_batch_holds_no_more_than_asked() {
1886        let m = meta(7);
1887        let frames = frames_of(ready((0..10).map(|n| data(&m, n)).chain([done(7)]).collect()), 4);
1888        assert_eq!(frames.iter().map(|f| items_of(f).len()).collect::<Vec<_>>(), vec![4, 4, 3]);
1889    }
1890}