Skip to main content

shard_telemetry/
native_client.rs

1use std::fmt;
2use std::net::SocketAddr;
3use std::sync::Arc;
4use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
5use std::time::{Duration, SystemTime, UNIX_EPOCH};
6
7use tokio::io::{AsyncReadExt, AsyncWriteExt};
8use tokio::net::TcpStream;
9use tokio::sync::{Mutex, MutexGuard};
10
11use crate::{
12    MAX_NATIVE_FRAME_BYTES, NATIVE_FRAME_HEADER_BYTES, NativeCapabilities, NativeFrame,
13    NativeFrameHeader, NativeLogQueryResult, NativeOpcode, NativeProtocolError, NativeQuery,
14    NativeStatus, NativeTelemetryAppendAck, NativeTelemetryBatch, decode_native_capabilities,
15    decode_native_log_query_result, decode_native_metric_query_result,
16    decode_native_trace_query_result, encode_native_metric_query, encode_native_query,
17    encode_native_trace_query,
18};
19
20/// Connection and bounded-request settings for [`ShardTelemetryClient`].
21#[derive(Debug, Clone)]
22pub struct NativeClientConfig {
23    /// Native TCP address of the ShardTelemetry server.
24    pub endpoint: SocketAddr,
25    /// Bearer token sent as the first native frame when configured.
26    pub auth_token: Option<Arc<str>>,
27    /// Largest accepted response payload.
28    pub max_frame_bytes: usize,
29    /// Deadline for one authenticate, append, query, or health exchange.
30    pub request_timeout: Duration,
31    /// Maximum persistent native connections to this endpoint.
32    ///
33    /// Each connection carries one exchange at a time. A pool larger than one
34    /// lets independent partition appends make progress concurrently while
35    /// bounding sockets, authentication handshakes, and server admission.
36    pub max_connections: usize,
37    /// Explicitly permits raw TCP to a non-loopback endpoint.
38    ///
39    /// Production callers should leave this false and use a local mTLS tunnel
40    /// or a TLS-capable transport endpoint. Native bearer authentication is a
41    /// protocol frame and must never traverse an untrusted network in clear
42    /// text.
43    pub allow_insecure_remote: bool,
44}
45
46impl NativeClientConfig {
47    /// Creates a client configuration with production frame and timeout defaults.
48    #[must_use]
49    pub fn new(endpoint: SocketAddr) -> Self {
50        Self {
51            endpoint,
52            auth_token: None,
53            max_frame_bytes: MAX_NATIVE_FRAME_BYTES,
54            request_timeout: Duration::from_secs(30),
55            max_connections: 1,
56            allow_insecure_remote: false,
57        }
58    }
59
60    /// Configures the bearer token required by production servers.
61    #[must_use]
62    pub fn with_auth_token(mut self, auth_token: impl Into<Arc<str>>) -> Self {
63        self.auth_token = Some(auth_token.into());
64        self
65    }
66
67    /// Changes the response-frame bound for this client.
68    #[must_use]
69    pub const fn with_max_frame_bytes(mut self, max_frame_bytes: usize) -> Self {
70        self.max_frame_bytes = max_frame_bytes;
71        self
72    }
73
74    /// Changes the deadline for one native exchange.
75    #[must_use]
76    pub const fn with_request_timeout(mut self, request_timeout: Duration) -> Self {
77        self.request_timeout = request_timeout;
78        self
79    }
80
81    /// Changes the maximum number of reusable connections to this endpoint.
82    #[must_use]
83    pub const fn with_max_connections(mut self, max_connections: usize) -> Self {
84        self.max_connections = max_connections;
85        self
86    }
87
88    /// Explicitly permits plaintext native TCP to a non-loopback endpoint.
89    ///
90    /// This exists for a locally administered TLS/service-mesh tunnel whose
91    /// remote encryption boundary is outside this client. Direct internet or
92    /// LAN use is unsafe; prefer the default fail-closed setting.
93    #[must_use]
94    pub const fn with_insecure_remote(mut self) -> Self {
95        self.allow_insecure_remote = true;
96        self
97    }
98
99    fn validate(&self) -> Result<(), NativeClientError> {
100        if self.max_frame_bytes == 0 || self.max_frame_bytes > MAX_NATIVE_FRAME_BYTES {
101            return Err(NativeClientError::new(format!(
102                "native client max_frame_bytes must be in 1..={MAX_NATIVE_FRAME_BYTES}"
103            )));
104        }
105        if self.request_timeout.is_zero() {
106            return Err(NativeClientError::new(
107                "native client request_timeout must be nonzero",
108            ));
109        }
110        if self.max_connections == 0 {
111            return Err(NativeClientError::new(
112                "native client max_connections must be nonzero",
113            ));
114        }
115        if !self.allow_insecure_remote && !self.endpoint.ip().is_loopback() {
116            return Err(NativeClientError::new(
117                "plaintext native transport is restricted to loopback; use a TLS tunnel or explicitly opt in to insecure remote transport",
118            ));
119        }
120        if self
121            .auth_token
122            .as_ref()
123            .is_some_and(|token| token.is_empty())
124        {
125            return Err(NativeClientError::new(
126                "native client authentication token must not be empty",
127            ));
128        }
129        Ok(())
130    }
131}
132
133/// Error returned by the reusable native client.
134#[derive(Debug, Clone, PartialEq, Eq)]
135pub struct NativeClientError {
136    message: Arc<str>,
137    /// Native status returned by the server, when a response was received.
138    pub status: Option<NativeStatus>,
139}
140
141impl NativeClientError {
142    fn new(message: impl Into<Arc<str>>) -> Self {
143        Self {
144            message: message.into(),
145            status: None,
146        }
147    }
148
149    fn server(status: NativeStatus, message: impl Into<Arc<str>>) -> Self {
150        Self {
151            message: message.into(),
152            status: Some(status),
153        }
154    }
155}
156
157impl fmt::Display for NativeClientError {
158    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
159        formatter.write_str(&self.message)
160    }
161}
162
163impl std::error::Error for NativeClientError {}
164
165impl From<NativeProtocolError> for NativeClientError {
166    fn from(error: NativeProtocolError) -> Self {
167        Self::new(error.to_string())
168    }
169}
170
171/// Persistent, bounded native-v1 client for ShardTelemetry.
172///
173/// One client uses a bounded pool of reusable TCP connections. Each connection
174/// serializes its exchanges; different callers may use different connections
175/// concurrently. A failed or timed-out exchange drops only its connection, so
176/// its next request starts with a clean authentication handshake instead of
177/// consuming a stale frame.
178#[derive(Debug)]
179pub struct ShardTelemetryClient {
180    config: NativeClientConfig,
181    connections: Box<[Mutex<Option<TcpStream>>]>,
182    request_id_prefix: u64,
183    next_request_id: AtomicU64,
184    next_connection: AtomicUsize,
185}
186
187/// A checked-out pool connection.
188///
189/// An incomplete native exchange makes framing state unknowable: cancellation
190/// may happen after bytes are sent but before the response is fully read. Drop
191/// such a connection instead of letting the next request consume that response.
192struct ConnectionLease<'a> {
193    connection: MutexGuard<'a, Option<TcpStream>>,
194    completed: bool,
195}
196
197impl<'a> ConnectionLease<'a> {
198    fn new(connection: MutexGuard<'a, Option<TcpStream>>) -> Self {
199        Self {
200            connection,
201            completed: false,
202        }
203    }
204
205    fn stream_mut(&mut self) -> &mut TcpStream {
206        self.connection
207            .as_mut()
208            .expect("native connection was opened before exchange")
209    }
210
211    fn preserve(&mut self) {
212        self.completed = true;
213    }
214}
215
216impl Drop for ConnectionLease<'_> {
217    fn drop(&mut self) {
218        if !self.completed {
219            *self.connection = None;
220        }
221    }
222}
223
224impl ShardTelemetryClient {
225    /// Builds a client. Connections are established lazily by the first request.
226    pub fn new(config: NativeClientConfig) -> Result<Self, NativeClientError> {
227        config.validate()?;
228        let connections: Box<[Mutex<Option<TcpStream>>]> = (0..config.max_connections)
229            .map(|_| Mutex::new(None))
230            .collect();
231        Ok(Self {
232            request_id_prefix: request_id_prefix(&config),
233            config,
234            connections,
235            next_request_id: AtomicU64::new(1),
236            next_connection: AtomicUsize::new(0),
237        })
238    }
239
240    /// Appends one or more signal-aware routed envelopes.
241    pub async fn append(
242        &self,
243        batch: &NativeTelemetryBatch,
244    ) -> Result<NativeTelemetryAppendAck, NativeClientError> {
245        self.append_with_request_id(batch, self.next_request_id())
246            .await
247    }
248
249    /// Appends logs with a client-generated request ID.
250    ///
251    /// Use [`Self::append_with_request_id`] when the caller may retry after an
252    /// indeterminate result and therefore needs a durable retry identity.
253    pub async fn append_logs(
254        &self,
255        batch: &NativeTelemetryBatch,
256    ) -> Result<NativeTelemetryAppendAck, NativeClientError> {
257        self.append(batch).await
258    }
259
260    /// Appends traces with a client-generated request ID.
261    ///
262    /// Use [`Self::append_with_request_id`] for durable retry identity.
263    pub async fn append_traces(
264        &self,
265        batch: &NativeTelemetryBatch,
266    ) -> Result<NativeTelemetryAppendAck, NativeClientError> {
267        self.append(batch).await
268    }
269
270    /// Appends metrics with a client-generated request ID.
271    ///
272    /// Use [`Self::append_with_request_id`] for durable retry identity.
273    pub async fn append_metrics(
274        &self,
275        batch: &NativeTelemetryBatch,
276    ) -> Result<NativeTelemetryAppendAck, NativeClientError> {
277        self.append(batch).await
278    }
279
280    /// Appends a batch using `request_id` as its durable idempotency identity.
281    pub async fn append_with_request_id(
282        &self,
283        batch: &NativeTelemetryBatch,
284        request_id: u128,
285    ) -> Result<NativeTelemetryAppendAck, NativeClientError> {
286        let payload = batch.encode_native_append()?;
287        let response = self
288            .request(NativeOpcode::Append, request_id, payload)
289            .await?;
290        NativeTelemetryAppendAck::decode(&response).map_err(Into::into)
291    }
292
293    /// Executes the existing bounded native indexed-log query.
294    pub async fn query_logs(
295        &self,
296        query: &NativeQuery,
297    ) -> Result<NativeLogQueryResult, NativeClientError> {
298        let payload = encode_native_query(query)?;
299        let response = self
300            .request(NativeOpcode::Query, self.next_request_id(), payload)
301            .await?;
302        decode_native_log_query_result(&response).map_err(Into::into)
303    }
304
305    /// Executes a bounded native metric query.
306    pub async fn query_metrics(
307        &self,
308        query: &crate::MetricQuery,
309    ) -> Result<Vec<crate::DurableMetricPoint>, NativeClientError> {
310        let payload = encode_native_metric_query(query)?;
311        let response = self
312            .request(NativeOpcode::QueryMetrics, self.next_request_id(), payload)
313            .await?;
314        decode_native_metric_query_result(&response).map_err(Into::into)
315    }
316
317    /// Executes a bounded native trace query.
318    pub async fn query_traces(
319        &self,
320        query: &crate::TraceQuery,
321    ) -> Result<Vec<crate::DurableSpan>, NativeClientError> {
322        let payload = encode_native_trace_query(query)?;
323        let response = self
324            .request(NativeOpcode::QueryTraces, self.next_request_id(), payload)
325            .await?;
326        decode_native_trace_query_result(&response).map_err(Into::into)
327    }
328
329    /// Returns the authenticated server's native protocol and signal support.
330    pub async fn describe(&self) -> Result<NativeCapabilities, NativeClientError> {
331        let response = self
332            .request(NativeOpcode::Describe, self.next_request_id(), Vec::new())
333            .await?;
334        decode_native_capabilities(&response).map_err(Into::into)
335    }
336
337    /// Checks native listener health and verifies a reusable round trip.
338    pub async fn health(&self) -> Result<(), NativeClientError> {
339        let payload = b"health".to_vec();
340        let response = self
341            .request(NativeOpcode::Ping, self.next_request_id(), payload.clone())
342            .await?;
343        if response != payload {
344            return Err(NativeClientError::new(
345                "native health response did not echo the request payload",
346            ));
347        }
348        Ok(())
349    }
350
351    fn next_request_id(&self) -> u128 {
352        let sequence = self.next_request_id.fetch_add(1, Ordering::Relaxed);
353        (u128::from(self.request_id_prefix) << 64) | u128::from(sequence)
354    }
355
356    async fn request(
357        &self,
358        opcode: NativeOpcode,
359        request_id: u128,
360        payload: Vec<u8>,
361    ) -> Result<Vec<u8>, NativeClientError> {
362        let request = NativeFrame::request(opcode, request_id, payload)?;
363        let connection_index =
364            self.next_connection.fetch_add(1, Ordering::Relaxed) % self.connections.len();
365        let mut connection = ConnectionLease::new(self.connections[connection_index].lock().await);
366        if connection.connection.is_none() {
367            *connection.connection = Some(self.open_connection().await?);
368        }
369        let exchange = exchange(
370            connection.stream_mut(),
371            request,
372            self.config.max_frame_bytes,
373        );
374        match tokio::time::timeout(self.config.request_timeout, exchange).await {
375            Ok(Ok(response)) => {
376                connection.preserve();
377                Ok(response)
378            }
379            Ok(Err(error)) => Err(error),
380            Err(_) => Err(NativeClientError::new("native request deadline exceeded")),
381        }
382    }
383
384    async fn open_connection(&self) -> Result<TcpStream, NativeClientError> {
385        let mut stream = tokio::time::timeout(
386            self.config.request_timeout,
387            TcpStream::connect(self.config.endpoint),
388        )
389        .await
390        .map_err(|_| NativeClientError::new("native connection deadline exceeded"))?
391        .map_err(|error| NativeClientError::new(format!("native connection failed: {error}")))?;
392        stream.set_nodelay(true).map_err(|error| {
393            NativeClientError::new(format!("native TCP_NODELAY failed: {error}"))
394        })?;
395        if let Some(token) = &self.config.auth_token {
396            let request = NativeFrame::request(
397                NativeOpcode::Authenticate,
398                self.next_request_id(),
399                token.as_bytes().to_vec(),
400            )?;
401            let response = tokio::time::timeout(
402                self.config.request_timeout,
403                exchange(&mut stream, request, self.config.max_frame_bytes),
404            )
405            .await
406            .map_err(|_| NativeClientError::new("native authentication deadline exceeded"))??;
407            if !response.is_empty() {
408                return Err(NativeClientError::new(
409                    "native authentication returned an unexpected payload",
410                ));
411            }
412        }
413        Ok(stream)
414    }
415}
416
417/// Produces a per-client namespace for one-shot request IDs. Retry-capable
418/// callers must still choose their own stable ID with `append_with_request_id`.
419/// The time, process, endpoint, and object-address inputs prevent independent
420/// client lifetimes from accidentally reusing the small monotonically
421/// increasing counter portion while preserving a lock-free hot path.
422fn request_id_prefix(config: &NativeClientConfig) -> u64 {
423    let mut hasher = blake3::Hasher::new();
424    hasher.update(b"shard-telemetry-native-client-request-id-v1\0");
425    hasher.update(config.endpoint.to_string().as_bytes());
426    hasher.update(&std::process::id().to_le_bytes());
427    hasher.update(
428        &SystemTime::now()
429            .duration_since(UNIX_EPOCH)
430            .unwrap_or_default()
431            .as_nanos()
432            .to_le_bytes(),
433    );
434    hasher.update(&(config as *const NativeClientConfig as usize).to_le_bytes());
435    u64::from_le_bytes(
436        hasher.finalize().as_bytes()[..8]
437            .try_into()
438            .expect("BLAKE3 digest contains eight bytes"),
439    )
440}
441
442async fn exchange(
443    stream: &mut TcpStream,
444    request: NativeFrame,
445    max_frame_bytes: usize,
446) -> Result<Vec<u8>, NativeClientError> {
447    stream
448        .write_all(&request.header.encode())
449        .await
450        .map_err(|error| {
451            NativeClientError::new(format!("native request header write failed: {error}"))
452        })?;
453    stream.write_all(&request.payload).await.map_err(|error| {
454        NativeClientError::new(format!("native request payload write failed: {error}"))
455    })?;
456
457    let mut encoded_header = [0_u8; NATIVE_FRAME_HEADER_BYTES];
458    stream
459        .read_exact(&mut encoded_header)
460        .await
461        .map_err(|error| {
462            NativeClientError::new(format!("native response header read failed: {error}"))
463        })?;
464    let header = NativeFrameHeader::decode(&encoded_header)?;
465    if !header.is_response
466        || header.opcode != request.header.opcode
467        || header.request_id != request.header.request_id
468    {
469        return Err(NativeClientError::new(
470            "native response does not match the active request",
471        ));
472    }
473    if header.payload_len as usize > max_frame_bytes {
474        return Err(NativeClientError::new(
475            "native response exceeds the configured client frame limit",
476        ));
477    }
478    let mut payload = vec![0; header.payload_len as usize];
479    stream.read_exact(&mut payload).await.map_err(|error| {
480        NativeClientError::new(format!("native response payload read failed: {error}"))
481    })?;
482    header.verify_payload(&payload)?;
483    if header.status != NativeStatus::Ok {
484        let message = std::str::from_utf8(&payload)
485            .map(str::to_owned)
486            .unwrap_or_else(|_| "native server returned a non-UTF-8 error payload".to_string());
487        return Err(NativeClientError::server(header.status, message));
488    }
489    Ok(payload)
490}
491
492#[cfg(test)]
493mod tests {
494    use std::collections::BTreeMap;
495    use std::sync::Arc;
496    use std::time::{Duration, SystemTime, UNIX_EPOCH};
497
498    use shard_stream_core::{LogicalPartitionId, TopicPartition};
499    use tokio::net::TcpListener;
500
501    use super::*;
502    use crate::{
503        DurableTelemetryConfig, DurableTelemetryStore, LokiEntry, NativePartitionAppend,
504        NativeQueryDirection, NativeRequestGate, NativeServerConfig, StripeConfig,
505        prepare_loki_log_envelope, serve_native,
506    };
507
508    #[derive(Debug)]
509    struct ParallelAppendGate {
510        state: std::sync::Mutex<ParallelAppendGateState>,
511        arrivals: std::sync::Condvar,
512    }
513
514    #[derive(Debug, Default)]
515    struct ParallelAppendGateState {
516        active: usize,
517        maximum_active: usize,
518    }
519
520    impl NativeRequestGate for ParallelAppendGate {
521        fn check(&self) -> Result<(), String> {
522            Ok(())
523        }
524
525        fn check_partitions(
526            &self,
527            _partitions: &[crate::NativePartitionAppend],
528        ) -> Result<(), String> {
529            let mut state = self.state.lock().map_err(|_| "test gate poisoned")?;
530            state.active = state.active.saturating_add(1);
531            state.maximum_active = state.maximum_active.max(state.active);
532            if state.active == 1 {
533                let (next, _) = self
534                    .arrivals
535                    .wait_timeout(state, Duration::from_secs(2))
536                    .map_err(|_| "test gate poisoned")?;
537                state = next;
538            } else {
539                self.arrivals.notify_one();
540            }
541            state.active = state.active.saturating_sub(1);
542            Ok(())
543        }
544    }
545
546    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
547    async fn client_reuses_a_connection_for_append_query_and_health() {
548        let nonce = SystemTime::now()
549            .duration_since(UNIX_EPOCH)
550            .expect("clock")
551            .as_nanos();
552        let directory = std::env::temp_dir().join(format!(
553            "shard-telemetry-native-client-{}-{nonce}",
554            std::process::id()
555        ));
556        let store = Arc::new(
557            DurableTelemetryStore::open(DurableTelemetryConfig {
558                data_directory: directory.clone(),
559                object_store_directory: None,
560                s3_object_store: None,
561                recovery_journal: false,
562                retention: None,
563                shard_count: 1,
564                tenant_partitions: 1,
565                append_linger: Duration::from_micros(250),
566                stripe: StripeConfig::default(),
567                indexed_ack_timeout: Duration::from_secs(30),
568            })
569            .expect("store"),
570        );
571        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
572        let address = listener.local_addr().expect("address");
573        let (stop, stopped) = tokio::sync::oneshot::channel();
574        let server_store = Arc::clone(&store);
575        let server = tokio::spawn(async move {
576            serve_native(
577                listener,
578                server_store,
579                NativeServerConfig::default(),
580                async {
581                    let _ = stopped.await;
582                },
583            )
584            .await
585        });
586
587        let client = ShardTelemetryClient::new(NativeClientConfig::new(address)).expect("client");
588        client.health().await.expect("health");
589        let capabilities = client.describe().await.expect("capabilities");
590        assert!(capabilities.append_v1);
591        assert!(capabilities.query_metrics);
592        assert!(capabilities.query_traces);
593        assert_eq!(capabilities.logical_partitions, [1, 1, 1]);
594        assert!(
595            client
596                .query_metrics(&crate::MetricQuery {
597                    tenant: Arc::from("tenant-a"),
598                    limit: 10,
599                    ..crate::MetricQuery::default()
600                })
601                .await
602                .expect("empty metric query")
603                .is_empty()
604        );
605        assert!(
606            client
607                .query_traces(&crate::TraceQuery {
608                    tenant: Arc::from("tenant-a"),
609                    limit: 10,
610                    ..crate::TraceQuery::default()
611                })
612                .await
613                .expect("empty trace query")
614                .is_empty()
615        );
616        let entry = LokiEntry {
617            timestamp_unix_nanos: 100,
618            labels: BTreeMap::from([("service".to_owned(), "api".to_owned())]),
619            line: "native client event".to_owned(),
620            structured_metadata: BTreeMap::new(),
621        };
622        let batch = NativeTelemetryBatch {
623            partitions: vec![NativePartitionAppend {
624                topic_partition: TopicPartition::new(
625                    crate::LOGS_TOPIC_ID,
626                    LogicalPartitionId::new(0),
627                ),
628                envelope: prepare_loki_log_envelope("tenant-a", vec![entry.clone()])
629                    .expect("envelope"),
630                transient_context: None,
631            }],
632        };
633        let first = client
634            .append_with_request_id(&batch, 42)
635            .await
636            .expect("append");
637        let replay = client
638            .append_with_request_id(&batch, 42)
639            .await
640            .expect("replay");
641        assert_eq!(first, replay);
642        let queried = client
643            .query_logs(&NativeQuery {
644                tenant: "tenant-a".to_owned(),
645                labels: BTreeMap::from([("service".to_owned(), "api".to_owned())]),
646                terms: vec!["client".to_owned()],
647                start_timestamp_unix_nanos: None,
648                end_timestamp_unix_nanos: None,
649                limit: 10,
650                direction: NativeQueryDirection::OldestFirst,
651            })
652            .await
653            .expect("query");
654        assert_eq!(queried.entries, vec![entry]);
655
656        drop(client);
657        stop.send(()).expect("stop");
658        server
659            .await
660            .expect("server joins")
661            .expect("server succeeds");
662        drop(store);
663        std::fs::remove_dir_all(directory).expect("cleanup");
664    }
665
666    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
667    async fn configured_connection_pool_executes_independent_appends_concurrently() {
668        let nonce = SystemTime::now()
669            .duration_since(UNIX_EPOCH)
670            .expect("clock")
671            .as_nanos();
672        let directory = std::env::temp_dir().join(format!(
673            "shard-telemetry-native-client-pool-{}-{nonce}",
674            std::process::id()
675        ));
676        let store = Arc::new(
677            DurableTelemetryStore::open(DurableTelemetryConfig {
678                data_directory: directory.clone(),
679                object_store_directory: None,
680                s3_object_store: None,
681                recovery_journal: false,
682                retention: None,
683                shard_count: 1,
684                tenant_partitions: 2,
685                append_linger: Duration::from_micros(250),
686                stripe: StripeConfig::default(),
687                indexed_ack_timeout: Duration::from_secs(30),
688            })
689            .expect("store"),
690        );
691        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
692        let address = listener.local_addr().expect("address");
693        let gate = Arc::new(ParallelAppendGate {
694            state: std::sync::Mutex::new(ParallelAppendGateState::default()),
695            arrivals: std::sync::Condvar::new(),
696        });
697        let (stop, stopped) = tokio::sync::oneshot::channel();
698        let server_store = Arc::clone(&store);
699        let server_gate: Arc<dyn NativeRequestGate> = gate.clone();
700        let server = tokio::spawn(async move {
701            serve_native(
702                listener,
703                server_store,
704                NativeServerConfig {
705                    request_gate: Some(server_gate),
706                    ..NativeServerConfig::default()
707                },
708                async {
709                    let _ = stopped.await;
710                },
711            )
712            .await
713        });
714        let client = Arc::new(
715            ShardTelemetryClient::new(NativeClientConfig::new(address).with_max_connections(2))
716                .expect("client"),
717        );
718        assert_eq!(client.connections.len(), 2);
719        let first = NativeTelemetryBatch {
720            partitions: vec![NativePartitionAppend {
721                topic_partition: TopicPartition::new(
722                    crate::LOGS_TOPIC_ID,
723                    LogicalPartitionId::new(0),
724                ),
725                envelope: prepare_loki_log_envelope(
726                    "tenant-a",
727                    vec![LokiEntry {
728                        timestamp_unix_nanos: 100,
729                        labels: BTreeMap::new(),
730                        line: "first pooled append".to_owned(),
731                        structured_metadata: BTreeMap::new(),
732                    }],
733                )
734                .expect("first envelope"),
735                transient_context: None,
736            }],
737        };
738        let second = NativeTelemetryBatch {
739            partitions: vec![NativePartitionAppend {
740                topic_partition: TopicPartition::new(
741                    crate::LOGS_TOPIC_ID,
742                    LogicalPartitionId::new(1),
743                ),
744                envelope: prepare_loki_log_envelope(
745                    "tenant-a",
746                    vec![LokiEntry {
747                        timestamp_unix_nanos: 101,
748                        labels: BTreeMap::new(),
749                        line: "second pooled append".to_owned(),
750                        structured_metadata: BTreeMap::new(),
751                    }],
752                )
753                .expect("second envelope"),
754                transient_context: None,
755            }],
756        };
757        let start = Arc::new(tokio::sync::Barrier::new(3));
758        let first_client = Arc::clone(&client);
759        let first_start = Arc::clone(&start);
760        let first_append = tokio::spawn(async move {
761            first_start.wait().await;
762            first_client.append_with_request_id(&first, 100).await
763        });
764        let second_client = Arc::clone(&client);
765        let second_start = Arc::clone(&start);
766        let second_append = tokio::spawn(async move {
767            second_start.wait().await;
768            second_client.append_with_request_id(&second, 101).await
769        });
770        start.wait().await;
771        first_append
772            .await
773            .expect("first append task")
774            .expect("first append");
775        second_append
776            .await
777            .expect("second append task")
778            .expect("second append");
779        assert_eq!(
780            gate.state.lock().expect("test gate state").maximum_active,
781            2
782        );
783
784        drop(client);
785        stop.send(()).expect("stop");
786        server
787            .await
788            .expect("server joins")
789            .expect("server succeeds");
790        drop(store);
791        std::fs::remove_dir_all(directory).expect("cleanup");
792    }
793
794    #[test]
795    fn client_rejects_an_empty_connection_pool() {
796        let endpoint = "127.0.0.1:3101".parse().expect("endpoint");
797        let error =
798            ShardTelemetryClient::new(NativeClientConfig::new(endpoint).with_max_connections(0))
799                .expect_err("empty pool must fail");
800        assert_eq!(
801            error.to_string(),
802            "native client max_connections must be nonzero"
803        );
804    }
805
806    #[test]
807    fn client_rejects_plaintext_remote_transport_by_default() {
808        let endpoint = "192.0.2.10:3101".parse().expect("address");
809        assert!(ShardTelemetryClient::new(NativeClientConfig::new(endpoint)).is_err());
810        assert!(
811            ShardTelemetryClient::new(NativeClientConfig::new(endpoint).with_insecure_remote())
812                .is_ok()
813        );
814    }
815
816    #[tokio::test]
817    async fn incomplete_connection_lease_discards_the_socket() {
818        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
819        let address = listener.local_addr().expect("address");
820        let client = TcpStream::connect(address).await.expect("connect");
821        let (server, _) = listener.accept().await.expect("accept");
822        let connection = Mutex::new(Some(client));
823        let lease = ConnectionLease::new(connection.lock().await);
824        drop(lease);
825        assert!(connection.lock().await.is_none());
826        drop(server);
827    }
828}