1use std::collections::{BTreeMap, BTreeSet};
2use std::fmt;
3use std::sync::Arc;
4
5use shard_stream_core::{LogicalPartitionId, TopicId, TopicPartition};
6
7use crate::{LokiEntry, TelemetryEnvelope};
8
9pub const NATIVE_FRAME_HEADER_BYTES: usize = 32;
11pub const MAX_NATIVE_FRAME_BYTES: usize = 64 * 1024 * 1024;
13
14const FRAME_MAGIC: [u8; 4] = *b"STNP";
15const FRAME_VERSION: u8 = 1;
16const FRAME_FLAG_RESPONSE: u8 = 1;
17const LOG_QUERY_RESULT_MAGIC: [u8; 4] = *b"STR1";
18const TELEMETRY_BATCH_MAGIC: [u8; 4] = *b"STB1";
19const TELEMETRY_ACK_MAGIC: [u8; 4] = *b"STM1";
20const QUERY_MAGIC: [u8; 4] = *b"STQ1";
21const METRIC_QUERY_MAGIC: [u8; 4] = *b"STQ2";
22const TRACE_QUERY_MAGIC: [u8; 4] = *b"STQ3";
23const METRIC_QUERY_RESULT_MAGIC: [u8; 4] = *b"STR2";
24const TRACE_QUERY_RESULT_MAGIC: [u8; 4] = *b"STR3";
25const CAPABILITIES_MAGIC: [u8; 4] = *b"STC1";
26const LOG_QUERY_RESULT_HEADER_BYTES: usize = 16;
27const QUERY_HEADER_BYTES: usize = 32;
28const MAX_TENANT_BYTES: usize = 1_024;
29const MAX_STREAMS: usize = 65_535;
30const MAX_LABELS_PER_STREAM: usize = 256;
31const MAX_METADATA_PER_ENTRY: usize = 256;
32const MAX_QUERY_TERMS: usize = 256;
33const MAX_QUERY_LIMIT: u32 = 1_000_000;
34
35#[derive(Debug, Clone, Copy, PartialEq, Eq)]
37#[repr(u8)]
38pub enum NativeOpcode {
39 Append = 1,
41 Query = 2,
43 Ping = 3,
45 Authenticate = 4,
47 QueryMetrics = 5,
49 QueryTraces = 6,
51 Describe = 7,
53}
54
55impl NativeOpcode {
56 fn from_byte(value: u8) -> Result<Self, NativeProtocolError> {
57 match value {
58 1 => Ok(Self::Append),
59 2 => Ok(Self::Query),
60 3 => Ok(Self::Ping),
61 4 => Ok(Self::Authenticate),
62 5 => Ok(Self::QueryMetrics),
63 6 => Ok(Self::QueryTraces),
64 7 => Ok(Self::Describe),
65 _ => Err(NativeProtocolError::new(format!(
66 "unsupported native opcode {value}"
67 ))),
68 }
69 }
70}
71
72#[derive(Debug, Clone, Copy, PartialEq, Eq)]
74#[repr(u8)]
75pub enum NativeStatus {
76 Ok = 0,
78 BadRequest = 1,
80 Internal = 2,
82 Unsupported = 3,
84 Unauthorized = 4,
86 Unavailable = 5,
88 TooManyRequests = 6,
90 Timeout = 7,
92}
93
94impl NativeStatus {
95 fn from_byte(value: u8) -> Result<Self, NativeProtocolError> {
96 match value {
97 0 => Ok(Self::Ok),
98 1 => Ok(Self::BadRequest),
99 2 => Ok(Self::Internal),
100 3 => Ok(Self::Unsupported),
101 4 => Ok(Self::Unauthorized),
102 5 => Ok(Self::Unavailable),
103 6 => Ok(Self::TooManyRequests),
104 7 => Ok(Self::Timeout),
105 _ => Err(NativeProtocolError::new(format!(
106 "unsupported native status {value}"
107 ))),
108 }
109 }
110}
111
112#[derive(Debug, Clone, Copy, PartialEq, Eq)]
114pub struct NativeFrameHeader {
115 pub opcode: NativeOpcode,
117 pub request_id: u128,
119 pub status: NativeStatus,
121 pub is_response: bool,
123 pub payload_len: u32,
125 payload_checksum: u32,
126}
127
128impl NativeFrameHeader {
129 pub fn request(
131 opcode: NativeOpcode,
132 request_id: u128,
133 payload: &[u8],
134 ) -> Result<Self, NativeProtocolError> {
135 Self::new(opcode, request_id, NativeStatus::Ok, false, payload)
136 }
137
138 pub fn response(
140 opcode: NativeOpcode,
141 request_id: u128,
142 status: NativeStatus,
143 payload: &[u8],
144 ) -> Result<Self, NativeProtocolError> {
145 Self::new(opcode, request_id, status, true, payload)
146 }
147
148 fn new(
149 opcode: NativeOpcode,
150 request_id: u128,
151 status: NativeStatus,
152 is_response: bool,
153 payload: &[u8],
154 ) -> Result<Self, NativeProtocolError> {
155 if payload.len() > MAX_NATIVE_FRAME_BYTES {
156 return Err(NativeProtocolError::new(format!(
157 "native frame payload is {} bytes, exceeding {MAX_NATIVE_FRAME_BYTES}",
158 payload.len()
159 )));
160 }
161 Ok(Self {
162 opcode,
163 request_id,
164 status,
165 is_response,
166 payload_len: u32::try_from(payload.len())
167 .map_err(|_| NativeProtocolError::new("native frame payload exceeds u32"))?,
168 payload_checksum: payload_checksum(payload),
169 })
170 }
171
172 #[must_use]
174 pub fn encode(self) -> [u8; NATIVE_FRAME_HEADER_BYTES] {
175 let mut bytes = [0; NATIVE_FRAME_HEADER_BYTES];
176 bytes[0..4].copy_from_slice(&FRAME_MAGIC);
177 bytes[4] = FRAME_VERSION;
178 bytes[5] = self.opcode as u8;
179 bytes[6] = u8::from(self.is_response) * FRAME_FLAG_RESPONSE;
180 bytes[7] = self.status as u8;
181 bytes[8..24].copy_from_slice(&self.request_id.to_le_bytes());
182 bytes[24..28].copy_from_slice(&self.payload_len.to_le_bytes());
183 bytes[28..32].copy_from_slice(&self.payload_checksum.to_le_bytes());
184 bytes
185 }
186
187 pub fn decode(bytes: &[u8; NATIVE_FRAME_HEADER_BYTES]) -> Result<Self, NativeProtocolError> {
189 if bytes[0..4] != FRAME_MAGIC {
190 return Err(NativeProtocolError::new("invalid native frame magic"));
191 }
192 if bytes[4] != FRAME_VERSION {
193 return Err(NativeProtocolError::new(format!(
194 "unsupported native frame version {}",
195 bytes[4]
196 )));
197 }
198 if bytes[6] & !FRAME_FLAG_RESPONSE != 0 {
199 return Err(NativeProtocolError::new(
200 "native frame contains unknown flags",
201 ));
202 }
203 let payload_len = u32::from_le_bytes(bytes[24..28].try_into().expect("fixed range"));
204 if payload_len as usize > MAX_NATIVE_FRAME_BYTES {
205 return Err(NativeProtocolError::new(format!(
206 "native frame payload is {payload_len} bytes, exceeding {MAX_NATIVE_FRAME_BYTES}"
207 )));
208 }
209 Ok(Self {
210 opcode: NativeOpcode::from_byte(bytes[5])?,
211 request_id: u128::from_le_bytes(bytes[8..24].try_into().expect("fixed range")),
212 status: NativeStatus::from_byte(bytes[7])?,
213 is_response: bytes[6] == FRAME_FLAG_RESPONSE,
214 payload_len,
215 payload_checksum: u32::from_le_bytes(bytes[28..32].try_into().expect("fixed range")),
216 })
217 }
218
219 pub fn verify_payload(self, payload: &[u8]) -> Result<(), NativeProtocolError> {
221 if payload.len() != self.payload_len as usize {
222 return Err(NativeProtocolError::new(
223 "native frame payload length disagrees with its header",
224 ));
225 }
226 if payload_checksum(payload) != self.payload_checksum {
227 return Err(NativeProtocolError::new(
228 "native frame payload checksum mismatch",
229 ));
230 }
231 Ok(())
232 }
233}
234
235#[derive(Debug, Clone, PartialEq, Eq)]
237pub struct NativeFrame {
238 pub header: NativeFrameHeader,
240 pub payload: Vec<u8>,
242}
243
244impl NativeFrame {
245 pub fn request(
247 opcode: NativeOpcode,
248 request_id: u128,
249 payload: Vec<u8>,
250 ) -> Result<Self, NativeProtocolError> {
251 Ok(Self {
252 header: NativeFrameHeader::request(opcode, request_id, &payload)?,
253 payload,
254 })
255 }
256
257 pub fn response(
259 opcode: NativeOpcode,
260 request_id: u128,
261 status: NativeStatus,
262 payload: Vec<u8>,
263 ) -> Result<Self, NativeProtocolError> {
264 Ok(Self {
265 header: NativeFrameHeader::response(opcode, request_id, status, &payload)?,
266 payload,
267 })
268 }
269
270 #[must_use]
272 pub fn encode(&self) -> Vec<u8> {
273 let mut encoded = Vec::with_capacity(NATIVE_FRAME_HEADER_BYTES + self.payload.len());
274 encoded.extend_from_slice(&self.header.encode());
275 encoded.extend_from_slice(&self.payload);
276 encoded
277 }
278}
279
280#[derive(Debug, Clone, PartialEq, Eq)]
282pub struct NativeLogQueryResult {
283 pub tenant: String,
285 pub entries: Vec<LokiEntry>,
287}
288
289#[derive(Debug, Clone, PartialEq, Eq)]
291pub struct NativePartitionAppend {
292 pub topic_partition: TopicPartition,
294 pub envelope: TelemetryEnvelope,
296 pub transient_context: Option<Arc<[u8]>>,
302}
303
304#[derive(Debug, Clone, PartialEq, Eq)]
306pub struct NativeTelemetryBatch {
307 pub partitions: Vec<NativePartitionAppend>,
309}
310
311impl NativeTelemetryBatch {
312 pub fn encode_native_append(&self) -> Result<Vec<u8>, NativeProtocolError> {
320 if self.partitions.len() != 1 {
321 return Err(NativeProtocolError::new(
322 "retryable native v1 append requires exactly one partition",
323 ));
324 }
325 self.encode()
326 }
327
328 pub fn encode(&self) -> Result<Vec<u8>, NativeProtocolError> {
330 if self.partitions.is_empty() || self.partitions.len() > 256 {
331 return Err(NativeProtocolError::new(
332 "native telemetry batch requires 1..=256 partitions",
333 ));
334 }
335 let mut seen = (self.partitions.len() > 1).then(BTreeSet::new);
336 let mut encoded = Vec::new();
337 encoded.extend_from_slice(&TELEMETRY_BATCH_MAGIC);
338 encoded.extend_from_slice(
339 &u16::try_from(self.partitions.len())
340 .expect("partition count was bounded")
341 .to_le_bytes(),
342 );
343 encoded.extend_from_slice(&[0; 2]);
344 for partition in &self.partitions {
345 if let Some(seen) = &mut seen
346 && !seen.insert(partition.topic_partition)
347 {
348 return Err(NativeProtocolError::new(
349 "native telemetry batch contains a duplicate partition",
350 ));
351 }
352 if partition.topic_partition.topic_id != partition.envelope.signal.topic_id() {
353 return Err(NativeProtocolError::new(
354 "native telemetry partition topic disagrees with its signal",
355 ));
356 }
357 let envelope = partition
358 .envelope
359 .encode()
360 .map_err(|error| NativeProtocolError::new(error.to_string()))?;
361 encoded.extend_from_slice(&partition.topic_partition.topic_id.get().to_le_bytes());
362 encoded.extend_from_slice(&partition.topic_partition.partition_id.get().to_le_bytes());
363 encoded.extend_from_slice(
364 &u32::try_from(envelope.len())
365 .map_err(|_| NativeProtocolError::new("STEL envelope exceeds u32"))?
366 .to_le_bytes(),
367 );
368 encoded.extend_from_slice(&envelope);
369 let transient_context = partition.transient_context.as_deref().unwrap_or_default();
370 encoded.extend_from_slice(
371 &u32::try_from(transient_context.len())
372 .map_err(|_| NativeProtocolError::new("transient context exceeds u32"))?
373 .to_le_bytes(),
374 );
375 encoded.extend_from_slice(transient_context);
376 }
377 if encoded.len() > MAX_NATIVE_FRAME_BYTES {
378 return Err(NativeProtocolError::new(
379 "native telemetry batch exceeds the frame limit",
380 ));
381 }
382 Ok(encoded)
383 }
384
385 pub fn decode(payload: &[u8]) -> Result<Self, NativeProtocolError> {
387 if payload.len() < 8 || payload[..4] != TELEMETRY_BATCH_MAGIC {
388 return Err(NativeProtocolError::new(
389 "invalid native telemetry batch header",
390 ));
391 }
392 let count = usize::from(u16::from_le_bytes(
393 payload[4..6].try_into().expect("fixed range"),
394 ));
395 if count == 0 || count > 256 || payload[6..8] != [0, 0] {
396 return Err(NativeProtocolError::new(
397 "invalid native telemetry partition count or flags",
398 ));
399 }
400 let mut cursor = Cursor::at(payload, 8);
401 let mut partitions = Vec::with_capacity(count);
402 let mut seen = (count > 1).then(BTreeSet::new);
403 for _ in 0..count {
404 let topic_partition = TopicPartition::new(
405 TopicId::new(cursor.u128("telemetry topic ID")?),
406 LogicalPartitionId::new(cursor.u32("telemetry partition ID")?),
407 );
408 let envelope_len = cursor.u32("telemetry envelope length")? as usize;
409 let envelope =
410 TelemetryEnvelope::decode(cursor.bytes(envelope_len, "telemetry envelope")?)
411 .map_err(|error| NativeProtocolError::new(error.to_string()))?;
412 let transient_len = cursor.u32("transient context length")? as usize;
413 let transient_context = if transient_len == 0 {
414 None
415 } else {
416 Some(Arc::<[u8]>::from(
417 cursor.bytes(transient_len, "transient context")?,
418 ))
419 };
420 if topic_partition.topic_id != envelope.signal.topic_id() {
421 return Err(NativeProtocolError::new(
422 "native telemetry partition topic disagrees with its signal",
423 ));
424 }
425 if let Some(seen) = &mut seen
426 && !seen.insert(topic_partition)
427 {
428 return Err(NativeProtocolError::new(
429 "native telemetry batch contains a duplicate partition",
430 ));
431 }
432 partitions.push(NativePartitionAppend {
433 topic_partition,
434 envelope,
435 transient_context,
436 });
437 }
438 cursor.finish()?;
439 Ok(Self { partitions })
440 }
441
442 pub fn decode_native_append(payload: &[u8]) -> Result<Self, NativeProtocolError> {
445 let batch = Self::decode(payload)?;
446 if batch.partitions.len() != 1 {
447 return Err(NativeProtocolError::new(
448 "retryable native v1 append requires exactly one partition",
449 ));
450 }
451 Ok(batch)
452 }
453}
454
455#[must_use]
457pub fn is_native_telemetry_batch(payload: &[u8]) -> bool {
458 payload.starts_with(&TELEMETRY_BATCH_MAGIC)
459}
460
461#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
463pub struct NativePartitionAck {
464 pub topic_partition: TopicPartition,
466 pub first_offset: u64,
468 pub last_offset: u64,
470}
471
472#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
474pub struct NativeTelemetryAppendAck {
475 pub partitions: Vec<NativePartitionAck>,
477}
478
479impl NativeTelemetryAppendAck {
480 pub fn encode(&self) -> Result<Vec<u8>, NativeProtocolError> {
482 if self.partitions.len() > 256 {
483 return Err(NativeProtocolError::new(
484 "native acknowledgement exceeds 256 partitions",
485 ));
486 }
487 let mut encoded = Vec::with_capacity(8 + self.partitions.len() * 36);
488 encoded.extend_from_slice(&TELEMETRY_ACK_MAGIC);
489 encoded.extend_from_slice(
490 &u16::try_from(self.partitions.len())
491 .expect("ack partition count was bounded")
492 .to_le_bytes(),
493 );
494 encoded.extend_from_slice(&[0; 2]);
495 for partition in &self.partitions {
496 encoded.extend_from_slice(&partition.topic_partition.topic_id.get().to_le_bytes());
497 encoded.extend_from_slice(&partition.topic_partition.partition_id.get().to_le_bytes());
498 encoded.extend_from_slice(&partition.first_offset.to_le_bytes());
499 encoded.extend_from_slice(&partition.last_offset.to_le_bytes());
500 }
501 Ok(encoded)
502 }
503
504 pub fn decode(payload: &[u8]) -> Result<Self, NativeProtocolError> {
506 if payload.len() < 8 || payload[..4] != TELEMETRY_ACK_MAGIC {
507 return Err(NativeProtocolError::new(
508 "invalid native telemetry acknowledgement",
509 ));
510 }
511 let count = usize::from(u16::from_le_bytes(
512 payload[4..6].try_into().expect("fixed range"),
513 ));
514 if count > 256 || payload[6..8] != [0, 0] || payload.len() != 8 + count * 36 {
515 return Err(NativeProtocolError::new(
516 "invalid native telemetry acknowledgement length",
517 ));
518 }
519 let mut cursor = Cursor::at(payload, 8);
520 let mut partitions = Vec::with_capacity(count);
521 for _ in 0..count {
522 partitions.push(NativePartitionAck {
523 topic_partition: TopicPartition::new(
524 TopicId::new(cursor.u128("ack topic ID")?),
525 LogicalPartitionId::new(cursor.u32("ack partition ID")?),
526 ),
527 first_offset: cursor.u64("ack first offset")?,
528 last_offset: cursor.u64("ack last offset")?,
529 });
530 }
531 cursor.finish()?;
532 Ok(Self { partitions })
533 }
534}
535
536#[derive(Debug, Clone, Copy, PartialEq, Eq)]
537struct NativeLogQueryResultInfo<'a> {
538 tenant: &'a str,
539 record_count: u32,
540}
541
542fn inspect_native_log_query_result(
543 payload: &[u8],
544) -> Result<NativeLogQueryResultInfo<'_>, NativeProtocolError> {
545 if payload.len() < LOG_QUERY_RESULT_HEADER_BYTES {
546 return Err(NativeProtocolError::new(
547 "native log query result is shorter than its header",
548 ));
549 }
550 if payload[0..4] != LOG_QUERY_RESULT_MAGIC {
551 return Err(NativeProtocolError::new(
552 "invalid native log query result magic",
553 ));
554 }
555 if payload[12..16] != [0; 4] {
556 return Err(NativeProtocolError::new(
557 "native log query result reserved bytes must be zero",
558 ));
559 }
560 let tenant_len = usize::from(u16::from_le_bytes(
561 payload[4..6].try_into().expect("fixed range"),
562 ));
563 if tenant_len > MAX_TENANT_BYTES {
564 return Err(NativeProtocolError::new(
565 "native log query result tenant exceeds its limit",
566 ));
567 }
568 let end = LOG_QUERY_RESULT_HEADER_BYTES
569 .checked_add(tenant_len)
570 .filter(|end| *end <= payload.len())
571 .ok_or_else(|| NativeProtocolError::new("native log query result tenant is truncated"))?;
572 let tenant = std::str::from_utf8(&payload[LOG_QUERY_RESULT_HEADER_BYTES..end])
573 .map_err(|_| NativeProtocolError::new("native log query result tenant is not UTF-8"))?;
574 if tenant.is_empty() {
575 return Err(NativeProtocolError::new(
576 "native log query result tenant must not be empty",
577 ));
578 }
579 Ok(NativeLogQueryResultInfo {
580 tenant,
581 record_count: u32::from_le_bytes(payload[8..12].try_into().expect("fixed range")),
582 })
583}
584
585pub fn encode_native_log_query_result(
587 tenant: &str,
588 entries: Vec<LokiEntry>,
589) -> Result<Vec<u8>, NativeProtocolError> {
590 validate_tenant(tenant)?;
591 let record_count = u32::try_from(entries.len())
592 .map_err(|_| NativeProtocolError::new("native batch contains more than u32 records"))?;
593 let mut streams = BTreeMap::<BTreeMap<String, String>, Vec<LokiEntry>>::new();
594 for entry in entries {
595 if entry.timestamp_unix_nanos < 0 {
596 return Err(NativeProtocolError::new(
597 "negative native log timestamps are unsupported",
598 ));
599 }
600 streams.entry(entry.labels.clone()).or_default().push(entry);
601 }
602 if streams.len() > MAX_STREAMS {
603 return Err(NativeProtocolError::new(
604 "native batch contains too many streams",
605 ));
606 }
607
608 let mut encoded = Vec::new();
609 encoded.extend_from_slice(&LOG_QUERY_RESULT_MAGIC);
610 put_u16(&mut encoded, tenant.len(), "tenant")?;
611 put_u16(&mut encoded, streams.len(), "stream count")?;
612 encoded.extend_from_slice(&record_count.to_le_bytes());
613 encoded.extend_from_slice(&0_u32.to_le_bytes());
614 encoded.extend_from_slice(tenant.as_bytes());
615 for (labels, entries) in streams {
616 if labels.len() > MAX_LABELS_PER_STREAM {
617 return Err(NativeProtocolError::new(
618 "native stream contains too many labels",
619 ));
620 }
621 put_u16(&mut encoded, labels.len(), "label count")?;
622 encoded.extend_from_slice(&0_u16.to_le_bytes());
623 let entry_count = u32::try_from(entries.len())
624 .map_err(|_| NativeProtocolError::new("native stream contains too many entries"))?;
625 encoded.extend_from_slice(&entry_count.to_le_bytes());
626 for (key, value) in &labels {
627 put_string16(&mut encoded, key, "label key")?;
628 put_string16(&mut encoded, value, "label value")?;
629 }
630 for entry in entries {
631 encoded.extend_from_slice(&(entry.timestamp_unix_nanos as u64).to_le_bytes());
632 put_u32(&mut encoded, entry.line.len(), "log line")?;
633 if entry.structured_metadata.len() > MAX_METADATA_PER_ENTRY {
634 return Err(NativeProtocolError::new(
635 "native entry contains too much structured metadata",
636 ));
637 }
638 put_u16(
639 &mut encoded,
640 entry.structured_metadata.len(),
641 "metadata count",
642 )?;
643 encoded.extend_from_slice(&0_u16.to_le_bytes());
644 encoded.extend_from_slice(entry.line.as_bytes());
645 for (key, value) in entry.structured_metadata {
646 put_string16(&mut encoded, &key, "metadata key")?;
647 put_string16(&mut encoded, &value, "metadata value")?;
648 }
649 }
650 }
651 if encoded.len() > MAX_NATIVE_FRAME_BYTES {
652 return Err(NativeProtocolError::new(format!(
653 "native batch is {} bytes, exceeding {MAX_NATIVE_FRAME_BYTES}",
654 encoded.len()
655 )));
656 }
657 Ok(encoded)
658}
659
660pub fn decode_native_log_query_result(
662 payload: &[u8],
663) -> Result<NativeLogQueryResult, NativeProtocolError> {
664 let info = inspect_native_log_query_result(payload)?;
665 let stream_count = usize::from(u16::from_le_bytes(
666 payload[6..8].try_into().expect("fixed range"),
667 ));
668 let mut cursor = Cursor::at(payload, LOG_QUERY_RESULT_HEADER_BYTES + info.tenant.len());
669 let mut entries = Vec::with_capacity(info.record_count as usize);
670 for _ in 0..stream_count {
671 let label_count = usize::from(cursor.u16("label count")?);
672 if label_count > MAX_LABELS_PER_STREAM {
673 return Err(NativeProtocolError::new(
674 "native stream contains too many labels",
675 ));
676 }
677 if cursor.u16("stream reserved bytes")? != 0 {
678 return Err(NativeProtocolError::new(
679 "native stream reserved bytes must be zero",
680 ));
681 }
682 let entry_count = cursor.u32("entry count")? as usize;
683 let mut labels = BTreeMap::new();
684 for _ in 0..label_count {
685 let key = cursor.string16("label key")?.to_owned();
686 let value = cursor.string16("label value")?.to_owned();
687 if key.is_empty() || labels.insert(key, value).is_some() {
688 return Err(NativeProtocolError::new(
689 "native stream contains an empty or duplicate label",
690 ));
691 }
692 }
693 entries
694 .len()
695 .checked_add(entry_count)
696 .filter(|count| *count <= info.record_count as usize)
697 .ok_or_else(|| {
698 NativeProtocolError::new("native stream counts exceed declared record count")
699 })?;
700 for _ in 0..entry_count {
701 let timestamp = cursor.u64("timestamp")?;
702 let line_len = cursor.u32("line length")? as usize;
703 let metadata_count = usize::from(cursor.u16("metadata count")?);
704 if metadata_count > MAX_METADATA_PER_ENTRY {
705 return Err(NativeProtocolError::new(
706 "native entry contains too much structured metadata",
707 ));
708 }
709 if cursor.u16("entry reserved bytes")? != 0 {
710 return Err(NativeProtocolError::new(
711 "native entry reserved bytes must be zero",
712 ));
713 }
714 let line = cursor.string(line_len, "log line")?.to_owned();
715 let mut structured_metadata = BTreeMap::new();
716 for _ in 0..metadata_count {
717 let key = cursor.string16("metadata key")?.to_owned();
718 let value = cursor.string16("metadata value")?.to_owned();
719 if key.is_empty() || structured_metadata.insert(key, value).is_some() {
720 return Err(NativeProtocolError::new(
721 "native entry contains empty or duplicate metadata",
722 ));
723 }
724 }
725 let timestamp_unix_nanos = i64::try_from(timestamp).map_err(|_| {
726 NativeProtocolError::new("native log timestamp exceeds the signed i64 range")
727 })?;
728 entries.push(LokiEntry {
729 timestamp_unix_nanos,
730 labels: labels.clone(),
731 line,
732 structured_metadata,
733 });
734 }
735 }
736 if entries.len() != info.record_count as usize {
737 return Err(NativeProtocolError::new(format!(
738 "native batch decoded {} records, expected {}",
739 entries.len(),
740 info.record_count
741 )));
742 }
743 cursor.finish()?;
744 Ok(NativeLogQueryResult {
745 tenant: info.tenant.to_owned(),
746 entries,
747 })
748}
749
750#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
752pub enum NativeQueryDirection {
753 #[default]
755 OldestFirst,
756 NewestFirst,
758}
759
760#[derive(Debug, Clone, PartialEq, Eq)]
762pub struct NativeQuery {
763 pub tenant: String,
765 pub labels: BTreeMap<String, String>,
767 pub terms: Vec<String>,
769 pub start_timestamp_unix_nanos: Option<u64>,
771 pub end_timestamp_unix_nanos: Option<u64>,
773 pub limit: u32,
775 pub direction: NativeQueryDirection,
777}
778
779#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
782pub struct NativeCapabilities {
783 pub protocol_version: u8,
785 pub append_v1: bool,
787 pub logical_partitions: [u16; 3],
789 pub query_logs: bool,
791 pub query_metrics: bool,
793 pub query_traces: bool,
795 pub query_series: bool,
797 pub append_queryable: bool,
799}
800
801pub fn encode_native_metric_query(
803 query: &crate::MetricQuery,
804) -> Result<Vec<u8>, NativeProtocolError> {
805 encode_messagepack(METRIC_QUERY_MAGIC, query, "metric query")
806}
807
808pub fn decode_native_metric_query(
810 payload: &[u8],
811) -> Result<crate::MetricQuery, NativeProtocolError> {
812 decode_messagepack(METRIC_QUERY_MAGIC, payload, "metric query")
813}
814
815pub fn encode_native_metric_query_result(
817 points: &[crate::DurableMetricPoint],
818) -> Result<Vec<u8>, NativeProtocolError> {
819 encode_messagepack(METRIC_QUERY_RESULT_MAGIC, points, "metric query result")
820}
821
822pub fn decode_native_metric_query_result(
824 payload: &[u8],
825) -> Result<Vec<crate::DurableMetricPoint>, NativeProtocolError> {
826 decode_messagepack(METRIC_QUERY_RESULT_MAGIC, payload, "metric query result")
827}
828
829pub fn encode_native_trace_query(
831 query: &crate::TraceQuery,
832) -> Result<Vec<u8>, NativeProtocolError> {
833 encode_messagepack(TRACE_QUERY_MAGIC, query, "trace query")
834}
835
836pub fn decode_native_trace_query(payload: &[u8]) -> Result<crate::TraceQuery, NativeProtocolError> {
838 decode_messagepack(TRACE_QUERY_MAGIC, payload, "trace query")
839}
840
841pub fn encode_native_trace_query_result(
843 spans: &[crate::DurableSpan],
844) -> Result<Vec<u8>, NativeProtocolError> {
845 encode_messagepack(TRACE_QUERY_RESULT_MAGIC, spans, "trace query result")
846}
847
848pub fn decode_native_trace_query_result(
850 payload: &[u8],
851) -> Result<Vec<crate::DurableSpan>, NativeProtocolError> {
852 decode_messagepack(TRACE_QUERY_RESULT_MAGIC, payload, "trace query result")
853}
854
855pub fn encode_native_capabilities(
857 capabilities: &NativeCapabilities,
858) -> Result<Vec<u8>, NativeProtocolError> {
859 encode_messagepack(CAPABILITIES_MAGIC, capabilities, "capabilities")
860}
861
862pub fn decode_native_capabilities(
864 payload: &[u8],
865) -> Result<NativeCapabilities, NativeProtocolError> {
866 decode_messagepack(CAPABILITIES_MAGIC, payload, "capabilities")
867}
868
869fn encode_messagepack<T: serde::Serialize + ?Sized>(
870 magic: [u8; 4],
871 value: &T,
872 kind: &str,
873) -> Result<Vec<u8>, NativeProtocolError> {
874 let encoded = rmp_serde::to_vec(value).map_err(|error| {
875 NativeProtocolError::new(format!("native {kind} encoding failed: {error}"))
876 })?;
877 if encoded.len().saturating_add(magic.len()) > MAX_NATIVE_FRAME_BYTES {
878 return Err(NativeProtocolError::new(format!(
879 "native {kind} exceeds the frame limit"
880 )));
881 }
882 let mut payload = Vec::with_capacity(magic.len() + encoded.len());
883 payload.extend_from_slice(&magic);
884 payload.extend_from_slice(&encoded);
885 Ok(payload)
886}
887
888fn decode_messagepack<T: serde::de::DeserializeOwned>(
889 magic: [u8; 4],
890 payload: &[u8],
891 kind: &str,
892) -> Result<T, NativeProtocolError> {
893 let Some(encoded) = payload.strip_prefix(&magic) else {
894 return Err(NativeProtocolError::new(format!(
895 "invalid native {kind} magic"
896 )));
897 };
898 rmp_serde::from_slice(encoded)
899 .map_err(|error| NativeProtocolError::new(format!("invalid native {kind}: {error}")))
900}
901
902pub fn encode_native_query(query: &NativeQuery) -> Result<Vec<u8>, NativeProtocolError> {
904 validate_query(query)?;
905 let mut encoded = Vec::new();
906 encoded.extend_from_slice(&QUERY_MAGIC);
907 put_u16(&mut encoded, query.tenant.len(), "query tenant")?;
908 put_u16(&mut encoded, query.labels.len(), "query label count")?;
909 put_u16(&mut encoded, query.terms.len(), "query term count")?;
910 encoded.push(match query.direction {
911 NativeQueryDirection::OldestFirst => 0,
912 NativeQueryDirection::NewestFirst => 1,
913 });
914 encoded.push(0);
915 encoded.extend_from_slice(&query.limit.to_le_bytes());
916 encoded.extend_from_slice(
917 &query
918 .start_timestamp_unix_nanos
919 .unwrap_or(u64::MAX)
920 .to_le_bytes(),
921 );
922 encoded.extend_from_slice(
923 &query
924 .end_timestamp_unix_nanos
925 .unwrap_or(u64::MAX)
926 .to_le_bytes(),
927 );
928 encoded.extend_from_slice(query.tenant.as_bytes());
929 for (key, value) in &query.labels {
930 put_string16(&mut encoded, key, "query label key")?;
931 put_string16(&mut encoded, value, "query label value")?;
932 }
933 for term in &query.terms {
934 put_string16(&mut encoded, term, "query term")?;
935 }
936 Ok(encoded)
937}
938
939pub fn decode_native_query(payload: &[u8]) -> Result<NativeQuery, NativeProtocolError> {
941 if payload.len() < QUERY_HEADER_BYTES || payload[0..4] != QUERY_MAGIC {
942 return Err(NativeProtocolError::new("invalid native query header"));
943 }
944 let tenant_len = usize::from(u16::from_le_bytes(
945 payload[4..6].try_into().expect("fixed range"),
946 ));
947 let label_count = usize::from(u16::from_le_bytes(
948 payload[6..8].try_into().expect("fixed range"),
949 ));
950 let term_count = usize::from(u16::from_le_bytes(
951 payload[8..10].try_into().expect("fixed range"),
952 ));
953 let direction = match payload[10] {
954 0 => NativeQueryDirection::OldestFirst,
955 1 => NativeQueryDirection::NewestFirst,
956 value => {
957 return Err(NativeProtocolError::new(format!(
958 "unsupported native query direction {value}"
959 )));
960 }
961 };
962 if payload[11] != 0 {
963 return Err(NativeProtocolError::new(
964 "native query reserved byte must be zero",
965 ));
966 }
967 let limit = u32::from_le_bytes(payload[12..16].try_into().expect("fixed range"));
968 let start = u64::from_le_bytes(payload[16..24].try_into().expect("fixed range"));
969 let end = u64::from_le_bytes(payload[24..32].try_into().expect("fixed range"));
970 let mut cursor = Cursor::at(payload, QUERY_HEADER_BYTES);
971 let tenant = cursor.string(tenant_len, "query tenant")?.to_owned();
972 let mut labels = BTreeMap::new();
973 for _ in 0..label_count {
974 let key = cursor.string16("query label key")?.to_owned();
975 let value = cursor.string16("query label value")?.to_owned();
976 if key.is_empty() || labels.insert(key, value).is_some() {
977 return Err(NativeProtocolError::new(
978 "native query contains an empty or duplicate label",
979 ));
980 }
981 }
982 let mut terms = Vec::with_capacity(term_count);
983 for _ in 0..term_count {
984 let term = cursor.string16("query term")?.to_owned();
985 if term.is_empty() {
986 return Err(NativeProtocolError::new(
987 "native query terms must not be empty",
988 ));
989 }
990 terms.push(term);
991 }
992 cursor.finish()?;
993 let query = NativeQuery {
994 tenant,
995 labels,
996 terms,
997 start_timestamp_unix_nanos: (start != u64::MAX).then_some(start),
998 end_timestamp_unix_nanos: (end != u64::MAX).then_some(end),
999 limit,
1000 direction,
1001 };
1002 validate_query(&query)?;
1003 Ok(query)
1004}
1005
1006#[derive(Debug, Clone, PartialEq, Eq)]
1008pub struct NativeProtocolError {
1009 message: String,
1010}
1011
1012impl NativeProtocolError {
1013 fn new(message: impl Into<String>) -> Self {
1014 Self {
1015 message: message.into(),
1016 }
1017 }
1018}
1019
1020impl fmt::Display for NativeProtocolError {
1021 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1022 formatter.write_str(&self.message)
1023 }
1024}
1025
1026impl std::error::Error for NativeProtocolError {}
1027
1028fn validate_tenant(tenant: &str) -> Result<(), NativeProtocolError> {
1029 if tenant.is_empty() {
1030 return Err(NativeProtocolError::new("native tenant must not be empty"));
1031 }
1032 if tenant.len() > MAX_TENANT_BYTES || tenant.len() > usize::from(u16::MAX) {
1033 return Err(NativeProtocolError::new(
1034 "native tenant exceeds its length limit",
1035 ));
1036 }
1037 Ok(())
1038}
1039
1040fn validate_query(query: &NativeQuery) -> Result<(), NativeProtocolError> {
1041 validate_tenant(&query.tenant)?;
1042 if query.labels.len() > MAX_LABELS_PER_STREAM {
1043 return Err(NativeProtocolError::new(
1044 "native query contains too many labels",
1045 ));
1046 }
1047 if query.terms.len() > MAX_QUERY_TERMS {
1048 return Err(NativeProtocolError::new(
1049 "native query contains too many terms",
1050 ));
1051 }
1052 if query.limit == 0 || query.limit > MAX_QUERY_LIMIT {
1053 return Err(NativeProtocolError::new(format!(
1054 "native query limit must be in 1..={MAX_QUERY_LIMIT}"
1055 )));
1056 }
1057 if let (Some(start), Some(end)) = (
1058 query.start_timestamp_unix_nanos,
1059 query.end_timestamp_unix_nanos,
1060 ) && start >= end
1061 {
1062 return Err(NativeProtocolError::new(
1063 "native query timestamp range must be nonempty",
1064 ));
1065 }
1066 for (key, value) in &query.labels {
1067 if key.is_empty() {
1068 return Err(NativeProtocolError::new(
1069 "native query label keys must not be empty",
1070 ));
1071 }
1072 validate_string16(key, "query label key")?;
1073 validate_string16(value, "query label value")?;
1074 }
1075 for term in &query.terms {
1076 if term.is_empty() {
1077 return Err(NativeProtocolError::new(
1078 "native query terms must not be empty",
1079 ));
1080 }
1081 validate_string16(term, "query term")?;
1082 }
1083 Ok(())
1084}
1085
1086fn payload_checksum(payload: &[u8]) -> u32 {
1087 u32::from_le_bytes(
1088 blake3::hash(payload).as_bytes()[0..4]
1089 .try_into()
1090 .expect("fixed range"),
1091 )
1092}
1093
1094fn put_u16(
1095 encoded: &mut Vec<u8>,
1096 value: usize,
1097 field: &'static str,
1098) -> Result<(), NativeProtocolError> {
1099 let value = u16::try_from(value)
1100 .map_err(|_| NativeProtocolError::new(format!("{field} exceeds u16")))?;
1101 encoded.extend_from_slice(&value.to_le_bytes());
1102 Ok(())
1103}
1104
1105fn put_u32(
1106 encoded: &mut Vec<u8>,
1107 value: usize,
1108 field: &'static str,
1109) -> Result<(), NativeProtocolError> {
1110 let value = u32::try_from(value)
1111 .map_err(|_| NativeProtocolError::new(format!("{field} exceeds u32")))?;
1112 encoded.extend_from_slice(&value.to_le_bytes());
1113 Ok(())
1114}
1115
1116fn validate_string16(value: &str, field: &'static str) -> Result<(), NativeProtocolError> {
1117 u16::try_from(value.len())
1118 .map(|_| ())
1119 .map_err(|_| NativeProtocolError::new(format!("{field} exceeds u16")))
1120}
1121
1122fn put_string16(
1123 encoded: &mut Vec<u8>,
1124 value: &str,
1125 field: &'static str,
1126) -> Result<(), NativeProtocolError> {
1127 put_u16(encoded, value.len(), field)?;
1128 encoded.extend_from_slice(value.as_bytes());
1129 Ok(())
1130}
1131
1132struct Cursor<'a> {
1133 bytes: &'a [u8],
1134 offset: usize,
1135}
1136
1137impl<'a> Cursor<'a> {
1138 fn at(bytes: &'a [u8], offset: usize) -> Self {
1139 Self { bytes, offset }
1140 }
1141
1142 fn bytes(&mut self, len: usize, field: &'static str) -> Result<&'a [u8], NativeProtocolError> {
1143 let end = self
1144 .offset
1145 .checked_add(len)
1146 .filter(|end| *end <= self.bytes.len())
1147 .ok_or_else(|| NativeProtocolError::new(format!("native {field} is truncated")))?;
1148 let value = &self.bytes[self.offset..end];
1149 self.offset = end;
1150 Ok(value)
1151 }
1152
1153 fn u16(&mut self, field: &'static str) -> Result<u16, NativeProtocolError> {
1154 Ok(u16::from_le_bytes(
1155 self.bytes(2, field)?.try_into().expect("fixed range"),
1156 ))
1157 }
1158
1159 fn u32(&mut self, field: &'static str) -> Result<u32, NativeProtocolError> {
1160 Ok(u32::from_le_bytes(
1161 self.bytes(4, field)?.try_into().expect("fixed range"),
1162 ))
1163 }
1164
1165 fn u64(&mut self, field: &'static str) -> Result<u64, NativeProtocolError> {
1166 Ok(u64::from_le_bytes(
1167 self.bytes(8, field)?.try_into().expect("fixed range"),
1168 ))
1169 }
1170
1171 fn u128(&mut self, field: &'static str) -> Result<u128, NativeProtocolError> {
1172 Ok(u128::from_le_bytes(
1173 self.bytes(16, field)?.try_into().expect("fixed range"),
1174 ))
1175 }
1176
1177 fn string(&mut self, len: usize, field: &'static str) -> Result<&'a str, NativeProtocolError> {
1178 std::str::from_utf8(self.bytes(len, field)?)
1179 .map_err(|_| NativeProtocolError::new(format!("native {field} is not UTF-8")))
1180 }
1181
1182 fn string16(&mut self, field: &'static str) -> Result<&'a str, NativeProtocolError> {
1183 let len = usize::from(self.u16(field)?);
1184 self.string(len, field)
1185 }
1186
1187 fn finish(self) -> Result<(), NativeProtocolError> {
1188 if self.offset == self.bytes.len() {
1189 Ok(())
1190 } else {
1191 Err(NativeProtocolError::new(
1192 "native payload contains trailing bytes",
1193 ))
1194 }
1195 }
1196}
1197
1198#[cfg(test)]
1199mod tests {
1200 use super::*;
1201
1202 #[test]
1203 fn signal_aware_batch_and_partition_ack_round_trip() {
1204 let topic_partition =
1205 TopicPartition::new(crate::TRACES_TOPIC_ID, LogicalPartitionId::new(7));
1206 let batch = NativeTelemetryBatch {
1207 partitions: vec![NativePartitionAppend {
1208 topic_partition,
1209 envelope: TelemetryEnvelope::new(
1210 crate::TelemetrySignal::Traces,
1211 "tenant-a",
1212 2,
1213 &b"route"[..],
1214 &b"payload"[..],
1215 )
1216 .unwrap(),
1217 transient_context: Some(Arc::<[u8]>::from(&b"context"[..])),
1218 }],
1219 };
1220 assert_eq!(
1221 NativeTelemetryBatch::decode(&batch.encode().unwrap()).unwrap(),
1222 batch
1223 );
1224
1225 let acknowledgement = NativeTelemetryAppendAck {
1226 partitions: vec![NativePartitionAck {
1227 topic_partition,
1228 first_offset: 10,
1229 last_offset: 11,
1230 }],
1231 };
1232 assert_eq!(
1233 NativeTelemetryAppendAck::decode(&acknowledgement.encode().unwrap()).unwrap(),
1234 acknowledgement
1235 );
1236 }
1237
1238 fn entries() -> Vec<LokiEntry> {
1239 vec![
1240 LokiEntry {
1241 timestamp_unix_nanos: 10,
1242 labels: BTreeMap::from([
1243 ("app".to_owned(), "api".to_owned()),
1244 ("region".to_owned(), "東京".to_owned()),
1245 ]),
1246 line: "request café".to_owned(),
1247 structured_metadata: BTreeMap::from([("trace_id".to_owned(), "abc".to_owned())]),
1248 },
1249 LokiEntry {
1250 timestamp_unix_nanos: 11,
1251 labels: BTreeMap::from([
1252 ("app".to_owned(), "api".to_owned()),
1253 ("region".to_owned(), "東京".to_owned()),
1254 ]),
1255 line: "request complete".to_owned(),
1256 structured_metadata: BTreeMap::new(),
1257 },
1258 ]
1259 }
1260
1261 #[test]
1262 fn log_query_result_round_trips_byte_exact_text_and_metadata() {
1263 let expected = entries();
1264 let encoded = encode_native_log_query_result("tenant-a", expected.clone())
1265 .expect("query result encodes");
1266 let decoded = decode_native_log_query_result(&encoded).expect("query result decodes");
1267 assert_eq!(decoded.tenant, "tenant-a");
1268 assert_eq!(decoded.entries, expected);
1269 }
1270
1271 #[test]
1272 fn log_query_result_rejects_truncation_count_mismatch_and_trailing_bytes() {
1273 let encoded = encode_native_log_query_result("tenant-a", entries()).expect("query result");
1274 for end in 0..encoded.len() {
1275 assert!(
1276 decode_native_log_query_result(&encoded[..end]).is_err(),
1277 "{end}"
1278 );
1279 }
1280 let mut count = encoded.clone();
1281 count[8..12].copy_from_slice(&3_u32.to_le_bytes());
1282 assert!(decode_native_log_query_result(&count).is_err());
1283 let mut reserved = encoded.clone();
1284 reserved[12] = 1;
1285 assert!(decode_native_log_query_result(&reserved).is_err());
1286 let mut trailing = encoded;
1287 trailing.push(0);
1288 assert!(decode_native_log_query_result(&trailing).is_err());
1289 }
1290
1291 #[test]
1292 fn frame_header_checks_magic_version_length_and_payload_checksum() {
1293 let frame = NativeFrame::request(NativeOpcode::Append, 42, vec![1, 2, 3]).expect("frame");
1294 let encoded = frame.encode();
1295 let header: [u8; NATIVE_FRAME_HEADER_BYTES] = encoded[..NATIVE_FRAME_HEADER_BYTES]
1296 .try_into()
1297 .expect("header");
1298 let decoded = NativeFrameHeader::decode(&header).expect("decode");
1299 assert_eq!(decoded.request_id, 42);
1300 decoded
1301 .verify_payload(&encoded[NATIVE_FRAME_HEADER_BYTES..])
1302 .expect("checksum");
1303 assert!(decoded.verify_payload(&[1, 2, 4]).is_err());
1304
1305 let mut invalid = header;
1306 invalid[4] = 3;
1307 assert!(NativeFrameHeader::decode(&invalid).is_err());
1308 }
1309
1310 #[test]
1311 fn indexed_query_round_trips_all_bounds_and_constraints() {
1312 let query = NativeQuery {
1313 tenant: "tenant-a".to_owned(),
1314 labels: BTreeMap::from([("app".to_owned(), "api".to_owned())]),
1315 terms: vec!["timeout".to_owned(), "error".to_owned()],
1316 start_timestamp_unix_nanos: Some(10),
1317 end_timestamp_unix_nanos: Some(20),
1318 limit: 100,
1319 direction: NativeQueryDirection::NewestFirst,
1320 };
1321 assert_eq!(
1322 decode_native_query(&encode_native_query(&query).expect("encode")).expect("decode"),
1323 query
1324 );
1325 }
1326
1327 #[test]
1328 fn signal_native_queries_and_capabilities_round_trip() {
1329 let metric = crate::MetricQuery {
1330 tenant: std::sync::Arc::from("tenant-a"),
1331 name: Some(std::sync::Arc::from("requests_total")),
1332 limit: 10,
1333 ..crate::MetricQuery::default()
1334 };
1335 assert_eq!(
1336 decode_native_metric_query(&encode_native_metric_query(&metric).expect("encode"))
1337 .expect("decode"),
1338 metric
1339 );
1340 let trace = crate::TraceQuery {
1341 tenant: std::sync::Arc::from("tenant-a"),
1342 name: Some(std::sync::Arc::from("checkout")),
1343 limit: 10,
1344 ..crate::TraceQuery::default()
1345 };
1346 assert_eq!(
1347 decode_native_trace_query(&encode_native_trace_query(&trace).expect("encode"))
1348 .expect("decode"),
1349 trace
1350 );
1351 let capabilities = NativeCapabilities {
1352 protocol_version: 1,
1353 append_v1: true,
1354 logical_partitions: [8, 8, 8],
1355 query_logs: true,
1356 query_metrics: true,
1357 query_traces: true,
1358 query_series: false,
1359 append_queryable: true,
1360 };
1361 assert_eq!(
1362 decode_native_capabilities(&encode_native_capabilities(&capabilities).expect("encode"))
1363 .expect("decode"),
1364 capabilities
1365 );
1366 }
1367}