Skip to main content

shard_telemetry/
types.rs

1use std::fmt;
2use std::sync::Arc;
3
4use regex::{Regex, RegexBuilder};
5use serde::{Deserialize, Serialize};
6use shard_stream_core::{LogicalOffset, ShardId, TopicPartition};
7
8use crate::{
9    CompressionCohortId, ResourceContext, ScopeContext, SpanId, TelemetryAttribute, TelemetryError,
10    TelemetryResult, TelemetrySignal, TelemetryValue, TraceId,
11};
12
13/// Stable address of one log record in shard-stream's ordered append log.
14#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
15pub struct TelemetryRecordRef {
16    /// Signal represented by this offset.
17    pub signal: TelemetrySignal,
18    /// Logical topic and partition containing the record.
19    pub topic_partition: TopicPartition,
20    /// Durable logical offset of the record.
21    pub offset: LogicalOffset,
22}
23
24impl TelemetryRecordRef {
25    /// Creates a record address.
26    #[must_use]
27    pub const fn new(topic_partition: TopicPartition, offset: LogicalOffset) -> Self {
28        Self {
29            signal: TelemetrySignal::Logs,
30            topic_partition,
31            offset,
32        }
33    }
34
35    /// Creates a signal-aware record address.
36    #[must_use]
37    pub const fn for_signal(
38        signal: TelemetrySignal,
39        topic_partition: TopicPartition,
40        offset: LogicalOffset,
41    ) -> Self {
42        Self {
43            signal,
44            topic_partition,
45            offset,
46        }
47    }
48}
49
50/// A normalized metadata field stored alongside a log message.
51#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
52pub struct MetadataField {
53    /// Metadata field name, for example `service` or `level`.
54    pub key: Arc<str>,
55    /// Exact field value.
56    pub value: Arc<str>,
57}
58
59impl MetadataField {
60    /// Creates a metadata field from owned or borrowed string-like values.
61    #[must_use]
62    pub fn new(key: impl Into<Arc<str>>, value: impl Into<Arc<str>>) -> Self {
63        Self {
64            key: key.into(),
65            value: value.into(),
66        }
67    }
68}
69
70/// One log event accepted after the corresponding shard-stream append is durable.
71#[derive(Debug, Clone, PartialEq, Eq)]
72pub struct DurableLog {
73    /// The physical shard-stream worker that owns this record.
74    pub stream_shard_id: ShardId,
75    /// Ordered log address assigned by shard-stream.
76    pub record_ref: TelemetryRecordRef,
77    /// Event timestamp in Unix nanoseconds.
78    pub timestamp_unix_nanos: u64,
79    /// Original observed timestamp, independently preserved from event time.
80    pub observed_timestamp_unix_nanos: u64,
81    /// Exact typed OTLP body. `None` is distinct from an empty body value.
82    pub body: Option<TelemetryValue>,
83    /// Original unstructured message body.
84    pub message: Arc<str>,
85    /// Extracted exact-match metadata.
86    pub fields: Arc<Vec<MetadataField>>,
87    /// Exact typed record attributes in wire order.
88    pub attributes: Arc<Vec<TelemetryAttribute>>,
89    /// Exact resource context and schema URL.
90    pub resource: Arc<ResourceContext>,
91    /// Exact instrumentation scope context and schema URL.
92    pub scope: Arc<ScopeContext>,
93    /// OTLP severity enum value, including unknown future values.
94    pub severity_number: i32,
95    /// Exact severity text.
96    pub severity_text: Arc<str>,
97    /// Number of record attributes dropped before export.
98    pub dropped_attributes_count: u32,
99    /// OTLP log flags, including unknown future bits.
100    pub flags: u32,
101    /// Binary trace ID when present.
102    pub trace_id: Option<TraceId>,
103    /// Binary span ID when present.
104    pub span_id: Option<SpanId>,
105    /// Exact event name.
106    pub event_name: Arc<str>,
107    /// Compression cohort selected by the producer or parser.
108    pub compression_cohort: CompressionCohortId,
109}
110
111impl DurableLog {
112    /// Returns the cross-signal identity of this log's resource context.
113    #[must_use]
114    pub fn resource_id(&self) -> crate::ResourceContextId {
115        self.resource.id()
116    }
117
118    /// Returns the cross-signal identity of this log's instrumentation scope.
119    #[must_use]
120    pub fn scope_id(&self) -> crate::ScopeContextId {
121        self.scope.id()
122    }
123
124    /// Constructs a durable log record with no extracted metadata.
125    #[must_use]
126    pub fn new(
127        stream_shard_id: ShardId,
128        topic_partition: TopicPartition,
129        offset: LogicalOffset,
130        timestamp_unix_nanos: u64,
131        message: impl Into<Arc<str>>,
132        compression_cohort: CompressionCohortId,
133    ) -> Self {
134        Self {
135            stream_shard_id,
136            record_ref: TelemetryRecordRef::new(topic_partition, offset),
137            timestamp_unix_nanos,
138            observed_timestamp_unix_nanos: 0,
139            body: None,
140            message: message.into(),
141            fields: Arc::new(Vec::new()),
142            attributes: Arc::new(Vec::new()),
143            resource: Arc::new(ResourceContext::default()),
144            scope: Arc::new(ScopeContext::default()),
145            severity_number: 0,
146            severity_text: Arc::from(""),
147            dropped_attributes_count: 0,
148            flags: 0,
149            trace_id: None,
150            span_id: None,
151            event_name: Arc::from(""),
152            compression_cohort,
153        }
154    }
155
156    /// Adds a normalized metadata field.
157    #[must_use]
158    pub fn with_field(mut self, key: impl Into<Arc<str>>, value: impl Into<Arc<str>>) -> Self {
159        Arc::make_mut(&mut self.fields).push(MetadataField::new(key, value));
160        self
161    }
162
163    /// Adds one exact typed OTLP record attribute.
164    #[must_use]
165    pub fn with_attribute(mut self, attribute: TelemetryAttribute) -> Self {
166        Arc::make_mut(&mut self.attributes).push(attribute);
167        self
168    }
169}
170
171/// Ordering applied to matching log records.
172#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
173pub enum QueryOrder {
174    /// Return the lowest durable offsets first.
175    #[default]
176    OldestFirst,
177    /// Return the highest durable offsets first.
178    NewestFirst,
179}
180
181/// Case handling for message and metadata text predicates.
182#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
183pub enum CaseSensitivity {
184    /// Compare Unicode text exactly.
185    Sensitive,
186    /// Compare Unicode text case-insensitively.
187    #[default]
188    Insensitive,
189}
190
191/// Non-regex text operation used by message and metadata predicates.
192#[derive(Debug, Clone, Copy, PartialEq, Eq)]
193pub enum TextMatchKind {
194    /// The complete value must match.
195    Exact,
196    /// The observed value must contain the requested text.
197    Contains,
198    /// The observed value must start with the requested text.
199    Prefix,
200    /// The observed value must end with the requested text.
201    Suffix,
202}
203
204/// Reusable literal text matcher.
205#[derive(Debug, Clone, PartialEq, Eq)]
206pub struct TextMatcher {
207    /// Requested literal text.
208    pub value: Arc<str>,
209    /// Literal operation to perform.
210    pub kind: TextMatchKind,
211    /// Case handling for the comparison.
212    pub case_sensitivity: CaseSensitivity,
213}
214
215impl TextMatcher {
216    /// Creates a literal text matcher.
217    #[must_use]
218    pub fn new(
219        value: impl Into<Arc<str>>,
220        kind: TextMatchKind,
221        case_sensitivity: CaseSensitivity,
222    ) -> Self {
223        Self {
224            value: value.into(),
225            kind,
226            case_sensitivity,
227        }
228    }
229}
230
231/// Validated regular expression used by a log predicate.
232#[derive(Clone)]
233pub struct LogRegex {
234    pattern: Arc<str>,
235    case_sensitivity: CaseSensitivity,
236    compiled: Arc<Regex>,
237}
238
239impl LogRegex {
240    /// Compiles one Unicode-aware regular expression.
241    pub fn new(
242        pattern: impl Into<Arc<str>>,
243        case_sensitivity: CaseSensitivity,
244    ) -> TelemetryResult<Self> {
245        let pattern = pattern.into();
246        let compiled = RegexBuilder::new(&pattern)
247            .case_insensitive(case_sensitivity == CaseSensitivity::Insensitive)
248            .build()
249            .map_err(|error| TelemetryError::InvalidQuery(error.to_string()))?;
250        Ok(Self {
251            pattern,
252            case_sensitivity,
253            compiled: Arc::new(compiled),
254        })
255    }
256
257    /// Returns the original regular-expression source.
258    #[must_use]
259    pub fn pattern(&self) -> &str {
260        &self.pattern
261    }
262
263    /// Returns the expression's case handling.
264    #[must_use]
265    pub const fn case_sensitivity(&self) -> CaseSensitivity {
266        self.case_sensitivity
267    }
268
269    pub(crate) fn is_match(&self, observed: &str) -> bool {
270        self.compiled.is_match(observed)
271    }
272}
273
274impl fmt::Debug for LogRegex {
275    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
276        formatter
277            .debug_struct("LogRegex")
278            .field("pattern", &self.pattern)
279            .field("case_sensitivity", &self.case_sensitivity)
280            .finish_non_exhaustive()
281    }
282}
283
284impl PartialEq for LogRegex {
285    fn eq(&self, other: &Self) -> bool {
286        self.pattern == other.pattern && self.case_sensitivity == other.case_sensitivity
287    }
288}
289
290impl Eq for LogRegex {}
291
292/// Integer comparison for normalized numeric metadata.
293#[derive(Debug, Clone, Copy, PartialEq, Eq)]
294pub enum NumericComparison {
295    /// Observed value is equal to the requested value after integer parsing.
296    Equal,
297    /// Observed value is not equal to the requested value after integer parsing.
298    NotEqual,
299    /// Observed value is less than the requested value.
300    LessThan,
301    /// Observed value is less than or equal to the requested value.
302    LessThanOrEqual,
303    /// Observed value is greater than the requested value.
304    GreaterThan,
305    /// Observed value is greater than or equal to the requested value.
306    GreaterThanOrEqual,
307}
308
309/// Exact Boolean predicate evaluated against one normalized log record.
310#[derive(Debug, Clone, Default, PartialEq, Eq)]
311pub enum LogPredicate {
312    /// Matches every record.
313    #[default]
314    MatchAll,
315    /// Matches no records.
316    MatchNone,
317    /// Matches one case-insensitive token produced by ShardTelemetry's scanner.
318    Term(Arc<str>),
319    /// Matches one message token with explicit case semantics and exact
320    /// ClickHouse-compatible ASCII token boundaries.
321    MessageToken {
322        /// Token bytes to match.
323        value: Arc<str>,
324        /// Whether ASCII letter case is significant.
325        case_sensitivity: CaseSensitivity,
326    },
327    /// Matches the complete message with a literal text operation.
328    Message(TextMatcher),
329    /// Matches the complete message with a validated regular expression.
330    MessageRegex(LogRegex),
331    /// Matches records containing at least one field with this key.
332    FieldExists(Arc<str>),
333    /// Matches a field key against one literal text matcher.
334    Field {
335        /// Exact normalized field key.
336        key: Arc<str>,
337        /// Matcher applied to every value under the key.
338        matcher: TextMatcher,
339    },
340    /// Matches when a field equals any value in the set.
341    FieldIn {
342        /// Exact normalized field key.
343        key: Arc<str>,
344        /// Exact case-sensitive values.
345        values: Vec<Arc<str>>,
346    },
347    /// Matches a field value with a validated regular expression.
348    FieldRegex {
349        /// Exact normalized field key.
350        key: Arc<str>,
351        /// Expression applied to every value under the key.
352        regex: LogRegex,
353    },
354    /// Parses field values as signed 128-bit integers and compares them.
355    FieldNumeric {
356        /// Exact normalized field key.
357        key: Arc<str>,
358        /// Integer comparison.
359        comparison: NumericComparison,
360        /// Requested comparison value.
361        value: i128,
362    },
363    /// Every child predicate must match.
364    And(Vec<LogPredicate>),
365    /// At least one child predicate must match.
366    Or(Vec<LogPredicate>),
367    /// Inverts one child predicate.
368    Not(Box<LogPredicate>),
369}
370
371impl LogPredicate {
372    /// Creates an exact case-insensitive token predicate.
373    #[must_use]
374    pub fn term(term: impl Into<Arc<str>>) -> Self {
375        Self::Term(term.into())
376    }
377
378    /// Creates an exact token predicate with ClickHouse-compatible ASCII
379    /// token boundaries.
380    #[must_use]
381    pub fn message_token(value: impl Into<Arc<str>>, case_sensitivity: CaseSensitivity) -> Self {
382        Self::MessageToken {
383            value: value.into(),
384            case_sensitivity,
385        }
386    }
387
388    /// Creates a literal message predicate.
389    #[must_use]
390    pub fn message(matcher: TextMatcher) -> Self {
391        Self::Message(matcher)
392    }
393
394    /// Creates a case-insensitive message substring predicate.
395    #[must_use]
396    pub fn message_contains(value: impl Into<Arc<str>>) -> Self {
397        Self::Message(TextMatcher::new(
398            value,
399            TextMatchKind::Contains,
400            CaseSensitivity::Insensitive,
401        ))
402    }
403
404    /// Creates a validated message regular-expression predicate.
405    pub fn message_regex(
406        pattern: impl Into<Arc<str>>,
407        case_sensitivity: CaseSensitivity,
408    ) -> TelemetryResult<Self> {
409        Ok(Self::MessageRegex(LogRegex::new(
410            pattern,
411            case_sensitivity,
412        )?))
413    }
414
415    /// Creates an exact metadata-key existence predicate.
416    #[must_use]
417    pub fn field_exists(key: impl Into<Arc<str>>) -> Self {
418        Self::FieldExists(key.into())
419    }
420
421    /// Creates an exact case-sensitive metadata equality predicate.
422    #[must_use]
423    pub fn field_equals(key: impl Into<Arc<str>>, value: impl Into<Arc<str>>) -> Self {
424        Self::Field {
425            key: key.into(),
426            matcher: TextMatcher::new(value, TextMatchKind::Exact, CaseSensitivity::Sensitive),
427        }
428    }
429
430    /// Creates a literal metadata predicate.
431    #[must_use]
432    pub fn field(key: impl Into<Arc<str>>, matcher: TextMatcher) -> Self {
433        Self::Field {
434            key: key.into(),
435            matcher,
436        }
437    }
438
439    /// Creates an exact case-sensitive metadata set-membership predicate.
440    #[must_use]
441    pub fn field_in(
442        key: impl Into<Arc<str>>,
443        values: impl IntoIterator<Item = impl Into<Arc<str>>>,
444    ) -> Self {
445        Self::FieldIn {
446            key: key.into(),
447            values: values.into_iter().map(Into::into).collect(),
448        }
449    }
450
451    /// Creates a validated metadata regular-expression predicate.
452    pub fn field_regex(
453        key: impl Into<Arc<str>>,
454        pattern: impl Into<Arc<str>>,
455        case_sensitivity: CaseSensitivity,
456    ) -> TelemetryResult<Self> {
457        Ok(Self::FieldRegex {
458            key: key.into(),
459            regex: LogRegex::new(pattern, case_sensitivity)?,
460        })
461    }
462
463    /// Creates an integer metadata comparison.
464    #[must_use]
465    pub fn field_numeric(
466        key: impl Into<Arc<str>>,
467        comparison: NumericComparison,
468        value: i128,
469    ) -> Self {
470        Self::FieldNumeric {
471            key: key.into(),
472            comparison,
473            value,
474        }
475    }
476
477    /// Creates a conjunction. An empty conjunction matches every record.
478    #[must_use]
479    pub fn and(predicates: Vec<Self>) -> Self {
480        if predicates.is_empty() {
481            Self::MatchAll
482        } else {
483            Self::And(predicates)
484        }
485    }
486
487    /// Creates a disjunction. An empty disjunction matches no records.
488    #[must_use]
489    pub fn or(predicates: Vec<Self>) -> Self {
490        if predicates.is_empty() {
491            Self::MatchNone
492        } else {
493            Self::Or(predicates)
494        }
495    }
496
497    /// Creates a negated predicate.
498    #[must_use]
499    pub fn negate(predicate: Self) -> Self {
500        Self::Not(Box::new(predicate))
501    }
502}
503
504/// Stable sort key used by a lookup and its cursor.
505#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
506pub enum QuerySort {
507    /// Sort by durable logical offset.
508    #[default]
509    Offset,
510    /// Sort by event timestamp, breaking ties with durable offset.
511    Timestamp,
512}
513
514/// Exclusive continuation point for deterministic lookup pagination.
515#[derive(Debug, Clone, Copy, PartialEq, Eq)]
516pub struct QueryCursor {
517    /// Event timestamp of the final result on the previous page.
518    pub timestamp_unix_nanos: u64,
519    /// Durable offset of the final result on the previous page.
520    pub offset: LogicalOffset,
521}
522
523impl QueryCursor {
524    /// Creates a continuation point from a normalized record position.
525    #[must_use]
526    pub const fn new(timestamp_unix_nanos: u64, offset: LogicalOffset) -> Self {
527        Self {
528            timestamp_unix_nanos,
529            offset,
530        }
531    }
532}
533
534/// Boolean log lookup against one logical partition.
535#[derive(Debug, Clone, PartialEq, Eq)]
536pub struct LogQuery {
537    /// Partition to search. Query fan-out is intentionally an explicit higher layer.
538    pub topic_partition: TopicPartition,
539    /// Case-insensitive message terms, combined with AND semantics.
540    pub terms: Vec<Arc<str>>,
541    /// Exact metadata fields, combined with AND semantics.
542    pub exact_fields: Vec<MetadataField>,
543    /// Additional exact Boolean predicate.
544    pub predicate: LogPredicate,
545    /// Inclusive lowest durable offset.
546    pub start_offset: Option<LogicalOffset>,
547    /// Exclusive highest durable offset.
548    pub end_offset: Option<LogicalOffset>,
549    /// Inclusive lowest event timestamp in Unix nanoseconds.
550    pub start_timestamp_unix_nanos: Option<u64>,
551    /// Exclusive highest event timestamp in Unix nanoseconds.
552    pub end_timestamp_unix_nanos: Option<u64>,
553    /// Maximum number of records to return.
554    pub limit: Option<usize>,
555    /// Durable-offset result order.
556    pub order: QueryOrder,
557    /// Stable record sort key.
558    pub sort: QuerySort,
559    /// Exclusive continuation point from the previous page.
560    pub after: Option<QueryCursor>,
561}
562
563impl LogQuery {
564    /// Creates an empty query for a logical partition.
565    #[must_use]
566    pub fn new(topic_partition: TopicPartition) -> Self {
567        Self {
568            topic_partition,
569            terms: Vec::new(),
570            exact_fields: Vec::new(),
571            predicate: LogPredicate::MatchAll,
572            start_offset: None,
573            end_offset: None,
574            start_timestamp_unix_nanos: None,
575            end_timestamp_unix_nanos: None,
576            limit: None,
577            order: QueryOrder::OldestFirst,
578            sort: QuerySort::Offset,
579            after: None,
580        }
581    }
582
583    /// Adds a case-insensitive exact token constraint.
584    #[must_use]
585    pub fn with_term(mut self, term: impl Into<Arc<str>>) -> Self {
586        self.terms.push(term.into());
587        self
588    }
589
590    /// Adds an exact metadata constraint.
591    #[must_use]
592    pub fn with_field(mut self, key: impl Into<Arc<str>>, value: impl Into<Arc<str>>) -> Self {
593        self.exact_fields.push(MetadataField::new(key, value));
594        self
595    }
596
597    /// Adds an arbitrary predicate with AND semantics.
598    #[must_use]
599    pub fn with_predicate(mut self, predicate: LogPredicate) -> Self {
600        self.predicate = match self.predicate {
601            LogPredicate::MatchAll => predicate,
602            current => LogPredicate::And(vec![current, predicate]),
603        };
604        self
605    }
606
607    /// Replaces the additional Boolean predicate.
608    #[must_use]
609    pub fn where_predicate(mut self, predicate: LogPredicate) -> Self {
610        self.predicate = predicate;
611        self
612    }
613
614    /// Restricts results to `start..end` durable offsets.
615    #[must_use]
616    pub fn with_offset_range(mut self, start: LogicalOffset, end: LogicalOffset) -> Self {
617        self.start_offset = Some(start);
618        self.end_offset = Some(end);
619        self
620    }
621
622    /// Restricts results to `start..end` event timestamps.
623    #[must_use]
624    pub fn with_timestamp_range(mut self, start: u64, end: u64) -> Self {
625        self.start_timestamp_unix_nanos = Some(start);
626        self.end_timestamp_unix_nanos = Some(end);
627        self
628    }
629
630    /// Caps the number of materialized results.
631    #[must_use]
632    pub fn with_limit(mut self, limit: usize) -> Self {
633        self.limit = Some(limit);
634        self
635    }
636
637    /// Returns the highest durable offsets first.
638    #[must_use]
639    pub fn newest_first(mut self) -> Self {
640        self.order = QueryOrder::NewestFirst;
641        self
642    }
643
644    /// Returns the lowest sort keys first.
645    #[must_use]
646    pub fn oldest_first(mut self) -> Self {
647        self.order = QueryOrder::OldestFirst;
648        self
649    }
650
651    /// Sorts by event timestamp with durable offset as the tie-breaker.
652    #[must_use]
653    pub fn sort_by_timestamp(mut self) -> Self {
654        self.sort = QuerySort::Timestamp;
655        self
656    }
657
658    /// Sorts by durable logical offset.
659    #[must_use]
660    pub fn sort_by_offset(mut self) -> Self {
661        self.sort = QuerySort::Offset;
662        self
663    }
664
665    /// Continues strictly after a previous page's final record in query order.
666    #[must_use]
667    pub fn after(mut self, cursor: QueryCursor) -> Self {
668        self.after = Some(cursor);
669        self
670    }
671}
672
673/// A visible log record returned by a [`LogQuery`].
674#[derive(Debug, Clone, PartialEq, Eq)]
675pub struct LogMatch {
676    /// The durable indexed record.
677    pub record: DurableLog,
678}