1use crate::codes::FILTER_OP_VERSION;
2use crate::error::InvalidError;
3use crate::filter::catalog::CatalogPosition;
4use crate::filter::expr::ConsumerFilter;
5use crate::limits::{
6 MAX_FILTER_PREVIEW_EXAMINED, MAX_FILTER_PREVIEW_RECORDS, MAX_FILTER_SAMPLE_BYTES,
7 MAX_FILTER_SAMPLE_HEADERS, MAX_FILTER_SOURCE_NAME_BYTES, MAX_FILTERED_PAGE_BYTES,
8 MAX_FILTERED_PAGE_EXAMINED, MAX_FILTERED_PAGE_RECORDS, MAX_FILTERED_WAIT_CANCEL_IDS,
9 MAX_FILTERED_WAIT_MS, MAX_FILTERED_WAIT_PARTITIONS,
10};
11use crate::result::ResultCode;
12use crate::schema::Digest32;
13use crate::validate::Validate;
14use serde::ser::SerializeStruct;
15use serde::{Deserialize, Deserializer, Serialize, Serializer};
16use std::collections::BTreeSet;
17use std::fmt;
18
19#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
21#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
22pub struct FilterSource {
23 pub stream: String,
24 pub topic: String,
25}
26
27#[derive(Clone, Debug, PartialEq, Eq)]
30pub enum FilterConsumer {
31 Consumer(String),
32 Group(String),
33 GroupId(u64),
34}
35
36impl fmt::Display for FilterConsumer {
37 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
38 match self {
39 Self::Consumer(name) | Self::Group(name) => formatter.write_str(name),
40 Self::GroupId(id) => write!(formatter, "{id}"),
41 }
42 }
43}
44
45impl FilterConsumer {
46 pub const fn is_group(&self) -> bool {
47 matches!(self, Self::Group(_) | Self::GroupId(_))
48 }
49}
50
51#[cfg(feature = "schemars")]
52impl schemars::JsonSchema for FilterConsumer {
53 fn schema_name() -> std::borrow::Cow<'static, str> {
54 "FilterConsumer".into()
55 }
56
57 fn json_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
58 schemars::json_schema!({
59 "oneOf": [
60 {
61 "type": "object",
62 "properties": {
63 "kind": { "enum": ["consumer", "group"] },
64 "name": { "type": "string" }
65 },
66 "required": ["kind", "name"]
67 },
68 {
69 "type": "object",
70 "properties": {
71 "kind": { "const": "group_id" },
72 "id": { "type": "integer", "minimum": 0 }
73 },
74 "required": ["kind", "id"]
75 }
76 ]
77 })
78 }
79}
80
81impl Serialize for FilterConsumer {
82 fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
83 let mut state = serializer.serialize_struct("FilterConsumer", 2)?;
84 match self {
85 Self::Consumer(name) => {
86 state.serialize_field("kind", "consumer")?;
87 state.serialize_field("name", name)?;
88 }
89 Self::Group(name) => {
90 state.serialize_field("kind", "group")?;
91 state.serialize_field("name", name)?;
92 }
93 Self::GroupId(id) => {
94 state.serialize_field("kind", "group_id")?;
95 state.serialize_field("id", id)?;
96 }
97 }
98 state.end()
99 }
100}
101
102impl<'de> Deserialize<'de> for FilterConsumer {
103 fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
104 #[derive(Deserialize)]
105 #[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
106 #[serde(tag = "kind", rename_all = "snake_case")]
107 enum Selector {
108 Consumer { name: String },
109 Group { name: String },
110 GroupId { id: u64 },
111 }
112 Ok(match Selector::deserialize(deserializer)? {
113 Selector::Consumer { name } => Self::Consumer(name),
114 Selector::Group { name } => Self::Group(name),
115 Selector::GroupId { id } => Self::GroupId(id),
116 })
117 }
118}
119
120#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
122#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
123#[serde(rename_all = "snake_case")]
124pub enum FilterRef {
125 Inline(ConsumerFilter),
128 Revision { filter_id: u32, revision: u32 },
130 Bound,
133 Group,
137}
138
139#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
141#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
142#[serde(rename_all = "snake_case")]
143pub enum FilteredStart {
144 Next,
146 First,
147 Last,
148 Offset(u64),
149 Timestamp(u64),
151 Continue(Continuation),
153}
154
155#[derive(
159 Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize, strum::Display,
160)]
161#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
162#[serde(rename_all = "snake_case")]
163#[strum(serialize_all = "snake_case")]
164pub enum ExecutionMode {
165 #[default]
166 Filtered,
167 Unfiltered,
168}
169
170impl ExecutionMode {
171 pub const fn is_filtered(&self) -> bool {
172 matches!(self, Self::Filtered)
173 }
174}
175
176fn validate_policy_identity(
178 mode: ExecutionMode,
179 digest: Option<&Digest32>,
180) -> Result<(), InvalidError> {
181 match (mode, digest) {
182 (ExecutionMode::Filtered, Some(digest)) => digest.validate(),
183 (ExecutionMode::Filtered, None) => {
184 Err(InvalidError::new("a filtered policy names its digest"))
185 }
186 (ExecutionMode::Unfiltered, None) => Ok(()),
187 (ExecutionMode::Unfiltered, Some(_)) => {
188 Err(InvalidError::new("an unfiltered policy carries no digest"))
189 }
190 }
191}
192
193#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
198#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
199pub struct SourceGeneration {
200 pub stream_id: u32,
201 pub stream_created_at_micros: u64,
202 pub topic_id: u32,
203 pub topic_created_at_micros: u64,
204 pub partition_id: u32,
205 pub partition_incarnation: u64,
206}
207
208#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
212#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
213pub struct AckContext {
214 pub owner_generation: u64,
216 pub metadata_op: u64,
218}
219
220#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
222#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
223pub struct Continuation {
224 #[serde(default, skip_serializing_if = "Option::is_none")]
225 pub group_id: Option<u64>,
226 pub next_scan_offset: u64,
227 pub generation: SourceGeneration,
228 #[serde(default, skip_serializing_if = "Option::is_none")]
230 pub digest: Option<Digest32>,
231 #[serde(default)]
232 pub mode: ExecutionMode,
233 #[serde(default)]
236 pub policy_generation: u64,
237}
238
239#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
245#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
246pub struct FilteredWaitRequest {
247 pub v: u32,
248 pub source: FilterSource,
249 pub consumer: FilterConsumer,
250 pub partitions: Vec<FilteredWaitPartition>,
251 pub max_wait_ms: u32,
252 pub wait_id: u64,
253}
254
255#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
258#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
259pub struct FilteredWaitCancel {
260 pub v: u32,
261 pub wait_ids: Vec<u64>,
262}
263
264#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
267#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
268pub struct FilteredWaitCancelled {
269 pub cancelled: Vec<u64>,
270}
271
272#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
274#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
275pub struct FilteredWaitPartition {
276 pub partition_id: u32,
277 pub offset: u64,
278}
279
280#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
283#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
284pub struct FilteredWaitResult {
285 pub ready: Vec<u32>,
286}
287
288#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
290#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
291pub struct FilteredPollRequest {
292 pub v: u32,
293 pub source: FilterSource,
294 pub partition_id: u32,
295 pub consumer: FilterConsumer,
296 pub filter: FilterRef,
297 pub start: FilteredStart,
298 pub count: u32,
300 pub max_reply_bytes: u32,
303 #[serde(default, skip_serializing_if = "Option::is_none")]
307 pub max_examined: Option<u32>,
308 #[serde(default, skip_serializing_if = "Option::is_none")]
311 pub min_catalog_position: Option<CatalogPosition>,
312}
313
314#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, strum::Display)]
316#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
317#[serde(rename_all = "snake_case")]
318#[strum(serialize_all = "snake_case")]
319pub enum StopReason {
320 Filled,
322 Budget,
324 EndOfVisible,
326 Fault,
328 OversizedRecord,
330}
331
332#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, strum::Display)]
334#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
335#[serde(rename_all = "snake_case")]
336#[strum(serialize_all = "snake_case")]
337pub enum FaultReason {
338 MissingSchema,
340 SchemaNotAllowed,
342 SchemaMismatch,
344 Malformed,
346 TooLarge,
348 TooDeep,
350 ForeignCodec,
352 TypeMismatch,
354}
355
356impl FaultReason {
357 pub const fn is_foreign(self) -> bool {
360 matches!(
361 self,
362 FaultReason::ForeignCodec
363 | FaultReason::MissingSchema
364 | FaultReason::SchemaNotAllowed
365 | FaultReason::SchemaMismatch
366 )
367 }
368}
369
370#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
372#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
373pub struct RecordFault {
374 pub offset: u64,
375 pub reason: FaultReason,
376}
377
378#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
380#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
381pub struct FilterDecodeLimits {
382 pub max_payload_bytes: u32,
383 pub max_depth: u32,
384}
385
386#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
388#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
389pub struct AppliedPolicy {
390 #[serde(default, skip_serializing_if = "Option::is_none")]
391 pub group_id: Option<u64>,
392 #[serde(default, skip_serializing_if = "Option::is_none")]
394 pub digest: Option<Digest32>,
395 #[serde(default, skip_serializing_if = "Option::is_none")]
396 pub filter_id: Option<u32>,
397 #[serde(default, skip_serializing_if = "Option::is_none")]
398 pub revision: Option<u32>,
399 #[serde(default)]
400 pub mode: ExecutionMode,
401 #[serde(default)]
403 pub policy_generation: u64,
404}
405
406impl AppliedPolicy {
407 #[must_use]
409 pub fn filtered(group_id: Option<u64>, digest: Digest32) -> Self {
410 Self {
411 group_id,
412 digest: Some(digest),
413 filter_id: None,
414 revision: None,
415 mode: ExecutionMode::Filtered,
416 policy_generation: 0,
417 }
418 }
419
420 #[must_use]
422 pub const fn unfiltered(group_id: u64, policy_generation: u64) -> Self {
423 Self {
424 group_id: Some(group_id),
425 digest: None,
426 filter_id: None,
427 revision: None,
428 mode: ExecutionMode::Unfiltered,
429 policy_generation,
430 }
431 }
432}
433
434impl Validate for AppliedPolicy {
435 fn validate(&self) -> Result<(), InvalidError> {
436 validate_policy_identity(self.mode, self.digest.as_ref())?;
437 if self.mode == ExecutionMode::Unfiltered {
438 if self.group_id.is_none() {
439 return Err(InvalidError::new("only a consumer group runs unfiltered"));
440 }
441 if self.filter_id.is_some() || self.revision.is_some() {
442 return Err(InvalidError::new(
443 "an unfiltered policy has no filter reference",
444 ));
445 }
446 }
447 Ok(())
448 }
449}
450
451#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
463#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
464pub struct FilteredPage {
465 pub v: u32,
466 pub partition_id: u32,
467 pub policy: AppliedPolicy,
468 pub generation: SourceGeneration,
469 pub ack_context: AckContext,
470 #[serde(default, skip_serializing_if = "Option::is_none")]
471 pub next_scan_offset: Option<u64>,
472 #[serde(default, skip_serializing_if = "Option::is_none")]
473 pub safe_ack_offset: Option<u64>,
474 pub frontier: u64,
477 pub examined: u32,
478 pub matched: u32,
479 pub stop: StopReason,
480 #[serde(default, skip_serializing_if = "Option::is_none")]
481 pub fault: Option<RecordFault>,
482 #[serde(default, skip_serializing_if = "Vec::is_empty")]
485 pub unevaluated: Vec<RecordFault>,
486 #[serde(default, skip_serializing_if = "Option::is_none")]
487 pub evaluation_limits: Option<Box<FilterDecodeLimits>>,
488 #[cfg_attr(
489 feature = "schemars",
490 schemars(schema_with = "crate::encoding::bytes_schema")
491 )]
492 #[serde(with = "crate::encoding::bin_bytes")]
493 pub records: Vec<u8>,
494}
495
496impl FilteredPage {
497 pub fn next_start(&self, original: &FilteredStart) -> FilteredStart {
500 match self.next_scan_offset {
501 Some(next_scan_offset) => FilteredStart::Continue(Continuation {
502 group_id: self.policy.group_id,
503 next_scan_offset,
504 generation: self.generation,
505 digest: self.policy.digest.clone(),
506 mode: self.policy.mode,
507 policy_generation: self.policy.policy_generation,
508 }),
509 None => original.clone(),
510 }
511 }
512}
513
514#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
518#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
519pub struct FilteredAck {
520 #[serde(default, skip_serializing_if = "Option::is_none")]
521 pub group_id: Option<u64>,
522 pub v: u32,
523 pub source: FilterSource,
524 pub partition_id: u32,
525 pub consumer: FilterConsumer,
526 pub generation: SourceGeneration,
527 pub ack_context: AckContext,
528 #[serde(default, skip_serializing_if = "Option::is_none")]
530 pub digest: Option<Digest32>,
531 pub offset: u64,
532 #[serde(default)]
533 pub mode: ExecutionMode,
534 #[serde(default)]
537 pub policy_generation: u64,
538}
539
540#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
542#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
543pub struct AckReceipt {
544 pub partition_id: u32,
545 pub offset: u64,
546 pub generation: SourceGeneration,
547}
548
549#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
552#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
553pub struct FilterPreviewRequest {
554 pub v: u32,
555 pub source: FilterSource,
556 pub partition_id: u32,
557 pub filter: FilterRef,
559 pub from_offset: u64,
560 pub max_examined: u32,
561 pub max_records: u32,
562 #[serde(default)]
564 pub explain: bool,
565}
566
567#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, strum::Display)]
569#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
570#[serde(rename_all = "snake_case")]
571#[strum(serialize_all = "snake_case")]
572pub enum Verdict {
573 Selected,
574 Rejected,
575 Fault,
576}
577
578#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
580#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
581pub struct PreviewRecord {
582 pub offset: u64,
583 pub timestamp_micros: u64,
584 pub verdict: Verdict,
585 #[serde(default, skip_serializing_if = "Option::is_none")]
586 pub fault: Option<FaultReason>,
587 pub payload_text: String,
589 #[serde(default)]
590 pub payload_truncated: bool,
591 #[serde(default, skip_serializing_if = "Option::is_none")]
592 pub explanation: Option<FilterExplanation>,
593}
594
595#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
597#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
598pub struct FilterPreview {
599 pub v: u32,
600 pub partition_id: u32,
601 pub policy: AppliedPolicy,
602 pub examined: u32,
603 pub matched: u32,
604 pub faults: u32,
605 pub stop: StopReason,
606 #[serde(default, skip_serializing_if = "Option::is_none")]
607 pub next_offset: Option<u64>,
608 pub frontier: u64,
609 pub records: Vec<PreviewRecord>,
612}
613
614#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
617#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
618#[serde(tag = "kind", content = "value", rename_all = "snake_case")]
619pub enum HeaderScalar {
620 Bool(bool),
621 Int(i64),
622 Uint(u64),
623 Float(f64),
624 String(String),
625 Raw(
626 #[cfg_attr(
627 feature = "schemars",
628 schemars(schema_with = "crate::encoding::bytes_schema")
629 )]
630 #[serde(with = "crate::encoding::bin_bytes")]
631 Vec<u8>,
632 ),
633}
634
635#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
637#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
638pub struct FilterHeader {
639 pub key: String,
640 pub value: HeaderScalar,
641}
642
643#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
646#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
647pub struct FilterTestRequest {
648 pub v: u32,
649 pub filter: FilterRef,
651 #[cfg_attr(
652 feature = "schemars",
653 schemars(schema_with = "crate::encoding::bytes_schema")
654 )]
655 #[serde(with = "crate::encoding::bin_bytes")]
656 pub payload: Vec<u8>,
657 #[serde(default, skip_serializing_if = "Vec::is_empty")]
658 pub headers: Vec<FilterHeader>,
659}
660
661#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, strum::Display)]
663#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
664#[serde(rename_all = "snake_case")]
665#[strum(serialize_all = "snake_case")]
666pub enum Truth {
667 Match,
668 NoMatch,
669 Unknown,
670}
671
672#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
675#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
676pub struct ExplainNode {
677 pub label: String,
678 #[serde(default, skip_serializing_if = "Option::is_none")]
679 pub truth: Option<Truth>,
680 #[serde(default, skip_serializing_if = "Vec::is_empty")]
681 pub children: Vec<ExplainNode>,
682}
683
684#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
686#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
687pub struct FilterExplanation {
688 pub verdict: Verdict,
689 #[serde(default, skip_serializing_if = "Option::is_none")]
690 pub fault: Option<FaultReason>,
691 pub root: ExplainNode,
692}
693
694#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
696#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
697pub struct FilterTestResult {
698 pub v: u32,
699 pub policy: AppliedPolicy,
700 pub explanation: FilterExplanation,
701}
702
703#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
705#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
706pub struct FilterValidation {
707 pub v: u32,
708 pub digest: Digest32,
709 pub reads_payload: bool,
711 pub reads_headers: bool,
713}
714
715#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, strum::Display)]
718#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
719#[serde(rename_all = "snake_case")]
720#[strum(serialize_all = "snake_case")]
721#[non_exhaustive]
722pub enum FilterErrorReason {
723 InvalidRequest,
725 Unsupported,
727 VersionSkew,
729 NotFound,
731 Conflict,
734 SourceChanged,
737 MembershipStale,
739 CatalogUnavailable,
741 RevisionDisabled,
743 TooLarge,
745 Unauthenticated,
746 Forbidden,
747 Unavailable,
749 CapacityExhausted,
751 Backend,
753 #[serde(other)]
756 Unknown,
757}
758
759impl FilterErrorReason {
760 pub const fn code(self) -> ResultCode {
761 match self {
762 Self::InvalidRequest => ResultCode::InvalidArgument,
763 Self::Unsupported => ResultCode::Unsupported,
764 Self::VersionSkew => ResultCode::VersionSkew,
765 Self::NotFound => ResultCode::NotFound,
766 Self::Conflict => ResultCode::Conflict,
767 Self::SourceChanged => ResultCode::StaleGeneration,
768 Self::MembershipStale => ResultCode::Stale,
769 Self::CatalogUnavailable | Self::Unavailable | Self::RevisionDisabled => {
770 ResultCode::Unavailable
771 }
772 Self::TooLarge => ResultCode::TooLarge,
773 Self::Unauthenticated => ResultCode::Unauthenticated,
774 Self::Forbidden => ResultCode::Forbidden,
775 Self::CapacityExhausted => ResultCode::ResourceLimit,
776 Self::Backend | Self::Unknown => ResultCode::Backend,
777 }
778 }
779}
780
781#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, thiserror::Error)]
783#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
784#[error("{reason}: {message}")]
785pub struct FilterError {
786 pub code: ResultCode,
787 pub reason: FilterErrorReason,
788 pub message: String,
789}
790
791impl FilterError {
792 pub fn new(reason: FilterErrorReason, message: impl Into<String>) -> Self {
793 Self {
794 code: reason.code(),
795 reason,
796 message: message.into(),
797 }
798 }
799
800 pub fn invalid(error: InvalidError) -> Self {
801 Self::new(FilterErrorReason::InvalidRequest, error.0)
802 }
803
804 pub fn check_version(v: u32) -> Result<(), Self> {
812 validate_version(v).map_err(|error| Self::new(FilterErrorReason::VersionSkew, error.0))
813 }
814}
815
816#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
818#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
819#[serde(rename_all = "snake_case")]
820pub enum FilterOutcome {
821 Page(FilteredPage),
822 Acknowledged(AckReceipt),
823 Preview(FilterPreview),
824 Tested(FilterTestResult),
825 Validated(FilterValidation),
826 Ready(FilteredWaitResult),
827 Cancelled(FilteredWaitCancelled),
828}
829
830#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
832#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
833#[serde(rename_all = "snake_case")]
834#[allow(clippy::large_enum_variant)]
836pub enum FilterReply {
837 Ok(FilterOutcome),
838 Err(FilterError),
839}
840
841fn validate_version(v: u32) -> Result<(), InvalidError> {
842 if v == FILTER_OP_VERSION {
843 Ok(())
844 } else {
845 Err(InvalidError::new(format!(
846 "filter request version {v} is not supported, expected {FILTER_OP_VERSION}"
847 )))
848 }
849}
850
851pub(crate) fn validate_name(label: &str, name: &str) -> Result<(), InvalidError> {
852 if name.is_empty() || name.len() > MAX_FILTER_SOURCE_NAME_BYTES {
853 return Err(InvalidError::new(format!(
854 "{label} must be 1..={MAX_FILTER_SOURCE_NAME_BYTES} bytes"
855 )));
856 }
857 Ok(())
858}
859
860impl Validate for FilterSource {
861 fn validate(&self) -> Result<(), InvalidError> {
862 validate_name("stream", &self.stream)?;
863 validate_name("topic", &self.topic)
864 }
865}
866
867impl Validate for FilterConsumer {
868 fn validate(&self) -> Result<(), InvalidError> {
869 match self {
870 Self::Consumer(name) => validate_name("consumer", name),
871 Self::Group(name) => validate_name("consumer group", name),
872 Self::GroupId(id) => u32::try_from(*id).map(|_| ()).map_err(|_| {
873 InvalidError::new("consumer group id exceeds the native offset-key range")
874 }),
875 }
876 }
877}
878
879impl Validate for FilterRef {
880 fn validate(&self) -> Result<(), InvalidError> {
881 match self {
882 Self::Inline(filter) => filter.validate(),
883 Self::Revision { .. } | Self::Bound | Self::Group => Ok(()),
884 }
885 }
886}
887
888impl Validate for FilteredWaitRequest {
889 fn validate(&self) -> Result<(), InvalidError> {
890 validate_version(self.v)?;
891 self.source.validate()?;
892 self.consumer.validate()?;
893 if self.partitions.is_empty() || self.partitions.len() > MAX_FILTERED_WAIT_PARTITIONS {
894 return Err(InvalidError::new(format!(
895 "a filtered wait names 1..={MAX_FILTERED_WAIT_PARTITIONS} partitions"
896 )));
897 }
898 let mut seen = BTreeSet::new();
899 if !self
900 .partitions
901 .iter()
902 .all(|partition| seen.insert(partition.partition_id))
903 {
904 return Err(InvalidError::new(
905 "a filtered wait names each partition once",
906 ));
907 }
908 if self.max_wait_ms > MAX_FILTERED_WAIT_MS {
909 return Err(InvalidError::new(format!(
910 "max_wait_ms must be 0..={MAX_FILTERED_WAIT_MS}"
911 )));
912 }
913 Ok(())
914 }
915}
916
917impl Validate for FilteredWaitCancel {
918 fn validate(&self) -> Result<(), InvalidError> {
919 validate_version(self.v)?;
920 if self.wait_ids.is_empty() || self.wait_ids.len() > MAX_FILTERED_WAIT_CANCEL_IDS {
921 return Err(InvalidError::new(format!(
922 "a filtered wait cancel names 1..={MAX_FILTERED_WAIT_CANCEL_IDS} waits"
923 )));
924 }
925 let mut seen = BTreeSet::new();
926 if !self.wait_ids.iter().all(|wait_id| seen.insert(*wait_id)) {
927 return Err(InvalidError::new(
928 "a filtered wait cancel names each wait once",
929 ));
930 }
931 Ok(())
932 }
933}
934
935impl Validate for FilteredPollRequest {
936 fn validate(&self) -> Result<(), InvalidError> {
937 validate_version(self.v)?;
938 self.source.validate()?;
939 self.consumer.validate()?;
940 self.filter.validate()?;
941 if matches!(self.filter, FilterRef::Bound | FilterRef::Group) && !self.consumer.is_group() {
942 return Err(InvalidError::new(
943 "a `bound` or `group` filter needs a consumer group",
944 ));
945 }
946 if self.count == 0 || self.count > MAX_FILTERED_PAGE_RECORDS {
947 return Err(InvalidError::new(format!(
948 "count must be 1..={MAX_FILTERED_PAGE_RECORDS}"
949 )));
950 }
951 if let Some(max_examined) = self.max_examined
952 && (max_examined == 0 || max_examined > MAX_FILTERED_PAGE_EXAMINED)
953 {
954 return Err(InvalidError::new(format!(
955 "max_examined must be 1..={MAX_FILTERED_PAGE_EXAMINED}"
956 )));
957 }
958 if self.max_reply_bytes == 0 || self.max_reply_bytes > MAX_FILTERED_PAGE_BYTES {
959 return Err(InvalidError::new(format!(
960 "max_reply_bytes must be 1..={MAX_FILTERED_PAGE_BYTES}"
961 )));
962 }
963 if let FilteredStart::Continue(continuation) = &self.start {
964 validate_policy_identity(continuation.mode, continuation.digest.as_ref())?;
965 if continuation.mode == ExecutionMode::Unfiltered && !self.consumer.is_group() {
966 return Err(InvalidError::new("only a consumer group runs unfiltered"));
967 }
968 if self.consumer.is_group() != continuation.group_id.is_some() {
969 return Err(InvalidError::new(
970 "a group continuation must retain its group identity",
971 ));
972 }
973 if continuation.generation.partition_id != self.partition_id {
974 return Err(InvalidError::new(
975 "a continuation belongs to another partition",
976 ));
977 }
978 }
979 Ok(())
980 }
981}
982
983impl Validate for FilteredAck {
984 fn validate(&self) -> Result<(), InvalidError> {
985 validate_version(self.v)?;
986 self.source.validate()?;
987 self.consumer.validate()?;
988 validate_policy_identity(self.mode, self.digest.as_ref())?;
989 if self.mode == ExecutionMode::Unfiltered && !self.consumer.is_group() {
990 return Err(InvalidError::new("only a consumer group runs unfiltered"));
991 }
992 if self.consumer.is_group() != self.group_id.is_some() {
993 return Err(InvalidError::new(
994 "a group acknowledgment must retain its group identity",
995 ));
996 }
997 if self.generation.partition_id != self.partition_id {
998 return Err(InvalidError::new(
999 "an acknowledgment names another partition's generation",
1000 ));
1001 }
1002 Ok(())
1003 }
1004}
1005
1006impl Validate for FilterPreviewRequest {
1007 fn validate(&self) -> Result<(), InvalidError> {
1008 validate_version(self.v)?;
1009 self.source.validate()?;
1010 self.filter.validate()?;
1011 if matches!(self.filter, FilterRef::Bound | FilterRef::Group) {
1012 return Err(InvalidError::new(
1013 "a preview executes an inline filter or a saved revision",
1014 ));
1015 }
1016 if self.max_examined == 0 || self.max_examined > MAX_FILTER_PREVIEW_EXAMINED {
1017 return Err(InvalidError::new(format!(
1018 "max_examined must be 1..={MAX_FILTER_PREVIEW_EXAMINED}"
1019 )));
1020 }
1021 if self.max_records == 0 || self.max_records > MAX_FILTER_PREVIEW_RECORDS {
1022 return Err(InvalidError::new(format!(
1023 "max_records must be 1..={MAX_FILTER_PREVIEW_RECORDS}"
1024 )));
1025 }
1026 Ok(())
1027 }
1028}
1029
1030impl Validate for FilterTestRequest {
1031 fn validate(&self) -> Result<(), InvalidError> {
1032 validate_version(self.v)?;
1033 self.filter.validate()?;
1034 if matches!(self.filter, FilterRef::Bound | FilterRef::Group) {
1035 return Err(InvalidError::new(
1036 "a sample test executes an inline filter or a saved revision",
1037 ));
1038 }
1039 if self.payload.len() > MAX_FILTER_SAMPLE_BYTES {
1040 return Err(InvalidError::new(format!(
1041 "a sample payload exceeds {MAX_FILTER_SAMPLE_BYTES}B"
1042 )));
1043 }
1044 if self.headers.len() > MAX_FILTER_SAMPLE_HEADERS {
1045 return Err(InvalidError::new(format!(
1046 "a sample carries more than {MAX_FILTER_SAMPLE_HEADERS} headers"
1047 )));
1048 }
1049 for header in &self.headers {
1050 validate_name("sample header key", &header.key)?;
1051 let bytes = match &header.value {
1052 HeaderScalar::String(value) => value.len(),
1053 HeaderScalar::Raw(value) => value.len(),
1054 _ => 0,
1055 };
1056 if bytes > crate::limits::MAX_FILTER_STRING_BYTES {
1057 return Err(InvalidError::new(
1058 "a sample header value exceeds its byte limit",
1059 ));
1060 }
1061 }
1062 Ok(())
1063 }
1064}
1065
1066#[cfg(test)]
1067mod tests {
1068 use super::*;
1069 use crate::filter::expr::FilterExpr;
1070 use crate::query::CmpOp;
1071
1072 fn wait_request() -> FilteredWaitRequest {
1073 FilteredWaitRequest {
1074 v: FILTER_OP_VERSION,
1075 source: FilterSource {
1076 stream: "orbit".to_owned(),
1077 topic: "fleet".to_owned(),
1078 },
1079 consumer: FilterConsumer::Group("desk".to_owned()),
1080 partitions: vec![
1081 FilteredWaitPartition {
1082 partition_id: 0,
1083 offset: 12,
1084 },
1085 FilteredWaitPartition {
1086 partition_id: 1,
1087 offset: 0,
1088 },
1089 ],
1090 max_wait_ms: 5_000,
1091 wait_id: 9,
1092 }
1093 }
1094
1095 #[test]
1096 fn given_a_wait_cancel_when_encoded_as_json_then_should_round_trip_with_its_reply() {
1097 let cancel = FilteredWaitCancel {
1098 v: FILTER_OP_VERSION,
1099 wait_ids: vec![9, 11],
1100 };
1101 cancel.validate().expect("valid");
1102 let json = serde_json::to_string(&cancel).expect("encodes");
1103 assert_eq!(
1104 serde_json::from_str::<FilteredWaitCancel>(&json).expect("decodes"),
1105 cancel
1106 );
1107 let cancelled = FilterReply::Ok(FilterOutcome::Cancelled(FilteredWaitCancelled {
1108 cancelled: vec![9],
1109 }));
1110 let json = serde_json::to_string(&cancelled).expect("encodes");
1111 assert_eq!(
1112 serde_json::from_str::<FilterReply>(&json).expect("decodes"),
1113 cancelled
1114 );
1115 }
1116
1117 #[test]
1118 fn given_wait_cancels_out_of_range_when_validated_then_should_be_rejected() {
1119 let cancel = |wait_ids: Vec<u64>| FilteredWaitCancel {
1120 v: FILTER_OP_VERSION,
1121 wait_ids,
1122 };
1123 let too_many = cancel((0..=MAX_FILTERED_WAIT_CANCEL_IDS as u64).collect());
1124 let old_version = FilteredWaitCancel {
1125 v: FILTER_OP_VERSION + 1,
1126 ..cancel(vec![1])
1127 };
1128 for request in [
1129 cancel(Vec::new()),
1130 cancel(vec![4, 4]),
1131 too_many,
1132 old_version,
1133 ] {
1134 assert!(request.validate().is_err(), "{request:?} must be rejected");
1135 }
1136 cancel((0..MAX_FILTERED_WAIT_CANCEL_IDS as u64).collect())
1137 .validate()
1138 .expect("the cap is accepted");
1139 }
1140
1141 #[test]
1142 fn given_a_wait_request_when_encoded_as_json_then_should_round_trip() {
1143 let request = wait_request();
1144 request.validate().expect("valid");
1145 let json = serde_json::to_string(&request).expect("encodes");
1146 assert_eq!(
1147 serde_json::from_str::<FilteredWaitRequest>(&json).expect("decodes"),
1148 request
1149 );
1150 let ready = FilterReply::Ok(FilterOutcome::Ready(FilteredWaitResult { ready: vec![1] }));
1151 let json = serde_json::to_string(&ready).expect("encodes");
1152 assert_eq!(
1153 serde_json::from_str::<FilterReply>(&json).expect("decodes"),
1154 ready
1155 );
1156 }
1157
1158 #[test]
1159 fn given_wait_requests_out_of_range_when_validated_then_should_be_rejected() {
1160 let no_partitions = FilteredWaitRequest {
1161 partitions: Vec::new(),
1162 ..wait_request()
1163 };
1164 let too_long = FilteredWaitRequest {
1165 max_wait_ms: MAX_FILTERED_WAIT_MS + 1,
1166 ..wait_request()
1167 };
1168 let mut repeated = wait_request();
1169 repeated.partitions[1].partition_id = 0;
1170 let too_many = FilteredWaitRequest {
1171 partitions: (0..=MAX_FILTERED_WAIT_PARTITIONS as u32)
1172 .map(|partition_id| FilteredWaitPartition {
1173 partition_id,
1174 offset: 0,
1175 })
1176 .collect(),
1177 ..wait_request()
1178 };
1179 for request in [no_partitions, too_long, repeated, too_many] {
1180 assert!(request.validate().is_err(), "{request:?} must be rejected");
1181 }
1182 FilteredWaitRequest {
1183 max_wait_ms: 0,
1184 ..wait_request()
1185 }
1186 .validate()
1187 .expect("a zero wait answers at once");
1188 }
1189
1190 fn generation() -> SourceGeneration {
1191 SourceGeneration {
1192 stream_id: 1,
1193 stream_created_at_micros: 10,
1194 topic_id: 2,
1195 topic_created_at_micros: 20,
1196 partition_id: 0,
1197 partition_incarnation: 30,
1198 }
1199 }
1200
1201 fn request() -> FilteredPollRequest {
1202 FilteredPollRequest {
1203 v: FILTER_OP_VERSION,
1204 source: FilterSource {
1205 stream: "orbit".to_owned(),
1206 topic: "fleet_changes".to_owned(),
1207 },
1208 partition_id: 0,
1209 consumer: FilterConsumer::Group("anomaly-desk".to_owned()),
1210 filter: FilterRef::Bound,
1211 start: FilteredStart::Next,
1212 count: 100,
1213 max_reply_bytes: 1024 * 1024,
1214 max_examined: None,
1215 min_catalog_position: None,
1216 }
1217 }
1218
1219 #[test]
1220 fn given_a_group_read_when_validated_then_should_pass() {
1221 request().validate().expect("valid");
1222 }
1223
1224 #[test]
1225 fn given_out_of_range_reads_when_validated_then_should_be_rejected() {
1226 let mut bound_consumer = request();
1227 bound_consumer.consumer = FilterConsumer::Consumer("one".to_owned());
1228 let mut zero = request();
1229 zero.count = 0;
1230 let mut huge = request();
1231 huge.max_reply_bytes = MAX_FILTERED_PAGE_BYTES + 1;
1232 let mut other_partition = request();
1233 other_partition.start = FilteredStart::Continue(Continuation {
1234 group_id: None,
1235 next_scan_offset: 7,
1236 generation: SourceGeneration {
1237 partition_id: 5,
1238 ..generation()
1239 },
1240 digest: Some(Digest32::new([1; 32])),
1241 mode: ExecutionMode::Filtered,
1242 policy_generation: 0,
1243 });
1244 let mut group_consumer = request();
1245 group_consumer.filter = FilterRef::Group;
1246 group_consumer.consumer = FilterConsumer::Consumer("one".to_owned());
1247 let mut over_examined = request();
1248 over_examined.max_examined = Some(MAX_FILTERED_PAGE_EXAMINED + 1);
1249 let mut zero_examined = request();
1250 zero_examined.max_examined = Some(0);
1251 let mut unfiltered_with_digest = request();
1252 unfiltered_with_digest.start = FilteredStart::Continue(Continuation {
1253 group_id: Some(3),
1254 next_scan_offset: 7,
1255 generation: generation(),
1256 digest: Some(Digest32::new([1; 32])),
1257 mode: ExecutionMode::Unfiltered,
1258 policy_generation: 1,
1259 });
1260 for request in [
1261 bound_consumer,
1262 zero,
1263 huge,
1264 other_partition,
1265 group_consumer,
1266 over_examined,
1267 zero_examined,
1268 unfiltered_with_digest,
1269 ] {
1270 assert!(request.validate().is_err(), "{request:?} must be rejected");
1271 }
1272 }
1273
1274 #[test]
1275 fn given_an_automatic_group_read_with_an_examined_bound_when_validated_then_should_pass() {
1276 let mut automatic = request();
1277 automatic.filter = FilterRef::Group;
1278 automatic.max_examined = Some(100);
1279 automatic.min_catalog_position = Some(CatalogPosition {
1280 partition_id: 0,
1281 offset: 41,
1282 operation_id: Some(11),
1283 revision: None,
1284 });
1285 automatic.validate().expect("valid");
1286 }
1287
1288 #[test]
1289 fn given_an_unfiltered_policy_when_validated_then_should_need_a_group_and_no_digest() {
1290 AppliedPolicy::unfiltered(3, 1).validate().expect("valid");
1291 assert!(
1292 AppliedPolicy::filtered(Some(3), Digest32::new([1; 32]))
1293 .validate()
1294 .is_ok()
1295 );
1296 let mut no_group = AppliedPolicy::unfiltered(3, 1);
1297 no_group.group_id = None;
1298 assert!(no_group.validate().is_err());
1299 let mut with_digest = AppliedPolicy::unfiltered(3, 1);
1300 with_digest.digest = Some(Digest32::new([1; 32]));
1301 assert!(with_digest.validate().is_err());
1302 let mut with_reference = AppliedPolicy::unfiltered(3, 1);
1303 with_reference.filter_id = Some(1);
1304 assert!(with_reference.validate().is_err());
1305 with_reference.filter_id = None;
1306 with_reference.revision = Some(1);
1307 assert!(with_reference.validate().is_err());
1308 let mut without_digest = AppliedPolicy::filtered(None, Digest32::new([1; 32]));
1309 without_digest.digest = None;
1310 assert!(without_digest.validate().is_err());
1311 }
1312
1313 #[test]
1314 fn given_a_page_when_asked_for_the_next_start_then_should_continue_or_repeat() {
1315 let page = FilteredPage {
1316 v: FILTER_OP_VERSION,
1317 partition_id: 0,
1318 policy: AppliedPolicy {
1319 group_id: None,
1320 digest: Some(Digest32::new([9; 32])),
1321 filter_id: Some(3),
1322 revision: Some(1),
1323 mode: ExecutionMode::Filtered,
1324 policy_generation: 0,
1325 },
1326 generation: generation(),
1327 ack_context: AckContext {
1328 owner_generation: 4,
1329 metadata_op: 40,
1330 },
1331 next_scan_offset: Some(1397),
1332 safe_ack_offset: Some(1396),
1333 frontier: 2000,
1334 examined: 397,
1335 matched: 3,
1336 stop: StopReason::Filled,
1337 fault: None,
1338 unevaluated: Vec::new(),
1339 evaluation_limits: None,
1340 records: Vec::new(),
1341 };
1342 assert_eq!(
1343 page.next_start(&FilteredStart::Next),
1344 FilteredStart::Continue(Continuation {
1345 group_id: None,
1346 next_scan_offset: 1397,
1347 generation: generation(),
1348 digest: Some(Digest32::new([9; 32])),
1349 mode: ExecutionMode::Filtered,
1350 policy_generation: 0,
1351 })
1352 );
1353 let empty = FilteredPage {
1354 next_scan_offset: None,
1355 safe_ack_offset: None,
1356 ..page
1357 };
1358 assert_eq!(empty.next_start(&FilteredStart::Last), FilteredStart::Last);
1359 }
1360
1361 #[test]
1362 fn given_error_reasons_when_classified_then_should_map_to_shared_codes() {
1363 assert_eq!(
1364 FilterError::new(FilterErrorReason::SourceChanged, "recreated").code,
1365 ResultCode::StaleGeneration
1366 );
1367 assert_eq!(FilterErrorReason::MembershipStale.code(), ResultCode::Stale);
1368 assert_eq!(FilterErrorReason::Conflict.code(), ResultCode::Conflict);
1369 assert_eq!(
1370 FilterErrorReason::CatalogUnavailable.code(),
1371 ResultCode::Unavailable
1372 );
1373 }
1374
1375 #[test]
1376 fn given_samples_and_previews_when_validated_then_should_reject_group_selectors() {
1377 let inline = FilterRef::Inline(ConsumerFilter::json(FilterExpr::pred(
1378 "kind",
1379 CmpOp::Eq,
1380 "satellites",
1381 )));
1382 let test = FilterTestRequest {
1383 v: FILTER_OP_VERSION,
1384 filter: inline.clone(),
1385 payload: br#"{"kind":"satellites"}"#.to_vec(),
1386 headers: Vec::new(),
1387 };
1388 test.validate().expect("valid sample");
1389 let mut preview = FilterPreviewRequest {
1390 v: FILTER_OP_VERSION,
1391 source: request().source,
1392 partition_id: 0,
1393 filter: inline,
1394 from_offset: 0,
1395 max_examined: 100,
1396 max_records: 10,
1397 explain: false,
1398 };
1399 preview.validate().expect("valid preview");
1400 for selector in [FilterRef::Bound, FilterRef::Group] {
1401 let sample = FilterTestRequest {
1402 filter: selector.clone(),
1403 ..test.clone()
1404 };
1405 assert!(sample.validate().is_err());
1406 preview.filter = selector;
1407 assert!(preview.validate().is_err());
1408 }
1409 }
1410
1411 #[test]
1412 fn given_oversized_sample_headers_when_validated_then_should_reject_before_evaluation() {
1413 let mut request = FilterTestRequest {
1414 v: FILTER_OP_VERSION,
1415 filter: FilterRef::Inline(ConsumerFilter::headers_only(FilterExpr::header(
1416 "priority",
1417 CmpOp::Eq,
1418 "critical",
1419 ))),
1420 payload: Vec::new(),
1421 headers: vec![FilterHeader {
1422 key: "priority".to_owned(),
1423 value: HeaderScalar::Raw(vec![0; crate::limits::MAX_FILTER_STRING_BYTES + 1]),
1424 }],
1425 };
1426 assert!(request.validate().is_err());
1427 request.headers[0].value = HeaderScalar::String("critical".to_owned());
1428 request.validate().expect("bounded header");
1429 request.headers[0].key.clear();
1430 assert!(request.validate().is_err());
1431 }
1432}