1use crate::content::ContentType;
2use crate::destination::BackendBinding;
3use crate::error::InvalidError;
4use crate::limits::{
5 MAX_FILTER_SCHEMA_BYTES, MAX_FILTER_SCHEMA_DEPTH, MAX_GRAPH_LABEL_BYTES, MAX_INDEX_FIELD_BYTES,
6 MAX_PROJECTION_ID_BYTES, MAX_PROJECTION_NAME_BYTES, MAX_SCHEMA_NAME_BYTES,
7};
8use crate::validate::{Validate, json_depth_within, validate_bounded_text};
9use serde::{Deserialize, Serialize};
10use std::str::FromStr;
11
12pub const RESERVED_PROJECTION_ID_PREFIX: &str = "index:";
15
16pub const JSON_SCHEMA_DIALECT: &str = "https://json-schema.org/draft/2020-12/schema";
18
19#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
31#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
32pub struct IndexField {
33 pub name: String,
34 pub pointer: String,
35 #[serde(default, skip_serializing_if = "Option::is_none")]
39 pub field_type: Option<FieldType>,
40}
41
42impl IndexField {
43 pub fn new(name: impl Into<String>, pointer: impl Into<String>) -> Self {
45 Self {
46 name: name.into(),
47 pointer: pointer.into(),
48 field_type: None,
49 }
50 }
51
52 pub fn typed(
54 name: impl Into<String>,
55 pointer: impl Into<String>,
56 field_type: FieldType,
57 ) -> Self {
58 Self {
59 name: name.into(),
60 pointer: pointer.into(),
61 field_type: Some(field_type),
62 }
63 }
64}
65
66#[derive(
69 Clone,
70 Copy,
71 Debug,
72 PartialEq,
73 Eq,
74 Serialize,
75 Deserialize,
76 strum::Display,
77 strum::EnumString,
78 strum::VariantArray,
79)]
80#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
81#[serde(rename_all = "snake_case")]
82#[strum(serialize_all = "snake_case")]
83#[non_exhaustive]
84pub enum FieldType {
85 Text,
86 Int,
87 Float,
88 Bool,
89}
90
91#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
93#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
94pub struct IndexSchema {
95 pub fields: Vec<IndexField>,
96 #[serde(default, skip_serializing_if = "Option::is_none")]
100 pub vector_field: Option<String>,
101 #[serde(default)]
106 pub inline_payload: bool,
107}
108
109impl IndexSchema {
110 pub fn builder() -> IndexSchemaBuilder {
112 IndexSchemaBuilder::default()
113 }
114}
115
116#[derive(Default)]
118pub struct IndexSchemaBuilder {
119 schema: IndexSchema,
120}
121
122impl IndexSchemaBuilder {
123 pub fn field(mut self, name: impl Into<String>) -> Self {
129 let name = name.into();
130 let pointer = format!("/{name}");
131 self.schema.fields.push(IndexField::new(name, pointer));
132 self
133 }
134
135 pub fn field_at(mut self, name: impl Into<String>, pointer: impl Into<String>) -> Self {
141 self.schema.fields.push(IndexField::new(name, pointer));
142 self
143 }
144
145 pub fn vector_field(mut self, pointer: impl Into<String>) -> Self {
148 self.schema.vector_field = Some(pointer.into());
149 self
150 }
151
152 pub fn inline_payload(mut self) -> Self {
155 self.schema.inline_payload = true;
156 self
157 }
158
159 pub fn build(self) -> IndexSchema {
161 self.schema
162 }
163}
164
165#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
171#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
172#[serde(transparent)]
173pub struct ProjectionId(String);
174
175impl ProjectionId {
176 pub fn new(value: impl Into<String>) -> Self {
178 Self(value.into())
179 }
180
181 pub fn as_str(&self) -> &str {
183 &self.0
184 }
185}
186
187impl std::fmt::Display for ProjectionId {
188 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
189 f.write_str(&self.0)
190 }
191}
192
193impl AsRef<str> for ProjectionId {
194 fn as_ref(&self) -> &str {
195 &self.0
196 }
197}
198
199impl std::borrow::Borrow<str> for ProjectionId {
200 fn borrow(&self) -> &str {
201 &self.0
202 }
203}
204
205impl FromStr for ProjectionId {
206 type Err = InvalidError;
207
208 fn from_str(s: &str) -> Result<Self, Self::Err> {
209 validate_projection_id(s)?;
210 Ok(Self(s.to_owned()))
211 }
212}
213
214impl From<&str> for ProjectionId {
215 fn from(value: &str) -> Self {
216 Self(value.to_owned())
217 }
218}
219
220impl From<String> for ProjectionId {
221 fn from(value: String) -> Self {
222 Self(value)
223 }
224}
225
226#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
227#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
228#[serde(tag = "kind", rename_all = "snake_case")]
229pub enum ProjectionHistory {
230 #[default]
231 Append,
232 Keyed {
233 key: ProjectionKey,
234 #[serde(default, skip_serializing_if = "Option::is_none")]
236 delete_header: Option<String>,
237 },
238}
239
240impl ProjectionHistory {
241 pub fn is_append(&self) -> bool {
242 matches!(self, Self::Append)
243 }
244}
245
246#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
247#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
248#[serde(tag = "kind", rename_all = "snake_case")]
249pub enum ProjectionKey {
250 Header { name: String },
251 Pointer { pointer: String },
252}
253
254#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
259#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
260#[serde(from = "u8", into = "u8")]
261pub enum ProjectionKind {
262 #[default]
264 Row,
265 Graph,
267 Record,
270 Unrecognized(u8),
272}
273
274impl ProjectionKind {
275 pub const fn code(self) -> u8 {
277 match self {
278 ProjectionKind::Row => 0,
279 ProjectionKind::Graph => 1,
280 ProjectionKind::Record => 2,
281 ProjectionKind::Unrecognized(code) => code,
282 }
283 }
284
285 pub const fn from_code(code: u8) -> Self {
288 match code {
289 0 => ProjectionKind::Row,
290 1 => ProjectionKind::Graph,
291 2 => ProjectionKind::Record,
292 other => ProjectionKind::Unrecognized(other),
293 }
294 }
295
296 pub const fn is_row(&self) -> bool {
298 matches!(self, ProjectionKind::Row)
299 }
300}
301
302impl From<u8> for ProjectionKind {
303 fn from(code: u8) -> Self {
304 Self::from_code(code)
305 }
306}
307
308impl From<ProjectionKind> for u8 {
309 fn from(kind: ProjectionKind) -> u8 {
310 kind.code()
311 }
312}
313
314#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
319#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
320pub struct EntitySchema {
321 pub nodes: Vec<NodeExtract>,
322 #[serde(default, skip_serializing_if = "Vec::is_empty")]
323 pub edges: Vec<EdgeExtract>,
324}
325
326#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
329#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
330pub struct NodeExtract {
331 pub label: String,
332 pub value_pointer: String,
333 #[serde(default, skip_serializing_if = "Option::is_none")]
334 pub embedding_pointer: Option<String>,
335}
336
337#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
340#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
341pub struct EdgeExtract {
342 pub edge_type: String,
343 pub from_pointer: String,
344 pub to_pointer: String,
345 #[serde(default, skip_serializing_if = "Option::is_none")]
348 pub valid_from_pointer: Option<String>,
349 #[serde(default, skip_serializing_if = "Option::is_none")]
352 pub valid_to_pointer: Option<String>,
353}
354
355#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
376#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
377pub struct Projection {
378 pub id: ProjectionId,
380 pub name: String,
383 pub version: u32,
386 #[serde(default, skip_serializing_if = "ProjectionKind::is_row")]
388 pub kind: ProjectionKind,
389 #[serde(default, skip_serializing_if = "ProjectionHistory::is_append")]
390 pub history: ProjectionHistory,
391 pub content_type: ContentType,
394 pub extraction: IndexSchema,
396 #[serde(default, skip_serializing_if = "Option::is_none")]
399 pub entity_schema: Option<EntitySchema>,
400 #[serde(default)]
403 pub inline_payload_default: bool,
404}
405
406impl Projection {
407 pub fn builder(id: impl Into<ProjectionId>) -> ProjectionBuilder {
409 ProjectionBuilder {
410 projection: Self {
411 id: id.into(),
412 name: String::new(),
413 version: 1,
414 kind: ProjectionKind::Row,
415 history: ProjectionHistory::Append,
416 content_type: ContentType::Any,
417 extraction: IndexSchema {
422 fields: Vec::new(),
423 vector_field: None,
424 inline_payload: true,
425 },
426 entity_schema: None,
427 inline_payload_default: true,
428 },
429 }
430 }
431}
432
433pub struct ProjectionBuilder {
435 projection: Projection,
436}
437
438impl ProjectionBuilder {
439 pub fn keyed(mut self, key: ProjectionKey, delete_header: Option<&str>) -> Self {
441 self.projection.history = ProjectionHistory::Keyed {
442 key,
443 delete_header: delete_header.map(str::to_owned),
444 };
445 self
446 }
447
448 pub fn name(mut self, value: impl Into<String>) -> Self {
450 self.projection.name = value.into();
451 self
452 }
453
454 pub fn version(mut self, value: u32) -> Self {
456 self.projection.version = value;
457 self
458 }
459
460 pub fn content_type(mut self, value: ContentType) -> Self {
462 self.projection.content_type = value;
463 self
464 }
465
466 pub fn extraction(mut self, value: IndexSchema) -> Self {
468 self.projection.extraction = value;
469 self
470 }
471
472 pub fn record(mut self) -> Self {
475 self.projection.kind = ProjectionKind::Record;
476 self.projection.extraction = IndexSchema::default();
477 self.projection.entity_schema = None;
478 self.projection.inline_payload_default = false;
479 self
480 }
481
482 pub fn graph(mut self, schema: EntitySchema) -> Self {
484 self.projection.kind = ProjectionKind::Graph;
485 self.projection.entity_schema = Some(schema);
486 self
487 }
488
489 pub fn field(mut self, name: impl Into<String>) -> Self {
491 let name = name.into();
492 let pointer = format!("/{name}");
493 self.projection
494 .extraction
495 .fields
496 .push(IndexField::new(name, pointer));
497 self
498 }
499
500 pub fn fields<I, S>(mut self, names: I) -> Self
504 where
505 I: IntoIterator<Item = S>,
506 S: Into<String>,
507 {
508 for name in names {
509 let name = name.into();
510 let pointer = format!("/{name}");
511 self.projection
512 .extraction
513 .fields
514 .push(IndexField::new(name, pointer));
515 }
516 self
517 }
518
519 pub fn field_at(mut self, name: impl Into<String>, pointer: impl Into<String>) -> Self {
521 self.projection
522 .extraction
523 .fields
524 .push(IndexField::new(name, pointer));
525 self
526 }
527
528 pub fn field_typed(mut self, name: impl Into<String>, field_type: FieldType) -> Self {
532 let name = name.into();
533 let pointer = format!("/{name}");
534 self.projection
535 .extraction
536 .fields
537 .push(IndexField::typed(name, pointer, field_type));
538 self
539 }
540
541 pub fn field_at_typed(
543 mut self,
544 name: impl Into<String>,
545 pointer: impl Into<String>,
546 field_type: FieldType,
547 ) -> Self {
548 self.projection
549 .extraction
550 .fields
551 .push(IndexField::typed(name, pointer, field_type));
552 self
553 }
554
555 pub fn vector_field(mut self, pointer: impl Into<String>) -> Self {
557 self.projection.extraction.vector_field = Some(pointer.into());
558 self
559 }
560
561 pub fn inline_payload(mut self) -> Self {
566 self.projection.inline_payload_default = true;
567 self.projection.extraction.inline_payload = true;
568 self
569 }
570
571 pub fn index_only(mut self) -> Self {
586 self.projection.inline_payload_default = false;
587 self.projection.extraction.inline_payload = false;
588 self
589 }
590
591 pub fn build(self) -> Projection {
593 self.projection
594 }
595}
596
597#[derive(Clone, Copy, Debug, PartialEq, Eq, Default, Serialize, Deserialize)]
605#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
606#[serde(tag = "kind", rename_all = "snake_case")]
607#[non_exhaustive]
608pub enum RetentionPolicy {
609 #[default]
613 MirrorLog,
614 Keep,
616 KeepUntilSourceDeleted,
620 TimeToLive { ttl_micros: u64 },
623 MaxRows { rows: u64 },
625 #[serde(other)]
631 Unknown,
632}
633
634#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
640#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
641pub struct ProjectionBinding {
642 pub source: SourceSelector,
643 #[serde(default)]
647 pub allowed_projections: Vec<ProjectionId>,
648 #[serde(default)]
651 pub default_projection: Option<ProjectionId>,
652 #[serde(default, skip_serializing_if = "Option::is_none")]
655 pub backend: Option<BackendBinding>,
656 pub index: String,
658 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
663 pub notify: bool,
664 #[serde(default, skip_serializing_if = "Option::is_none")]
669 pub retention: Option<RetentionPolicy>,
670 #[serde(default, skip_serializing_if = "InvalidRecordPolicy::is_block")]
678 pub invalid_records: InvalidRecordPolicy,
679 #[serde(default, skip_serializing_if = "OversizePolicy::is_block")]
685 pub oversize: OversizePolicy,
686}
687
688#[derive(
690 Clone,
691 Copy,
692 Debug,
693 Default,
694 PartialEq,
695 Eq,
696 Serialize,
697 Deserialize,
698 strum::Display,
699 strum::EnumString,
700)]
701#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
702#[serde(rename_all = "snake_case")]
703#[strum(serialize_all = "snake_case")]
704#[non_exhaustive]
705pub enum InvalidRecordPolicy {
706 #[default]
708 Block,
709 DeadLetter,
711 #[serde(other)]
713 #[strum(disabled)]
714 Unknown,
715}
716
717impl InvalidRecordPolicy {
718 pub fn is_block(&self) -> bool {
720 matches!(self, Self::Block)
721 }
722}
723
724#[derive(
726 Clone,
727 Copy,
728 Debug,
729 Default,
730 PartialEq,
731 Eq,
732 Serialize,
733 Deserialize,
734 strum::Display,
735 strum::EnumString,
736)]
737#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
738#[serde(rename_all = "snake_case")]
739#[strum(serialize_all = "snake_case")]
740#[non_exhaustive]
741pub enum OversizePolicy {
742 #[default]
744 Block,
745 Reference,
747 #[serde(other)]
749 #[strum(disabled)]
750 Unknown,
751}
752
753impl OversizePolicy {
754 pub fn is_block(&self) -> bool {
756 matches!(self, Self::Block)
757 }
758}
759
760impl ProjectionBinding {
761 pub fn builder() -> ProjectionBindingBuilder {
763 ProjectionBindingBuilder::default()
764 }
765}
766
767#[derive(Default)]
769pub struct ProjectionBindingBuilder {
770 source: Option<SourceSelector>,
771 allowed: Vec<ProjectionId>,
772 default_projection: Option<ProjectionId>,
773 backend: Option<BackendBinding>,
774 index: Option<String>,
775 notify: bool,
776 retention: Option<RetentionPolicy>,
777 invalid_records: InvalidRecordPolicy,
778 oversize: OversizePolicy,
779}
780
781impl ProjectionBindingBuilder {
782 pub fn source(mut self, stream: impl Into<String>, topic: impl Into<String>) -> Self {
785 self.source = Some(SourceSelector::new(stream, topic));
786 self
787 }
788
789 pub fn selector(mut self, source: SourceSelector) -> Self {
791 self.source = Some(source);
792 self
793 }
794
795 pub fn allow(mut self, projection: impl Into<ProjectionId>) -> Self {
797 self.allowed.push(projection.into());
798 self
799 }
800
801 pub fn default_projection(mut self, projection: impl Into<ProjectionId>) -> Self {
803 self.default_projection = Some(projection.into());
804 self
805 }
806
807 pub fn backend(mut self, backend: BackendBinding) -> Self {
810 self.backend = Some(backend);
811 self
812 }
813
814 pub fn retention(mut self, retention: RetentionPolicy) -> Self {
828 self.retention = Some(retention);
829 self
830 }
831
832 pub fn index(mut self, index: impl Into<String>) -> Self {
834 self.index = Some(index.into());
835 self
836 }
837
838 #[must_use]
841 pub fn notify(mut self) -> Self {
842 self.notify = true;
843 self
844 }
845
846 #[must_use]
849 pub fn invalid_records(mut self, policy: InvalidRecordPolicy) -> Self {
850 self.invalid_records = policy;
851 self
852 }
853
854 #[must_use]
857 pub fn oversize(mut self, policy: OversizePolicy) -> Self {
858 self.oversize = policy;
859 self
860 }
861
862 pub fn build(self) -> ProjectionBinding {
866 self.try_build()
867 .expect("ProjectionBinding requires a source - call .source(stream, topic)")
868 }
869
870 pub fn try_build(self) -> Result<ProjectionBinding, InvalidError> {
873 let source = self
874 .source
875 .ok_or_else(|| InvalidError::new("ProjectionBinding requires a source"))?;
876 let index = self.index.unwrap_or_else(|| source.topic.clone());
877 Ok(ProjectionBinding {
878 source,
879 allowed_projections: self.allowed,
880 default_projection: self.default_projection,
881 backend: self.backend,
882 index,
883 notify: self.notify,
884 retention: self.retention,
885 invalid_records: self.invalid_records,
886 oversize: self.oversize,
887 })
888 }
889}
890
891#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
896#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
897pub struct SourceSelector {
898 pub stream: String,
899 pub topic: String,
900}
901
902impl SourceSelector {
903 pub fn new(stream: impl Into<String>, topic: impl Into<String>) -> Self {
905 Self {
906 stream: stream.into(),
907 topic: topic.into(),
908 }
909 }
910}
911
912#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
919#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
920pub struct SchemaDef {
921 pub id: u32,
922 pub source: SchemaSource,
923 #[serde(default, skip_serializing_if = "Option::is_none")]
927 pub name: Option<String>,
928 #[serde(default, skip_serializing_if = "Option::is_none")]
933 pub version: Option<u32>,
934}
935
936#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
941#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
942pub struct SchemaAllocation {
943 pub source: SchemaSource,
944 #[serde(default, skip_serializing_if = "Option::is_none")]
945 pub name: Option<String>,
946 #[serde(default, skip_serializing_if = "Option::is_none")]
947 pub version: Option<u32>,
948}
949
950impl SchemaAllocation {
951 pub fn definition(&self, id: u32) -> SchemaDef {
953 SchemaDef {
954 id,
955 source: self.source.clone(),
956 name: self.name.clone(),
957 version: self.version,
958 }
959 }
960}
961
962impl SchemaDef {
963 pub fn content_type(&self) -> ContentType {
966 match self.source {
967 SchemaSource::Avro { .. } => ContentType::Avro,
968 SchemaSource::Protobuf { .. } => ContentType::Protobuf,
969 SchemaSource::JsonSchema { .. } => ContentType::Json,
970 SchemaSource::Unknown => ContentType::Any,
972 }
973 }
974}
975
976#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
981#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
982#[serde(tag = "kind", rename_all = "snake_case")]
983#[non_exhaustive]
984pub enum SchemaSource {
985 Avro { schema: String },
987 Protobuf {
990 #[cfg_attr(
991 feature = "schemars",
992 schemars(schema_with = "crate::encoding::bytes_schema")
993 )]
994 #[serde(with = "crate::encoding::bin_bytes")]
995 descriptor_set: Vec<u8>,
996 message_type: String,
997 },
998 JsonSchema { schema: String },
1003 #[serde(other)]
1009 Unknown,
1010}
1011
1012#[derive(Clone, Debug, Serialize, Deserialize)]
1016#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
1017pub enum ControlCommand {
1018 RegisterProjection(Projection),
1019 DropProjection(String),
1020 ApplyBinding(ProjectionBinding),
1021 RemoveBinding {
1022 source: SourceSelector,
1023 projection_ref: Option<String>,
1024 #[serde(default, skip_serializing_if = "Option::is_none")]
1028 generation: Option<ControlStreamGeneration>,
1029 },
1030 RegisterSchema(SchemaDef),
1032 DropSchema(u32),
1034 AllocateSchema(SchemaAllocation),
1037 RegisterGraph(Projection),
1042 DropGraph(String),
1044 RegisterSessionSource {
1046 stream: String,
1047 topics: SessionTopics,
1048 },
1049 RemoveSessionSource {
1051 stream: String,
1052 },
1053 FilterCatalog(Box<crate::filter::FilterCatalogCommand>),
1056 AcceptGap(GapAcceptance),
1062 SkipRecord(RecordSkip),
1068 SourceRetained(SourceRetention),
1074}
1075
1076#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
1078#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
1079pub struct SourceRetention {
1080 pub source: SourceSelector,
1081 pub partition_id: u32,
1082 pub topic_created_at_micros: u64,
1084 pub first_offset: u64,
1087}
1088
1089#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
1091#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
1092pub struct RecordSkip {
1093 pub source: SourceSelector,
1094 pub partition_id: u32,
1095 pub topic_created_at_micros: u64,
1098 pub offset: u64,
1099 pub target: String,
1102 pub reason: String,
1104}
1105
1106impl RecordSkip {
1107 pub fn covers(
1110 &self,
1111 stream: &str,
1112 topic: &str,
1113 partition_id: u32,
1114 generation: u64,
1115 offset: u64,
1116 target: &str,
1117 ) -> bool {
1118 self.source.stream == stream
1119 && self.source.topic == topic
1120 && self.partition_id == partition_id
1121 && self.topic_created_at_micros == generation
1122 && self.offset == offset
1123 && self.target == target
1124 }
1125}
1126
1127#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
1129#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
1130pub struct GapAcceptance {
1131 pub source: SourceSelector,
1132 pub partition_id: u32,
1133 pub topic_created_at_micros: u64,
1136 pub from_offset: u64,
1138 pub to_offset: u64,
1140}
1141
1142impl GapAcceptance {
1143 pub fn covers(&self, generation: u64, cursor: u64, first_available: u64) -> bool {
1146 self.topic_created_at_micros == generation
1147 && self.from_offset <= cursor
1148 && self.to_offset >= first_available
1149 }
1150}
1151
1152#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
1154#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
1155pub enum SessionTopics {
1156 All,
1157 Named(Vec<String>),
1158}
1159
1160#[derive(Clone, Debug, Serialize, Deserialize)]
1162#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
1163pub struct ControlEnvelope {
1164 pub v: u32,
1165 pub timestamp_micros: u64,
1166 pub command: ControlCommand,
1167 #[serde(default, skip_serializing_if = "Option::is_none")]
1173 pub stream: Option<String>,
1174 #[serde(default, skip_serializing_if = "Option::is_none")]
1179 pub stamp: Option<ControlStamp>,
1180}
1181
1182#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
1184#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
1185pub struct ControlStamp {
1186 pub deployment_id: u64,
1189 pub operation_id: u128,
1192 #[serde(default, skip_serializing_if = "Option::is_none")]
1195 pub stream_generation: Option<ControlStreamGeneration>,
1196 pub key_id: u64,
1198 #[serde(default, skip_serializing_if = "Option::is_none")]
1202 pub caller: Option<crate::forward::Caller>,
1203}
1204
1205#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
1208#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
1209pub struct ControlStreamGeneration {
1210 pub stream_id: u32,
1211 pub created_at_micros: u64,
1212}
1213
1214impl Validate for ProjectionId {
1215 fn validate(&self) -> Result<(), InvalidError> {
1216 validate_projection_id(&self.0)
1217 }
1218}
1219
1220impl Validate for Projection {
1221 fn validate(&self) -> Result<(), InvalidError> {
1222 self.id.validate()?;
1223 if let ProjectionHistory::Keyed { key, delete_header } = &self.history {
1224 if self.kind == ProjectionKind::Graph {
1225 return Err(InvalidError::new(
1226 "graph projections do not support keyed history",
1227 ));
1228 }
1229 if let Some(delete_header) = delete_header {
1230 validate_bounded_text("deletion header", delete_header, MAX_INDEX_FIELD_BYTES)?;
1231 }
1232 match key {
1233 ProjectionKey::Header { name } => {
1234 validate_bounded_text("key header", name, MAX_INDEX_FIELD_BYTES)?
1235 }
1236 ProjectionKey::Pointer { pointer } => {
1237 if self.kind != ProjectionKind::Row {
1238 return Err(InvalidError::new(
1239 "payload key pointers require a row projection",
1240 ));
1241 }
1242 if !pointer.is_empty() {
1243 validate_bounded_text("key pointer", pointer, MAX_INDEX_FIELD_BYTES)?;
1244 if !pointer.starts_with('/') {
1245 return Err(InvalidError::new("key pointer must be a JSON pointer"));
1246 }
1247 }
1248 let mut bytes = pointer.bytes();
1249 while let Some(byte) = bytes.next() {
1250 if byte == b'~' && !matches!(bytes.next(), Some(b'0' | b'1')) {
1251 return Err(InvalidError::new("key pointer has an invalid escape"));
1252 }
1253 }
1254 }
1255 }
1256 }
1257
1258 if self.kind == ProjectionKind::Record
1259 && (!self.extraction.fields.is_empty()
1260 || self.extraction.vector_field.is_some()
1261 || self.entity_schema.is_some())
1262 {
1263 return Err(InvalidError::new(
1264 "a whole-record projection extracts nothing, so it declares no fields, vector field or entity schema",
1265 ));
1266 }
1267 if !self.name.is_empty() {
1268 validate_bounded_text("projection name", &self.name, MAX_PROJECTION_NAME_BYTES)?;
1269 }
1270 for field in &self.extraction.fields {
1271 validate_bounded_text("index field name", &field.name, MAX_INDEX_FIELD_BYTES)?;
1272 let mut characters = field.name.chars();
1273 let leads = characters
1274 .next()
1275 .is_some_and(|first| first.is_ascii_alphabetic() || first == '_');
1276 if !leads || !characters.all(|rest| rest.is_ascii_alphanumeric() || rest == '_') {
1277 return Err(InvalidError::new(format!(
1278 "index field name `{}` must start with an ASCII letter or `_` and hold only ASCII letters, digits and `_`",
1279 field.name
1280 )));
1281 }
1282 validate_bounded_text("index field pointer", &field.pointer, MAX_INDEX_FIELD_BYTES)?;
1283 }
1284 if let Some(vector_field) = &self.extraction.vector_field {
1285 validate_bounded_text("index vector field", vector_field, MAX_INDEX_FIELD_BYTES)?;
1286 }
1287 if let Some(entity_schema) = &self.entity_schema {
1288 for node in &entity_schema.nodes {
1289 validate_bounded_text("graph node label", &node.label, MAX_GRAPH_LABEL_BYTES)?;
1290 }
1291 for edge in &entity_schema.edges {
1292 validate_bounded_text("graph edge type", &edge.edge_type, MAX_GRAPH_LABEL_BYTES)?;
1293 }
1294 }
1295 Ok(())
1296 }
1297}
1298
1299impl Validate for ProjectionBinding {
1300 fn validate(&self) -> Result<(), InvalidError> {
1301 for projection in &self.allowed_projections {
1302 projection.validate()?;
1303 }
1304 if let Some(projection) = &self.default_projection {
1305 projection.validate()?;
1306 }
1307 if self.invalid_records == InvalidRecordPolicy::Unknown
1308 || self.oversize == OversizePolicy::Unknown
1309 || self.retention == Some(RetentionPolicy::Unknown)
1310 {
1311 return Err(InvalidError::new(
1312 "a binding cannot apply a policy this build does not know",
1313 ));
1314 }
1315 if matches!(
1316 self.retention,
1317 Some(
1318 RetentionPolicy::TimeToLive { ttl_micros: 0 }
1319 | RetentionPolicy::MaxRows { rows: 0 }
1320 )
1321 ) {
1322 return Err(InvalidError::new(
1323 "a time to live retention needs a lifetime above zero, and a max rows retention at least one row",
1324 ));
1325 }
1326 if self.oversize == OversizePolicy::Reference
1327 && self
1328 .retention
1329 .is_some_and(|retention| retention != RetentionPolicy::MirrorLog)
1330 {
1331 return Err(InvalidError::new(
1332 "a binding that keeps body references needs mirror_log retention, so no row outlives its record",
1333 ));
1334 }
1335 Ok(())
1336 }
1337}
1338
1339impl Validate for SchemaDef {
1340 fn validate(&self) -> Result<(), InvalidError> {
1341 if self.id == 0 {
1342 return Err(InvalidError::new("writer schema ids start at 1"));
1343 }
1344 validate_schema_metadata(&self.source, self.name.as_deref())
1345 }
1346}
1347
1348impl Validate for SchemaAllocation {
1349 fn validate(&self) -> Result<(), InvalidError> {
1350 validate_schema_metadata(&self.source, self.name.as_deref())
1351 }
1352}
1353
1354impl Validate for SchemaSource {
1355 fn validate(&self) -> Result<(), InvalidError> {
1356 match self {
1357 Self::Avro { schema } => validate_schema_text("Avro schema", schema),
1358 Self::JsonSchema { schema } => {
1359 validate_schema_text("JSON Schema", schema)?;
1360 let value: serde_json::Value = serde_json::from_str(schema).map_err(|error| {
1361 InvalidError::new(format!("JSON Schema is not valid JSON: {error}"))
1362 })?;
1363 match value.get("$schema") {
1364 None => Ok(()),
1365 Some(serde_json::Value::String(dialect))
1366 if dialect.trim_end_matches('#') == JSON_SCHEMA_DIALECT =>
1367 {
1368 Ok(())
1369 }
1370 Some(other) => Err(InvalidError::new(format!(
1371 "a JSON Schema uses draft 2020-12, not {other}"
1372 ))),
1373 }
1374 }
1375 Self::Protobuf {
1376 descriptor_set,
1377 message_type,
1378 } => {
1379 if descriptor_set.len() > MAX_FILTER_SCHEMA_BYTES {
1380 return Err(InvalidError::new(format!(
1381 "Protobuf descriptor set exceeds {MAX_FILTER_SCHEMA_BYTES} bytes"
1382 )));
1383 }
1384 validate_bounded_text("Protobuf message type", message_type, MAX_SCHEMA_NAME_BYTES)
1385 }
1386 Self::Unknown => Err(InvalidError::new(
1387 "a schema source of an unknown kind cannot be registered",
1388 )),
1389 }
1390 }
1391}
1392
1393impl Validate for ControlCommand {
1394 fn validate(&self) -> Result<(), InvalidError> {
1395 match self {
1396 Self::RegisterProjection(projection) | Self::RegisterGraph(projection) => {
1397 projection.validate()
1398 }
1399 Self::DropProjection(id) | Self::DropGraph(id) => validate_projection_id(id),
1400 Self::ApplyBinding(binding) => binding.validate(),
1401 Self::RemoveBinding { projection_ref, .. } => projection_ref
1402 .as_deref()
1403 .map_or(Ok(()), validate_projection_id),
1404 Self::RegisterSchema(schema) => schema.validate(),
1405 Self::AllocateSchema(allocation) => allocation.validate(),
1406 Self::FilterCatalog(command) => command.mutation.validate(),
1407 Self::DropSchema(_)
1408 | Self::RegisterSessionSource { .. }
1409 | Self::RemoveSessionSource { .. }
1410 | Self::AcceptGap(_)
1411 | Self::SkipRecord(_)
1412 | Self::SourceRetained(_) => Ok(()),
1413 }
1414 }
1415}
1416
1417fn validate_projection_id(id: &str) -> Result<(), InvalidError> {
1418 validate_bounded_text("projection id", id, MAX_PROJECTION_ID_BYTES)?;
1419 if id.starts_with(RESERVED_PROJECTION_ID_PREFIX) {
1420 return Err(InvalidError::new(format!(
1421 "projection id must not start with {RESERVED_PROJECTION_ID_PREFIX:?}"
1422 )));
1423 }
1424 Ok(())
1425}
1426
1427fn validate_schema_metadata(source: &SchemaSource, name: Option<&str>) -> Result<(), InvalidError> {
1428 if let Some(name) = name {
1429 validate_bounded_text("schema name", name, MAX_SCHEMA_NAME_BYTES)?;
1430 }
1431 source.validate()
1432}
1433
1434fn validate_schema_text(label: &str, text: &str) -> Result<(), InvalidError> {
1435 if text.len() > MAX_FILTER_SCHEMA_BYTES {
1436 return Err(InvalidError::new(format!(
1437 "{label} exceeds {MAX_FILTER_SCHEMA_BYTES} bytes"
1438 )));
1439 }
1440 if !json_depth_within(text.as_bytes(), MAX_FILTER_SCHEMA_DEPTH) {
1441 return Err(InvalidError::new(format!(
1442 "{label} nests deeper than {MAX_FILTER_SCHEMA_DEPTH}"
1443 )));
1444 }
1445 Ok(())
1446}
1447
1448#[cfg(test)]
1449mod tests {
1450 use super::*;
1451 use crate::limits::{
1452 MAX_GRAPH_LABEL_BYTES, MAX_INDEX_FIELD_BYTES, MAX_PROJECTION_ID_BYTES,
1453 MAX_PROJECTION_NAME_BYTES, MAX_SCHEMA_NAME_BYTES,
1454 };
1455 use std::borrow::Borrow;
1456
1457 #[test]
1458 fn given_keyed_projection_when_validated_then_should_check_kind_and_pointer() {
1459 for pointer in ["", "/id", "/a~1b/~0"] {
1460 Projection::builder("keyed")
1461 .keyed(
1462 ProjectionKey::Pointer {
1463 pointer: pointer.to_owned(),
1464 },
1465 Some("deleted"),
1466 )
1467 .build()
1468 .validate()
1469 .expect("valid pointer");
1470 }
1471 for pointer in ["id", "/a~", "/a~2"] {
1472 assert!(
1473 Projection::builder("keyed")
1474 .keyed(
1475 ProjectionKey::Pointer {
1476 pointer: pointer.to_owned()
1477 },
1478 Some("deleted")
1479 )
1480 .build()
1481 .validate()
1482 .is_err()
1483 );
1484 }
1485 assert!(
1486 Projection::builder("keyed")
1487 .record()
1488 .keyed(
1489 ProjectionKey::Pointer {
1490 pointer: "/id".to_owned()
1491 },
1492 Some("deleted")
1493 )
1494 .build()
1495 .validate()
1496 .is_err()
1497 );
1498 assert!(
1499 Projection::builder("keyed")
1500 .graph(EntitySchema::default())
1501 .keyed(
1502 ProjectionKey::Header {
1503 name: "id".to_owned()
1504 },
1505 Some("deleted")
1506 )
1507 .build()
1508 .validate()
1509 .is_err()
1510 );
1511 Projection::builder("keyed")
1512 .record()
1513 .keyed(
1514 ProjectionKey::Header {
1515 name: "id".to_owned(),
1516 },
1517 Some("deleted"),
1518 )
1519 .build()
1520 .validate()
1521 .expect("record header key");
1522 }
1523
1524 #[test]
1525 fn given_an_invalid_projection_id_when_any_command_names_it_then_should_refuse_it() {
1526 let projection = |id: &str| {
1527 Projection::builder(id)
1528 .name("reading")
1529 .field("host")
1530 .build()
1531 };
1532 let at_cap = "p".repeat(MAX_PROJECTION_ID_BYTES);
1533 projection(&at_cap)
1534 .validate()
1535 .expect("a projection id at the cap is valid");
1536 ControlCommand::DropProjection(at_cap.clone())
1537 .validate()
1538 .expect("a drop of a projection id at the cap is valid");
1539 ControlCommand::RemoveBinding {
1540 source: SourceSelector::new("fleet", "readings"),
1541 projection_ref: None,
1542 generation: None,
1543 }
1544 .validate()
1545 .expect("a whole-binding removal names no projection");
1546 for invalid in [
1547 String::new(),
1548 "p".repeat(MAX_PROJECTION_ID_BYTES + 1),
1549 "reading\n.v1".to_owned(),
1550 format!("{RESERVED_PROJECTION_ID_PREFIX}reading"),
1551 ] {
1552 assert!(projection(&invalid).validate().is_err(), "{invalid:?}");
1553 assert!(
1554 ControlCommand::RegisterProjection(projection(&invalid))
1555 .validate()
1556 .is_err()
1557 );
1558 assert!(
1559 ControlCommand::DropProjection(invalid.clone())
1560 .validate()
1561 .is_err()
1562 );
1563 assert!(
1564 ControlCommand::DropGraph(invalid.clone())
1565 .validate()
1566 .is_err()
1567 );
1568 let allowed = ProjectionBinding::builder()
1569 .source("fleet", "readings")
1570 .allow(invalid.as_str())
1571 .build();
1572 assert!(ControlCommand::ApplyBinding(allowed).validate().is_err());
1573 let defaulted = ProjectionBinding::builder()
1574 .source("fleet", "readings")
1575 .default_projection(invalid.as_str())
1576 .build();
1577 assert!(defaulted.validate().is_err());
1578 assert!(
1579 ControlCommand::RemoveBinding {
1580 source: SourceSelector::new("fleet", "readings"),
1581 projection_ref: Some(invalid.clone()),
1582 generation: None,
1583 }
1584 .validate()
1585 .is_err()
1586 );
1587 assert!(invalid.parse::<ProjectionId>().is_err());
1588 }
1589 }
1590
1591 #[test]
1592 fn given_a_projection_name_over_the_cap_when_validated_then_should_refuse_it_and_accept_none() {
1593 let named = |name: &str| Projection::builder("reading.v1").name(name).build();
1594 named("").validate().expect("a projection name is optional");
1595 named(&"n".repeat(MAX_PROJECTION_NAME_BYTES))
1596 .validate()
1597 .expect("a projection name at the cap is valid");
1598 assert!(
1599 named(&"n".repeat(MAX_PROJECTION_NAME_BYTES + 1))
1600 .validate()
1601 .is_err()
1602 );
1603 assert!(named("read\ting").validate().is_err());
1604 }
1605
1606 #[test]
1607 fn given_an_invalid_index_field_when_a_projection_is_validated_then_should_refuse_it() {
1608 let with_field = |name: &str, pointer: &str| {
1609 Projection::builder("reading.v1")
1610 .field_at(name, pointer)
1611 .build()
1612 };
1613 with_field("host", "/host")
1614 .validate()
1615 .expect("a plain index field is valid");
1616 let long = "f".repeat(MAX_INDEX_FIELD_BYTES + 1);
1617 assert!(with_field(&long, "/host").validate().is_err());
1618 assert!(with_field("host", &long).validate().is_err());
1619 assert!(with_field("", "/host").validate().is_err());
1620 assert!(with_field("host", "").validate().is_err());
1621 assert!(with_field("ho\u{7}st", "/host").validate().is_err());
1622 assert!(
1623 Projection::builder("reading.v1")
1624 .vector_field(long)
1625 .build()
1626 .validate()
1627 .is_err()
1628 );
1629 assert!(
1630 Projection::builder("reading.v1")
1631 .vector_field("")
1632 .build()
1633 .validate()
1634 .is_err()
1635 );
1636 }
1637
1638 #[test]
1639 fn given_an_invalid_schema_name_when_registered_then_should_refuse_it() {
1640 let schema = |name: Option<String>| SchemaDef {
1641 id: 7,
1642 source: SchemaSource::JsonSchema {
1643 schema: "{}".to_owned(),
1644 },
1645 name,
1646 version: None,
1647 };
1648 schema(None)
1649 .validate()
1650 .expect("a schema without a name is valid");
1651 ControlCommand::RegisterSchema(schema(Some("s".repeat(MAX_SCHEMA_NAME_BYTES))))
1652 .validate()
1653 .expect("a schema name at the cap is valid");
1654 for invalid in [
1655 String::new(),
1656 "s".repeat(MAX_SCHEMA_NAME_BYTES + 1),
1657 "read\ning".to_owned(),
1658 ] {
1659 assert!(schema(Some(invalid.clone())).validate().is_err());
1660 assert!(
1661 ControlCommand::RegisterSchema(schema(Some(invalid)))
1662 .validate()
1663 .is_err()
1664 );
1665 }
1666 }
1667
1668 #[test]
1669 fn given_an_invalid_graph_label_when_a_graph_projection_is_validated_then_should_refuse_it() {
1670 let graph = |label: &str, edge_type: &str| {
1671 Projection::builder("people.v1")
1672 .graph(EntitySchema {
1673 nodes: vec![NodeExtract {
1674 label: label.to_owned(),
1675 value_pointer: "/name".to_owned(),
1676 embedding_pointer: None,
1677 }],
1678 edges: vec![EdgeExtract {
1679 edge_type: edge_type.to_owned(),
1680 from_pointer: "/name".to_owned(),
1681 to_pointer: "/company".to_owned(),
1682 valid_from_pointer: None,
1683 valid_to_pointer: None,
1684 }],
1685 })
1686 .build()
1687 };
1688 let at_cap = "l".repeat(MAX_GRAPH_LABEL_BYTES);
1689 ControlCommand::RegisterGraph(graph(&at_cap, &at_cap))
1690 .validate()
1691 .expect("a label and an edge type at the cap are valid");
1692 let long = "l".repeat(MAX_GRAPH_LABEL_BYTES + 1);
1693 assert!(graph(&long, "works_at").validate().is_err());
1694 assert!(graph("Person", &long).validate().is_err());
1695 assert!(graph("", "works_at").validate().is_err());
1696 assert!(graph("Person", "").validate().is_err());
1697 assert!(
1698 ControlCommand::RegisterGraph(graph("Per\nson", "works_at"))
1699 .validate()
1700 .is_err()
1701 );
1702 }
1703
1704 #[test]
1705 fn given_binding_policies_when_encoded_then_should_carry_them_only_when_not_the_default() {
1706 let plain = ProjectionBinding::builder().source("s", "t").build();
1707 assert_eq!(plain.invalid_records, InvalidRecordPolicy::Block);
1708 assert_eq!(plain.oversize, OversizePolicy::Block);
1709 let plain_json = serde_json::to_value(&plain).expect("encode a default binding");
1710 assert!(plain_json.get("invalid_records").is_none());
1711 assert!(plain_json.get("oversize").is_none());
1712 let opted = ProjectionBinding::builder()
1713 .source("s", "t")
1714 .invalid_records(InvalidRecordPolicy::DeadLetter)
1715 .oversize(OversizePolicy::Reference)
1716 .build();
1717 let json = serde_json::to_value(&opted).expect("encode an opted binding");
1718 assert_eq!(json["invalid_records"], "dead_letter");
1719 assert_eq!(json["oversize"], "reference");
1720 let decoded: ProjectionBinding =
1721 serde_json::from_value(json).expect("decode an opted binding");
1722 assert_eq!(decoded, opted);
1723 }
1724
1725 #[test]
1726 fn given_an_unknown_binding_policy_when_decoded_then_should_degrade_and_refuse_to_apply_it() {
1727 let binding: ProjectionBinding = serde_json::from_value(serde_json::json!({
1728 "source": {"stream": "s", "topic": "t"},
1729 "index": "t",
1730 "invalid_records": "quarantine",
1731 "oversize": "truncate",
1732 }))
1733 .expect("an unknown policy decodes to Unknown");
1734 assert_eq!(binding.invalid_records, InvalidRecordPolicy::Unknown);
1735 assert_eq!(binding.oversize, OversizePolicy::Unknown);
1736 assert!(binding.validate().is_err());
1737 }
1738
1739 #[test]
1740 fn given_body_references_when_retention_outlives_the_log_then_should_refuse_the_binding() {
1741 let binding = |retention: Option<RetentionPolicy>| {
1742 let builder = ProjectionBinding::builder()
1743 .source("s", "t")
1744 .oversize(OversizePolicy::Reference);
1745 match retention {
1746 Some(retention) => builder.retention(retention),
1747 None => builder,
1748 }
1749 .build()
1750 };
1751 binding(None)
1752 .validate()
1753 .expect("the fleet default is checked by the backend");
1754 binding(Some(RetentionPolicy::MirrorLog))
1755 .validate()
1756 .expect("references live as long as their records");
1757 assert!(binding(Some(RetentionPolicy::Keep)).validate().is_err());
1758 assert!(
1759 binding(Some(RetentionPolicy::TimeToLive { ttl_micros: 1 }))
1760 .validate()
1761 .is_err()
1762 );
1763 }
1764
1765 #[test]
1766 fn given_a_record_projection_when_validated_then_should_refuse_extraction_rules() {
1767 let record = Projection::builder("orders").record().build();
1768 assert_eq!(record.kind, ProjectionKind::Record);
1769 assert_eq!(ProjectionKind::from_code(2), ProjectionKind::Record);
1770 assert_eq!(ProjectionKind::Record.code(), 2);
1771 record
1772 .validate()
1773 .expect("a bare record projection is valid");
1774 let with_field = Projection::builder("orders").record().field("id").build();
1775 assert!(with_field.validate().is_err());
1776 let with_vector = Projection::builder("orders")
1777 .record()
1778 .vector_field("/embedding")
1779 .build();
1780 assert!(with_vector.validate().is_err());
1781 }
1782
1783 #[test]
1784 fn given_a_record_skip_when_matched_then_should_cover_only_its_exact_record() {
1785 let skip = RecordSkip {
1786 source: SourceSelector::new("s", "t"),
1787 partition_id: 2,
1788 topic_created_at_micros: 7,
1789 offset: 40,
1790 target: "rows".to_owned(),
1791 reason: "rejected".to_owned(),
1792 };
1793 assert!(skip.covers("s", "t", 2, 7, 40, "rows"));
1794 assert!(
1795 !skip.covers("s", "t", 2, 8, 40, "rows"),
1796 "another incarnation"
1797 );
1798 assert!(!skip.covers("s", "t", 2, 7, 41, "rows"));
1799 assert!(!skip.covers("s", "t", 3, 7, 40, "rows"));
1800 assert!(!skip.covers("s", "t", 2, 7, 40, "other"));
1801 assert!(!skip.covers("s", "u", 2, 7, 40, "rows"));
1802 }
1803
1804 #[test]
1805 fn given_an_unknown_schema_source_kind_when_decoded_then_should_degrade_not_fail() {
1806 let source: SchemaSource =
1810 serde_json::from_str(r#"{"kind":"parquet_descriptor","blob":[1,2,3]}"#)
1811 .expect("an unknown schema kind decodes to Unknown, not an error");
1812 assert_eq!(source, SchemaSource::Unknown);
1813 let def = SchemaDef {
1815 id: 1,
1816 source,
1817 name: None,
1818 version: None,
1819 };
1820 assert_eq!(def.content_type(), ContentType::Any);
1821 }
1822
1823 #[test]
1824 fn given_an_unknown_retention_kind_when_decoded_then_should_degrade_not_fail() {
1825 let policy: RetentionPolicy = serde_json::from_str(r#"{"kind":"keep_for_eras","eras":3}"#)
1826 .expect("an unknown retention kind decodes to Unknown, not an error");
1827 assert_eq!(policy, RetentionPolicy::Unknown);
1828 }
1829
1830 #[test]
1831 fn given_an_explicit_operational_index_when_built_then_should_keep_one_index() {
1832 let binding = ProjectionBinding::builder()
1833 .source("fleet", "readings")
1834 .allow("reading.v1")
1835 .index("readings_rows")
1836 .build();
1837 assert_eq!(binding.index, "readings_rows");
1838 assert_eq!(binding.backend, None);
1839 }
1840
1841 #[test]
1842 fn given_an_operational_backend_when_built_then_should_keep_exact_generation() {
1843 let backend = BackendBinding {
1844 resource_id: crate::destination::BackendResourceId::from_u128(7),
1845 generation: 3,
1846 };
1847 let binding = ProjectionBinding::builder()
1848 .source("fleet", "readings")
1849 .backend(backend.clone())
1850 .build();
1851 assert_eq!(binding.backend, Some(backend));
1852 }
1853
1854 #[test]
1855 fn given_no_operational_index_when_built_then_should_use_the_topic_name() {
1856 let binding = ProjectionBinding::builder()
1857 .source("fleet", "readings")
1858 .allow("reading.v1")
1859 .build();
1860 assert_eq!(binding.index, "readings");
1861 }
1862
1863 #[test]
1864 fn given_no_retention_when_built_then_should_default_to_none() {
1865 let binding = ProjectionBinding::builder()
1866 .source("fleet", "readings")
1867 .index("readings_rows")
1868 .build();
1869 assert_eq!(binding.retention, None);
1870 }
1871
1872 #[test]
1873 fn given_no_source_when_try_built_then_should_error() {
1874 assert!(ProjectionBinding::builder().try_build().is_err());
1875 }
1876
1877 #[test]
1878 fn given_a_projection_built_with_the_default_when_inspected_then_should_inline_payload() {
1879 let projection = Projection::builder("api.call.v1")
1880 .name("api.call")
1881 .version(1)
1882 .fields(["endpoint", "status"])
1883 .build();
1884 assert!(
1885 projection.inline_payload_default,
1886 "Projection::builder default should inline payload"
1887 );
1888 assert!(
1889 projection.extraction.inline_payload,
1890 "Projection::builder default should mark extraction.inline_payload too"
1891 );
1892 }
1893
1894 #[test]
1895 fn given_a_projection_with_index_only_when_inspected_then_should_skip_inlining() {
1896 let projection = Projection::builder("api.call.v1")
1897 .name("api.call")
1898 .version(1)
1899 .fields(["endpoint"])
1900 .index_only()
1901 .build();
1902 assert!(
1903 !projection.inline_payload_default,
1904 "index_only must clear inline_payload_default"
1905 );
1906 assert!(
1907 !projection.extraction.inline_payload,
1908 "index_only must clear extraction.inline_payload"
1909 );
1910 }
1911
1912 #[test]
1913 fn given_an_empty_projection_id_when_parsed_then_should_error() {
1914 assert!("".parse::<ProjectionId>().is_err());
1915 assert_eq!(
1916 "reading.v1"
1917 .parse::<ProjectionId>()
1918 .expect("non-empty id parses")
1919 .as_str(),
1920 "reading.v1"
1921 );
1922 }
1923
1924 #[test]
1925 fn given_an_index_schema_builder_when_built_then_should_keep_every_declared_rule() {
1926 let schema = IndexSchema::builder()
1927 .field("host")
1928 .field_at("cpu_pct", "/cpu/value")
1929 .vector_field("/vectors/text")
1930 .inline_payload()
1931 .build();
1932 assert_eq!(
1933 schema,
1934 IndexSchema {
1935 fields: vec![
1936 IndexField::new("host", "/host"),
1937 IndexField::new("cpu_pct", "/cpu/value"),
1938 ],
1939 vector_field: Some("/vectors/text".to_owned()),
1940 inline_payload: true,
1941 }
1942 );
1943 }
1944
1945 #[test]
1946 fn given_a_projection_id_from_a_string_when_read_as_text_then_should_expose_that_string() {
1947 let id = ProjectionId::from("reading.v1".to_owned());
1948 let viewed: &str = id.as_ref();
1949 let borrowed: &str = id.borrow();
1950 assert_eq!(id, ProjectionId::new("reading.v1"));
1951 assert_eq!(
1952 (id.to_string(), viewed, borrowed),
1953 ("reading.v1".to_owned(), "reading.v1", "reading.v1")
1954 );
1955 }
1956
1957 #[test]
1958 fn given_a_keyed_record_projection_when_encoded_then_should_round_trip_and_omit_defaults() {
1959 let keyed = Projection::builder("orders.v1")
1960 .record()
1961 .keyed(
1962 ProjectionKey::Header {
1963 name: "order_id".to_owned(),
1964 },
1965 Some("deleted"),
1966 )
1967 .build();
1968 let json = serde_json::to_value(&keyed).expect("encode a keyed record projection");
1969 let decoded: Projection =
1970 serde_json::from_value(json).expect("decode a keyed record projection");
1971 assert_eq!(decoded, keyed);
1972 let plain = serde_json::to_value(Projection::builder("orders.v1").build())
1973 .expect("encode a default projection");
1974 assert_eq!((plain.get("kind"), plain.get("history")), (None, None));
1975 }
1976
1977 #[test]
1978 fn given_a_keyed_projection_without_a_delete_header_when_decoded_then_should_validate_and_encode_without_it()
1979 {
1980 let keyed = Projection::builder("orders.v1")
1981 .record()
1982 .keyed(
1983 ProjectionKey::Header {
1984 name: "order_id".to_owned(),
1985 },
1986 Some("deleted"),
1987 )
1988 .build();
1989 let mut json = serde_json::to_value(&keyed).expect("encode a keyed projection");
1990 json["history"]
1991 .as_object_mut()
1992 .expect("a keyed history object")
1993 .remove("delete_header");
1994 let decoded: Projection = serde_json::from_value(json.clone())
1995 .expect("decode a keyed projection without a delete header");
1996 decoded
1997 .validate()
1998 .expect("a keyed view may name no delete header");
1999 assert_eq!(
2000 serde_json::to_value(&decoded).expect("encode it again"),
2001 json
2002 );
2003 json["history"]["delete_header"] = serde_json::Value::String(String::new());
2004 let empty: Projection =
2005 serde_json::from_value(json).expect("decode an empty delete header");
2006 assert!(
2007 empty.validate().is_err(),
2008 "a named delete header must not be empty"
2009 );
2010 }
2011
2012 #[test]
2013 fn given_a_selector_and_notify_when_a_binding_is_built_then_should_keep_both() {
2014 let binding = ProjectionBinding::builder()
2015 .notify()
2016 .selector(SourceSelector::new("fleet", "readings"))
2017 .try_build()
2018 .expect("a selector is a source");
2019 assert_eq!(
2020 binding,
2021 ProjectionBinding {
2022 source: SourceSelector::new("fleet", "readings"),
2023 allowed_projections: Vec::new(),
2024 default_projection: None,
2025 backend: None,
2026 index: "readings".to_owned(),
2027 notify: true,
2028 retention: None,
2029 invalid_records: InvalidRecordPolicy::Block,
2030 oversize: OversizePolicy::Block,
2031 }
2032 );
2033 }
2034
2035 #[test]
2036 fn given_a_gap_acceptance_when_matched_then_should_cover_only_its_incarnation_and_range() {
2037 let gap = GapAcceptance {
2038 source: SourceSelector::new("s", "t"),
2039 partition_id: 2,
2040 topic_created_at_micros: 7,
2041 from_offset: 10,
2042 to_offset: 20,
2043 };
2044 assert!(gap.covers(7, 10, 20), "a resume at both accepted edges");
2045 assert!(!gap.covers(8, 10, 20), "another incarnation");
2046 assert!(!gap.covers(7, 9, 20), "a cursor before the accepted range");
2047 assert!(
2048 !gap.covers(7, 10, 21),
2049 "a first record past the accepted range"
2050 );
2051 }
2052
2053 #[test]
2054 fn given_an_index_field_name_that_is_not_an_identifier_when_validated_then_should_refuse_it() {
2055 for name in ["user-id", "order.total", "9lives", "caf\u{e9}"] {
2056 let projection = Projection::builder("orders")
2057 .field_at(name, "/value")
2058 .build();
2059 assert!(projection.validate().is_err(), "`{name}` is refused");
2060 }
2061 for name in ["user_id", "_private", "total2"] {
2062 let projection = Projection::builder("orders")
2063 .field_at(name, "/value")
2064 .build();
2065 assert!(projection.validate().is_ok(), "`{name}` is accepted");
2066 }
2067 }
2068
2069 #[test]
2070 fn given_a_retention_that_never_or_always_prunes_when_validated_then_should_refuse_the_binding()
2071 {
2072 for retention in [
2073 RetentionPolicy::Unknown,
2074 RetentionPolicy::TimeToLive { ttl_micros: 0 },
2075 RetentionPolicy::MaxRows { rows: 0 },
2076 ] {
2077 let binding = ProjectionBinding::builder()
2078 .source("s", "t")
2079 .retention(retention)
2080 .build();
2081 assert!(binding.validate().is_err(), "{retention:?} is refused");
2082 }
2083 let bounded = ProjectionBinding::builder()
2084 .source("s", "t")
2085 .retention(RetentionPolicy::MaxRows { rows: 1 })
2086 .build();
2087 assert!(bounded.validate().is_ok());
2088 }
2089
2090 #[test]
2091 fn given_one_unknown_binding_policy_when_validated_then_should_refuse_the_binding() {
2092 let binding = ProjectionBinding {
2093 invalid_records: InvalidRecordPolicy::Unknown,
2094 ..ProjectionBinding::builder().source("s", "t").build()
2095 };
2096 assert_eq!(binding.oversize, OversizePolicy::Block);
2097 assert!(binding.validate().is_err());
2098 }
2099}
2100
2101#[cfg(all(test, feature = "cbor"))]
2102mod wire_tests {
2103 use super::*;
2104 use crate::codes::CONTROL_OP_VERSION;
2105 use crate::content::ContentType;
2106 use crate::framing::{decode_named, encode_named};
2107
2108 #[test]
2109 fn given_an_apply_binding_when_round_tripped_then_should_preserve_index_and_version() {
2110 let binding = ProjectionBinding::builder()
2111 .source("fleet", "readings")
2112 .allow("reading.v1")
2113 .default_projection("reading.v1")
2114 .index("readings_rows")
2115 .build();
2116 let envelope = ControlEnvelope {
2117 v: CONTROL_OP_VERSION,
2118 timestamp_micros: 42,
2119 command: ControlCommand::ApplyBinding(binding),
2120 stream: None,
2121 stamp: None,
2122 };
2123 let bytes = encode_named(&envelope).expect("envelope serializes");
2124 let back: ControlEnvelope = decode_named(&bytes).expect("envelope deserializes");
2125 assert_eq!(back.v, CONTROL_OP_VERSION);
2126 let ControlCommand::ApplyBinding(decoded) = back.command else {
2127 panic!("expected ApplyBinding");
2128 };
2129 assert_eq!(decoded.index, "readings_rows");
2130 assert_eq!(decoded.backend, None);
2131 assert_eq!(decoded.retention, None);
2134 }
2135
2136 #[test]
2137 fn given_a_register_schema_when_round_tripped_then_should_preserve_source() {
2138 let envelope = ControlEnvelope {
2139 v: CONTROL_OP_VERSION,
2140 timestamp_micros: 7,
2141 command: ControlCommand::RegisterSchema(SchemaDef {
2142 id: 11,
2143 source: SchemaSource::Avro {
2144 schema: r#"{"type":"record","name":"Reading","fields":[]}"#.to_owned(),
2145 },
2146 name: None,
2147 version: None,
2148 }),
2149 stream: None,
2150 stamp: None,
2151 };
2152 let bytes = encode_named(&envelope).expect("envelope serializes");
2153 let back: ControlEnvelope = decode_named(&bytes).expect("envelope deserializes");
2154 assert_eq!(back.stream, None);
2155 let ControlCommand::RegisterSchema(decoded) = back.command else {
2156 panic!("expected RegisterSchema");
2157 };
2158 assert_eq!(decoded.id, 11);
2159 assert_eq!(decoded.content_type(), ContentType::Avro);
2160 }
2161
2162 #[test]
2163 fn given_a_stream_scoped_drop_schema_when_round_tripped_then_should_preserve_the_stream() {
2164 let envelope = ControlEnvelope {
2165 v: CONTROL_OP_VERSION,
2166 timestamp_micros: 7,
2167 command: ControlCommand::DropSchema(11),
2168 stream: Some("acme".to_owned()),
2169 stamp: None,
2170 };
2171 let back: ControlEnvelope =
2172 decode_named(&encode_named(&envelope).expect("envelope serializes"))
2173 .expect("envelope deserializes");
2174 assert_eq!(back.stream.as_deref(), Some("acme"));
2175 assert!(matches!(back.command, ControlCommand::DropSchema(11)));
2176 let unscoped = ControlEnvelope {
2177 stream: None,
2178 ..envelope
2179 };
2180 let json = serde_json::to_string(&unscoped).expect("envelope serializes");
2181 assert!(
2182 !json.contains("stream"),
2183 "an unset stream is omitted: {json}"
2184 );
2185 }
2186
2187 #[test]
2188 fn given_a_stamped_envelope_when_round_tripped_then_should_keep_the_identity_and_omit_an_unset_stamp()
2189 {
2190 let envelope = ControlEnvelope {
2191 v: CONTROL_OP_VERSION,
2192 timestamp_micros: 7,
2193 command: ControlCommand::DropSchema(11),
2194 stream: Some("acme".to_owned()),
2195 stamp: Some(ControlStamp {
2196 deployment_id: 42,
2197 operation_id: u128::MAX - 1,
2198 stream_generation: Some(ControlStreamGeneration {
2199 stream_id: 9,
2200 created_at_micros: 1_700_000_000_000_000,
2201 }),
2202 key_id: u64::MAX,
2203 caller: Some(crate::forward::Caller {
2204 user_id: 0,
2205 agent: None,
2206 on_behalf_of: None,
2207 }),
2208 }),
2209 };
2210 let back: ControlEnvelope =
2211 decode_named(&encode_named(&envelope).expect("envelope serializes"))
2212 .expect("envelope deserializes");
2213 assert_eq!(back.stamp, envelope.stamp);
2214 let unstamped = ControlEnvelope {
2215 stamp: None,
2216 ..envelope
2217 };
2218 let json = serde_json::to_string(&unstamped).expect("envelope serializes");
2219 assert!(
2220 !json.contains("\"stamp\""),
2221 "an unset stamp is omitted: {json}"
2222 );
2223 }
2224
2225 #[test]
2226 fn given_a_protobuf_schema_source_when_round_tripped_then_should_preserve_bytes() {
2227 let source = SchemaSource::Protobuf {
2228 descriptor_set: vec![10, 20, 30],
2229 message_type: "fleet.Reading".to_owned(),
2230 };
2231 let bytes = encode_named(&source).expect("serializes");
2232 let back: SchemaSource = decode_named(&bytes).expect("deserializes");
2233 assert_eq!(back, source);
2234 }
2235
2236 #[test]
2237 fn given_retention_set_when_round_tripped_then_should_preserve_policy() {
2238 for policy in [
2239 RetentionPolicy::MirrorLog,
2243 RetentionPolicy::Keep,
2244 RetentionPolicy::KeepUntilSourceDeleted,
2245 RetentionPolicy::TimeToLive {
2246 ttl_micros: 3_600_000_000,
2247 },
2248 RetentionPolicy::MaxRows { rows: 10_000 },
2249 ] {
2250 let binding = ProjectionBinding::builder()
2251 .source("fleet", "telemetry")
2252 .allow("telemetry.v1")
2253 .index("telemetry_rows")
2254 .retention(policy)
2255 .build();
2256 assert_eq!(binding.retention, Some(policy));
2257 let bytes = encode_named(&binding).expect("binding serializes");
2258 let back: ProjectionBinding = decode_named(&bytes).expect("binding deserializes");
2259 assert_eq!(back.retention, Some(policy));
2260 }
2261 }
2262}
2263
2264#[cfg(all(test, feature = "codecs"))]
2265mod schema_tests {
2266 use super::*;
2267 use crate::framing::{decode_named, encode_named};
2268
2269 #[test]
2270 fn given_a_schema_def_with_version_when_round_tripped_then_should_preserve_it() {
2271 let def = SchemaDef {
2272 id: 7,
2273 source: SchemaSource::Avro {
2274 schema: "{}".to_owned(),
2275 },
2276 name: Some("readings".to_owned()),
2277 version: Some(2),
2278 };
2279 let bytes = encode_named(&def).expect("serializes");
2280 let back: SchemaDef = decode_named(&bytes).expect("deserializes");
2281 assert_eq!(back.version, Some(2));
2282 let unversioned = SchemaDef {
2285 name: None,
2286 version: None,
2287 ..def
2288 };
2289 let json = serde_json::to_string(&unversioned).expect("serializes");
2290 assert!(
2291 !json.contains("version"),
2292 "unset version must be omitted: {json}"
2293 );
2294 }
2295}