1use std::fmt::{self, Debug, Formatter};
3use std::sync::Arc;
4
5use serde::{Deserialize, Serialize};
6use uuid::Uuid;
7
8use crate::api::catalog::{
9 AdvanceResourceChange, AssertNamespaceChange, AssertResourceChange, BindNamespaceChange,
10 CatalogError, CommitId, CreateResourceChange, NamespaceChange, OperationId, ProjectOperation,
11 Publication, PublicationConflict, PublishOperationRequest, PublishOperationResponse,
12 PublishRequest, ResourceChange, ResourceKind, RetireResourceChange, UnbindNamespaceChange,
13};
14use crate::api::connections::{
15 Connection as ExternalConnection, CreateConnectionRequest, Credential,
16};
17#[cfg(feature = "tables")]
18use crate::api::enrichments::{
19 CreateEnrichmentRequest, CreateVersionRequest as CreateEnrichmentVersionRequest,
20};
21use crate::catalog_types::{
22 IdempotencyKey, NamespaceParent, NamespaceSegment, ProjectCommitId, ProjectTransactionId,
23 ResourceDescriptor, ResourceId, ResourceServiceId, VersionId,
24};
25use crate::connections::require_connection;
26#[cfg(feature = "tables")]
27use crate::enrichment::{
28 ENRICHMENT_KIND, ENRICHMENT_VALUE_COLUMN, EnrichmentDefinition, enrichment_coverage,
29};
30use crate::namespace::{Namespace, resource_path};
31#[cfg(feature = "tables")]
32use crate::table::CatalogBase;
33#[cfg(feature = "tables")]
34use crate::vector_index::{VECTOR_INDEX_KIND, VECTOR_INDEX_SERVICE};
35use crate::{BranchHead, CallError, Error, Project};
36#[cfg(feature = "tables")]
37use crate::{Expression, NamedExpression, ScanOptions, Table, VectorIndex};
38
39#[derive(Clone)]
40struct ResourceMutation {
41 resource_id: ResourceId,
42 expected: Option<VersionId>,
43 replacement: Option<VersionId>,
44 create: Option<ResourceDescriptor>,
45}
46
47impl ResourceMutation {
48 fn wire(&self) -> ResourceChange {
49 match (&self.create, self.expected, self.replacement) {
50 (Some(descriptor), None, Some(version_id)) => {
51 ResourceChange::Create(CreateResourceChange {
52 descriptor: descriptor.clone(),
53 version_id,
54 })
55 }
56 (None, Some(expected), Some(replacement)) if expected != replacement => {
57 ResourceChange::Advance(AdvanceResourceChange {
58 resource_id: self.resource_id,
59 expected,
60 replacement,
61 })
62 }
63 (None, Some(expected), None) => ResourceChange::Retire(RetireResourceChange {
64 resource_id: self.resource_id,
65 expected,
66 }),
67 (None, expected, replacement) if expected == replacement => {
68 ResourceChange::Assert(AssertResourceChange {
69 resource_id: self.resource_id,
70 expected,
71 })
72 }
73 _ => unreachable!("SDK staging only creates valid resource mutations"),
74 }
75 }
76}
77
78#[derive(Clone, Debug, PartialEq, Eq)]
79pub(crate) struct BindingMutation {
80 pub(crate) parent: NamespaceParent,
81 pub(crate) name: NamespaceSegment,
82 pub(crate) expected: Option<ResourceId>,
83 pub(crate) replacement: Option<ResourceId>,
84}
85
86impl BindingMutation {
87 fn wire(&self) -> NamespaceChange {
88 match (self.expected, self.replacement) {
89 (expected, Some(replacement)) if expected != Some(replacement) => {
90 NamespaceChange::Bind(BindNamespaceChange {
91 parent: self.parent.clone(),
92 name: self.name.clone(),
93 expected,
94 replacement,
95 })
96 }
97 (Some(expected), None) => NamespaceChange::Unbind(UnbindNamespaceChange {
98 parent: self.parent.clone(),
99 name: self.name.clone(),
100 expected,
101 }),
102 (expected, _) => NamespaceChange::Assert(AssertNamespaceChange {
103 parent: self.parent.clone(),
104 name: self.name.clone(),
105 expected,
106 }),
107 }
108 }
109}
110
111pub struct Commit {
117 head: BranchHead,
118 operation_id: Uuid,
119 message: Option<String>,
120 resources: Vec<ResourceMutation>,
121 namespace: Vec<BindingMutation>,
122}
123
124impl Commit {
125 pub(crate) fn new(head: BranchHead) -> Self {
126 Self {
127 head,
128 operation_id: Uuid::now_v7(),
129 message: None,
130 resources: Vec::new(),
131 namespace: Vec::new(),
132 }
133 }
134
135 pub fn head(&self) -> &BranchHead {
137 &self.head
138 }
139
140 pub fn operation_id(&self) -> Uuid {
142 self.operation_id
143 }
144
145 pub fn message(mut self, message: impl Into<String>) -> Self {
147 self.message = Some(message.into());
148 self
149 }
150
151 fn require_unstaged(&self, id: ResourceId) -> Result<(), Error> {
152 if self.resources.iter().any(|change| change.resource_id == id) {
153 return Err(Error::invalid("a resource may change once per commit"));
154 }
155 Ok(())
156 }
157
158 #[cfg(feature = "tables")]
160 fn head_base<'t>(&self, table: &'t Table) -> Result<&'t CatalogBase, Error> {
161 table
162 .base
163 .as_ref()
164 .filter(|base| {
165 Arc::ptr_eq(&table.connection, &self.head.project.client.connection)
166 && base.project_id == self.head.project.id
167 && base.commit_id == self.head.commit_id
168 })
169 .ok_or_else(|| Error::invalid("table was not opened at this commit's branch head"))
170 }
171
172 #[cfg(feature = "tables")]
174 fn unclaimed_root_name(&self, name: &str) -> Result<NamespaceSegment, Error> {
175 let name = NamespaceSegment::new(name).map_err(Error::invalid)?;
176 if self.namespace.iter().any(|staged| {
177 staged.parent == NamespaceParent::project_root()
178 && staged.name == name
179 && staged.replacement.is_some()
180 }) {
181 return Err(Error::invalid(format!("name `{name}` is already staged")));
182 }
183 Ok(name)
184 }
185
186 fn stage_binding(&mut self, change: BindingMutation) -> Result<(), Error> {
189 let staged = self
190 .namespace
191 .iter_mut()
192 .find(|staged| staged.parent == change.parent && staged.name == change.name);
193 match staged {
194 None => self.namespace.push(change),
195 Some(staged) if staged.expected.is_none() && change.replacement.is_none() => {
196 staged.expected = change.expected;
197 }
198 Some(staged) if staged.replacement.is_none() && change.expected.is_none() => {
199 staged.replacement = change.replacement;
200 }
201 Some(staged) => {
202 return Err(Error::invalid(format!(
203 "binding `{}` is already staged",
204 staged.name
205 )));
206 }
207 }
208 Ok(())
209 }
210
211 #[cfg(feature = "tables")]
217 pub fn update_table(&mut self, table: &Table) -> Result<(), Error> {
218 let base = self.head_base(table)?;
219 let resource_id = ResourceId::new(table.identity().resource_id).map_err(Error::invalid)?;
220 self.require_unstaged(resource_id)?;
221 self.resources.push(ResourceMutation {
222 resource_id,
223 expected: Some(VersionId::new(base.version_id).map_err(Error::invalid)?),
224 replacement: Some(VersionId::new(table.identity().version_id).map_err(Error::invalid)?),
225 create: None,
226 });
227 Ok(())
228 }
229
230 #[cfg(feature = "tables")]
236 pub fn create_table(&mut self, name: &str, table: &Table) -> Result<(), Error> {
237 if table.base.is_some() {
238 return Err(Error::invalid(
239 "table is already in the Catalog; rename it to change its name",
240 ));
241 }
242 if !Arc::ptr_eq(&table.connection, &self.head.project.client.connection) {
243 return Err(Error::invalid("table was written through another client"));
244 }
245 let identity = table.identity();
246 self.stage_root_creation(
247 name,
248 identity.resource_id,
249 identity.version_id,
250 "arbor",
251 "table",
252 )
253 }
254
255 #[cfg(feature = "tables")]
258 pub fn create_vector_index(&mut self, name: &str, index: &VectorIndex) -> Result<(), Error> {
259 if !index.is_unbound() {
260 return Err(Error::invalid(
261 "The Vector Index already belongs to a snapshot",
262 ));
263 }
264 if index.project.id() != self.head.project.id()
265 || !Arc::ptr_eq(
266 &index.project.client.connection,
267 &self.head.project.client.connection,
268 )
269 {
270 return Err(Error::invalid(
271 "The Vector Index belongs to another project or client",
272 ));
273 }
274 self.stage_root_creation(
275 name,
276 index.resource_id(),
277 index.version_id(),
278 VECTOR_INDEX_SERVICE,
279 VECTOR_INDEX_KIND,
280 )
281 }
282
283 #[cfg(feature = "tables")]
287 fn stage_root_creation(
288 &mut self,
289 name: &str,
290 resource_id: Uuid,
291 version_id: Uuid,
292 service: &str,
293 kind: &str,
294 ) -> Result<(), Error> {
295 let name = self.unclaimed_root_name(name)?;
296 let resource_id = ResourceId::new(resource_id).map_err(Error::invalid)?;
297 self.require_unstaged(resource_id)?;
298 let creation = ResourceMutation {
299 resource_id,
300 expected: None,
301 replacement: Some(VersionId::new(version_id).map_err(Error::invalid)?),
302 create: Some(ResourceDescriptor {
303 namespace_root: None,
304 namespace_managed: false,
305 resource_id,
306 service: ResourceServiceId::new(service).map_err(Error::invalid)?,
307 kind: ResourceKind(kind.to_owned()),
308 }),
309 };
310 self.stage_binding(BindingMutation {
311 parent: NamespaceParent::project_root(),
312 name,
313 expected: None,
314 replacement: Some(resource_id),
315 })?;
316 self.resources.push(creation);
317 Ok(())
318 }
319
320 #[cfg(feature = "tables")]
328 pub async fn create_enrichment(
329 &mut self,
330 source: &Table,
331 name: &str,
332 expression: Expression,
333 ) -> Result<(), Error> {
334 let base = self.head_base(source)?;
335 let source_id = ResourceId::new(source.identity().resource_id).map_err(Error::invalid)?;
336 let parent = NamespaceParent::resource(source_id);
337 let name = NamespaceSegment::new(name).map_err(Error::invalid)?;
338 if self
339 .namespace
340 .iter()
341 .any(|staged| staged.parent == parent && staged.name == name)
342 {
343 return Err(Error::invalid(format!("name `{name}` is already staged")));
344 }
345 if source
346 .schema()
347 .await?
348 .field_with_name(name.as_str())
349 .is_ok()
350 {
351 return Err(Error::invalid(format!(
352 "`{name}` is a column of the source table"
353 )));
354 }
355 let output = source
356 .scan(ScanOptions {
357 projection: Some(vec![NamedExpression::new(
358 ENRICHMENT_VALUE_COLUMN,
359 expression.clone(),
360 )]),
361 ..ScanOptions::default()
362 })
363 .await?
364 .schema()
365 .field(0)
366 .clone();
367 let definition = EnrichmentDefinition { expression, output }.to_wire()?;
368 let arbor = self.head.project.client.tables();
369 let root = arbor
370 .create_enrichment(&CreateEnrichmentRequest {
371 project_id: self.head.project.id.to_string(),
372 source_table_id: source.identity().resource_id,
373 })
374 .await?
375 .root;
376 let version = arbor
377 .create_enrichment_version(
378 root.enrichment_id,
379 &CreateEnrichmentVersionRequest {
380 base_version_id: None,
381 source_version_id: base.version_id,
382 source_project_commit_id: self.head.commit_id.as_uuid(),
383 definition,
384 coverage: enrichment_coverage(false)?,
385 native_state: None,
386 },
387 )
388 .await?
389 .version;
390 let resource_id = ResourceId::new(root.enrichment_id).map_err(Error::invalid)?;
391 self.stage_binding(BindingMutation {
392 parent,
393 name,
394 expected: None,
395 replacement: Some(resource_id),
396 })?;
397 self.resources.push(ResourceMutation {
398 resource_id,
399 expected: None,
400 replacement: Some(VersionId::new(version.version_id).map_err(Error::invalid)?),
401 create: Some(ResourceDescriptor {
402 namespace_root: None,
403 namespace_managed: false,
404 resource_id,
405 service: ResourceServiceId::new("arbor").map_err(Error::invalid)?,
406 kind: ResourceKind(ENRICHMENT_KIND.into()),
407 }),
408 });
409 Ok(())
410 }
411
412 #[cfg(feature = "tables")]
418 pub fn rename_table(&mut self, table: &Table, name: &str) -> Result<(), Error> {
419 self.head_base(table)?;
420 let name = self.unclaimed_root_name(name)?;
421 let unbinding = table.unbinding()?;
422 if unbinding.parent == NamespaceParent::project_root() && unbinding.name == name {
423 return Err(Error::invalid("table already has this name"));
424 }
425 let binding = BindingMutation {
426 parent: NamespaceParent::project_root(),
427 name,
428 expected: None,
429 replacement: unbinding.expected,
430 };
431 self.stage_binding(unbinding)?;
433 self.stage_binding(binding)
434 }
435
436 #[cfg(feature = "tables")]
442 pub fn drop_table(&mut self, table: &Table) -> Result<(), Error> {
443 let base = self.head_base(table)?;
444 let resource_id = ResourceId::new(table.identity().resource_id).map_err(Error::invalid)?;
445 self.require_unstaged(resource_id)?;
446 let retirement = ResourceMutation {
447 resource_id,
448 expected: Some(VersionId::new(base.version_id).map_err(Error::invalid)?),
449 replacement: None,
450 create: None,
451 };
452 self.stage_binding(table.unbinding()?)?;
453 self.resources.push(retirement);
454 Ok(())
455 }
456
457 pub async fn create_connection(
463 &mut self,
464 path: &str,
465 uri: String,
466 credential: Credential,
467 description: Option<String>,
468 ) -> Result<ExternalConnection, Error> {
469 let path = resource_path(path)?;
470 self.head.require_main_branch().await?;
471 let mut namespace = Namespace::read(&self.head).await?;
472 namespace.apply(
473 self.resources.iter().map(ResourceMutation::wire),
474 self.namespace.iter().map(BindingMutation::wire),
475 )?;
476 if namespace.resolve(&path)?.is_some() {
477 return Err(Error::invalid(
478 "a resource already has this Connection name",
479 ));
480 }
481 let (parent, name) = namespace.parent(&path)?;
482 let created = self
483 .head
484 .project
485 .client
486 .connections()
487 .create_connection(CreateConnectionRequest {
488 project_id: self.head.project.id.as_str().into(),
489 uri,
490 credential,
491 description,
492 })
493 .await?;
494 let id = ResourceId::new(created.connection.connection_id).map_err(Error::invalid)?;
495 let version = VersionId::new(created.version_id).map_err(Error::invalid)?;
496 self.stage_binding(BindingMutation {
497 parent,
498 name,
499 expected: None,
500 replacement: Some(id),
501 })?;
502 self.resources.push(ResourceMutation {
503 resource_id: id,
504 expected: None,
505 replacement: Some(version),
506 create: Some(ResourceDescriptor {
507 resource_id: id,
508 service: ResourceServiceId::new("external-connection").map_err(Error::invalid)?,
509 kind: ResourceKind("connection".into()),
510 namespace_managed: false,
511 namespace_root: None,
512 }),
513 });
514 Ok(created.connection)
515 }
516
517 pub async fn drop_connection(&mut self, path: &str) -> Result<(), Error> {
523 let path = resource_path(path)?;
524 self.head.require_main_branch().await?;
525 let mut namespace = Namespace::read(&self.head).await?;
526 namespace.apply(
527 self.resources.iter().map(ResourceMutation::wire),
528 self.namespace.iter().map(BindingMutation::wire),
529 )?;
530 let binding = namespace
531 .resolve(&path)?
532 .ok_or_else(|| Error::NotFound("Connection".into()))?;
533 require_connection(&binding.resource)?;
534 let id = binding.resource.descriptor.resource_id;
535 self.require_unstaged(id)?;
536 self.stage_binding(BindingMutation {
537 parent: binding.parent,
538 name: binding.name,
539 expected: Some(id),
540 replacement: None,
541 })?;
542 self.resources.push(ResourceMutation {
543 resource_id: id,
544 expected: Some(binding.resource.version_id),
545 replacement: None,
546 create: None,
547 });
548 Ok(())
549 }
550
551 pub fn freeze(self) -> Result<FrozenCommit, Error> {
553 let namespace = self
556 .namespace
557 .iter()
558 .filter(|binding| binding.expected != binding.replacement)
559 .map(BindingMutation::wire)
560 .collect::<Vec<_>>();
561 if self.resources.is_empty() && namespace.is_empty() {
562 return Err(Error::invalid("commit has no staged changes"));
563 }
564 let publication = PublishRequest {
565 branch_id: self.head.branch_id(),
566 base_commit_id: self.head.commit_id,
567 expected_head_commit_id: Some(self.head.commit_id),
568 transaction_id: ProjectTransactionId::new(Uuid::now_v7()).map_err(Error::invalid)?,
569 idempotency_key: IdempotencyKey::new(self.operation_id.to_string())
570 .map_err(Error::invalid)?,
571 message: self.message,
572 resources: self.resources.iter().map(ResourceMutation::wire).collect(),
573 namespace,
574 };
575 publication.validate().map_err(Error::invalid)?;
576 let input = serde_json::to_value(&publication).map_err(Error::invalid)?;
577 Ok(FrozenCommit {
578 head: self.head,
579 request: Arc::new(PublishOperationRequest {
580 operation: ProjectOperation {
581 operation_id: OperationId(self.operation_id),
582 input,
583 result: serde_json::Value::Null,
584 },
585 publication,
586 }),
587 })
588 }
589
590 pub async fn publish(self) -> Result<BranchHead, PublishError> {
594 self.freeze()?.publish().await
595 }
596}
597
598#[derive(Clone)]
601pub struct FrozenCommit {
602 head: BranchHead,
603 request: Arc<PublishOperationRequest>,
604}
605
606impl Debug for FrozenCommit {
607 fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result {
608 formatter
609 .debug_struct("FrozenCommit")
610 .field("operation_id", &self.operation_id())
611 .field("base_commit_id", &self.head.commit_id)
612 .finish_non_exhaustive()
613 }
614}
615
616impl FrozenCommit {
617 pub(crate) fn resume(project: &Project, saved: SavedCommit) -> Result<Self, Error> {
618 let SavedCommit(request) = saved;
619 request.publication.validate().map_err(Error::invalid)?;
620 if request.publication.expected_head_commit_id != Some(request.publication.base_commit_id)
621 || request.operation.input
622 != serde_json::to_value(&request.publication).map_err(Error::invalid)?
623 || request.publication.idempotency_key.as_str()
624 != request.operation.operation_id.to_string()
625 || request.operation.result != serde_json::Value::Null
626 {
627 return Err(Error::invalid(
628 "saved commit does not match its base or operation identity",
629 ));
630 }
631 Ok(Self {
632 head: BranchHead::new(
633 project.clone(),
634 request.publication.base_commit_id,
635 request.publication.branch_id,
636 ),
637 request: Arc::new(request),
638 })
639 }
640
641 pub fn operation_id(&self) -> Uuid {
643 self.request.operation.operation_id.0
644 }
645
646 pub fn head(&self) -> &BranchHead {
648 &self.head
649 }
650
651 pub fn save(&self) -> SavedCommit {
653 SavedCommit(PublishOperationRequest::clone(&self.request))
654 }
655
656 pub async fn publish(&self) -> Result<BranchHead, PublishError> {
660 let project = &self.head.project;
661 let response = project
662 .client
663 .connection
664 .catalog
665 .publish_operation(&project.id, &self.request)
666 .await;
667 match response {
668 Ok(PublishOperationResponse::Published(receipt)) => {
669 Ok(self.published(receipt.publication))
670 }
671 Ok(PublishOperationResponse::Conflict(conflict)) => {
672 Err(PublishError::Conflict(Box::new(conflict)))
673 }
674 Ok(PublishOperationResponse::Unknown { .. }) => {
675 Err(self.unknown("unrecognized publication response"))
676 }
677 Err(Error::Catalog(error)) => Err(match moved_head(&error) {
678 Some(current) => PublishError::BranchMoved { current },
679 None if publication_outcome_uncertain(&error) => self.unknown(error),
680 None => PublishError::Failed(Error::Catalog(error)),
681 }),
682 Err(error) => Err(PublishError::Failed(error)),
683 }
684 }
685
686 pub async fn recover(&self) -> Result<Option<BranchHead>, Error> {
690 let project = &self.head.project;
691 let receipt = project
692 .client
693 .connection
694 .catalog
695 .read_operation(&project.id, OperationId(self.operation_id()))
696 .await?;
697 let Some(receipt) = receipt.receipt else {
698 return Ok(None);
699 };
700 if receipt.operation != self.request.operation {
701 return Err(Error::invalid(
702 "operation receipt does not match this commit",
703 ));
704 }
705 Ok(Some(self.published(receipt.publication)))
706 }
707
708 fn published(&self, publication: Publication) -> BranchHead {
709 BranchHead::new(
710 self.head.project.clone(),
711 publication.commit_id,
712 self.head.branch_id(),
713 )
714 }
715
716 fn unknown(&self, message: impl fmt::Display) -> PublishError {
717 PublishError::Unknown {
718 commit: Box::new(self.clone()),
719 message: message.to_string(),
720 }
721 }
722}
723
724#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
727#[serde(transparent)]
728pub struct SavedCommit(PublishOperationRequest);
729
730#[derive(Debug, thiserror::Error)]
732pub enum PublishError {
733 #[error("the branch moved to {current} before publication")]
736 BranchMoved {
737 current: ProjectCommitId,
739 },
740 #[error("a staged change conflicts with the branch head")]
743 Conflict(Box<PublicationConflict>),
744 #[error("publication outcome unknown for {}: {message}", .commit.operation_id())]
747 Unknown {
748 commit: Box<FrozenCommit>,
749 message: String,
750 },
751 #[error(transparent)]
753 Failed(#[from] Error),
754}
755
756impl From<PublishError> for Error {
757 fn from(error: PublishError) -> Self {
758 match error {
759 PublishError::BranchMoved { current } => Self::BranchMoved { current },
760 PublishError::Conflict(conflict) => Self::Conflict(conflict),
761 PublishError::Unknown { commit, message } => Self::OutcomeUnknown {
762 operation_id: commit.operation_id(),
763 message,
764 },
765 PublishError::Failed(error) => error,
766 }
767 }
768}
769
770fn moved_head(error: &CallError<CatalogError>) -> Option<ProjectCommitId> {
772 let CallError::Service(failure) = error else {
773 return None;
774 };
775 let Some(CatalogError::StaleHead { current, .. }) = failure.declared() else {
776 return None;
777 };
778 current.parse().ok().map(CommitId)
779}
780
781fn publication_outcome_uncertain(error: &CallError<CatalogError>) -> bool {
782 let status = match error {
783 CallError::Service(failure) => Some(failure.status()),
784 CallError::UnknownRemote(failure) => Some(failure.status()),
785 CallError::Transport(failure) => failure.observed_status(),
786 CallError::Protocol(failure) => failure.observed_status(),
787 CallError::Internal { .. } => None,
788 };
789 !status.is_some_and(|status| (400..500).contains(&status))
790}
791
792#[cfg(test)]
793#[cfg(feature = "tables")]
794mod tests {
795 use catalog_api::types::*;
796 use catalog_model::journal_model::ProjectJournalModel;
797 use uuid::Uuid;
798
799 use super::{Commit, publication_outcome_uncertain};
800 use crate::api::catalog::{
801 BranchId as WireBranchId, CatalogError, CommitId as WireCommitId, NamespaceChange,
802 NamespaceParent as WireParent, PublishRequest as WirePublishRequest, ResourceChange,
803 };
804 use crate::scandal::types::TableWriteResult;
805 use crate::table::CatalogBase;
806 use crate::{BranchHead, CallError, Client, Table};
807
808 fn legacy_parent(parent: &WireParent) -> NamespaceParent {
809 match parent {
810 WireParent::ProjectRoot(_) => NamespaceParent::ProjectRoot,
811 WireParent::Resource(parent) => NamespaceParent::Resource {
812 resource_id: ResourceId::new(parent.resource_id.0).unwrap(),
813 },
814 WireParent::Prefix(parent) => NamespaceParent::Prefix {
815 resource_id: parent.resource_id.map(|id| ResourceId::new(id.0).unwrap()),
816 path: NamespacePath::new(parent.path.0.iter().map(|s| segment(&s.0)).collect())
817 .unwrap(),
818 },
819 WireParent::Unknown { .. } => panic!("unexpected namespace parent"),
820 }
821 }
822
823 fn legacy_publication(request: &WirePublishRequest) -> PublishProjectRequest {
824 PublishProjectRequest {
825 branch_id: ProjectBranchId::new(request.branch_id.0).unwrap(),
826 base_commit_id: ProjectCommitId::new(request.base_commit_id.0).unwrap(),
827 expected_head_commit_id: request
828 .expected_head_commit_id
829 .map(|id| ProjectCommitId::new(id.0).unwrap()),
830 transaction_id: ProjectTransactionId::new(request.transaction_id.0).unwrap(),
831 idempotency_key: IdempotencyKey::new(request.idempotency_key.0.clone()).unwrap(),
832 message: request.message.clone(),
833 resources: request
834 .resources
835 .iter()
836 .map(|change| {
837 let (id, expected, replacement, create) = match change {
838 ResourceChange::Create(c) => (
839 c.descriptor.resource_id,
840 None,
841 Some(c.version_id),
842 Some(ResourceDescriptor {
843 resource_id: ResourceId::new(c.descriptor.resource_id.0).unwrap(),
844 service: ResourceServiceId::new(c.descriptor.service.0.clone())
845 .unwrap(),
846 kind: ResourceKindName::new(c.descriptor.kind.0.clone()).unwrap(),
847 namespace_root: c
848 .descriptor
849 .namespace_root
850 .map(|id| ResourceId::new(id.0).unwrap()),
851 namespace_managed: c.descriptor.namespace_managed,
852 }),
853 ),
854 ResourceChange::Advance(c) => {
855 (c.resource_id, Some(c.expected), Some(c.replacement), None)
856 }
857 ResourceChange::Retire(c) => (c.resource_id, Some(c.expected), None, None),
858 ResourceChange::Assert(c) => (c.resource_id, c.expected, c.expected, None),
859 ResourceChange::Unknown { .. } => panic!("unexpected resource change"),
860 };
861 ResourceCas {
862 resource_id: ResourceId::new(id.0).unwrap(),
863 expected: expected.map(|id| VersionId::new(id.0).unwrap()),
864 replacement: replacement.map(|id| VersionId::new(id.0).unwrap()),
865 create,
866 }
867 })
868 .collect(),
869 namespace: request
870 .namespace
871 .iter()
872 .map(|change| {
873 let (parent, name, expected, replacement) = match change {
874 NamespaceChange::Bind(c) => {
875 (&c.parent, &c.name, c.expected, Some(c.replacement))
876 }
877 NamespaceChange::Unbind(c) => (&c.parent, &c.name, Some(c.expected), None),
878 NamespaceChange::Assert(c) => (&c.parent, &c.name, c.expected, c.expected),
879 NamespaceChange::Unknown { .. } => panic!("unexpected namespace change"),
880 };
881 NamespaceCas {
882 parent: legacy_parent(parent),
883 name: segment(&name.0),
884 expected: expected.map(|id| ResourceId::new(id.0).unwrap()),
885 replacement: replacement.map(|id| ResourceId::new(id.0).unwrap()),
886 }
887 })
888 .collect(),
889 }
890 }
891
892 #[test]
893 fn publication_distinguishes_rejections_from_uncertain_outcomes() {
894 for status in [400, 401, 403, 404, 409, 422] {
895 let error = CallError::<CatalogError>::unknown_remote(status, None, vec![], false);
896 assert!(!publication_outcome_uncertain(&error));
897 }
898 for error in [
899 CallError::<CatalogError>::unknown_remote(500, None, vec![], false),
900 CallError::protocol_response(200, "invalid acknowledgement"),
901 CallError::transport("connection closed"),
902 ] {
903 assert!(publication_outcome_uncertain(&error));
904 }
905 }
906
907 fn actor() -> ActorId {
908 ActorId::new("user:test").unwrap()
909 }
910
911 fn segment(value: &str) -> NamespaceSegment {
912 NamespaceSegment::new(value).unwrap()
913 }
914
915 fn prefix(path: &[&str]) -> NamespaceParent {
916 NamespaceParent::Prefix {
917 resource_id: None,
918 path: NamespacePath::new(path.iter().copied().map(segment).collect()).unwrap(),
919 }
920 }
921
922 fn resource() -> ResourceId {
923 ResourceId::new(Uuid::new_v4()).unwrap()
924 }
925
926 fn catalog(bindings: &[(NamespaceParent, &str, ResourceId)]) -> ProjectJournalModel {
929 let mut model = ProjectJournalModel::new(actor(), 0);
930 let main = model.branch(&ProjectBranchName::main()).unwrap().clone();
931 let mut resources = Vec::<ResourceCas>::new();
932 for (_, _, id) in bindings {
933 if resources.iter().all(|created| created.resource_id != *id) {
934 resources.push(ResourceCas {
935 resource_id: *id,
936 expected: None,
937 replacement: Some(VersionId::new(Uuid::now_v7()).unwrap()),
938 create: Some(ResourceDescriptor {
939 namespace_root: None,
940 namespace_managed: false,
941 resource_id: *id,
942 service: ResourceServiceId::new("arbor").unwrap(),
943 kind: ResourceKind::Table.name(),
944 }),
945 });
946 }
947 }
948 let seed = PublishProjectRequest {
949 branch_id: main.branch_id,
950 base_commit_id: main.head_commit_id,
951 expected_head_commit_id: Some(main.head_commit_id),
952 transaction_id: ProjectTransactionId::new(Uuid::now_v7()).unwrap(),
953 idempotency_key: IdempotencyKey::new("seed").unwrap(),
954 message: None,
955 resources,
956 namespace: bindings
957 .iter()
958 .map(|(parent, name, id)| NamespaceCas {
959 parent: parent.clone(),
960 name: segment(name),
961 expected: None,
962 replacement: Some(*id),
963 })
964 .collect(),
965 };
966 assert!(matches!(
967 model.publish(actor(), 0, seed).unwrap(),
968 PublishProjectResponse::Published { .. }
969 ));
970 model
971 }
972
973 fn commit(catalog: &ProjectJournalModel) -> Commit {
974 let client = Client::builder()
975 .api_url("https://api.example.test")
976 .bearer_token("test-token")
977 .build()
978 .unwrap();
979 let main = catalog.branch(&ProjectBranchName::main()).unwrap();
980 BranchHead::new(
981 client.project("project-demo").unwrap(),
982 WireCommitId(main.head_commit_id.as_uuid()),
983 WireBranchId(main.branch_id.as_uuid()),
984 )
985 .commit()
986 }
987
988 fn table_at(
990 commit: &Commit,
991 catalog: &ProjectJournalModel,
992 path: &str,
993 id: ResourceId,
994 ) -> Table {
995 let head = commit.head();
996 let version = catalog
997 .state(ProjectCommitId::new(head.commit_id.as_uuid()).unwrap())
998 .unwrap()
999 .resource(&id)
1000 .unwrap()
1001 .version_id;
1002 let mut table = head
1003 .project
1004 .client
1005 .table_by_id(id.as_uuid(), version.as_uuid());
1006 table.base = Some(CatalogBase {
1007 project_id: head.project.id.clone(),
1008 commit_id: head.commit_id,
1009 branch_id: Some(head.branch_id()),
1010 version_id: version.as_uuid(),
1011 address: Some(path.parse().unwrap()),
1012 });
1013 table.address = Some(path.parse().unwrap());
1014 table
1015 }
1016
1017 fn written(commit: &Commit) -> Table {
1019 commit
1020 .head()
1021 .project
1022 .client
1023 .table_by_id(Uuid::new_v4(), Uuid::now_v7())
1024 }
1025
1026 fn next_version(table: &Table) -> Table {
1028 table
1029 .successor(TableWriteResult {
1030 resource_id: table.identity().resource_id,
1031 version_id: Uuid::now_v7(),
1032 })
1033 .unwrap()
1034 }
1035
1036 fn retirement(table: &Table) -> ResourceCas {
1037 ResourceCas {
1038 resource_id: ResourceId::new(table.identity().resource_id).unwrap(),
1039 expected: Some(VersionId::new(table.identity().version_id).unwrap()),
1040 replacement: None,
1041 create: None,
1042 }
1043 }
1044
1045 fn publication(commit: Commit) -> PublishProjectRequest {
1046 legacy_publication(&commit.freeze().unwrap().request.publication)
1047 }
1048
1049 fn published_head(response: PublishProjectResponse) -> ProjectCommitId {
1050 match response {
1051 PublishProjectResponse::Published { commit_id, .. } => commit_id,
1052 conflict => panic!("unexpected conflict: {conflict:?}"),
1053 }
1054 }
1055
1056 #[test]
1057 fn drops_retire_resources_and_unbind_root_and_prefixed_paths() {
1058 let (flat_id, nested_id, kept_id) = (resource(), resource(), resource());
1059 let mut catalog = catalog(&[
1060 (
1061 NamespaceParent::ProjectRoot,
1062 "datasets.warp_sim_bottles.episodes",
1063 flat_id,
1064 ),
1065 (prefix(&["datasets", "x"]), "episodes", nested_id),
1066 (NamespaceParent::ProjectRoot, "kept", kept_id),
1067 ]);
1068 let mut commit = commit(&catalog);
1069 let flat = table_at(
1070 &commit,
1071 &catalog,
1072 "datasets.warp_sim_bottles.episodes",
1073 flat_id,
1074 );
1075 let nested = table_at(&commit, &catalog, "datasets/x/episodes", nested_id);
1076 commit.drop_table(&flat).unwrap();
1077 commit.drop_table(&nested).unwrap();
1078 let publication = publication(commit);
1079 assert_eq!(
1080 publication.resources,
1081 [retirement(&flat), retirement(&nested)]
1082 );
1083 assert_eq!(
1084 publication.namespace,
1085 [
1086 NamespaceCas {
1087 parent: NamespaceParent::ProjectRoot,
1088 name: segment("datasets.warp_sim_bottles.episodes"),
1089 expected: Some(flat_id),
1090 replacement: None,
1091 },
1092 NamespaceCas {
1093 parent: prefix(&["datasets", "x"]),
1094 name: segment("episodes"),
1095 expected: Some(nested_id),
1096 replacement: None,
1097 },
1098 ]
1099 );
1100 let head = published_head(catalog.publish(actor(), 1, publication).unwrap());
1101 let state = catalog.state(head).unwrap();
1102 assert_eq!(state.namespace().values().collect::<Vec<_>>(), [&kept_id]);
1103 assert_eq!(state.resources().keys().collect::<Vec<_>>(), [&kept_id]);
1104 }
1105
1106 #[test]
1107 fn dropping_and_creating_one_name_publishes_one_rebinding_in_either_order() {
1108 for drop_first in [true, false] {
1109 let old_id = resource();
1110 let mut catalog = catalog(&[(NamespaceParent::ProjectRoot, "episodes", old_id)]);
1111 let mut commit = commit(&catalog);
1112 let old = table_at(&commit, &catalog, "episodes", old_id);
1113 if drop_first {
1114 commit.drop_table(&old).unwrap();
1115 }
1116 let new = written(&commit);
1117 commit.create_table("episodes", &new).unwrap();
1118 if !drop_first {
1119 commit.drop_table(&old).unwrap();
1120 }
1121 assert!(commit.unclaimed_root_name("episodes").is_err());
1122 let new_id = ResourceId::new(new.identity().resource_id).unwrap();
1123 let publication = publication(commit);
1124 assert_eq!(
1125 publication.namespace,
1126 [NamespaceCas {
1127 parent: NamespaceParent::ProjectRoot,
1128 name: segment("episodes"),
1129 expected: Some(old_id),
1130 replacement: Some(new_id),
1131 }]
1132 );
1133 let head = published_head(catalog.publish(actor(), 1, publication).unwrap());
1134 let state = catalog.state(head).unwrap();
1135 assert_eq!(
1136 state.binding(&NamespaceParent::ProjectRoot, &segment("episodes")),
1137 Some(&new_id)
1138 );
1139 assert_eq!(state.resources().keys().collect::<Vec<_>>(), [&new_id]);
1140 }
1141 }
1142
1143 #[test]
1144 fn drops_that_leave_the_table_bound_or_miss_its_binding_conflict() {
1145 let (aliased, anchor, anchored) = (resource(), resource(), resource());
1146 let mut catalog = catalog(&[
1147 (NamespaceParent::ProjectRoot, "episodes", aliased),
1148 (NamespaceParent::ProjectRoot, "episodes_alias", aliased),
1149 (NamespaceParent::ProjectRoot, "datasets", anchor),
1150 (
1151 NamespaceParent::Resource {
1152 resource_id: anchor,
1153 },
1154 "frames",
1155 anchored,
1156 ),
1157 ]);
1158 let mut alias = commit(&catalog);
1159 alias
1160 .drop_table(&table_at(&alias, &catalog, "episodes", aliased))
1161 .unwrap();
1162 match catalog.publish(actor(), 1, publication(alias)).unwrap() {
1163 PublishProjectResponse::Conflict { constraints, .. } => assert!(matches!(
1164 constraints.as_slice(),
1165 [NamespaceConstraintConflict::BindingTargetsRetiredResource { .. }]
1166 )),
1167 published => panic!("an aliased drop must conflict: {published:?}"),
1168 }
1169 let mut beneath = commit(&catalog);
1170 beneath
1171 .drop_table(&table_at(&beneath, &catalog, "datasets/frames", anchored))
1172 .unwrap();
1173 match catalog.publish(actor(), 1, publication(beneath)).unwrap() {
1174 PublishProjectResponse::Conflict { namespace, .. } => assert_eq!(namespace.len(), 1),
1175 published => panic!("a resource-anchored drop must conflict: {published:?}"),
1176 }
1177 }
1178
1179 #[test]
1180 fn a_commit_changes_each_table_once_from_its_head() {
1181 let id = resource();
1182 let catalog = catalog(&[(NamespaceParent::ProjectRoot, "episodes", id)]);
1183 let mut commit = commit(&catalog);
1184 let dropped = table_at(&commit, &catalog, "episodes", id);
1185 commit.drop_table(&dropped).unwrap();
1186 for error in [
1187 commit.drop_table(&dropped).unwrap_err(),
1188 commit.update_table(&next_version(&dropped)).unwrap_err(),
1189 ] {
1190 assert!(error.to_string().contains("once per commit"), "{error}");
1191 }
1192
1193 let created = written(&commit);
1194 commit.create_table("created", &created).unwrap();
1195 let other = self::commit(&catalog);
1196 let foreign = table_at(&other, &catalog, "episodes", id);
1197 for table in [created, foreign] {
1198 let error = commit.drop_table(&table).unwrap_err();
1199 assert!(error.to_string().contains("branch head"), "{error}");
1200 }
1201 }
1202
1203 #[test]
1204 fn updates_swap_the_head_version_for_the_newest_written_version() {
1205 let id = resource();
1206 let mut catalog = catalog(&[(NamespaceParent::ProjectRoot, "episodes", id)]);
1207 let mut commit = commit(&catalog);
1208 let opened = table_at(&commit, &catalog, "episodes", id);
1209 let newest = next_version(&next_version(&opened));
1210 commit.update_table(&newest).unwrap();
1211 let publication = publication(commit);
1212 assert_eq!(
1213 publication.resources,
1214 [ResourceCas {
1215 resource_id: id,
1216 expected: Some(VersionId::new(opened.identity().version_id).unwrap()),
1217 replacement: Some(VersionId::new(newest.identity().version_id).unwrap()),
1218 create: None,
1219 }]
1220 );
1221 let head = published_head(catalog.publish(actor(), 1, publication).unwrap());
1222 assert_eq!(
1223 catalog
1224 .state(head)
1225 .unwrap()
1226 .resource(&id)
1227 .unwrap()
1228 .version_id
1229 .as_uuid(),
1230 newest.identity().version_id
1231 );
1232 }
1233
1234 fn binding(
1235 parent: NamespaceParent,
1236 name: &str,
1237 change: [Option<ResourceId>; 2],
1238 ) -> NamespaceCas {
1239 NamespaceCas {
1240 parent,
1241 name: segment(name),
1242 expected: change[0],
1243 replacement: change[1],
1244 }
1245 }
1246
1247 #[test]
1248 fn renames_move_root_and_prefixed_bindings_and_keep_resources() {
1249 let (root, episodes, stream) = (resource(), resource(), resource());
1250 let mut catalog = catalog(&[
1251 (NamespaceParent::ProjectRoot, "recordings", root),
1252 (prefix(&["datasets", "x"]), "episodes", episodes),
1253 (prefix(&["datasets", "x", "messages"]), "front", stream),
1254 ]);
1255 let mut commit = commit(&catalog);
1256 for (path, id, name) in [
1257 ("recordings", root, "datasets.x.recordings"),
1258 ("datasets/x/episodes", episodes, "datasets.x.episodes"),
1259 (
1260 "datasets/x/messages/front",
1261 stream,
1262 "datasets.x.messages.front",
1263 ),
1264 ] {
1265 let table = table_at(&commit, &catalog, path, id);
1266 commit.rename_table(&table, name).unwrap();
1267 }
1268 let base = commit.head().commit_id;
1269 let publication = publication(commit);
1270 assert!(publication.resources.is_empty());
1271 assert_eq!(
1272 publication.namespace,
1273 [
1274 binding(
1275 NamespaceParent::ProjectRoot,
1276 "recordings",
1277 [Some(root), None]
1278 ),
1279 binding(
1280 NamespaceParent::ProjectRoot,
1281 "datasets.x.recordings",
1282 [None, Some(root)]
1283 ),
1284 binding(
1285 prefix(&["datasets", "x"]),
1286 "episodes",
1287 [Some(episodes), None]
1288 ),
1289 binding(
1290 NamespaceParent::ProjectRoot,
1291 "datasets.x.episodes",
1292 [None, Some(episodes)]
1293 ),
1294 binding(
1295 prefix(&["datasets", "x", "messages"]),
1296 "front",
1297 [Some(stream), None]
1298 ),
1299 binding(
1300 NamespaceParent::ProjectRoot,
1301 "datasets.x.messages.front",
1302 [None, Some(stream)]
1303 ),
1304 ]
1305 );
1306 let head = published_head(catalog.publish(actor(), 1, publication).unwrap());
1307 let state = catalog.state(head).unwrap();
1308 assert_eq!(
1309 state.resources(),
1310 catalog
1311 .state(ProjectCommitId::new(base.as_uuid()).unwrap())
1312 .unwrap()
1313 .resources()
1314 );
1315 assert_eq!(
1316 state
1317 .namespace()
1318 .iter()
1319 .map(|(key, id)| (&key.parent, key.name.as_str(), *id))
1320 .collect::<Vec<_>>(),
1321 [
1322 (
1323 &NamespaceParent::ProjectRoot,
1324 "datasets.x.episodes",
1325 episodes
1326 ),
1327 (
1328 &NamespaceParent::ProjectRoot,
1329 "datasets.x.messages.front",
1330 stream
1331 ),
1332 (&NamespaceParent::ProjectRoot, "datasets.x.recordings", root),
1333 ]
1334 );
1335 }
1336
1337 #[test]
1338 fn renames_compose_with_writes_drops_and_creates_in_one_commit() {
1339 let (a, b, c) = (resource(), resource(), resource());
1340 let mut catalog = catalog(&[
1341 (NamespaceParent::ProjectRoot, "a", a),
1342 (NamespaceParent::ProjectRoot, "b", b),
1343 (NamespaceParent::ProjectRoot, "c", c),
1344 ]);
1345 let mut commit = commit(&catalog);
1346 let table_a = table_at(&commit, &catalog, "a", a);
1347 commit.rename_table(&table_a, "b").unwrap();
1348 let updated = next_version(&table_a);
1350 commit.update_table(&updated).unwrap();
1351 let table_b = table_at(&commit, &catalog, "b", b);
1352 commit.rename_table(&table_b, "c").unwrap();
1353 commit
1354 .drop_table(&table_at(&commit, &catalog, "c", c))
1355 .unwrap();
1356 let created = written(&commit);
1357 commit.create_table("a", &created).unwrap();
1358 let created = ResourceId::new(created.identity().resource_id).unwrap();
1359 let publication = publication(commit);
1360 assert_eq!(
1361 publication.namespace,
1362 [
1363 binding(NamespaceParent::ProjectRoot, "a", [Some(a), Some(created)]),
1364 binding(NamespaceParent::ProjectRoot, "b", [Some(b), Some(a)]),
1365 binding(NamespaceParent::ProjectRoot, "c", [Some(c), Some(b)]),
1366 ]
1367 );
1368 let head = published_head(catalog.publish(actor(), 1, publication).unwrap());
1369 let state = catalog.state(head).unwrap();
1370 for (name, id) in [("a", created), ("b", a), ("c", b)] {
1371 assert_eq!(
1372 state.binding(&NamespaceParent::ProjectRoot, &segment(name)),
1373 Some(&id)
1374 );
1375 }
1376 assert_eq!(
1377 state.resource(&a).unwrap().version_id.as_uuid(),
1378 updated.identity().version_id
1379 );
1380 assert!(state.resource(&c).is_none());
1381 }
1382
1383 #[test]
1384 fn renames_reject_or_conflict_without_staging_partial_changes() {
1385 let (renamed, taken, dropped) = (resource(), resource(), resource());
1386 let mut catalog = catalog(&[
1387 (NamespaceParent::ProjectRoot, "renamed", renamed),
1388 (NamespaceParent::ProjectRoot, "taken", taken),
1389 (NamespaceParent::ProjectRoot, "dropped", dropped),
1390 ]);
1391 let mut commit = commit(&catalog);
1392 let renamed = table_at(&commit, &catalog, "renamed", renamed);
1393 let dropped = table_at(&commit, &catalog, "dropped", dropped);
1394 commit.rename_table(&renamed, "taken").unwrap();
1395 commit.drop_table(&dropped).unwrap();
1396 let staged = commit.namespace.clone();
1397 let created = written(&commit);
1398 commit.create_table("created", &created).unwrap();
1399 for (error, message) in [
1400 (
1401 commit.rename_table(&renamed, "elsewhere"),
1402 "binding `renamed` is already staged",
1403 ),
1404 (
1405 commit.rename_table(&dropped, "elsewhere"),
1406 "binding `dropped` is already staged",
1407 ),
1408 (
1409 commit.drop_table(&renamed),
1410 "binding `renamed` is already staged",
1411 ),
1412 (commit.rename_table(&created, "elsewhere"), "branch head"),
1413 (
1414 commit.rename_table(&dropped, "created"),
1415 "name `created` is already staged",
1416 ),
1417 (
1418 commit.rename_table(&dropped, "dropped"),
1419 "already has this name",
1420 ),
1421 ] {
1422 let error = error.unwrap_err().to_string();
1423 assert!(error.contains(message), "{error}");
1424 }
1425 assert_eq!(&commit.namespace[..staged.len()], staged.as_slice());
1426 assert_eq!(commit.namespace.len(), staged.len() + 1);
1427 let publication = publication(commit);
1428 match catalog.publish(actor(), 1, publication).unwrap() {
1429 PublishProjectResponse::Conflict { namespace, .. } => assert!(matches!(
1430 namespace.as_slice(),
1431 [NamespaceCasConflict {
1432 expected: None,
1433 current: Some(_),
1434 ..
1435 }]
1436 )),
1437 published => panic!("renaming onto a bound name must conflict: {published:?}"),
1438 }
1439 }
1440
1441 #[test]
1442 fn renaming_through_an_alias_publishes_only_changed_bindings() {
1443 let id = resource();
1444 let mut catalog = catalog(&[
1445 (NamespaceParent::ProjectRoot, "a", id),
1446 (NamespaceParent::ProjectRoot, "b", id),
1447 ]);
1448 let mut commit = commit(&catalog);
1449 let through_a = table_at(&commit, &catalog, "a", id);
1450 let through_b = table_at(&commit, &catalog, "b", id);
1451 commit.rename_table(&through_a, "b").unwrap();
1452 commit.rename_table(&through_b, "c").unwrap();
1453 let publication = publication(commit);
1454 assert_eq!(
1455 publication.namespace,
1456 [
1457 binding(NamespaceParent::ProjectRoot, "a", [Some(id), None]),
1458 binding(NamespaceParent::ProjectRoot, "c", [None, Some(id)]),
1459 ]
1460 );
1461 let head = published_head(catalog.publish(actor(), 1, publication).unwrap());
1462 let state = catalog.state(head).unwrap();
1463 for name in ["b", "c"] {
1464 assert_eq!(
1465 state.binding(&NamespaceParent::ProjectRoot, &segment(name)),
1466 Some(&id)
1467 );
1468 }
1469 }
1470}