1use std::collections::HashMap;
77use std::str::FromStr;
78use std::sync::Arc;
79
80use bytes::Bytes;
81use chrono::Utc;
82use conflict_checker::ConflictChecker;
83use delta_kernel::table_properties::TableProperties;
84use futures::future::BoxFuture;
85use object_store::Error as ObjectStoreError;
86use object_store::ObjectStoreExt as _;
87use object_store::path::Path;
88use serde_json::Value;
89use tracing::*;
90use uuid::Uuid;
91
92use delta_kernel::table_features::TableFeature;
93use serde::{Deserialize, Serialize};
94
95use self::conflict_checker::{TransactionInfo, WinningCommitSummary};
96use crate::errors::DeltaTableError;
97use crate::kernel::{
98 Action, CommitInfo, EagerSnapshot, IsolationLevel, Metadata, Protocol, Transaction, Version,
99};
100use crate::logstore::ObjectStoreRef;
101use crate::logstore::{CommitOrBytes, LogStoreRef};
102use crate::operations::CustomExecuteHandler;
103use crate::protocol::{DeltaOperation, operation_parameter_value};
104use crate::protocol::{cleanup_expired_logs_for, create_checkpoint_for};
105use crate::table::config::TablePropertiesExt as _;
106use crate::table::state::DeltaTableState;
107use crate::{DeltaResult, crate_version};
108
109pub use self::conflict_checker::CommitConflictError;
110pub use self::protocol::INSTANCE as PROTOCOL;
111
112#[cfg(test)]
113pub(crate) mod application;
114mod conflict_checker;
115mod protocol;
116#[cfg(feature = "datafusion")]
117mod state;
118
119const DELTA_LOG_FOLDER: &str = "_delta_log";
120pub(crate) const DEFAULT_RETRIES: usize = 15;
121const RESERVED_COMMIT_INFO_KEYS: &[&str] = &[
125 "timestamp",
126 "userId",
127 "userName",
128 "operation",
129 "operationParameters",
130 "readVersion",
131 "isolationLevel",
132 "isBlindAppend",
133 "engineInfo",
134 "userMetadata",
135];
136
137#[derive(Default, Debug, PartialEq, Clone, Serialize, Deserialize)]
138#[serde(rename_all = "camelCase")]
140pub struct CommitMetrics {
141 pub num_retries: u64,
143}
144
145#[derive(Default, Debug, PartialEq, Clone, Serialize, Deserialize)]
146#[serde(rename_all = "camelCase")]
148pub struct PostCommitMetrics {
149 pub new_checkpoint_created: bool,
151
152 pub num_log_files_cleaned_up: u64,
154}
155
156#[derive(Default, Debug, PartialEq, Clone, Serialize, Deserialize)]
157#[serde(rename_all = "camelCase")]
159pub struct Metrics {
160 pub num_retries: u64,
162
163 pub new_checkpoint_created: bool,
165
166 pub num_log_files_cleaned_up: u64,
168}
169
170#[derive(thiserror::Error, Debug)]
172pub enum TransactionError {
173 #[error("Tried committing existing table version: {0}")]
175 VersionAlreadyExists(Version),
176
177 #[error("Error serializing commit log to json: {json_err}")]
179 SerializeLogJson {
180 json_err: serde_json::error::Error,
182 },
183
184 #[error("Log storage error: {}", .source)]
186 ObjectStore {
187 #[from]
189 source: ObjectStoreError,
190 },
191
192 #[error("Failed to commit transaction: {0}")]
194 CommitConflict(#[from] CommitConflictError),
195
196 #[error("Failed to commit transaction: {0}")]
198 MaxCommitAttempts(i32),
199
200 #[error(
202 "The transaction includes Remove action with data change but Delta table is append-only"
203 )]
204 DeltaTableAppendOnly,
205
206 #[error("Unsupported table features required: {0:?}")]
208 UnsupportedTableFeatures(Vec<TableFeature>),
209
210 #[error("Table features must be specified, please specify: {0:?}")]
212 TableFeaturesRequired(TableFeature),
213
214 #[error("Transaction failed: {msg}")]
217 LogStoreError {
218 msg: String,
220 source: Box<dyn std::error::Error + Send + Sync + 'static>,
222 },
223}
224
225impl From<TransactionError> for DeltaTableError {
226 fn from(err: TransactionError) -> Self {
227 match err {
228 TransactionError::VersionAlreadyExists(version) => {
229 DeltaTableError::VersionAlreadyExists(version)
230 }
231 TransactionError::SerializeLogJson { json_err } => {
232 DeltaTableError::SerializeLogJson { json_err }
233 }
234 TransactionError::ObjectStore { source } => DeltaTableError::ObjectStore { source },
235 other => DeltaTableError::Transaction { source: other },
236 }
237 }
238}
239
240#[derive(thiserror::Error, Debug)]
242pub enum CommitBuilderError {}
243
244impl From<CommitBuilderError> for DeltaTableError {
245 fn from(err: CommitBuilderError) -> Self {
246 DeltaTableError::CommitValidation { source: err }
247 }
248}
249
250pub trait TableReference: Send + Sync {
252 fn config(&self) -> &TableProperties;
254
255 fn protocol(&self) -> &Protocol;
257
258 fn metadata(&self) -> &Metadata;
260
261 fn eager_snapshot(&self) -> &EagerSnapshot;
263}
264
265impl TableReference for EagerSnapshot {
266 fn protocol(&self) -> &Protocol {
267 EagerSnapshot::protocol(self)
268 }
269
270 fn metadata(&self) -> &Metadata {
271 EagerSnapshot::metadata(self)
272 }
273
274 fn config(&self) -> &TableProperties {
275 self.table_properties()
276 }
277
278 fn eager_snapshot(&self) -> &EagerSnapshot {
279 self
280 }
281}
282
283impl TableReference for DeltaTableState {
284 fn config(&self) -> &TableProperties {
285 self.snapshot.config()
286 }
287
288 fn protocol(&self) -> &Protocol {
289 self.snapshot.protocol()
290 }
291
292 fn metadata(&self) -> &Metadata {
293 self.snapshot.metadata()
294 }
295
296 fn eager_snapshot(&self) -> &EagerSnapshot {
297 &self.snapshot
298 }
299}
300
301#[derive(Debug)]
303pub struct CommitData {
304 pub actions: Vec<Action>,
306 pub operation: DeltaOperation,
308 pub app_metadata: HashMap<String, Value>,
310 pub app_transactions: Vec<Transaction>,
312}
313
314fn normalize_reserved_commit_metadata(
322 commit_info: &mut CommitInfo,
323 app_metadata: &mut HashMap<String, Value>,
324) {
325 match app_metadata.remove("operationParameters") {
326 Some(Value::Object(operation_parameters)) => {
327 let generated_parameters = commit_info
328 .operation_parameters
329 .get_or_insert_with(HashMap::new);
330 for (key, value) in operation_parameters {
331 generated_parameters
332 .entry(key)
333 .or_insert_with(|| operation_parameter_value(value));
334 }
335 }
336 Some(value) => log_unexpected_reserved_metadata_type(
337 "operationParameters",
338 &value,
339 "object with string-compatible values",
340 ),
341 None => {}
342 }
343
344 if let Some(value) = app_metadata.remove("readVersion") {
345 if let Some(value) = value.as_u64() {
346 if commit_info.read_version.is_none() {
347 commit_info.read_version = Some(value);
348 }
349 } else {
350 log_unexpected_reserved_metadata_type("readVersion", &value, "non-negative integer");
351 }
352 }
353
354 promote_string_reserved_metadata(app_metadata, "userId", &mut commit_info.user_id);
355 promote_string_reserved_metadata(app_metadata, "userName", &mut commit_info.user_name);
356 promote_string_reserved_metadata(app_metadata, "userMetadata", &mut commit_info.user_metadata);
357
358 if let Some(value) = app_metadata.remove("isolationLevel") {
359 if let Some(value) = value
360 .as_str()
361 .and_then(|value| IsolationLevel::from_str(value).ok())
362 {
363 if commit_info.isolation_level.is_none() {
364 commit_info.isolation_level = Some(value);
365 }
366 } else {
367 log_unexpected_reserved_metadata_type(
368 "isolationLevel",
369 &value,
370 "valid IsolationLevel string",
371 );
372 }
373 }
374
375 if let Some(value) = app_metadata.remove("isBlindAppend") {
376 if let Some(value) = value.as_bool() {
377 if commit_info.is_blind_append.is_none() {
378 commit_info.is_blind_append = Some(value);
379 }
380 } else {
381 log_unexpected_reserved_metadata_type("isBlindAppend", &value, "boolean");
382 }
383 }
384
385 for key in RESERVED_COMMIT_INFO_KEYS {
386 app_metadata.remove(*key);
387 }
388}
389
390fn promote_string_reserved_metadata(
391 metadata: &mut HashMap<String, Value>,
392 key: &'static str,
393 target: &mut Option<String>,
394) {
395 let Some(value) = metadata.remove(key) else {
396 return;
397 };
398
399 if let Value::String(value) = value {
400 if target.is_none() {
401 *target = Some(value);
402 }
403 } else {
404 log_unexpected_reserved_metadata_type(key, &value, "string");
405 }
406}
407
408fn log_unexpected_reserved_metadata_type(key: &str, value: &Value, expected: &str) {
409 debug!(
410 key,
411 expected,
412 actual = json_value_kind(value),
413 "Ignoring reserved commit metadata key with unexpected type"
414 );
415}
416
417fn json_value_kind(value: &Value) -> &'static str {
418 match value {
419 Value::Null => "null",
420 Value::Bool(_) => "boolean",
421 Value::Number(_) => "number",
422 Value::String(_) => "string",
423 Value::Array(_) => "array",
424 Value::Object(_) => "object",
425 }
426}
427
428fn assign_commit_info_metadata(
429 commit_info: &mut CommitInfo,
430 app_metadata: &mut HashMap<String, Value>,
431) {
432 normalize_reserved_commit_metadata(commit_info, app_metadata);
433
434 let mut existing_info = std::mem::take(&mut commit_info.info);
435 normalize_reserved_commit_metadata(commit_info, &mut existing_info);
436 for (key, value) in existing_info {
439 app_metadata.entry(key).or_insert(value);
440 }
441 debug_assert!(
442 RESERVED_COMMIT_INFO_KEYS
443 .iter()
444 .all(|key| !app_metadata.contains_key(*key))
445 );
446
447 commit_info.info = app_metadata.clone();
448}
449
450impl CommitData {
451 pub fn new(
453 mut actions: Vec<Action>,
454 operation: DeltaOperation,
455 mut app_metadata: HashMap<String, Value>,
456 app_transactions: Vec<Transaction>,
457 ) -> Self {
458 if let Some(Action::CommitInfo(commit_info)) = actions
459 .iter_mut()
460 .find(|action| matches!(action, Action::CommitInfo(..)))
461 {
462 assign_commit_info_metadata(commit_info, &mut app_metadata);
466 } else {
467 let mut commit_info = operation.get_commit_info();
468 commit_info.timestamp = Some(Utc::now().timestamp_millis());
469 app_metadata
472 .entry("clientVersion".to_string())
473 .or_insert(Value::String(format!("delta-rs.{}", crate_version())));
474 assign_commit_info_metadata(&mut commit_info, &mut app_metadata);
475 actions.insert(0, Action::CommitInfo(commit_info));
477 }
478
479 for txn in &app_transactions {
480 actions.push(Action::Txn(txn.clone()))
481 }
482
483 CommitData {
484 actions,
485 operation,
486 app_metadata,
487 app_transactions,
488 }
489 }
490
491 pub fn get_bytes(&self) -> Result<bytes::Bytes, TransactionError> {
493 let mut jsons = Vec::<String>::new();
494 for action in &self.actions {
495 let json = serde_json::to_string(action)
496 .map_err(|e| TransactionError::SerializeLogJson { json_err: e })?;
497 jsons.push(json);
498 }
499 Ok(bytes::Bytes::from(jsons.join("\n")))
500 }
501
502 pub fn update_retry_count_in_bytes(
505 bytes: &Bytes,
506 num_retries: u64,
507 ) -> Result<Bytes, TransactionError> {
508 let bytes_str = std::str::from_utf8(bytes.as_ref())
509 .expect("Delta log bytes should always be valid UTF-8");
510
511 let lines: Vec<&str> = bytes_str.split('\n').collect();
512 let mut updated_lines = Vec::with_capacity(lines.len());
513
514 for line in lines {
515 if line.contains("\"commitInfo\"") {
516 let mut action: Value = serde_json::from_str(line)
517 .map_err(|e| TransactionError::SerializeLogJson { json_err: e })?;
518
519 if let Some(commit_info) = action.get_mut("commitInfo") {
520 if let Some(Value::Object(metrics)) = commit_info.get_mut("operationMetrics") {
521 metrics
522 .insert("num_retries".to_string(), Value::Number(num_retries.into()));
523 }
524 }
525 let updated_line = serde_json::to_string(&action)
527 .map_err(|e| TransactionError::SerializeLogJson { json_err: e })?;
528 updated_lines.push(updated_line);
529 } else {
530 updated_lines.push(line.to_string());
532 }
533 }
534
535 Ok(Bytes::from(updated_lines.join("\n")))
536 }
537}
538
539#[derive(Clone, Debug, Copy)]
540pub struct PostCommitHookProperties {
542 create_checkpoint: bool,
543 cleanup_expired_logs: Option<bool>,
545}
546
547#[derive(Clone, Debug)]
548pub struct CommitProperties {
551 pub(crate) app_metadata: HashMap<String, Value>,
552 pub(crate) app_transaction: Vec<Transaction>,
553 max_retries: usize,
554 create_checkpoint: bool,
555 cleanup_expired_logs: Option<bool>,
556}
557
558impl Default for CommitProperties {
559 fn default() -> Self {
560 Self {
561 app_metadata: Default::default(),
562 app_transaction: Vec::new(),
563 max_retries: DEFAULT_RETRIES,
564 create_checkpoint: true,
565 cleanup_expired_logs: None,
566 }
567 }
568}
569
570impl CommitProperties {
571 pub fn with_metadata(
573 mut self,
574 metadata: impl IntoIterator<Item = (String, serde_json::Value)>,
575 ) -> Self {
576 self.app_metadata = HashMap::from_iter(metadata);
577 self
578 }
579
580 pub fn with_max_retries(mut self, max_retries: usize) -> Self {
582 self.max_retries = max_retries;
583 self
584 }
585
586 pub fn with_create_checkpoint(mut self, create_checkpoint: bool) -> Self {
588 self.create_checkpoint = create_checkpoint;
589 self
590 }
591
592 pub fn with_application_transaction(mut self, txn: Transaction) -> Self {
594 self.app_transaction.push(txn);
595 self
596 }
597
598 pub fn with_application_transactions(mut self, txn: Vec<Transaction>) -> Self {
600 self.app_transaction = txn;
601 self
602 }
603
604 pub fn with_cleanup_expired_logs(mut self, cleanup_expired_logs: Option<bool>) -> Self {
606 self.cleanup_expired_logs = cleanup_expired_logs;
607 self
608 }
609}
610
611impl From<CommitProperties> for CommitBuilder {
612 fn from(value: CommitProperties) -> Self {
613 CommitBuilder {
614 max_retries: value.max_retries,
615 app_metadata: value.app_metadata,
616 post_commit_hook: Some(PostCommitHookProperties {
617 create_checkpoint: value.create_checkpoint,
618 cleanup_expired_logs: value.cleanup_expired_logs,
619 }),
620 app_transaction: value.app_transaction,
621 ..Default::default()
622 }
623 }
624}
625
626pub struct CommitBuilder {
628 actions: Vec<Action>,
629 app_metadata: HashMap<String, Value>,
630 app_transaction: Vec<Transaction>,
631 max_retries: usize,
632 post_commit_hook: Option<PostCommitHookProperties>,
633 post_commit_hook_handler: Option<Arc<dyn CustomExecuteHandler>>,
634 operation_id: Uuid,
635}
636
637impl Default for CommitBuilder {
638 fn default() -> Self {
639 CommitBuilder {
640 actions: Vec::new(),
641 app_metadata: HashMap::new(),
642 app_transaction: Vec::new(),
643 max_retries: DEFAULT_RETRIES,
644 post_commit_hook: None,
645 post_commit_hook_handler: None,
646 operation_id: Uuid::new_v4(),
647 }
648 }
649}
650
651impl<'a> CommitBuilder {
652 pub fn with_actions(mut self, actions: Vec<Action>) -> Self {
654 self.actions = actions;
655 self
656 }
657
658 pub fn with_app_metadata(mut self, app_metadata: HashMap<String, Value>) -> Self {
660 self.app_metadata = app_metadata;
661 self
662 }
663
664 pub fn with_max_retries(mut self, max_retries: usize) -> Self {
666 self.max_retries = max_retries;
667 self
668 }
669
670 pub fn with_post_commit_hook(mut self, post_commit_hook: PostCommitHookProperties) -> Self {
672 self.post_commit_hook = Some(post_commit_hook);
673 self
674 }
675
676 pub fn with_operation_id(mut self, operation_id: Uuid) -> Self {
678 self.operation_id = operation_id;
679 self
680 }
681
682 pub fn with_post_commit_hook_handler(
684 mut self,
685 handler: Option<Arc<dyn CustomExecuteHandler>>,
686 ) -> Self {
687 self.post_commit_hook_handler = handler;
688 self
689 }
690
691 pub fn build(
693 self,
694 table_data: Option<&'a dyn TableReference>,
695 log_store: LogStoreRef,
696 operation: DeltaOperation,
697 ) -> PreCommit<'a> {
698 let data = CommitData::new(
699 self.actions,
700 operation,
701 self.app_metadata,
702 self.app_transaction,
703 );
704 PreCommit {
705 log_store,
706 table_data,
707 max_retries: self.max_retries,
708 data,
709 post_commit_hook: self.post_commit_hook,
710 post_commit_hook_handler: self.post_commit_hook_handler,
711 operation_id: self.operation_id,
712 }
713 }
714}
715
716pub struct PreCommit<'a> {
718 log_store: LogStoreRef,
719 table_data: Option<&'a dyn TableReference>,
720 data: CommitData,
721 max_retries: usize,
722 post_commit_hook: Option<PostCommitHookProperties>,
723 post_commit_hook_handler: Option<Arc<dyn CustomExecuteHandler>>,
724 operation_id: Uuid,
725}
726
727impl<'a> std::future::IntoFuture for PreCommit<'a> {
728 type Output = DeltaResult<FinalizedCommit>;
729 type IntoFuture = BoxFuture<'a, Self::Output>;
730
731 fn into_future(self) -> Self::IntoFuture {
732 Box::pin(async move { self.into_prepared_commit_future().await?.await?.await })
733 }
734}
735
736impl<'a> PreCommit<'a> {
737 pub fn into_prepared_commit_future(self) -> BoxFuture<'a, DeltaResult<PreparedCommit<'a>>> {
739 let this = self;
740
741 async fn write_tmp_commit(
744 log_entry: Bytes,
745 store: ObjectStoreRef,
746 ) -> DeltaResult<CommitOrBytes> {
747 let token = uuid::Uuid::new_v4().to_string();
748 let path = Path::from_iter([DELTA_LOG_FOLDER, &format!("_commit_{token}.json.tmp")]);
749 store.put(&path, log_entry.into()).await?;
750 Ok(CommitOrBytes::TmpCommit(path))
751 }
752
753 Box::pin(async move {
754 if let Some(table_reference) = this.table_data {
755 PROTOCOL.can_commit(table_reference, &this.data.actions, &this.data.operation)?;
756 }
757 let log_entry = this.data.get_bytes()?;
758
759 let commit_or_bytes = if ["LakeFSLogStore", "DefaultLogStore"]
762 .contains(&this.log_store.name().as_str())
763 {
764 CommitOrBytes::LogBytes(log_entry)
765 } else {
766 write_tmp_commit(
767 log_entry,
768 this.log_store.object_store(Some(this.operation_id)),
769 )
770 .await?
771 };
772
773 Ok(PreparedCommit {
774 commit_or_bytes,
775 log_store: this.log_store,
776 table_data: this.table_data,
777 max_retries: this.max_retries,
778 data: this.data,
779 post_commit: this.post_commit_hook,
780 post_commit_hook_handler: this.post_commit_hook_handler,
781 operation_id: this.operation_id,
782 })
783 })
784 }
785}
786
787pub struct PreparedCommit<'a> {
789 commit_or_bytes: CommitOrBytes,
790 log_store: LogStoreRef,
791 data: CommitData,
792 table_data: Option<&'a dyn TableReference>,
793 max_retries: usize,
794 post_commit: Option<PostCommitHookProperties>,
795 post_commit_hook_handler: Option<Arc<dyn CustomExecuteHandler>>,
796 operation_id: Uuid,
797}
798
799impl PreparedCommit<'_> {
800 pub fn commit_or_bytes(&self) -> &CommitOrBytes {
802 &self.commit_or_bytes
803 }
804}
805
806impl<'a> std::future::IntoFuture for PreparedCommit<'a> {
807 type Output = DeltaResult<PostCommit>;
808 type IntoFuture = BoxFuture<'a, Self::Output>;
809
810 fn into_future(self) -> Self::IntoFuture {
811 let this = self;
812
813 Box::pin(async move {
814 let commit_or_bytes = this.commit_or_bytes;
815
816 let mut attempt_number: usize = 1;
817
818 let read_snapshot: EagerSnapshot = if let Some(table_data) = this.table_data {
820 table_data.eager_snapshot().clone()
821 } else {
822 debug!("committing initial table version 0");
823 match this
824 .log_store
825 .write_commit_entry(0, commit_or_bytes.clone(), this.operation_id)
826 .await
827 {
828 Ok(_) => {
829 return Ok(PostCommit {
830 version: 0,
831 data: this.data,
832 create_checkpoint: false,
833 cleanup_expired_logs: None,
834 log_store: this.log_store,
835 table_data: None,
836 custom_execute_handler: this.post_commit_hook_handler,
837 metrics: CommitMetrics { num_retries: 0 },
838 });
839 }
840 Err(TransactionError::VersionAlreadyExists(0)) => {
841 debug!("version 0 already exists, loading table state for retry");
844 attempt_number = 2;
845 let latest_version: Version = this.log_store.get_latest_version(0).await?;
846 EagerSnapshot::try_new(this.log_store.as_ref(), Some(latest_version))
847 .await?
848 }
849 Err(e) => return Err(e.into()),
850 }
851 };
852
853 let mut read_snapshot = read_snapshot;
854
855 let commit_span = info_span!(
856 "commit_with_retries",
857 base_version = read_snapshot.version(),
858 max_retries = this.max_retries,
859 attempt = field::Empty,
860 target_version = field::Empty,
861 conflicts_checked = 0
862 );
863
864 async move {
865 let total_retries = this.max_retries + 1;
866 while attempt_number <= total_retries {
867 Span::current().record("attempt", attempt_number);
868 let latest_version = this
869 .log_store
870 .get_latest_version(read_snapshot.version())
871 .await?;
872
873 if latest_version > read_snapshot.version() {
874 if this.max_retries == 0 {
877 warn!(
878 base_version = read_snapshot.version(),
879 latest_version = latest_version,
880 "table updated but max_retries is 0, failing immediately"
881 );
882 return Err(TransactionError::MaxCommitAttempts(
883 this.max_retries as i32,
884 )
885 .into());
886 }
887 warn!(
888 base_version = read_snapshot.version(),
889 latest_version = latest_version,
890 versions_behind = latest_version - read_snapshot.version(),
891 "table updated during transaction, checking for conflicts"
892 );
893 let mut steps = latest_version - read_snapshot.version();
894 let mut conflicts_checked = 0;
895
896 while steps != 0 {
899 conflicts_checked += 1;
900 let summary = WinningCommitSummary::try_new(
901 this.log_store.as_ref(),
902 latest_version - steps,
903 (latest_version - steps) + 1,
904 )
905 .await?;
906 let conflict_read_set = read_snapshot
907 .snapshot()
908 .conflict_read_set(this.log_store.as_ref())
909 .await?;
910 let transaction_info = TransactionInfo::try_new(
911 conflict_read_set,
912 this.data.operation.read_predicate(),
913 &this.data.actions,
914 this.data.operation.read_whole_table(),
915 )?;
916 let conflict_checker = ConflictChecker::new(
917 transaction_info,
918 summary,
919 Some(&this.data.operation),
920 );
921
922 match conflict_checker.check_conflicts() {
923 Ok(_) => {}
924 Err(err) => {
925 error!(
926 conflicts_checked = conflicts_checked,
927 error = %err,
928 "conflict detected, aborting transaction"
929 );
930 return Err(TransactionError::CommitConflict(err).into());
931 }
932 }
933 steps -= 1;
934 }
935 Span::current().record("conflicts_checked", conflicts_checked);
936 debug!(
937 conflicts_checked = conflicts_checked,
938 "all conflicts resolved, updating snapshot"
939 );
940 read_snapshot
942 .update(&this.log_store, Some(latest_version))
943 .await?;
944 }
945 let version: Version = latest_version + 1;
946 Span::current().record("target_version", version);
947
948 let current_retries = (attempt_number - 1) as u64;
950
951 let updated_commit_or_bytes = match &commit_or_bytes {
953 CommitOrBytes::LogBytes(bytes) => {
954 let updated_bytes = if current_retries > 0 {
955 CommitData::update_retry_count_in_bytes(bytes, current_retries)?
956 } else {
957 bytes.clone()
958 };
959 CommitOrBytes::LogBytes(updated_bytes)
960 }
961 CommitOrBytes::TmpCommit(_path) => {
962 commit_or_bytes.clone()
965 }
966 };
967
968 match this
969 .log_store
970 .write_commit_entry(version, updated_commit_or_bytes, this.operation_id)
971 .await
972 {
973 Ok(()) => {
974 info!(
975 version = version,
976 num_retries = attempt_number - 1,
977 "transaction committed successfully"
978 );
979 return Ok(PostCommit {
980 version,
981 data: this.data,
982 create_checkpoint: this
983 .post_commit
984 .map(|v| v.create_checkpoint)
985 .unwrap_or_default(),
986 cleanup_expired_logs: this
987 .post_commit
988 .map(|v| v.cleanup_expired_logs)
989 .unwrap_or_default(),
990 log_store: this.log_store,
991 table_data: Some(Box::new(read_snapshot)),
992 custom_execute_handler: this.post_commit_hook_handler,
993 metrics: CommitMetrics {
994 num_retries: (attempt_number - 1) as u64,
995 },
996 });
997 }
998 Err(TransactionError::VersionAlreadyExists(version)) => {
999 warn!(
1000 version = version,
1001 attempt = attempt_number,
1002 "version already exists, will retry"
1003 );
1004 attempt_number += 1;
1007 }
1008 Err(err) => {
1009 error!(
1010 version = version,
1011 error = %err,
1012 "commit failed, aborting"
1013 );
1014 this.log_store
1015 .abort_commit_entry(version, commit_or_bytes, this.operation_id)
1016 .await?;
1017 return Err(err.into());
1018 }
1019 }
1020 }
1021
1022 error!(
1023 max_retries = this.max_retries,
1024 "exceeded maximum commit attempts"
1025 );
1026 Err(TransactionError::MaxCommitAttempts(this.max_retries as i32).into())
1027 }
1028 .instrument(commit_span)
1029 .await
1030 })
1031 }
1032}
1033
1034pub struct PostCommit {
1036 pub version: Version,
1038 pub data: CommitData,
1040 create_checkpoint: bool,
1041 cleanup_expired_logs: Option<bool>,
1042 log_store: LogStoreRef,
1043 table_data: Option<Box<dyn TableReference>>,
1044 custom_execute_handler: Option<Arc<dyn CustomExecuteHandler>>,
1045 metrics: CommitMetrics,
1046}
1047
1048impl PostCommit {
1049 async fn run_post_commit_hook(&self) -> DeltaResult<(DeltaTableState, PostCommitMetrics)> {
1051 if let Some(table) = &self.table_data {
1052 let post_commit_operation_id = Uuid::new_v4();
1053 let mut snapshot = table.eager_snapshot().clone();
1054 if self.version != snapshot.version() {
1055 snapshot.update(&self.log_store, Some(self.version)).await?;
1056 }
1057
1058 let mut state = DeltaTableState { snapshot };
1059
1060 let cleanup_logs = if let Some(cleanup_logs) = self.cleanup_expired_logs {
1061 cleanup_logs
1062 } else {
1063 state.table_config().enable_expired_log_cleanup()
1064 };
1065
1066 if let Some(custom_execute_handler) = &self.custom_execute_handler {
1068 custom_execute_handler
1069 .before_post_commit_hook(
1070 &self.log_store,
1071 cleanup_logs || self.create_checkpoint,
1072 post_commit_operation_id,
1073 )
1074 .await?
1075 }
1076
1077 let mut new_checkpoint_created = false;
1078 if self.create_checkpoint {
1079 new_checkpoint_created = self
1081 .create_checkpoint(
1082 &state,
1083 &self.log_store,
1084 self.version,
1085 post_commit_operation_id,
1086 )
1087 .await?;
1088 }
1089
1090 let mut num_log_files_cleaned_up: u64 = 0;
1091 if cleanup_logs {
1092 num_log_files_cleaned_up = cleanup_expired_logs_for(
1094 self.version,
1095 self.log_store.as_ref(),
1096 Utc::now().timestamp_millis()
1097 - state.table_config().log_retention_duration().as_millis() as i64,
1098 Some(post_commit_operation_id),
1099 )
1100 .await? as u64;
1101 if num_log_files_cleaned_up > 0 {
1102 state = DeltaTableState::try_new(&self.log_store, Some(self.version)).await?;
1103 }
1104 }
1105
1106 if let Some(custom_execute_handler) = &self.custom_execute_handler {
1108 custom_execute_handler
1109 .after_post_commit_hook(
1110 &self.log_store,
1111 cleanup_logs || self.create_checkpoint,
1112 post_commit_operation_id,
1113 )
1114 .await?
1115 }
1116 Ok((
1117 state,
1118 PostCommitMetrics {
1119 new_checkpoint_created,
1120 num_log_files_cleaned_up,
1121 },
1122 ))
1123 } else {
1124 let state = DeltaTableState::try_new(&self.log_store, Some(self.version)).await?;
1125 Ok((
1126 state,
1127 PostCommitMetrics {
1128 new_checkpoint_created: false,
1129 num_log_files_cleaned_up: 0,
1130 },
1131 ))
1132 }
1133 }
1134 async fn create_checkpoint(
1135 &self,
1136 table_state: &DeltaTableState,
1137 log_store: &LogStoreRef,
1138 version: Version,
1139 operation_id: Uuid,
1140 ) -> DeltaResult<bool> {
1141 let checkpoint_interval = table_state.config().checkpoint_interval().get();
1142 if (version + 1).is_multiple_of(checkpoint_interval) {
1143 create_checkpoint_for(version, log_store.as_ref(), Some(operation_id)).await?;
1144 Ok(true)
1145 } else {
1146 Ok(false)
1147 }
1148 }
1149}
1150
1151pub struct FinalizedCommit {
1153 pub snapshot: DeltaTableState,
1155
1156 pub version: Version,
1158
1159 pub metrics: Metrics,
1161}
1162
1163impl std::fmt::Debug for FinalizedCommit {
1164 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1165 f.debug_struct("FinalizedCommit")
1166 .field("version", &self.version)
1167 .field("metrics", &self.metrics)
1168 .finish()
1169 }
1170}
1171
1172impl FinalizedCommit {
1173 pub fn snapshot(&self) -> DeltaTableState {
1175 self.snapshot.clone()
1176 }
1177 pub fn version(&self) -> Version {
1179 self.version
1180 }
1181}
1182
1183impl std::future::IntoFuture for PostCommit {
1184 type Output = DeltaResult<FinalizedCommit>;
1185 type IntoFuture = BoxFuture<'static, Self::Output>;
1186
1187 fn into_future(self) -> Self::IntoFuture {
1188 let this = self;
1189
1190 Box::pin(async move {
1191 match this.run_post_commit_hook().await {
1192 Ok((snapshot, post_commit_metrics)) => Ok(FinalizedCommit {
1193 snapshot,
1194 version: this.version,
1195 metrics: Metrics {
1196 num_retries: this.metrics.num_retries,
1197 new_checkpoint_created: post_commit_metrics.new_checkpoint_created,
1198 num_log_files_cleaned_up: post_commit_metrics.num_log_files_cleaned_up,
1199 },
1200 }),
1201 Err(err) => Err(err),
1202 }
1203 })
1204 }
1205}
1206
1207#[cfg(test)]
1208mod tests {
1209 use std::sync::Arc;
1210
1211 use super::*;
1212 use crate::kernel::IsolationLevel;
1213 use crate::logstore::{LogStore, StorageConfig, default_logstore::DefaultLogStore};
1214 use crate::protocol::SaveMode;
1215 use object_store::{PutPayload, memory::InMemory};
1216 use serde_json::json;
1217 use url::Url;
1218
1219 #[tokio::test]
1220 async fn test_try_commit_transaction() {
1221 let store = Arc::new(InMemory::new());
1222 let url = Url::parse("mem://what/is/this").unwrap();
1223 let log_store = DefaultLogStore::new(
1224 store.clone(),
1225 store.clone(),
1226 crate::logstore::LogStoreConfig::new(&url, StorageConfig::default()),
1227 );
1228 let version_path = Path::from("_delta_log/00000000000000000000.json");
1229 store.put(&version_path, PutPayload::new()).await.unwrap();
1230
1231 let res = log_store
1232 .write_commit_entry(
1233 0,
1234 CommitOrBytes::LogBytes(PutPayload::new().into()),
1235 Uuid::new_v4(),
1236 )
1237 .await;
1238 assert!(res.is_err());
1240
1241 log_store
1243 .write_commit_entry(
1244 1,
1245 CommitOrBytes::LogBytes(PutPayload::new().into()),
1246 Uuid::new_v4(),
1247 )
1248 .await
1249 .unwrap();
1250 }
1251
1252 #[test]
1253 fn test_commit_with_retries_tracing_span() {
1254 let span = info_span!(
1255 "commit_with_retries",
1256 base_version = 5,
1257 max_retries = 10,
1258 attempt = field::Empty,
1259 target_version = field::Empty,
1260 conflicts_checked = 0
1261 );
1262
1263 let metadata = span.metadata().expect("span should have metadata");
1264 assert_eq!(metadata.name(), "commit_with_retries");
1265 assert_eq!(metadata.level(), &Level::INFO);
1266 assert!(metadata.is_span());
1267
1268 span.record("attempt", 1);
1269 span.record("target_version", 6);
1270 span.record("conflicts_checked", 2);
1271 }
1272
1273 #[test]
1274 fn test_commit_properties_with_retries() {
1275 let props = CommitProperties::default()
1276 .with_max_retries(5)
1277 .with_create_checkpoint(false);
1278
1279 assert_eq!(props.max_retries, 5);
1280 assert!(!props.create_checkpoint);
1281 }
1282
1283 #[test]
1284 fn test_commit_metrics() {
1285 let metrics = CommitMetrics { num_retries: 3 };
1286 assert_eq!(metrics.num_retries, 3);
1287 }
1288
1289 #[test]
1290 fn test_commit_data_client_version() {
1291 let no_metadata = CommitData::new(
1292 vec![],
1293 DeltaOperation::FileSystemCheck {},
1294 HashMap::new(),
1295 vec![],
1296 );
1297 assert_eq!(
1298 *no_metadata.app_metadata.get("clientVersion").unwrap(),
1299 json!(format!("delta-rs.{}", crate_version()))
1300 );
1301
1302 let with_metadata = CommitData::new(
1303 vec![],
1304 DeltaOperation::FileSystemCheck {},
1305 HashMap::from([("clientVersion".to_owned(), json!("test-client.0.0.1"))]),
1306 vec![],
1307 );
1308 assert_eq!(
1309 *with_metadata.app_metadata.get("clientVersion").unwrap(),
1310 json!("test-client.0.0.1")
1311 );
1312 }
1313
1314 fn commit_info(data: &CommitData) -> &CommitInfo {
1315 match &data.actions[0] {
1316 Action::CommitInfo(info) => info,
1317 action => panic!("expected first action to be commitInfo, got {action:?}"),
1318 }
1319 }
1320
1321 #[test]
1322 fn test_commit_data_strips_and_promotes_reserved_metadata() {
1323 let data = CommitData::new(
1324 vec![],
1325 DeltaOperation::FileSystemCheck {},
1326 HashMap::from([
1327 ("readVersion".to_owned(), json!(7)),
1328 ("userId".to_owned(), json!("user-1")),
1329 ("userName".to_owned(), json!("Jane Doe")),
1330 ("userMetadata".to_owned(), json!("metadata")),
1331 ("isolationLevel".to_owned(), json!("SnapshotIsolation")),
1332 ("isBlindAppend".to_owned(), json!(true)),
1333 ("timestamp".to_owned(), json!(1)),
1334 ("operation".to_owned(), json!("CUSTOM OPERATION")),
1335 ("engineInfo".to_owned(), json!("custom-engine")),
1336 ("custom".to_owned(), json!({"kept": true})),
1337 ]),
1338 vec![],
1339 );
1340
1341 let info = commit_info(&data);
1342 assert_eq!(info.read_version, Some(7));
1343 assert_eq!(info.user_id.as_deref(), Some("user-1"));
1344 assert_eq!(info.user_name.as_deref(), Some("Jane Doe"));
1345 assert_eq!(info.user_metadata.as_deref(), Some("metadata"));
1346 assert_eq!(
1347 info.isolation_level,
1348 Some(IsolationLevel::SnapshotIsolation)
1349 );
1350 assert_eq!(info.is_blind_append, Some(true));
1351 assert_eq!(info.operation.as_deref(), Some("FSCK"));
1352 assert_ne!(info.engine_info.as_deref(), Some("custom-engine"));
1353 assert_eq!(info.info.get("custom"), Some(&json!({"kept": true})));
1354
1355 for key in [
1356 "timestamp",
1357 "userId",
1358 "userName",
1359 "operation",
1360 "operationParameters",
1361 "readVersion",
1362 "isolationLevel",
1363 "isBlindAppend",
1364 "engineInfo",
1365 "userMetadata",
1366 ] {
1367 assert!(
1368 !info.info.contains_key(key),
1369 "reserved key {key} must not remain in flattened commit info"
1370 );
1371 }
1372 }
1373
1374 #[test]
1375 fn test_commit_data_merges_operation_parameters_generated_keys_win() {
1376 let data = CommitData::new(
1377 vec![],
1378 DeltaOperation::Write {
1379 mode: SaveMode::Overwrite,
1380 partition_by: Some(vec!["id".to_owned()]),
1381 predicate: None,
1382 },
1383 HashMap::from([(
1384 "operationParameters".to_owned(),
1385 json!({
1386 "mode": "custom-mode",
1387 "partitionBy": "custom-partitioning",
1388 "customParameter": {"kept": true},
1389 "customBoolean": true,
1390 "customNumber": 7,
1391 }),
1392 )]),
1393 vec![],
1394 );
1395
1396 let info = commit_info(&data);
1397 let operation_parameters = info
1398 .operation_parameters
1399 .as_ref()
1400 .expect("operation parameters should be present");
1401 assert_eq!(operation_parameters.get("mode"), Some(&json!("Overwrite")));
1402 assert_eq!(
1403 operation_parameters.get("partitionBy"),
1404 Some(&json!("[\"id\"]"))
1405 );
1406 assert_eq!(
1407 operation_parameters.get("customParameter"),
1408 Some(&json!("{\"kept\":true}"))
1409 );
1410 assert_eq!(
1411 operation_parameters.get("customBoolean"),
1412 Some(&json!("true"))
1413 );
1414 assert_eq!(operation_parameters.get("customNumber"), Some(&json!("7")));
1415 assert!(!info.info.contains_key("operationParameters"));
1416 }
1417
1418 #[test]
1419 fn test_commit_data_normalizes_reserved_keys_from_existing_commit_info_info() {
1420 let data = CommitData::new(
1421 vec![Action::CommitInfo(CommitInfo {
1422 info: HashMap::from([
1423 ("userName".to_owned(), json!("shadow-user")),
1424 (
1425 "operationParameters".to_owned(),
1426 json!({"custom": {"kept": true}}),
1427 ),
1428 ("custom".to_owned(), json!("kept")),
1429 ]),
1430 ..Default::default()
1431 })],
1432 DeltaOperation::FileSystemCheck {},
1433 HashMap::from([("custom".to_owned(), json!("metadata-wins"))]),
1434 vec![],
1435 );
1436
1437 let info = commit_info(&data);
1438 assert_eq!(info.user_name.as_deref(), Some("shadow-user"));
1439 let operation_parameters = info
1440 .operation_parameters
1441 .as_ref()
1442 .expect("operation parameters should be promoted");
1443 assert_eq!(
1444 operation_parameters.get("custom"),
1445 Some(&json!("{\"kept\":true}"))
1446 );
1447 assert_eq!(info.info.get("custom"), Some(&json!("metadata-wins")));
1448 assert!(!info.info.contains_key("userName"));
1449 assert!(!info.info.contains_key("operationParameters"));
1450 }
1451
1452 #[test]
1453 fn test_commit_data_does_not_promote_reserved_metadata_over_typed_fields() {
1454 let data = CommitData::new(
1455 vec![Action::CommitInfo(CommitInfo {
1456 user_name: Some("typed-user".to_owned()),
1457 read_version: Some(10),
1458 ..Default::default()
1459 })],
1460 DeltaOperation::FileSystemCheck {},
1461 HashMap::from([
1462 ("userName".to_owned(), json!("metadata-user")),
1463 ("readVersion".to_owned(), json!(11)),
1464 ("custom".to_owned(), json!("kept")),
1465 ]),
1466 vec![],
1467 );
1468
1469 let info = commit_info(&data);
1470 assert_eq!(info.user_name.as_deref(), Some("typed-user"));
1471 assert_eq!(info.read_version, Some(10));
1472 assert!(!info.info.contains_key("userName"));
1473 assert!(!info.info.contains_key("readVersion"));
1474 assert_eq!(info.info.get("custom"), Some(&json!("kept")));
1475 }
1476
1477 #[test]
1478 fn test_commit_data_strips_wrong_typed_reserved_metadata_without_clobbering_typed_fields() {
1479 let data = CommitData::new(
1480 vec![Action::CommitInfo(CommitInfo {
1481 user_name: Some("typed-user".to_owned()),
1482 read_version: Some(10),
1483 ..Default::default()
1484 })],
1485 DeltaOperation::FileSystemCheck {},
1486 HashMap::from([
1487 ("userName".to_owned(), json!(123)),
1488 ("readVersion".to_owned(), json!("abc")),
1489 ("isBlindAppend".to_owned(), json!("not-a-bool")),
1490 ("custom".to_owned(), json!("kept")),
1491 ]),
1492 vec![],
1493 );
1494
1495 let info = commit_info(&data);
1496 assert_eq!(info.user_name.as_deref(), Some("typed-user"));
1497 assert_eq!(info.read_version, Some(10));
1498 assert_eq!(info.info.get("custom"), Some(&json!("kept")));
1499 assert!(!info.info.contains_key("userName"));
1500 assert!(!info.info.contains_key("readVersion"));
1501 assert!(!info.info.contains_key("isBlindAppend"));
1502 }
1503}