Skip to main content

deltalake_core/kernel/transaction/
mod.rs

1//! Add a commit entry to the Delta Table.
2//! This module provides a unified interface for modifying commit behavior and attributes
3//!
4//! [`CommitProperties`] provides an unified client interface for all Delta operations.
5//! Internally this is used to initialize a [`CommitBuilder`].
6//!
7//! For advanced use cases [`CommitBuilder`] can be used which allows
8//! finer control over the commit process. The builder can be converted
9//! into a future the yield either a [`PreparedCommit`] or a [`FinalizedCommit`].
10//!
11//! A [`PreparedCommit`] represents a temporary commit marker written to storage.
12//! To convert to a [`FinalizedCommit`] an atomic rename is attempted. If the rename fails
13//! then conflict resolution is performed and the atomic rename is tried for the latest version.
14//!
15//!<pre>
16//!                                          Client Interface
17//!        ┌─────────────────────────────┐
18//!        │      Commit Properties      │
19//!        │                             │
20//!        │ Public commit interface for │
21//!        │     all Delta Operations    │
22//!        │                             │
23//!        └─────────────┬───────────────┘
24//!                      │
25//! ─────────────────────┼────────────────────────────────────
26//!                      │
27//!                      ▼                  Advanced Interface
28//!        ┌─────────────────────────────┐
29//!        │       Commit Builder        │
30//!        │                             │
31//!        │   Advanced entry point for  │
32//!        │     creating a commit       │
33//!        └─────────────┬───────────────┘
34//!                      │
35//!                      ▼
36//!     ┌───────────────────────────────────┐
37//!     │                                   │
38//!     │ ┌───────────────────────────────┐ │
39//!     │ │        Prepared Commit        │ │
40//!     │ │                               │ │
41//!     │ │     Represents a temporary    │ │
42//!     │ │   commit marker written to    │ │
43//!     │ │           storage             │ │
44//!     │ └──────────────┬────────────────┘ │
45//!     │                │                  │
46//!     │                ▼                  │
47//!     │ ┌───────────────────────────────┐ │
48//!     │ │       Finalize Commit         │ │
49//!     │ │                               │ │
50//!     │ │   Convert the commit marker   │ │
51//!     │ │   to a commit using atomic    │ │
52//!     │ │         operations            │ │
53//!     │ │                               │ │
54//!     │ └───────────────────────────────┘ │
55//!     │                                   │
56//!     └────────────────┬──────────────────┘
57//!                      │
58//!                      ▼
59//!       ┌───────────────────────────────┐
60//!       │          Post Commit          │
61//!       │                               │
62//!       │ Commit that was materialized  │
63//!       │ to storage with post commit   │
64//!       │      hooks to be executed     │
65//!       └──────────────┬────────────────┘
66//!                      │
67//!                      ▼
68//!       ┌───────────────────────────────┐
69//!       │        Finalized Commit       │
70//!       │                               │
71//!       │ Commit that was materialized  │
72//!       │         to storage            │
73//!       │                               │
74//!       └───────────────────────────────┘
75//!</pre>
76use std::collections::HashMap;
77use std::str::FromStr;
78use std::sync::Arc;
79
80use bytes::Bytes;
81use chrono::Utc;
82use conflict_checker::ConflictChecker;
83use delta_kernel::table_properties::TableProperties;
84use futures::future::BoxFuture;
85use object_store::Error as ObjectStoreError;
86use object_store::ObjectStoreExt as _;
87use object_store::path::Path;
88use serde_json::Value;
89use tracing::*;
90use uuid::Uuid;
91
92use delta_kernel::table_features::TableFeature;
93use serde::{Deserialize, Serialize};
94
95use self::conflict_checker::{TransactionInfo, WinningCommitSummary};
96use crate::errors::DeltaTableError;
97use crate::kernel::{
98    Action, CommitInfo, EagerSnapshot, IsolationLevel, Metadata, Protocol, Transaction, Version,
99};
100use crate::logstore::ObjectStoreRef;
101use crate::logstore::{CommitOrBytes, LogStoreRef};
102use crate::operations::CustomExecuteHandler;
103use crate::protocol::{DeltaOperation, operation_parameter_value};
104use crate::protocol::{cleanup_expired_logs_for, create_checkpoint_for};
105use crate::table::config::TablePropertiesExt as _;
106use crate::table::state::DeltaTableState;
107use crate::{DeltaResult, crate_version};
108
109pub use self::conflict_checker::CommitConflictError;
110pub use self::protocol::INSTANCE as PROTOCOL;
111
112#[cfg(test)]
113pub(crate) mod application;
114mod conflict_checker;
115mod protocol;
116#[cfg(feature = "datafusion")]
117mod state;
118
119const DELTA_LOG_FOLDER: &str = "_delta_log";
120pub(crate) const DEFAULT_RETRIES: usize = 15;
121// These keys map to typed CommitInfo fields used by common Spark compatible writers. They are not
122// required by the protocol, but leaving them in flattened metadata can produce duplicate JSON keys.
123// Keep this list aligned with validate_reserved_commit_metadata in python/src/lib.rs.
124const RESERVED_COMMIT_INFO_KEYS: &[&str] = &[
125    "timestamp",
126    "userId",
127    "userName",
128    "operation",
129    "operationParameters",
130    "readVersion",
131    "isolationLevel",
132    "isBlindAppend",
133    "engineInfo",
134    "userMetadata",
135];
136
137#[derive(Default, Debug, PartialEq, Clone, Serialize, Deserialize)]
138/// Metrics describing the work performed to land a single commit.
139#[serde(rename_all = "camelCase")]
140pub struct CommitMetrics {
141    /// Number of retries before a successful commit
142    pub num_retries: u64,
143}
144
145#[derive(Default, Debug, PartialEq, Clone, Serialize, Deserialize)]
146/// Metrics describing work performed by post-commit hooks (checkpointing, log cleanup).
147#[serde(rename_all = "camelCase")]
148pub struct PostCommitMetrics {
149    /// Whether a new checkpoint was created as part of this commit
150    pub new_checkpoint_created: bool,
151
152    /// Number of log files cleaned up
153    pub num_log_files_cleaned_up: u64,
154}
155
156#[derive(Default, Debug, PartialEq, Clone, Serialize, Deserialize)]
157/// Aggregate metrics for a commit, combining commit-time and post-commit measurements.
158#[serde(rename_all = "camelCase")]
159pub struct Metrics {
160    /// Number of retries before a successful commit
161    pub num_retries: u64,
162
163    /// Whether a new checkpoint was created as part of this commit
164    pub new_checkpoint_created: bool,
165
166    /// Number of log files cleaned up
167    pub num_log_files_cleaned_up: u64,
168}
169
170/// Error raised while commititng transaction
171#[derive(thiserror::Error, Debug)]
172pub enum TransactionError {
173    /// Version already exists
174    #[error("Tried committing existing table version: {0}")]
175    VersionAlreadyExists(Version),
176
177    /// Error returned when reading the delta log object failed.
178    #[error("Error serializing commit log to json: {json_err}")]
179    SerializeLogJson {
180        /// Commit log record JSON serialization error.
181        json_err: serde_json::error::Error,
182    },
183
184    /// Error returned when reading the delta log object failed.
185    #[error("Log storage error: {}", .source)]
186    ObjectStore {
187        /// Storage error details when reading the delta log object failed.
188        #[from]
189        source: ObjectStoreError,
190    },
191
192    /// Error returned when a commit conflict occurred
193    #[error("Failed to commit transaction: {0}")]
194    CommitConflict(#[from] CommitConflictError),
195
196    /// Error returned when maximum number of commit trioals is exceeded
197    #[error("Failed to commit transaction: {0}")]
198    MaxCommitAttempts(i32),
199
200    /// The transaction includes Remove action with data change but Delta table is append-only
201    #[error(
202        "The transaction includes Remove action with data change but Delta table is append-only"
203    )]
204    DeltaTableAppendOnly,
205
206    /// Error returned when unsupported table features are required
207    #[error("Unsupported table features required: {0:?}")]
208    UnsupportedTableFeatures(Vec<TableFeature>),
209
210    /// Error returned when table features are required but not specified
211    #[error("Table features must be specified, please specify: {0:?}")]
212    TableFeaturesRequired(TableFeature),
213
214    /// The transaction failed to commit due to an error in an implementation-specific layer.
215    /// Currently used by DynamoDb-backed S3 log store when database operations fail.
216    #[error("Transaction failed: {msg}")]
217    LogStoreError {
218        /// Detailed message for the commit failure.
219        msg: String,
220        /// underlying error in the log store transactional layer.
221        source: Box<dyn std::error::Error + Send + Sync + 'static>,
222    },
223}
224
225impl From<TransactionError> for DeltaTableError {
226    fn from(err: TransactionError) -> Self {
227        match err {
228            TransactionError::VersionAlreadyExists(version) => {
229                DeltaTableError::VersionAlreadyExists(version)
230            }
231            TransactionError::SerializeLogJson { json_err } => {
232                DeltaTableError::SerializeLogJson { json_err }
233            }
234            TransactionError::ObjectStore { source } => DeltaTableError::ObjectStore { source },
235            other => DeltaTableError::Transaction { source: other },
236        }
237    }
238}
239
240/// Error raised while commititng transaction
241#[derive(thiserror::Error, Debug)]
242pub enum CommitBuilderError {}
243
244impl From<CommitBuilderError> for DeltaTableError {
245    fn from(err: CommitBuilderError) -> Self {
246        DeltaTableError::CommitValidation { source: err }
247    }
248}
249
250/// Reference to some structure that contains mandatory attributes for performing a commit.
251pub trait TableReference: Send + Sync {
252    /// Well known table configuration
253    fn config(&self) -> &TableProperties;
254
255    /// Get the table protocol of the snapshot
256    fn protocol(&self) -> &Protocol;
257
258    /// Get the table metadata of the snapshot
259    fn metadata(&self) -> &Metadata;
260
261    /// Try to cast this table reference to a `EagerSnapshot`
262    fn eager_snapshot(&self) -> &EagerSnapshot;
263}
264
265impl TableReference for EagerSnapshot {
266    fn protocol(&self) -> &Protocol {
267        EagerSnapshot::protocol(self)
268    }
269
270    fn metadata(&self) -> &Metadata {
271        EagerSnapshot::metadata(self)
272    }
273
274    fn config(&self) -> &TableProperties {
275        self.table_properties()
276    }
277
278    fn eager_snapshot(&self) -> &EagerSnapshot {
279        self
280    }
281}
282
283impl TableReference for DeltaTableState {
284    fn config(&self) -> &TableProperties {
285        self.snapshot.config()
286    }
287
288    fn protocol(&self) -> &Protocol {
289        self.snapshot.protocol()
290    }
291
292    fn metadata(&self) -> &Metadata {
293        self.snapshot.metadata()
294    }
295
296    fn eager_snapshot(&self) -> &EagerSnapshot {
297        &self.snapshot
298    }
299}
300
301/// Data that was actually written to the log store.
302#[derive(Debug)]
303pub struct CommitData {
304    /// The actions
305    pub actions: Vec<Action>,
306    /// The Operation
307    pub operation: DeltaOperation,
308    /// The Metadata
309    pub app_metadata: HashMap<String, Value>,
310    /// Application specific transaction
311    pub app_transactions: Vec<Transaction>,
312}
313
314/// Moves reserved commit metadata into typed CommitInfo fields.
315///
316/// This prevents serde flatten from serializing duplicate commitInfo keys when callers pass fields
317/// such as readVersion or operationParameters through custom metadata. Rust callers keep the
318/// existing tolerant behavior: invalid reserved metadata is logged and dropped. Python validates
319/// the same key set early in validate_reserved_commit_metadata and raises ValueError instead.
320/// See issue 4443 for the Python custom metadata request that introduced this normalization path.
321fn normalize_reserved_commit_metadata(
322    commit_info: &mut CommitInfo,
323    app_metadata: &mut HashMap<String, Value>,
324) {
325    match app_metadata.remove("operationParameters") {
326        Some(Value::Object(operation_parameters)) => {
327            let generated_parameters = commit_info
328                .operation_parameters
329                .get_or_insert_with(HashMap::new);
330            for (key, value) in operation_parameters {
331                generated_parameters
332                    .entry(key)
333                    .or_insert_with(|| operation_parameter_value(value));
334            }
335        }
336        Some(value) => log_unexpected_reserved_metadata_type(
337            "operationParameters",
338            &value,
339            "object with string-compatible values",
340        ),
341        None => {}
342    }
343
344    if let Some(value) = app_metadata.remove("readVersion") {
345        if let Some(value) = value.as_u64() {
346            if commit_info.read_version.is_none() {
347                commit_info.read_version = Some(value);
348            }
349        } else {
350            log_unexpected_reserved_metadata_type("readVersion", &value, "non-negative integer");
351        }
352    }
353
354    promote_string_reserved_metadata(app_metadata, "userId", &mut commit_info.user_id);
355    promote_string_reserved_metadata(app_metadata, "userName", &mut commit_info.user_name);
356    promote_string_reserved_metadata(app_metadata, "userMetadata", &mut commit_info.user_metadata);
357
358    if let Some(value) = app_metadata.remove("isolationLevel") {
359        if let Some(value) = value
360            .as_str()
361            .and_then(|value| IsolationLevel::from_str(value).ok())
362        {
363            if commit_info.isolation_level.is_none() {
364                commit_info.isolation_level = Some(value);
365            }
366        } else {
367            log_unexpected_reserved_metadata_type(
368                "isolationLevel",
369                &value,
370                "valid IsolationLevel string",
371            );
372        }
373    }
374
375    if let Some(value) = app_metadata.remove("isBlindAppend") {
376        if let Some(value) = value.as_bool() {
377            if commit_info.is_blind_append.is_none() {
378                commit_info.is_blind_append = Some(value);
379            }
380        } else {
381            log_unexpected_reserved_metadata_type("isBlindAppend", &value, "boolean");
382        }
383    }
384
385    for key in RESERVED_COMMIT_INFO_KEYS {
386        app_metadata.remove(*key);
387    }
388}
389
390fn promote_string_reserved_metadata(
391    metadata: &mut HashMap<String, Value>,
392    key: &'static str,
393    target: &mut Option<String>,
394) {
395    let Some(value) = metadata.remove(key) else {
396        return;
397    };
398
399    if let Value::String(value) = value {
400        if target.is_none() {
401            *target = Some(value);
402        }
403    } else {
404        log_unexpected_reserved_metadata_type(key, &value, "string");
405    }
406}
407
408fn log_unexpected_reserved_metadata_type(key: &str, value: &Value, expected: &str) {
409    debug!(
410        key,
411        expected,
412        actual = json_value_kind(value),
413        "Ignoring reserved commit metadata key with unexpected type"
414    );
415}
416
417fn json_value_kind(value: &Value) -> &'static str {
418    match value {
419        Value::Null => "null",
420        Value::Bool(_) => "boolean",
421        Value::Number(_) => "number",
422        Value::String(_) => "string",
423        Value::Array(_) => "array",
424        Value::Object(_) => "object",
425    }
426}
427
428fn assign_commit_info_metadata(
429    commit_info: &mut CommitInfo,
430    app_metadata: &mut HashMap<String, Value>,
431) {
432    normalize_reserved_commit_metadata(commit_info, app_metadata);
433
434    let mut existing_info = std::mem::take(&mut commit_info.info);
435    normalize_reserved_commit_metadata(commit_info, &mut existing_info);
436    // app_metadata is the override layer for flattened commit info. It fills missing typed fields
437    // above, and wins over provided CommitInfo.info keys for custom fields.
438    for (key, value) in existing_info {
439        app_metadata.entry(key).or_insert(value);
440    }
441    debug_assert!(
442        RESERVED_COMMIT_INFO_KEYS
443            .iter()
444            .all(|key| !app_metadata.contains_key(*key))
445    );
446
447    commit_info.info = app_metadata.clone();
448}
449
450impl CommitData {
451    /// Create new data to be committed
452    pub fn new(
453        mut actions: Vec<Action>,
454        operation: DeltaOperation,
455        mut app_metadata: HashMap<String, Value>,
456        app_transactions: Vec<Transaction>,
457    ) -> Self {
458        if let Some(Action::CommitInfo(commit_info)) = actions
459            .iter_mut()
460            .find(|action| matches!(action, Action::CommitInfo(..)))
461        {
462            // Callers that provide a CommitInfo action own generated typed fields such as
463            // timestamp and engineInfo. Flattened app metadata overrides custom keys, and callers
464            // may provide clientVersion for compatibility.
465            assign_commit_info_metadata(commit_info, &mut app_metadata);
466        } else {
467            let mut commit_info = operation.get_commit_info();
468            commit_info.timestamp = Some(Utc::now().timestamp_millis());
469            // clientVersion is provenance metadata, but it is not reserved. Callers can override it
470            // while engineInfo remains a generated typed CommitInfo field.
471            app_metadata
472                .entry("clientVersion".to_string())
473                .or_insert(Value::String(format!("delta-rs.{}", crate_version())));
474            assign_commit_info_metadata(&mut commit_info, &mut app_metadata);
475            // commit info should be the first action to support in-commit timestamps.
476            actions.insert(0, Action::CommitInfo(commit_info));
477        }
478
479        for txn in &app_transactions {
480            actions.push(Action::Txn(txn.clone()))
481        }
482
483        CommitData {
484            actions,
485            operation,
486            app_metadata,
487            app_transactions,
488        }
489    }
490
491    /// Obtain the byte representation of the commit.
492    pub fn get_bytes(&self) -> Result<bytes::Bytes, TransactionError> {
493        let mut jsons = Vec::<String>::new();
494        for action in &self.actions {
495            let json = serde_json::to_string(action)
496                .map_err(|e| TransactionError::SerializeLogJson { json_err: e })?;
497            jsons.push(json);
498        }
499        Ok(bytes::Bytes::from(jsons.join("\n")))
500    }
501
502    /// Update num_retries in operationMetrics within serialized bytes.
503    /// This allows updating the retry count without re-serializing all actions.
504    pub fn update_retry_count_in_bytes(
505        bytes: &Bytes,
506        num_retries: u64,
507    ) -> Result<Bytes, TransactionError> {
508        let bytes_str = std::str::from_utf8(bytes.as_ref())
509            .expect("Delta log bytes should always be valid UTF-8");
510
511        let lines: Vec<&str> = bytes_str.split('\n').collect();
512        let mut updated_lines = Vec::with_capacity(lines.len());
513
514        for line in lines {
515            if line.contains("\"commitInfo\"") {
516                let mut action: Value = serde_json::from_str(line)
517                    .map_err(|e| TransactionError::SerializeLogJson { json_err: e })?;
518
519                if let Some(commit_info) = action.get_mut("commitInfo") {
520                    if let Some(Value::Object(metrics)) = commit_info.get_mut("operationMetrics") {
521                        metrics
522                            .insert("num_retries".to_string(), Value::Number(num_retries.into()));
523                    }
524                }
525                // Serialize just this updated line
526                let updated_line = serde_json::to_string(&action)
527                    .map_err(|e| TransactionError::SerializeLogJson { json_err: e })?;
528                updated_lines.push(updated_line);
529            } else {
530                // Keep other action lines unchanged (Add, Remove, etc.)
531                updated_lines.push(line.to_string());
532            }
533        }
534
535        Ok(Bytes::from(updated_lines.join("\n")))
536    }
537}
538
539#[derive(Clone, Debug, Copy)]
540/// Properties for post commit hook.
541pub struct PostCommitHookProperties {
542    create_checkpoint: bool,
543    /// Override the EnableExpiredLogCleanUp setting, if None config setting is used
544    cleanup_expired_logs: Option<bool>,
545}
546
547#[derive(Clone, Debug)]
548/// End user facing interface to be used by operations on the table.
549/// Enable controlling commit behaviour and modifying metadata that is written during a commit.
550pub struct CommitProperties {
551    pub(crate) app_metadata: HashMap<String, Value>,
552    pub(crate) app_transaction: Vec<Transaction>,
553    max_retries: usize,
554    create_checkpoint: bool,
555    cleanup_expired_logs: Option<bool>,
556}
557
558impl Default for CommitProperties {
559    fn default() -> Self {
560        Self {
561            app_metadata: Default::default(),
562            app_transaction: Vec::new(),
563            max_retries: DEFAULT_RETRIES,
564            create_checkpoint: true,
565            cleanup_expired_logs: None,
566        }
567    }
568}
569
570impl CommitProperties {
571    /// Specify metadata the be committed
572    pub fn with_metadata(
573        mut self,
574        metadata: impl IntoIterator<Item = (String, serde_json::Value)>,
575    ) -> Self {
576        self.app_metadata = HashMap::from_iter(metadata);
577        self
578    }
579
580    /// Specify maximum number of times to retry the transaction before failing to commit
581    pub fn with_max_retries(mut self, max_retries: usize) -> Self {
582        self.max_retries = max_retries;
583        self
584    }
585
586    /// Specify if it should create a checkpoint when the commit interval condition is met
587    pub fn with_create_checkpoint(mut self, create_checkpoint: bool) -> Self {
588        self.create_checkpoint = create_checkpoint;
589        self
590    }
591
592    /// Add an additional application transaction to the commit
593    pub fn with_application_transaction(mut self, txn: Transaction) -> Self {
594        self.app_transaction.push(txn);
595        self
596    }
597
598    /// Override application transactions for the commit
599    pub fn with_application_transactions(mut self, txn: Vec<Transaction>) -> Self {
600        self.app_transaction = txn;
601        self
602    }
603
604    /// Specify if it should clean up the logs when the logRetentionDuration interval is met
605    pub fn with_cleanup_expired_logs(mut self, cleanup_expired_logs: Option<bool>) -> Self {
606        self.cleanup_expired_logs = cleanup_expired_logs;
607        self
608    }
609}
610
611impl From<CommitProperties> for CommitBuilder {
612    fn from(value: CommitProperties) -> Self {
613        CommitBuilder {
614            max_retries: value.max_retries,
615            app_metadata: value.app_metadata,
616            post_commit_hook: Some(PostCommitHookProperties {
617                create_checkpoint: value.create_checkpoint,
618                cleanup_expired_logs: value.cleanup_expired_logs,
619            }),
620            app_transaction: value.app_transaction,
621            ..Default::default()
622        }
623    }
624}
625
626/// Prepare data to be committed to the Delta log and control how the commit is performed
627pub struct CommitBuilder {
628    actions: Vec<Action>,
629    app_metadata: HashMap<String, Value>,
630    app_transaction: Vec<Transaction>,
631    max_retries: usize,
632    post_commit_hook: Option<PostCommitHookProperties>,
633    post_commit_hook_handler: Option<Arc<dyn CustomExecuteHandler>>,
634    operation_id: Uuid,
635}
636
637impl Default for CommitBuilder {
638    fn default() -> Self {
639        CommitBuilder {
640            actions: Vec::new(),
641            app_metadata: HashMap::new(),
642            app_transaction: Vec::new(),
643            max_retries: DEFAULT_RETRIES,
644            post_commit_hook: None,
645            post_commit_hook_handler: None,
646            operation_id: Uuid::new_v4(),
647        }
648    }
649}
650
651impl<'a> CommitBuilder {
652    /// Actions to be included in the commit
653    pub fn with_actions(mut self, actions: Vec<Action>) -> Self {
654        self.actions = actions;
655        self
656    }
657
658    /// Metadata for the operation performed like metrics, user, and notebook
659    pub fn with_app_metadata(mut self, app_metadata: HashMap<String, Value>) -> Self {
660        self.app_metadata = app_metadata;
661        self
662    }
663
664    /// Maximum number of times to retry the transaction before failing to commit
665    pub fn with_max_retries(mut self, max_retries: usize) -> Self {
666        self.max_retries = max_retries;
667        self
668    }
669
670    /// Specify all the post commit hook properties
671    pub fn with_post_commit_hook(mut self, post_commit_hook: PostCommitHookProperties) -> Self {
672        self.post_commit_hook = Some(post_commit_hook);
673        self
674    }
675
676    /// Propagate operation id to log store
677    pub fn with_operation_id(mut self, operation_id: Uuid) -> Self {
678        self.operation_id = operation_id;
679        self
680    }
681
682    /// Set a custom execute handler, for pre and post execution
683    pub fn with_post_commit_hook_handler(
684        mut self,
685        handler: Option<Arc<dyn CustomExecuteHandler>>,
686    ) -> Self {
687        self.post_commit_hook_handler = handler;
688        self
689    }
690
691    /// Prepare a Commit operation using the configured builder
692    pub fn build(
693        self,
694        table_data: Option<&'a dyn TableReference>,
695        log_store: LogStoreRef,
696        operation: DeltaOperation,
697    ) -> PreCommit<'a> {
698        let data = CommitData::new(
699            self.actions,
700            operation,
701            self.app_metadata,
702            self.app_transaction,
703        );
704        PreCommit {
705            log_store,
706            table_data,
707            max_retries: self.max_retries,
708            data,
709            post_commit_hook: self.post_commit_hook,
710            post_commit_hook_handler: self.post_commit_hook_handler,
711            operation_id: self.operation_id,
712        }
713    }
714}
715
716/// Represents a commit that has not yet started but all details are finalized
717pub struct PreCommit<'a> {
718    log_store: LogStoreRef,
719    table_data: Option<&'a dyn TableReference>,
720    data: CommitData,
721    max_retries: usize,
722    post_commit_hook: Option<PostCommitHookProperties>,
723    post_commit_hook_handler: Option<Arc<dyn CustomExecuteHandler>>,
724    operation_id: Uuid,
725}
726
727impl<'a> std::future::IntoFuture for PreCommit<'a> {
728    type Output = DeltaResult<FinalizedCommit>;
729    type IntoFuture = BoxFuture<'a, Self::Output>;
730
731    fn into_future(self) -> Self::IntoFuture {
732        Box::pin(async move { self.into_prepared_commit_future().await?.await?.await })
733    }
734}
735
736impl<'a> PreCommit<'a> {
737    /// Prepare the commit but do not finalize it
738    pub fn into_prepared_commit_future(self) -> BoxFuture<'a, DeltaResult<PreparedCommit<'a>>> {
739        let this = self;
740
741        // Write delta log entry as temporary file to storage. For the actual commit,
742        // the temporary file is moved (atomic rename) to the delta log folder within `commit` function.
743        async fn write_tmp_commit(
744            log_entry: Bytes,
745            store: ObjectStoreRef,
746        ) -> DeltaResult<CommitOrBytes> {
747            let token = uuid::Uuid::new_v4().to_string();
748            let path = Path::from_iter([DELTA_LOG_FOLDER, &format!("_commit_{token}.json.tmp")]);
749            store.put(&path, log_entry.into()).await?;
750            Ok(CommitOrBytes::TmpCommit(path))
751        }
752
753        Box::pin(async move {
754            if let Some(table_reference) = this.table_data {
755                PROTOCOL.can_commit(table_reference, &this.data.actions, &this.data.operation)?;
756            }
757            let log_entry = this.data.get_bytes()?;
758
759            // With the DefaultLogStore & LakeFSLogstore, we just pass the bytes around, since we use conditionalPuts
760            // Other stores will use tmp_commits
761            let commit_or_bytes = if ["LakeFSLogStore", "DefaultLogStore"]
762                .contains(&this.log_store.name().as_str())
763            {
764                CommitOrBytes::LogBytes(log_entry)
765            } else {
766                write_tmp_commit(
767                    log_entry,
768                    this.log_store.object_store(Some(this.operation_id)),
769                )
770                .await?
771            };
772
773            Ok(PreparedCommit {
774                commit_or_bytes,
775                log_store: this.log_store,
776                table_data: this.table_data,
777                max_retries: this.max_retries,
778                data: this.data,
779                post_commit: this.post_commit_hook,
780                post_commit_hook_handler: this.post_commit_hook_handler,
781                operation_id: this.operation_id,
782            })
783        })
784    }
785}
786
787/// Represents a inflight commit
788pub struct PreparedCommit<'a> {
789    commit_or_bytes: CommitOrBytes,
790    log_store: LogStoreRef,
791    data: CommitData,
792    table_data: Option<&'a dyn TableReference>,
793    max_retries: usize,
794    post_commit: Option<PostCommitHookProperties>,
795    post_commit_hook_handler: Option<Arc<dyn CustomExecuteHandler>>,
796    operation_id: Uuid,
797}
798
799impl PreparedCommit<'_> {
800    /// The temporary commit file created
801    pub fn commit_or_bytes(&self) -> &CommitOrBytes {
802        &self.commit_or_bytes
803    }
804}
805
806impl<'a> std::future::IntoFuture for PreparedCommit<'a> {
807    type Output = DeltaResult<PostCommit>;
808    type IntoFuture = BoxFuture<'a, Self::Output>;
809
810    fn into_future(self) -> Self::IntoFuture {
811        let this = self;
812
813        Box::pin(async move {
814            let commit_or_bytes = this.commit_or_bytes;
815
816            let mut attempt_number: usize = 1;
817
818            // Handle the case where table doesn't exist yet (initial table creation)
819            let read_snapshot: EagerSnapshot = if let Some(table_data) = this.table_data {
820                table_data.eager_snapshot().clone()
821            } else {
822                debug!("committing initial table version 0");
823                match this
824                    .log_store
825                    .write_commit_entry(0, commit_or_bytes.clone(), this.operation_id)
826                    .await
827                {
828                    Ok(_) => {
829                        return Ok(PostCommit {
830                            version: 0,
831                            data: this.data,
832                            create_checkpoint: false,
833                            cleanup_expired_logs: None,
834                            log_store: this.log_store,
835                            table_data: None,
836                            custom_execute_handler: this.post_commit_hook_handler,
837                            metrics: CommitMetrics { num_retries: 0 },
838                        });
839                    }
840                    Err(TransactionError::VersionAlreadyExists(0)) => {
841                        // Table was created by another writer since the `this.table_data.is_none()` check.
842                        // Load the current table state and continue with the retry loop.
843                        debug!("version 0 already exists, loading table state for retry");
844                        attempt_number = 2;
845                        let latest_version: Version = this.log_store.get_latest_version(0).await?;
846                        EagerSnapshot::try_new(this.log_store.as_ref(), Some(latest_version))
847                            .await?
848                    }
849                    Err(e) => return Err(e.into()),
850                }
851            };
852
853            let mut read_snapshot = read_snapshot;
854
855            let commit_span = info_span!(
856                "commit_with_retries",
857                base_version = read_snapshot.version(),
858                max_retries = this.max_retries,
859                attempt = field::Empty,
860                target_version = field::Empty,
861                conflicts_checked = 0
862            );
863
864            async move {
865                let total_retries = this.max_retries + 1;
866                while attempt_number <= total_retries {
867                    Span::current().record("attempt", attempt_number);
868                    let latest_version = this
869                        .log_store
870                        .get_latest_version(read_snapshot.version())
871                        .await?;
872
873                    if latest_version > read_snapshot.version() {
874                        // If max_retries are set to 0, do not try to use the conflict checker to resolve the conflict
875                        // and throw immediately
876                        if this.max_retries == 0 {
877                            warn!(
878                                base_version = read_snapshot.version(),
879                                latest_version = latest_version,
880                                "table updated but max_retries is 0, failing immediately"
881                            );
882                            return Err(TransactionError::MaxCommitAttempts(
883                                this.max_retries as i32,
884                            )
885                            .into());
886                        }
887                        warn!(
888                            base_version = read_snapshot.version(),
889                            latest_version = latest_version,
890                            versions_behind = latest_version - read_snapshot.version(),
891                            "table updated during transaction, checking for conflicts"
892                        );
893                        let mut steps = latest_version - read_snapshot.version();
894                        let mut conflicts_checked = 0;
895
896                        // Need to check for conflicts with each version between the read_snapshot and
897                        // the latest!
898                        while steps != 0 {
899                            conflicts_checked += 1;
900                            let summary = WinningCommitSummary::try_new(
901                                this.log_store.as_ref(),
902                                latest_version - steps,
903                                (latest_version - steps) + 1,
904                            )
905                            .await?;
906                            let conflict_read_set = read_snapshot
907                                .snapshot()
908                                .conflict_read_set(this.log_store.as_ref())
909                                .await?;
910                            let transaction_info = TransactionInfo::try_new(
911                                conflict_read_set,
912                                this.data.operation.read_predicate(),
913                                &this.data.actions,
914                                this.data.operation.read_whole_table(),
915                            )?;
916                            let conflict_checker = ConflictChecker::new(
917                                transaction_info,
918                                summary,
919                                Some(&this.data.operation),
920                            );
921
922                            match conflict_checker.check_conflicts() {
923                                Ok(_) => {}
924                                Err(err) => {
925                                    error!(
926                                        conflicts_checked = conflicts_checked,
927                                        error = %err,
928                                        "conflict detected, aborting transaction"
929                                    );
930                                    return Err(TransactionError::CommitConflict(err).into());
931                                }
932                            }
933                            steps -= 1;
934                        }
935                        Span::current().record("conflicts_checked", conflicts_checked);
936                        debug!(
937                            conflicts_checked = conflicts_checked,
938                            "all conflicts resolved, updating snapshot"
939                        );
940                        // Update snapshot to latest version after successful conflict check
941                        read_snapshot
942                            .update(&this.log_store, Some(latest_version))
943                            .await?;
944                    }
945                    let version: Version = latest_version + 1;
946                    Span::current().record("target_version", version);
947
948                    // Calculate current retry count (attempt 1 = 0 retries, attempt 2 = 1 retry, etc.)
949                    let current_retries = (attempt_number - 1) as u64;
950
951                    // Update operationMetrics.numRetries in the serialized bytes
952                    let updated_commit_or_bytes = match &commit_or_bytes {
953                        CommitOrBytes::LogBytes(bytes) => {
954                            let updated_bytes = if current_retries > 0 {
955                                CommitData::update_retry_count_in_bytes(bytes, current_retries)?
956                            } else {
957                                bytes.clone()
958                            };
959                            CommitOrBytes::LogBytes(updated_bytes)
960                        }
961                        CommitOrBytes::TmpCommit(_path) => {
962                            // For TmpCommit stores, keep original behavior for now
963                            // (these stores write to tmp file first, then rename)
964                            commit_or_bytes.clone()
965                        }
966                    };
967
968                    match this
969                        .log_store
970                        .write_commit_entry(version, updated_commit_or_bytes, this.operation_id)
971                        .await
972                    {
973                        Ok(()) => {
974                            info!(
975                                version = version,
976                                num_retries = attempt_number - 1,
977                                "transaction committed successfully"
978                            );
979                            return Ok(PostCommit {
980                                version,
981                                data: this.data,
982                                create_checkpoint: this
983                                    .post_commit
984                                    .map(|v| v.create_checkpoint)
985                                    .unwrap_or_default(),
986                                cleanup_expired_logs: this
987                                    .post_commit
988                                    .map(|v| v.cleanup_expired_logs)
989                                    .unwrap_or_default(),
990                                log_store: this.log_store,
991                                table_data: Some(Box::new(read_snapshot)),
992                                custom_execute_handler: this.post_commit_hook_handler,
993                                metrics: CommitMetrics {
994                                    num_retries: (attempt_number - 1) as u64,
995                                },
996                            });
997                        }
998                        Err(TransactionError::VersionAlreadyExists(version)) => {
999                            warn!(
1000                                version = version,
1001                                attempt = attempt_number,
1002                                "version already exists, will retry"
1003                            );
1004                            // If the version already exists, loop through again and re-check
1005                            // conflicts
1006                            attempt_number += 1;
1007                        }
1008                        Err(err) => {
1009                            error!(
1010                                version = version,
1011                                error = %err,
1012                                "commit failed, aborting"
1013                            );
1014                            this.log_store
1015                                .abort_commit_entry(version, commit_or_bytes, this.operation_id)
1016                                .await?;
1017                            return Err(err.into());
1018                        }
1019                    }
1020                }
1021
1022                error!(
1023                    max_retries = this.max_retries,
1024                    "exceeded maximum commit attempts"
1025                );
1026                Err(TransactionError::MaxCommitAttempts(this.max_retries as i32).into())
1027            }
1028            .instrument(commit_span)
1029            .await
1030        })
1031    }
1032}
1033
1034/// Represents items for the post commit hook
1035pub struct PostCommit {
1036    /// The winning version number of the commit
1037    pub version: Version,
1038    /// The data that was committed to the log store
1039    pub data: CommitData,
1040    create_checkpoint: bool,
1041    cleanup_expired_logs: Option<bool>,
1042    log_store: LogStoreRef,
1043    table_data: Option<Box<dyn TableReference>>,
1044    custom_execute_handler: Option<Arc<dyn CustomExecuteHandler>>,
1045    metrics: CommitMetrics,
1046}
1047
1048impl PostCommit {
1049    /// Runs the post commit activities
1050    async fn run_post_commit_hook(&self) -> DeltaResult<(DeltaTableState, PostCommitMetrics)> {
1051        if let Some(table) = &self.table_data {
1052            let post_commit_operation_id = Uuid::new_v4();
1053            let mut snapshot = table.eager_snapshot().clone();
1054            if self.version != snapshot.version() {
1055                snapshot.update(&self.log_store, Some(self.version)).await?;
1056            }
1057
1058            let mut state = DeltaTableState { snapshot };
1059
1060            let cleanup_logs = if let Some(cleanup_logs) = self.cleanup_expired_logs {
1061                cleanup_logs
1062            } else {
1063                state.table_config().enable_expired_log_cleanup()
1064            };
1065
1066            // Run arbitrary before_post_commit_hook code
1067            if let Some(custom_execute_handler) = &self.custom_execute_handler {
1068                custom_execute_handler
1069                    .before_post_commit_hook(
1070                        &self.log_store,
1071                        cleanup_logs || self.create_checkpoint,
1072                        post_commit_operation_id,
1073                    )
1074                    .await?
1075            }
1076
1077            let mut new_checkpoint_created = false;
1078            if self.create_checkpoint {
1079                // Execute create checkpoint hook
1080                new_checkpoint_created = self
1081                    .create_checkpoint(
1082                        &state,
1083                        &self.log_store,
1084                        self.version,
1085                        post_commit_operation_id,
1086                    )
1087                    .await?;
1088            }
1089
1090            let mut num_log_files_cleaned_up: u64 = 0;
1091            if cleanup_logs {
1092                // Execute clean up logs hook
1093                num_log_files_cleaned_up = cleanup_expired_logs_for(
1094                    self.version,
1095                    self.log_store.as_ref(),
1096                    Utc::now().timestamp_millis()
1097                        - state.table_config().log_retention_duration().as_millis() as i64,
1098                    Some(post_commit_operation_id),
1099                )
1100                .await? as u64;
1101                if num_log_files_cleaned_up > 0 {
1102                    state = DeltaTableState::try_new(&self.log_store, Some(self.version)).await?;
1103                }
1104            }
1105
1106            // Run arbitrary after_post_commit_hook code
1107            if let Some(custom_execute_handler) = &self.custom_execute_handler {
1108                custom_execute_handler
1109                    .after_post_commit_hook(
1110                        &self.log_store,
1111                        cleanup_logs || self.create_checkpoint,
1112                        post_commit_operation_id,
1113                    )
1114                    .await?
1115            }
1116            Ok((
1117                state,
1118                PostCommitMetrics {
1119                    new_checkpoint_created,
1120                    num_log_files_cleaned_up,
1121                },
1122            ))
1123        } else {
1124            let state = DeltaTableState::try_new(&self.log_store, Some(self.version)).await?;
1125            Ok((
1126                state,
1127                PostCommitMetrics {
1128                    new_checkpoint_created: false,
1129                    num_log_files_cleaned_up: 0,
1130                },
1131            ))
1132        }
1133    }
1134    async fn create_checkpoint(
1135        &self,
1136        table_state: &DeltaTableState,
1137        log_store: &LogStoreRef,
1138        version: Version,
1139        operation_id: Uuid,
1140    ) -> DeltaResult<bool> {
1141        let checkpoint_interval = table_state.config().checkpoint_interval().get();
1142        if (version + 1).is_multiple_of(checkpoint_interval) {
1143            create_checkpoint_for(version, log_store.as_ref(), Some(operation_id)).await?;
1144            Ok(true)
1145        } else {
1146            Ok(false)
1147        }
1148    }
1149}
1150
1151/// A commit that successfully completed
1152pub struct FinalizedCommit {
1153    /// The new table state after a commit
1154    pub snapshot: DeltaTableState,
1155
1156    /// Version of the finalized commit
1157    pub version: Version,
1158
1159    /// Metrics associated with the commit operation
1160    pub metrics: Metrics,
1161}
1162
1163impl std::fmt::Debug for FinalizedCommit {
1164    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1165        f.debug_struct("FinalizedCommit")
1166            .field("version", &self.version)
1167            .field("metrics", &self.metrics)
1168            .finish()
1169    }
1170}
1171
1172impl FinalizedCommit {
1173    /// The new table state after a commit
1174    pub fn snapshot(&self) -> DeltaTableState {
1175        self.snapshot.clone()
1176    }
1177    /// Version of the finalized commit
1178    pub fn version(&self) -> Version {
1179        self.version
1180    }
1181}
1182
1183impl std::future::IntoFuture for PostCommit {
1184    type Output = DeltaResult<FinalizedCommit>;
1185    type IntoFuture = BoxFuture<'static, Self::Output>;
1186
1187    fn into_future(self) -> Self::IntoFuture {
1188        let this = self;
1189
1190        Box::pin(async move {
1191            match this.run_post_commit_hook().await {
1192                Ok((snapshot, post_commit_metrics)) => Ok(FinalizedCommit {
1193                    snapshot,
1194                    version: this.version,
1195                    metrics: Metrics {
1196                        num_retries: this.metrics.num_retries,
1197                        new_checkpoint_created: post_commit_metrics.new_checkpoint_created,
1198                        num_log_files_cleaned_up: post_commit_metrics.num_log_files_cleaned_up,
1199                    },
1200                }),
1201                Err(err) => Err(err),
1202            }
1203        })
1204    }
1205}
1206
1207#[cfg(test)]
1208mod tests {
1209    use std::sync::Arc;
1210
1211    use super::*;
1212    use crate::kernel::IsolationLevel;
1213    use crate::logstore::{LogStore, StorageConfig, default_logstore::DefaultLogStore};
1214    use crate::protocol::SaveMode;
1215    use object_store::{PutPayload, memory::InMemory};
1216    use serde_json::json;
1217    use url::Url;
1218
1219    #[tokio::test]
1220    async fn test_try_commit_transaction() {
1221        let store = Arc::new(InMemory::new());
1222        let url = Url::parse("mem://what/is/this").unwrap();
1223        let log_store = DefaultLogStore::new(
1224            store.clone(),
1225            store.clone(),
1226            crate::logstore::LogStoreConfig::new(&url, StorageConfig::default()),
1227        );
1228        let version_path = Path::from("_delta_log/00000000000000000000.json");
1229        store.put(&version_path, PutPayload::new()).await.unwrap();
1230
1231        let res = log_store
1232            .write_commit_entry(
1233                0,
1234                CommitOrBytes::LogBytes(PutPayload::new().into()),
1235                Uuid::new_v4(),
1236            )
1237            .await;
1238        // fails if file version already exists
1239        assert!(res.is_err());
1240
1241        // succeeds for next version
1242        log_store
1243            .write_commit_entry(
1244                1,
1245                CommitOrBytes::LogBytes(PutPayload::new().into()),
1246                Uuid::new_v4(),
1247            )
1248            .await
1249            .unwrap();
1250    }
1251
1252    #[test]
1253    fn test_commit_with_retries_tracing_span() {
1254        let span = info_span!(
1255            "commit_with_retries",
1256            base_version = 5,
1257            max_retries = 10,
1258            attempt = field::Empty,
1259            target_version = field::Empty,
1260            conflicts_checked = 0
1261        );
1262
1263        let metadata = span.metadata().expect("span should have metadata");
1264        assert_eq!(metadata.name(), "commit_with_retries");
1265        assert_eq!(metadata.level(), &Level::INFO);
1266        assert!(metadata.is_span());
1267
1268        span.record("attempt", 1);
1269        span.record("target_version", 6);
1270        span.record("conflicts_checked", 2);
1271    }
1272
1273    #[test]
1274    fn test_commit_properties_with_retries() {
1275        let props = CommitProperties::default()
1276            .with_max_retries(5)
1277            .with_create_checkpoint(false);
1278
1279        assert_eq!(props.max_retries, 5);
1280        assert!(!props.create_checkpoint);
1281    }
1282
1283    #[test]
1284    fn test_commit_metrics() {
1285        let metrics = CommitMetrics { num_retries: 3 };
1286        assert_eq!(metrics.num_retries, 3);
1287    }
1288
1289    #[test]
1290    fn test_commit_data_client_version() {
1291        let no_metadata = CommitData::new(
1292            vec![],
1293            DeltaOperation::FileSystemCheck {},
1294            HashMap::new(),
1295            vec![],
1296        );
1297        assert_eq!(
1298            *no_metadata.app_metadata.get("clientVersion").unwrap(),
1299            json!(format!("delta-rs.{}", crate_version()))
1300        );
1301
1302        let with_metadata = CommitData::new(
1303            vec![],
1304            DeltaOperation::FileSystemCheck {},
1305            HashMap::from([("clientVersion".to_owned(), json!("test-client.0.0.1"))]),
1306            vec![],
1307        );
1308        assert_eq!(
1309            *with_metadata.app_metadata.get("clientVersion").unwrap(),
1310            json!("test-client.0.0.1")
1311        );
1312    }
1313
1314    fn commit_info(data: &CommitData) -> &CommitInfo {
1315        match &data.actions[0] {
1316            Action::CommitInfo(info) => info,
1317            action => panic!("expected first action to be commitInfo, got {action:?}"),
1318        }
1319    }
1320
1321    #[test]
1322    fn test_commit_data_strips_and_promotes_reserved_metadata() {
1323        let data = CommitData::new(
1324            vec![],
1325            DeltaOperation::FileSystemCheck {},
1326            HashMap::from([
1327                ("readVersion".to_owned(), json!(7)),
1328                ("userId".to_owned(), json!("user-1")),
1329                ("userName".to_owned(), json!("Jane Doe")),
1330                ("userMetadata".to_owned(), json!("metadata")),
1331                ("isolationLevel".to_owned(), json!("SnapshotIsolation")),
1332                ("isBlindAppend".to_owned(), json!(true)),
1333                ("timestamp".to_owned(), json!(1)),
1334                ("operation".to_owned(), json!("CUSTOM OPERATION")),
1335                ("engineInfo".to_owned(), json!("custom-engine")),
1336                ("custom".to_owned(), json!({"kept": true})),
1337            ]),
1338            vec![],
1339        );
1340
1341        let info = commit_info(&data);
1342        assert_eq!(info.read_version, Some(7));
1343        assert_eq!(info.user_id.as_deref(), Some("user-1"));
1344        assert_eq!(info.user_name.as_deref(), Some("Jane Doe"));
1345        assert_eq!(info.user_metadata.as_deref(), Some("metadata"));
1346        assert_eq!(
1347            info.isolation_level,
1348            Some(IsolationLevel::SnapshotIsolation)
1349        );
1350        assert_eq!(info.is_blind_append, Some(true));
1351        assert_eq!(info.operation.as_deref(), Some("FSCK"));
1352        assert_ne!(info.engine_info.as_deref(), Some("custom-engine"));
1353        assert_eq!(info.info.get("custom"), Some(&json!({"kept": true})));
1354
1355        for key in [
1356            "timestamp",
1357            "userId",
1358            "userName",
1359            "operation",
1360            "operationParameters",
1361            "readVersion",
1362            "isolationLevel",
1363            "isBlindAppend",
1364            "engineInfo",
1365            "userMetadata",
1366        ] {
1367            assert!(
1368                !info.info.contains_key(key),
1369                "reserved key {key} must not remain in flattened commit info"
1370            );
1371        }
1372    }
1373
1374    #[test]
1375    fn test_commit_data_merges_operation_parameters_generated_keys_win() {
1376        let data = CommitData::new(
1377            vec![],
1378            DeltaOperation::Write {
1379                mode: SaveMode::Overwrite,
1380                partition_by: Some(vec!["id".to_owned()]),
1381                predicate: None,
1382            },
1383            HashMap::from([(
1384                "operationParameters".to_owned(),
1385                json!({
1386                    "mode": "custom-mode",
1387                    "partitionBy": "custom-partitioning",
1388                    "customParameter": {"kept": true},
1389                    "customBoolean": true,
1390                    "customNumber": 7,
1391                }),
1392            )]),
1393            vec![],
1394        );
1395
1396        let info = commit_info(&data);
1397        let operation_parameters = info
1398            .operation_parameters
1399            .as_ref()
1400            .expect("operation parameters should be present");
1401        assert_eq!(operation_parameters.get("mode"), Some(&json!("Overwrite")));
1402        assert_eq!(
1403            operation_parameters.get("partitionBy"),
1404            Some(&json!("[\"id\"]"))
1405        );
1406        assert_eq!(
1407            operation_parameters.get("customParameter"),
1408            Some(&json!("{\"kept\":true}"))
1409        );
1410        assert_eq!(
1411            operation_parameters.get("customBoolean"),
1412            Some(&json!("true"))
1413        );
1414        assert_eq!(operation_parameters.get("customNumber"), Some(&json!("7")));
1415        assert!(!info.info.contains_key("operationParameters"));
1416    }
1417
1418    #[test]
1419    fn test_commit_data_normalizes_reserved_keys_from_existing_commit_info_info() {
1420        let data = CommitData::new(
1421            vec![Action::CommitInfo(CommitInfo {
1422                info: HashMap::from([
1423                    ("userName".to_owned(), json!("shadow-user")),
1424                    (
1425                        "operationParameters".to_owned(),
1426                        json!({"custom": {"kept": true}}),
1427                    ),
1428                    ("custom".to_owned(), json!("kept")),
1429                ]),
1430                ..Default::default()
1431            })],
1432            DeltaOperation::FileSystemCheck {},
1433            HashMap::from([("custom".to_owned(), json!("metadata-wins"))]),
1434            vec![],
1435        );
1436
1437        let info = commit_info(&data);
1438        assert_eq!(info.user_name.as_deref(), Some("shadow-user"));
1439        let operation_parameters = info
1440            .operation_parameters
1441            .as_ref()
1442            .expect("operation parameters should be promoted");
1443        assert_eq!(
1444            operation_parameters.get("custom"),
1445            Some(&json!("{\"kept\":true}"))
1446        );
1447        assert_eq!(info.info.get("custom"), Some(&json!("metadata-wins")));
1448        assert!(!info.info.contains_key("userName"));
1449        assert!(!info.info.contains_key("operationParameters"));
1450    }
1451
1452    #[test]
1453    fn test_commit_data_does_not_promote_reserved_metadata_over_typed_fields() {
1454        let data = CommitData::new(
1455            vec![Action::CommitInfo(CommitInfo {
1456                user_name: Some("typed-user".to_owned()),
1457                read_version: Some(10),
1458                ..Default::default()
1459            })],
1460            DeltaOperation::FileSystemCheck {},
1461            HashMap::from([
1462                ("userName".to_owned(), json!("metadata-user")),
1463                ("readVersion".to_owned(), json!(11)),
1464                ("custom".to_owned(), json!("kept")),
1465            ]),
1466            vec![],
1467        );
1468
1469        let info = commit_info(&data);
1470        assert_eq!(info.user_name.as_deref(), Some("typed-user"));
1471        assert_eq!(info.read_version, Some(10));
1472        assert!(!info.info.contains_key("userName"));
1473        assert!(!info.info.contains_key("readVersion"));
1474        assert_eq!(info.info.get("custom"), Some(&json!("kept")));
1475    }
1476
1477    #[test]
1478    fn test_commit_data_strips_wrong_typed_reserved_metadata_without_clobbering_typed_fields() {
1479        let data = CommitData::new(
1480            vec![Action::CommitInfo(CommitInfo {
1481                user_name: Some("typed-user".to_owned()),
1482                read_version: Some(10),
1483                ..Default::default()
1484            })],
1485            DeltaOperation::FileSystemCheck {},
1486            HashMap::from([
1487                ("userName".to_owned(), json!(123)),
1488                ("readVersion".to_owned(), json!("abc")),
1489                ("isBlindAppend".to_owned(), json!("not-a-bool")),
1490                ("custom".to_owned(), json!("kept")),
1491            ]),
1492            vec![],
1493        );
1494
1495        let info = commit_info(&data);
1496        assert_eq!(info.user_name.as_deref(), Some("typed-user"));
1497        assert_eq!(info.read_version, Some(10));
1498        assert_eq!(info.info.get("custom"), Some(&json!("kept")));
1499        assert!(!info.info.contains_key("userName"));
1500        assert!(!info.info.contains_key("readVersion"));
1501        assert!(!info.info.contains_key("isBlindAppend"));
1502    }
1503}