Skip to main content

rustfs_ecstore/store/
peer.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use super::*;
16use crate::bucket::utils::has_bad_path_component;
17use crate::disk::error::{DiskError, Result as DiskResult};
18use crate::disk::{DeleteOptions, Disk, RenameDataGuards, RenameDataResp};
19use crate::runtime::instance::{InstanceContext, NamespaceCommitGuard};
20use crate::runtime::sources as runtime_sources;
21use tracing::{debug, error};
22
23const LOG_COMPONENT_ECSTORE: &str = "ecstore";
24const LOG_SUBSYSTEM_DISK_STARTUP: &str = "disk_startup";
25const EVENT_LOCAL_DISK_ID_PREWARM_SKIPPED: &str = "local_disk_id_prewarm_skipped";
26const EVENT_LOCK_CLIENT_INITIALIZATION_FAILED: &str = "lock_client_initialization_failed";
27
28/// An instance-bound capability for internal writes before ECStore/IAM startup.
29/// Its private context and volume checks cannot be replaced by a caller guard.
30#[derive(Clone)]
31pub struct BootstrapLocalTarget {
32    ctx: Arc<InstanceContext>,
33}
34
35impl BootstrapLocalTarget {
36    pub fn new(ctx: Arc<InstanceContext>) -> Self {
37        Self { ctx }
38    }
39
40    pub fn is_for_store(&self, store: &ECStore) -> bool {
41        Arc::ptr_eq(&self.ctx, &store.ctx)
42    }
43
44    pub async fn rename_local_data(
45        &self,
46        disk_ref: &str,
47        source: (&str, &str),
48        fi: &FileInfo,
49        destination: (&str, &str),
50        scanner_token: Option<Uuid>,
51    ) -> DiskResult<RenameDataResp> {
52        if scanner_token.is_some() {
53            return Err(DiskError::other("bootstrap rename cannot use a scanner publication lease"));
54        }
55        validate_bootstrap_volume(source.0)?;
56        validate_bootstrap_volume(destination.0)?;
57        rename_local_data_with_ctx(&self.ctx, disk_ref, source, fi, destination, RenameDataGuards::default()).await
58    }
59
60    pub async fn undo_local_write(
61        &self,
62        disk_ref: &str,
63        volume: &str,
64        path: &str,
65        fi: FileInfo,
66        opts: DeleteOptions,
67    ) -> DiskResult<()> {
68        validate_bootstrap_volume(volume)?;
69        undo_local_write_with_ctx(&self.ctx, disk_ref, volume, path, fi, opts).await
70    }
71}
72
73fn validate_bootstrap_volume(volume: &str) -> DiskResult<()> {
74    // Prefix membership alone permits aliases such as .rustfs.sys/../bucket.
75    // Validate both raw rename volumes before any disk lookup or admission.
76    if has_bad_path_component(volume) || !is_meta_bucketname(volume) {
77        return Err(DiskError::FileAccessDenied);
78    }
79    Ok(())
80}
81
82impl ECStore {
83    pub async fn write_local_metadata_at_incarnation(
84        self: &Arc<Self>,
85        disk_ref: &str,
86        target: (&str, &str),
87        fi: FileInfo,
88        expected: Uuid,
89    ) -> DiskResult<()> {
90        let disk_ref = disk_ref.to_owned();
91        let path = target.1.to_owned();
92        self.run_bucket_heal_at_incarnation(target.0, expected, &Default::default(), move |store, volume, _| async move {
93            let (disk, id) = local_disk_candidate(&store.ctx, &disk_ref).await?;
94            let _owner = admit_local_disk(&store.ctx, &disk, id, true).await?;
95            disk.write_metadata("", &volume, &path, fi).await.map_err(Into::into)
96        })
97        .await
98        .map_err(|error| DiskError::other(error.to_string()))
99    }
100
101    pub async fn delete_local_path_at_incarnation(
102        self: &Arc<Self>,
103        disk_ref: &str,
104        target: (&str, &str),
105        opts: DeleteOptions,
106        expected: Uuid,
107    ) -> DiskResult<()> {
108        let disk_ref = disk_ref.to_owned();
109        let path = target.1.to_owned();
110        self.run_bucket_heal_at_incarnation(target.0, expected, &Default::default(), move |store, volume, _| async move {
111            let (disk, id) = local_disk_candidate(&store.ctx, &disk_ref).await?;
112            let owner = admit_local_disk(&store.ctx, &disk, id, true).await?;
113            let scope = super::bucket_heal_scope(&volume).ok_or_else(|| StorageError::other("missing bucket heal scope"))?;
114            scope.check()?;
115            disk.delete_with_namespace_owner(&volume, &path, opts, Some(Arc::new((scope, owner))))
116                .await
117                .map_err(Into::into)
118        })
119        .await
120        .map_err(|error| DiskError::other(error.to_string()))
121    }
122
123    pub async fn delete_local_version_at_incarnation(
124        &self,
125        disk_ref: &str,
126        target: (&str, &str),
127        fi: FileInfo,
128        force_del_marker: bool,
129        opts: DeleteOptions,
130        expected: Uuid,
131    ) -> DiskResult<()> {
132        if expected.is_nil() || opts.undo_write || self.ctx.lock_manager().is_disabled() {
133            return Err(DiskError::other("invalid incarnation-bound heal deletion"));
134        }
135        let fence = Arc::new(
136            self.acquire_bucket_incarnation_fence(target.0, expected)
137                .await
138                .map_err(|error| DiskError::other(error.to_string()))?,
139        );
140        let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
141        let owner = admit_local_disk(&self.ctx, &disk, id, true).await?;
142        if fence.is_lock_lost() {
143            return Err(DiskError::other("bucket heal incarnation fence was lost"));
144        }
145        disk.delete_version_with_namespace_owner(target.0, target.1, fi, force_del_marker, opts, Some(Arc::new((fence, owner))))
146            .await
147    }
148
149    /// Target-side admission for an incarnation-bound heal rename. The physical
150    /// namespace operations retain the lifecycle owner after RPC cancellation.
151    pub async fn rename_local_data_at_incarnation(
152        &self,
153        disk_ref: &str,
154        source: (&str, &str),
155        fi: &FileInfo,
156        destination: (&str, &str),
157        expected: Uuid,
158    ) -> DiskResult<RenameDataResp> {
159        if expected.is_nil() || self.ctx.lock_manager().is_disabled() {
160            return Err(DiskError::other(
161                "incarnation-bound rename requires namespace locking and a non-nil identity",
162            ));
163        }
164        let bucket = if destination.0 == RUSTFS_META_BUCKET {
165            super::heal::bucket_metadata_owner(destination.1).ok_or(DiskError::FileAccessDenied)?
166        } else {
167            destination.0
168        };
169        let guard = Arc::new(
170            self.acquire_bucket_incarnation_fence(bucket, expected)
171                .await
172                .map_err(|error| DiskError::other(error.to_string()))?,
173        );
174        if guard.is_lock_lost() {
175            return Err(DiskError::other("bucket heal incarnation fence was lost"));
176        }
177        let transaction = if destination.0 == RUSTFS_META_BUCKET {
178            // The receiver must retain the config fence after an RPC timeout.
179            // A delayed request must also reject a newer config in the same
180            // bucket incarnation before publishing its older repair payload.
181            let transaction = crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket)
182                .await
183                .map_err(|error| DiskError::other(error.to_string()))?;
184            let (current, _) = self
185                .get_pool_info_for_delete_marker(
186                    destination.0,
187                    destination.1,
188                    &ObjectOptions {
189                        no_lock: true,
190                        ..Default::default()
191                    },
192                )
193                .await
194                .map_err(|error| DiskError::other(error.to_string()))?;
195            let current = current.object_info;
196            if transaction.is_lock_lost()
197                || guard.is_lock_lost()
198                || fi.mod_time.is_none()
199                || current.mod_time != fi.mod_time
200                || current.data_dir != fi.data_dir
201                || current.size != fi.size
202                || current.etag != fi.get_etag()
203            {
204                return Err(DiskError::FileAccessDenied);
205            }
206            Some(transaction)
207        } else {
208            None
209        };
210        rename_local_data_with_ctx(
211            &self.ctx,
212            disk_ref,
213            source,
214            fi,
215            destination,
216            RenameDataGuards {
217                external_guard: Some(Arc::new((guard, transaction))),
218                ..Default::default()
219            },
220        )
221        .await
222    }
223
224    /// Execute on this instance's active local disk through the physical owner.
225    pub async fn rename_local_data(
226        &self,
227        disk_ref: &str,
228        source: (&str, &str),
229        fi: &FileInfo,
230        destination: (&str, &str),
231        scanner_token: Option<Uuid>,
232    ) -> DiskResult<RenameDataResp> {
233        let external_guard: Option<Arc<dyn Send + Sync>> = if let Some(token) = scanner_token {
234            Some(Arc::new(
235                self.acquire_scanner_publication_lease_guard(token)
236                    .await
237                    .map_err(|err| DiskError::other(err.to_string()))?,
238            ))
239        } else {
240            None
241        };
242        rename_local_data_with_ctx(
243            &self.ctx,
244            disk_ref,
245            source,
246            fi,
247            destination,
248            RenameDataGuards {
249                scanner_publication_lease_token: scanner_token,
250                external_guard,
251                namespace_owner: None,
252            },
253        )
254        .await
255    }
256
257    pub async fn undo_local_write(
258        &self,
259        disk_ref: &str,
260        volume: &str,
261        path: &str,
262        fi: FileInfo,
263        opts: DeleteOptions,
264    ) -> DiskResult<()> {
265        undo_local_write_with_ctx(&self.ctx, disk_ref, volume, path, fi, opts).await
266    }
267}
268
269// The optional ID is a cold lookup to cache only after final admission.
270async fn local_disk_candidate(ctx: &Arc<InstanceContext>, disk_ref: &str) -> DiskResult<(DiskStore, Option<Uuid>)> {
271    let map = ctx.local_disk_map();
272    if let Some(disk) = map.read().await.get(disk_ref).and_then(Option::as_ref).cloned() {
273        return Ok((disk, None));
274    }
275    let disk_id = Uuid::parse_str(disk_ref).map_err(|_| DiskError::DiskNotFound)?;
276    let cached_path = ctx.local_disk_id_map().read().await.get(&disk_id).cloned();
277    if let Some(path) = cached_path {
278        let cached_disk = map.read().await.get(&path).and_then(Option::as_ref).cloned();
279        if let Some(disk) = cached_disk
280            && matches!(disk.as_ref(), Disk::Local(_))
281            && disk.get_disk_id().await? == Some(disk_id)
282        {
283            return Ok((disk, None));
284        }
285    }
286    let disks: Vec<_> = map.read().await.values().filter_map(Clone::clone).collect();
287    // Disk identity may perform format I/O. No registry guard spans this await.
288    for disk in disks {
289        if matches!(disk.as_ref(), Disk::Local(_)) && disk.get_disk_id().await.ok().flatten() == Some(disk_id) {
290            return Ok((disk, Some(disk_id)));
291        }
292    }
293    Err(DiskError::DiskNotFound)
294}
295
296async fn admit_local_disk(
297    ctx: &Arc<InstanceContext>,
298    disk: &DiskStore,
299    disk_id: Option<Uuid>,
300    mutates_namespace: bool,
301) -> DiskResult<Option<Arc<NamespaceCommitGuard>>> {
302    if !matches!(disk.as_ref(), Disk::Local(_)) {
303        return Err(DiskError::DiskNotFound);
304    }
305    let map = ctx.local_disk_map();
306    let active = map.read().await;
307    if !active
308        .get(&disk.endpoint().to_string())
309        .and_then(Option::as_ref)
310        .is_some_and(|current| Arc::ptr_eq(current, disk))
311    {
312        return Err(DiskError::DiskNotFound);
313    }
314    // Preserve registry -> ID-cache lock order; no filesystem I/O under either.
315    if let Some(disk_id) = disk_id {
316        ctx.local_disk_id_map()
317            .write()
318            .await
319            .insert(disk_id, disk.endpoint().to_string());
320    }
321    // Admission linearizes under the registry read: replacement/quarantine
322    // before this point rejects; later changes do not revoke physical I/O.
323    Ok(mutates_namespace.then(|| ctx.begin_namespace_commit()))
324}
325
326async fn rename_local_data_with_ctx(
327    ctx: &Arc<InstanceContext>,
328    disk_ref: &str,
329    source: (&str, &str),
330    fi: &FileInfo,
331    destination: (&str, &str),
332    mut guards: RenameDataGuards,
333) -> DiskResult<RenameDataResp> {
334    let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?;
335    let mutates_namespace = !is_meta_bucketname(source.0) || !is_meta_bucketname(destination.0);
336    let owner = admit_local_disk(ctx, &disk, disk_id, mutates_namespace).await?;
337    guards.namespace_owner = owner.as_ref().map(|owner| owner.clone() as Arc<dyn Send + Sync>);
338    let result = disk
339        .rename_data_borrowed_with_fence_observed(source.0, source.1, fi, destination.0, destination.1, guards)
340        .await
341        .result;
342    drop(owner);
343    result
344}
345
346async fn undo_local_write_with_ctx(
347    ctx: &Arc<InstanceContext>,
348    disk_ref: &str,
349    volume: &str,
350    path: &str,
351    fi: FileInfo,
352    opts: DeleteOptions,
353) -> DiskResult<()> {
354    if !opts.undo_write {
355        return Err(DiskError::other("target undo requires undo_write"));
356    }
357    let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?;
358    let owner = admit_local_disk(ctx, &disk, disk_id, !is_meta_bucketname(volume)).await?;
359    let physical_owner = owner.as_ref().map(|owner| owner.clone() as Arc<dyn Send + Sync>);
360    let result = disk
361        .undo_write_with_namespace_owner(volume, path, fi, opts, physical_owner)
362        .await;
363    drop(owner);
364    result
365}
366
367async fn remember_local_disk_id(disk: &DiskStore) -> Option<Uuid> {
368    remember_local_disk_id_with_instance_ctx(&crate::runtime::global::current_ctx(), disk).await
369}
370
371async fn remember_local_disk_id_with_instance_ctx(instance_ctx: &Arc<InstanceContext>, disk: &DiskStore) -> Option<Uuid> {
372    let disk_id = disk.get_disk_id().await.ok().flatten()?;
373    record_local_disk_id_if_active(instance_ctx, disk, disk_id)
374        .await
375        .then_some(disk_id)
376}
377
378async fn record_local_disk_id_if_active(instance_ctx: &Arc<InstanceContext>, disk: &DiskStore, disk_id: Uuid) -> bool {
379    let endpoint = disk.endpoint().to_string();
380    let local_disk_map = instance_ctx.local_disk_map();
381    let local_disks = local_disk_map.read().await;
382    let Some(active_disk) = local_disks.get(&endpoint).and_then(Option::as_ref) else {
383        return false;
384    };
385    if !Arc::ptr_eq(active_disk, disk) {
386        return false;
387    }
388
389    // Lock order is local_disk_map -> local_disk_id_map so quarantine is the
390    // linearization point for rejecting an in-flight stale disk snapshot.
391    instance_ctx.local_disk_id_map().write().await.insert(disk_id, endpoint);
392    true
393}
394
395pub async fn find_local_disk(disk_path: &str) -> Option<DiskStore> {
396    runtime_sources::local_disk_by_path(disk_path).await
397}
398
399pub async fn find_local_disk_by_ref(disk_ref: &str) -> Option<DiskStore> {
400    if let Some(disk) = find_local_disk(disk_ref).await {
401        let _ = remember_local_disk_id(&disk).await;
402        return Some(disk);
403    }
404
405    let Ok(disk_id) = Uuid::parse_str(disk_ref) else {
406        return None;
407    };
408
409    if let Some(disk_path) = runtime_sources::local_disk_path_by_id(&disk_id).await
410        && let Some(disk) = find_local_disk(&disk_path).await
411    {
412        return Some(disk);
413    }
414
415    for disk in all_local_disk().await {
416        if remember_local_disk_id(&disk).await == Some(disk_id) {
417            return Some(disk);
418        }
419    }
420
421    None
422}
423
424pub async fn all_local_disk_path() -> Vec<String> {
425    runtime_sources::local_disk_paths().await
426}
427
428pub async fn all_local_disk() -> Vec<DiskStore> {
429    runtime_sources::local_disks().await
430}
431
432pub async fn prewarm_local_disk_id_map() {
433    prewarm_local_disk_id_map_with_instance_ctx(&crate::runtime::global::current_ctx()).await
434}
435
436/// Prewarm the disk-id map of an explicit instance context (Phase 5 follow-up,
437/// backlog#1052): startup passes the context whose disk map it just populated
438/// instead of resolving the process-level default.
439pub async fn prewarm_local_disk_id_map_with_instance_ctx(instance_ctx: &Arc<InstanceContext>) {
440    let disks: Vec<DiskStore> = instance_ctx
441        .local_disk_map()
442        .read()
443        .await
444        .values()
445        .filter_map(|v| v.as_ref().cloned())
446        .collect();
447    for disk in disks {
448        if let Err(err) = disk.get_disk_id().await {
449            debug!(
450                event = EVENT_LOCAL_DISK_ID_PREWARM_SKIPPED,
451                component = LOG_COMPONENT_ECSTORE,
452                subsystem = LOG_SUBSYSTEM_DISK_STARTUP,
453                disk_endpoint = %disk.endpoint(),
454                error = %err,
455                "Skipped local disk id prewarm"
456            );
457            continue;
458        }
459
460        let _ = remember_local_disk_id_with_instance_ctx(instance_ctx, &disk).await;
461    }
462}
463
464pub async fn init_local_disks(endpoint_pools: EndpointServerPools) -> Result<()> {
465    init_local_disks_with_instance_ctx(&crate::runtime::global::current_ctx(), endpoint_pools).await
466}
467
468/// Register the pools' local disks into an explicit instance context (Phase 5
469/// follow-up, backlog#1052). The legacy [`init_local_disks`] entry resolves the
470/// process-level default context; startup paths that own a context pass it here
471/// so a future second instance's disks cannot leak into the first one's registry.
472pub async fn init_local_disks_with_instance_ctx(
473    instance_ctx: &Arc<InstanceContext>,
474    endpoint_pools: EndpointServerPools,
475) -> Result<()> {
476    let opt = &DiskOption {
477        cleanup: true,
478        health_check: true,
479    };
480
481    runtime_sources::initialize_local_disk_maps(instance_ctx, endpoint_pools, opt).await
482}
483
484pub fn init_lock_clients(endpoint_pools: EndpointServerPools) {
485    let mut unique_endpoints: HashMap<String, &Endpoint> = HashMap::new();
486
487    for pool_eps in endpoint_pools.as_ref().iter() {
488        for ep in pool_eps.endpoints.as_ref().iter() {
489            unique_endpoints.insert(ep.host_port(), ep);
490        }
491    }
492
493    let mut clients = HashMap::new();
494    let mut first_local_client_set = false;
495
496    for (key, endpoint) in unique_endpoints {
497        if endpoint.is_local {
498            let local_client = Arc::new(LocalClient::new()) as Arc<dyn LockClient>;
499
500            // Store the first LocalClient globally for use by other modules
501            if !first_local_client_set {
502                if let Err(e) = runtime_sources::set_primary_lock_client(local_client.clone()) {
503                    // If already set, ignore the error (another thread may have set it)
504                    debug!(
505                        event = EVENT_LOCK_CLIENT_INITIALIZATION_FAILED,
506                        component = LOG_COMPONENT_ECSTORE,
507                        subsystem = LOG_SUBSYSTEM_DISK_STARTUP,
508                        error = ?e,
509                        reason = "global_lock_client_already_set",
510                        "Skipped global lock client publication"
511                    );
512                } else {
513                    first_local_client_set = true;
514                }
515            }
516
517            clients.insert(key, local_client);
518        } else {
519            clients.insert(key, Arc::new(RemoteClient::new(endpoint.url.to_string())) as Arc<dyn LockClient>);
520        }
521    }
522
523    // Store the lock clients map globally
524    if runtime_sources::set_lock_clients(clients).is_err() {
525        error!(
526            event = EVENT_LOCK_CLIENT_INITIALIZATION_FAILED,
527            component = LOG_COMPONENT_ECSTORE,
528            subsystem = LOG_SUBSYSTEM_DISK_STARTUP,
529            reason = "set_global_lock_clients_failed",
530            "Failed to initialize lock clients"
531        );
532    }
533}
534
535fn endpoint_rpc_authority(endpoint: &Endpoint) -> Option<String> {
536    let host = endpoint.url.host_str()?;
537    let host = if host.contains(':') && !host.starts_with('[') {
538        format!("[{host}]")
539    } else {
540        host.to_string()
541    };
542    Some(match endpoint.url.port() {
543        Some(port) => format!("{host}:{port}"),
544        None => host,
545    })
546}
547
548pub(super) async fn init_local_peer(endpoint_pools: &EndpointServerPools, host: &String, port: &String) {
549    let mut peer_set = Vec::new();
550    endpoint_pools.as_ref().iter().for_each(|endpoints| {
551        endpoints.endpoints.as_ref().iter().for_each(|endpoint| {
552            if endpoint.get_type() == EndpointType::Url
553                && endpoint.is_local
554                && let Some(authority) = endpoint_rpc_authority(endpoint)
555            {
556                peer_set.push(authority);
557            }
558        });
559    });
560
561    if peer_set.is_empty() {
562        if !host.is_empty() {
563            runtime_sources::set_local_node_name(format!("{host}:{port}")).await;
564            return;
565        }
566
567        runtime_sources::set_local_node_name(format!("127.0.0.1:{port}")).await;
568        return;
569    }
570
571    runtime_sources::set_local_node_name(peer_set[0].clone()).await;
572}
573
574pub async fn get_disk_infos(disks: &[Option<DiskStore>]) -> Vec<Option<DiskInfo>> {
575    let opts = &DiskInfoOptions::default();
576    let mut res = vec![None; disks.len()];
577    for (idx, disk_op) in disks.iter().enumerate() {
578        if let Some(disk) = disk_op
579            && let Ok(info) = disk.disk_info(opts).await
580        {
581            res[idx] = Some(info);
582        }
583    }
584
585    res
586}
587
588#[cfg(test)]
589mod tests {
590    use super::*;
591    use crate::disk::new_disk;
592    use crate::layout::endpoints::{Endpoints, PoolEndpoints};
593
594    fn single_local_disk_pools(dir: &std::path::Path) -> EndpointServerPools {
595        let mut endpoint = Endpoint::try_from(dir.to_str().expect("temp dir path should be utf-8")).expect("local endpoint");
596        endpoint.set_pool_index(0);
597        endpoint.set_set_index(0);
598        endpoint.set_disk_index(0);
599
600        EndpointServerPools(vec![PoolEndpoints {
601            legacy: false,
602            set_count: 1,
603            drives_per_set: 1,
604            endpoints: Endpoints::from(vec![endpoint]),
605            cmd_line: "instance-ctx-disk-registry-test".to_string(),
606            platform: "test".to_string(),
607        }])
608    }
609
610    async fn target_disk(ctx: &Arc<InstanceContext>, root: &std::path::Path, id: Uuid) -> DiskStore {
611        let mut format = crate::layout::format::FormatV3::new(1, 1);
612        format.erasure.this = id;
613        format.erasure.sets[0][0] = id;
614        let meta = root.join(crate::disk::RUSTFS_META_BUCKET);
615        tokio::fs::create_dir_all(&meta).await.expect("create format volume");
616        tokio::fs::write(
617            meta.join(crate::disk::FORMAT_CONFIG_FILE),
618            serde_json::to_vec(&format).expect("encode format"),
619        )
620        .await
621        .expect("write real disk identity");
622        let mut endpoint = Endpoint::try_from(root.to_str().expect("UTF-8 root")).expect("endpoint");
623        endpoint.set_pool_index(0);
624        endpoint.set_set_index(0);
625        endpoint.set_disk_index(0);
626        let disk = new_disk(
627            &endpoint,
628            &DiskOption {
629                cleanup: false,
630                health_check: false,
631            },
632        )
633        .await
634        .expect("open real local disk");
635        assert_eq!(disk.get_disk_id().await.expect("read disk format identity"), Some(id));
636        ctx.local_disk_map()
637            .write()
638            .await
639            .insert(disk.endpoint().to_string(), Some(disk.clone()));
640        disk
641    }
642
643    fn target_file_info(object: &str, version: Uuid, body: &'static [u8]) -> FileInfo {
644        let mut fi = FileInfo::new(object, 1, 0);
645        fi.erasure.index = 1;
646        fi.version_id = Some(version);
647        fi.mod_time = Some(OffsetDateTime::now_utc());
648        fi.size = i64::try_from(body.len()).expect("fixture length");
649        fi.parts = vec![rustfs_filemeta::ObjectPartInfo {
650            number: 1,
651            size: body.len(),
652            actual_size: fi.size,
653            ..Default::default()
654        }];
655        fi.data = Some(bytes::Bytes::from_static(body));
656        fi.set_inline_data();
657        fi
658    }
659
660    async fn seed_target(disk: &DiskStore, volume: &str, object: &str, fi: FileInfo) -> Vec<u8> {
661        let dir = disk.path().join(volume);
662        tokio::fs::create_dir_all(&dir).await.expect("real fixture volume");
663        disk.write_metadata(volume, volume, object, fi.clone())
664            .await
665            .expect("seed real metadata");
666        let read = disk
667            .read_version(
668                volume,
669                volume,
670                object,
671                &fi.version_id.expect("fixture version").to_string(),
672                &crate::disk::ReadOptions {
673                    read_data: true,
674                    ..Default::default()
675                },
676            )
677            .await
678            .expect("read fixture before mutation");
679        assert_eq!(read.data, fi.data, "fixture must contain readable inline bytes");
680        tokio::fs::read(dir.join(object).join(crate::disk::STORAGE_FORMAT_FILE))
681            .await
682            .expect("seeded metadata bytes")
683    }
684
685    #[tokio::test]
686    async fn target_uuid_lookup_binds_real_disk_and_owner_to_one_instance() {
687        for warm in [false, true] {
688            let ctx_a = Arc::new(InstanceContext::new());
689            let ctx_b = Arc::new(InstanceContext::new());
690            let a = tempfile::tempdir().expect("A root");
691            let b = tempfile::tempdir().expect("B root");
692            let id = Uuid::new_v4();
693            let disk_a = target_disk(&ctx_a, a.path(), id).await;
694            let disk_b = target_disk(&ctx_b, b.path(), id).await;
695            if warm {
696                assert!(record_local_disk_id_if_active(&ctx_a, &disk_a, id).await);
697                assert!(record_local_disk_id_if_active(&ctx_b, &disk_b, id).await);
698            }
699            let version = Uuid::new_v4();
700            let fi = target_file_info("destination", version, b"new-A");
701            for disk in [&disk_a, &disk_b] {
702                seed_target(disk, "target-bucket", "staged", fi.clone()).await;
703            }
704            let b_before = seed_target(
705                &disk_b,
706                "target-bucket",
707                "destination",
708                target_file_info("destination", version, b"old-B"),
709            )
710            .await;
711            let store = super::super::tests::build_store_with_ctx(ctx_a.clone());
712            store
713                .rename_local_data(&id.to_string(), ("target-bucket", "staged"), &fi, ("target-bucket", "destination"), None)
714                .await
715                .expect("rename on A");
716            let read = disk_a
717                .read_version(
718                    "target-bucket",
719                    "target-bucket",
720                    "destination",
721                    &version.to_string(),
722                    &crate::disk::ReadOptions {
723                        read_data: true,
724                        ..Default::default()
725                    },
726                )
727                .await
728                .expect("read committed A");
729            assert_eq!(read.data, fi.data, "warm={warm}");
730            assert_eq!(
731                tokio::fs::read(b.path().join("target-bucket/destination/xl.meta"))
732                    .await
733                    .expect("B metadata"),
734                b_before
735            );
736            assert!(b.path().join("target-bucket/staged/xl.meta").exists());
737            assert!(ctx_a.namespace_commit_generation() > 0);
738            assert_eq!(ctx_b.namespace_commit_generation(), 0);
739            assert!(!ctx_a.namespace_commits_pending());
740            assert!(!ctx_b.namespace_commits_pending());
741            assert_eq!(ctx_a.local_disk_id_map().read().await.get(&id), Some(&disk_a.endpoint().to_string()));
742        }
743    }
744
745    #[tokio::test]
746    async fn target_admission_rejects_removed_quarantined_and_replaced_arcs() {
747        let ctx = Arc::new(InstanceContext::new());
748        let root = tempfile::tempdir().expect("root");
749        let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
750        let endpoint = disk.endpoint().to_string();
751        for state in ["removed", "quarantined", "replaced"] {
752            let replacement = new_disk(
753                &disk.endpoint(),
754                &DiskOption {
755                    cleanup: false,
756                    health_check: false,
757                },
758            )
759            .await
760            .expect("separate active Arc");
761            let map = ctx.local_disk_map();
762            let mut entries = map.write().await;
763            match state {
764                "removed" => {
765                    entries.remove(&endpoint);
766                }
767                "quarantined" => {
768                    entries.insert(endpoint.clone(), None);
769                }
770                _ => {
771                    entries.insert(endpoint.clone(), Some(replacement));
772                }
773            }
774            drop(entries);
775            assert!(
776                matches!(admit_local_disk(&ctx, &disk, None, true).await, Err(DiskError::DiskNotFound)),
777                "{state}"
778            );
779            assert!(!ctx.namespace_commits_pending());
780            assert_eq!(ctx.namespace_commit_generation(), 0);
781        }
782    }
783
784    #[tokio::test]
785    async fn target_uuid_cache_cannot_admit_a_different_format_at_the_same_path() {
786        let ctx = Arc::new(InstanceContext::new());
787        let root = tempfile::tempdir().expect("root");
788        let old_id = Uuid::new_v4();
789        let old = target_disk(&ctx, root.path(), old_id).await;
790        assert!(record_local_disk_id_if_active(&ctx, &old, old_id).await);
791        let replacement_id = Uuid::new_v4();
792        let replacement = target_disk(&ctx, root.path(), replacement_id).await;
793        assert!(!Arc::ptr_eq(&old, &replacement));
794        assert!(matches!(
795            local_disk_candidate(&ctx, &old_id.to_string()).await,
796            Err(DiskError::DiskNotFound)
797        ));
798        let (candidate, verified) = local_disk_candidate(&ctx, &replacement_id.to_string())
799            .await
800            .expect("replacement UUID");
801        assert!(Arc::ptr_eq(&candidate, &replacement));
802        assert_eq!(verified, Some(replacement_id));
803        assert!(!ctx.namespace_commits_pending());
804    }
805
806    #[tokio::test]
807    async fn bootstrap_rejects_user_volumes_aliases_and_scanner_tokens_without_mutation() {
808        let ctx = Arc::new(InstanceContext::new());
809        let root = tempfile::tempdir().expect("root");
810        let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
811        let target = BootstrapLocalTarget::new(ctx.clone());
812        let fi = target_file_info("destination", Uuid::new_v4(), b"body");
813        let user_before = seed_target(&disk, "victim", "staged", fi.clone()).await;
814        let meta_before = seed_target(&disk, ".rustfs.sys/tmp", "staged", fi.clone()).await;
815        for invalid in [
816            "victim",
817            ".rustfs.sys/../victim",
818            ".rustfs.sys/./tmp",
819            ".rustfs.sys/ .. /victim",
820            ".rustfs.sys\\..\\victim",
821            ".minio.sys/../victim",
822        ] {
823            for (src, dst) in [(invalid, ".rustfs.sys/tmp"), (".rustfs.sys/tmp", invalid)] {
824                assert!(
825                    target
826                        .rename_local_data(&disk.endpoint().to_string(), (src, "staged"), &fi, (dst, "destination"), None)
827                        .await
828                        .is_err(),
829                    "src={src}, dst={dst}"
830                );
831            }
832            assert!(
833                target
834                    .undo_local_write(
835                        &disk.endpoint().to_string(),
836                        invalid,
837                        "staged",
838                        fi.clone(),
839                        DeleteOptions {
840                            undo_write: true,
841                            ..Default::default()
842                        }
843                    )
844                    .await
845                    .is_err(),
846                "{invalid}"
847            );
848        }
849        assert!(
850            target
851                .rename_local_data(
852                    &disk.endpoint().to_string(),
853                    (".rustfs.sys/tmp", "staged"),
854                    &fi,
855                    (".rustfs.sys/tmp", "destination"),
856                    Some(Uuid::new_v4())
857                )
858                .await
859                .is_err()
860        );
861        assert_eq!(
862            tokio::fs::read(root.path().join("victim/staged/xl.meta"))
863                .await
864                .expect("user source"),
865            user_before
866        );
867        assert_eq!(
868            tokio::fs::read(root.path().join(".rustfs.sys/tmp/staged/xl.meta"))
869                .await
870                .expect("metadata source"),
871            meta_before
872        );
873        assert!(!root.path().join("victim/destination").exists());
874        assert!(!root.path().join(".rustfs.sys/tmp/destination").exists());
875        assert_eq!(ctx.namespace_commit_generation(), 0);
876        assert!(!ctx.namespace_commits_pending());
877    }
878
879    #[tokio::test]
880    async fn bootstrap_allows_internal_multisegment_rename_without_namespace_owner() {
881        for volume in [".rustfs.sys/tmp", ".rustfs.sys/multipart", ".minio.sys/config"] {
882            let ctx = Arc::new(InstanceContext::new());
883            let root = tempfile::tempdir().expect("root");
884            let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
885            let fi = target_file_info("destination", Uuid::new_v4(), b"internal-CAS-body");
886            seed_target(&disk, volume, "staged", fi.clone()).await;
887            BootstrapLocalTarget::new(ctx.clone())
888                .rename_local_data(&disk.endpoint().to_string(), (volume, "staged"), &fi, (volume, "destination"), None)
889                .await
890                .expect("legitimate bootstrap metadata write");
891            let read = disk
892                .read_version(
893                    volume,
894                    volume,
895                    "destination",
896                    &fi.version_id.expect("version").to_string(),
897                    &crate::disk::ReadOptions {
898                        read_data: true,
899                        ..Default::default()
900                    },
901                )
902                .await
903                .expect("read bootstrap result");
904            assert_eq!(read.data, fi.data);
905            assert_eq!(ctx.namespace_commit_generation(), 0);
906            assert!(!ctx.namespace_commits_pending());
907        }
908    }
909
910    #[cfg(not(windows))]
911    #[tokio::test]
912    async fn target_user_source_to_internal_destination_retains_owner_after_cancellation() {
913        use crate::disk::os::prepared_publication_test_hooks as hooks;
914        use futures::FutureExt;
915        use std::time::Duration;
916
917        let ctx = Arc::new(InstanceContext::new());
918        let root = tempfile::tempdir().expect("source-volume root");
919        let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
920        let store = super::super::tests::build_store_with_ctx(ctx.clone());
921        let version = Uuid::new_v4();
922        let fi = target_file_info("object", version, b"user-source-inline-body");
923        fi.validate_for_metadata_read().expect("valid real inline metadata");
924        let source_before = seed_target(&disk, "photos", "object", fi.clone()).await;
925        assert!(!source_before.is_empty());
926        tokio::fs::create_dir_all(root.path().join(crate::disk::RUSTFS_META_TMP_BUCKET))
927            .await
928            .expect("internal staging volume");
929        let destination = disk
930            .get_object_path_for_io_if_local(crate::disk::RUSTFS_META_TMP_BUCKET, "object")
931            .expect("local disk")
932            .expect("actual destination object key");
933        let destination_metadata = destination.join(crate::disk::STORAGE_FORMAT_FILE);
934        assert!(!destination_metadata.exists());
935        assert!(!ctx.namespace_commits_pending());
936        let generation = ctx.namespace_commit_generation();
937        let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
938        let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
939        let _hook = hooks::install(&destination_metadata, move || {
940            let _ = entered_tx.send(());
941            let _ = release_rx.recv();
942        });
943        let disk_ref = disk.endpoint().to_string();
944        let rename_fi = fi.clone();
945        let mut rename = tokio::spawn(async move {
946            store
947                .rename_local_data(
948                    &disk_ref,
949                    ("photos", "object"),
950                    &rename_fi,
951                    (crate::disk::RUSTFS_META_TMP_BUCKET, "object"),
952                    None,
953                )
954                .await
955        });
956        let mut entered = false;
957        let mut joined = false;
958        let observations = std::panic::AssertUnwindSafe(async {
959            tokio::time::timeout(Duration::from_secs(10), async {
960                tokio::select! {
961                    result = &mut rename => {
962                        joined = true;
963                        panic!("rename completed before prepared publication: {result:?}");
964                    }
965                    result = entered_rx => {
966                        result.expect("real prepared rename must enter");
967                        entered = true;
968                    }
969                }
970            })
971            .await
972            .expect("bounded physical entry");
973            let at_entry = (ctx.namespace_commits_pending(), ctx.namespace_commit_generation());
974            assert!(!rename.is_finished(), "caller must still await the paused physical rename");
975            rename.abort();
976            let cancelled = tokio::time::timeout(Duration::from_secs(5), &mut rename)
977                .await
978                .expect("caller cancellation must finish while physical publication is paused");
979            joined = true;
980            let after_cancel = (ctx.namespace_commits_pending(), ctx.namespace_commit_generation());
981            (at_entry, after_cancel, cancelled)
982        })
983        .catch_unwind()
984        .await;
985
986        // Release on every observation failure. Pending alone is not a drain
987        // oracle: the implementation under test can fail to create the owner.
988        drop(release_tx);
989        if !joined {
990            rename.abort();
991            joined = tokio::time::timeout(Duration::from_secs(5), &mut rename).await.is_ok();
992        }
993        let physical_drained = tokio::time::timeout(Duration::from_secs(10), hooks::drain_namespace_key(&destination)).await;
994        let owner_drained = tokio::time::timeout(Duration::from_secs(10), async {
995            while ctx.namespace_commits_pending() {
996                tokio::task::yield_now().await;
997            }
998        })
999        .await;
1000        if !entered || !joined || physical_drained.is_err() || owner_drained.is_err() {
1001            // Without proven physical entry/drain, keep the root instead of
1002            // deleting files that a detached local executor may still use.
1003            let retained = root.keep();
1004            eprintln!("source-volume cleanup incomplete: entered={entered}, joined={joined}, retained={retained:?}");
1005            if let Err(panic) = observations {
1006                std::panic::resume_unwind(panic);
1007            }
1008            panic!("source-volume physical cleanup did not finish: retained={retained:?}");
1009        }
1010        let (at_entry, after_cancel, cancelled) = match observations {
1011            Ok(observations) => observations,
1012            Err(panic) => std::panic::resume_unwind(panic),
1013        };
1014        let latest = tokio::time::timeout(
1015            Duration::from_secs(5),
1016            disk.read_version(
1017                crate::disk::RUSTFS_META_TMP_BUCKET,
1018                crate::disk::RUSTFS_META_TMP_BUCKET,
1019                "object",
1020                "",
1021                &crate::disk::ReadOptions {
1022                    read_data: true,
1023                    ..Default::default()
1024                },
1025            ),
1026        )
1027        .await;
1028        let source =
1029            tokio::time::timeout(Duration::from_secs(5), tokio::fs::read(root.path().join("photos/object/xl.meta"))).await;
1030        let after_drain = (ctx.namespace_commits_pending(), ctx.namespace_commit_generation());
1031        assert!(cancelled.expect_err("caller must return cancellation").is_cancelled());
1032        let latest = latest
1033            .expect("latest read must finish")
1034            .expect("late physical commit must be readable");
1035        assert_eq!(latest.data, fi.data);
1036        assert_eq!(latest.version_id, Some(version));
1037        assert_eq!(
1038            source
1039                .expect("source observation must finish")
1040                .expect_err("user source metadata must have moved")
1041                .kind(),
1042            std::io::ErrorKind::NotFound
1043        );
1044        assert_eq!(
1045            (at_entry, after_cancel, after_drain),
1046            ((true, generation + 1), (true, generation + 1), (false, generation + 2)),
1047            "a real user source mutation must remain counted through its cancelled caller and physical drain"
1048        );
1049    }
1050
1051    #[cfg(not(windows))]
1052    #[tokio::test]
1053    async fn target_rename_cancellation_retains_real_namespace_and_scanner_owners() {
1054        use crate::disk::os::prepared_publication_test_hooks as hooks;
1055        let ctx = Arc::new(InstanceContext::new());
1056        let sibling = Arc::new(InstanceContext::new());
1057        let root = tempfile::tempdir().expect("root");
1058        let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
1059        let store = super::super::tests::build_store_with_ctx(ctx.clone());
1060        let fi = target_file_info("destination", Uuid::new_v4(), b"physically-owned");
1061        seed_target(&disk, "target-bucket", "staged", fi.clone()).await;
1062        let (token, _) = store
1063            .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
1064            .await
1065            .expect("real scanner token in A");
1066        let destination = disk
1067            .get_object_path_for_io_if_local("target-bucket", "destination/xl.meta")
1068            .expect("local disk")
1069            .expect("destination IO path");
1070        let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
1071        let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
1072        let _hook = hooks::install(&destination, move || {
1073            let _ = entered_tx.send(());
1074            let _ = release_rx.recv();
1075        });
1076        let disk_ref = disk.endpoint().to_string();
1077        let mut rename = Box::pin(store.rename_local_data(
1078            &disk_ref,
1079            ("target-bucket", "staged"),
1080            &fi,
1081            ("target-bucket", "destination"),
1082            Some(token),
1083        ));
1084        tokio::time::timeout(std::time::Duration::from_secs(10), async {
1085            tokio::select! {
1086                result = &mut rename => panic!("rename completed before physical pause: {result:?}"),
1087                entered = entered_rx => entered.expect("physical rename entered"),
1088            }
1089        })
1090        .await
1091        .expect("bounded physical entry");
1092        drop(rename);
1093        assert!(store.scanner_data_usage_publication_blocked().await);
1094        assert!(ctx.namespace_commits_pending());
1095        assert!(!sibling.namespace_commits_pending());
1096        assert!(
1097            store
1098                .rename_local_data(&disk_ref, ("target-bucket", "staged"), &fi, ("target-bucket", "another"), Some(token))
1099                .await
1100                .is_err(),
1101            "real pending rename blocks another scanner publication"
1102        );
1103        assert!(store.release_scanner_publication_lease(token).await, "remove registered token");
1104        let gate = ctx.data_movement_operation_gate();
1105        assert!(
1106            gate.clone().try_write_owned().is_err(),
1107            "physical operation still owns the scanner read guard"
1108        );
1109        drop(release_tx);
1110        let _drained = tokio::time::timeout(std::time::Duration::from_secs(10), gate.write_owned())
1111            .await
1112            .expect("physical tail must release scanner guard");
1113        tokio::time::timeout(std::time::Duration::from_secs(10), async {
1114            while ctx.namespace_commits_pending() {
1115                tokio::task::yield_now().await;
1116            }
1117        })
1118        .await
1119        .expect("namespace owner drains");
1120        let read = disk
1121            .read_version(
1122                "target-bucket",
1123                "target-bucket",
1124                "destination",
1125                &fi.version_id.expect("version").to_string(),
1126                &crate::disk::ReadOptions {
1127                    read_data: true,
1128                    ..Default::default()
1129                },
1130            )
1131            .await
1132            .expect("read actual late commit");
1133        assert_eq!(read.data, fi.data);
1134        assert!(ctx.namespace_commit_generation() >= 2);
1135        assert_eq!(sibling.namespace_commit_generation(), 0);
1136    }
1137
1138    #[tokio::test]
1139    async fn target_ready_rejects_unknown_foreign_released_and_expired_scanner_tokens() {
1140        let ctx = Arc::new(InstanceContext::new());
1141        let other = Arc::new(InstanceContext::new());
1142        let store = super::super::tests::build_store_with_ctx(ctx.clone());
1143        let other_store = super::super::tests::build_store_with_ctx(other);
1144        let root = tempfile::tempdir().expect("root");
1145        let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
1146        let fi = target_file_info("destination", Uuid::new_v4(), b"unchanged");
1147        let before = seed_target(&disk, "target-bucket", "staged", fi.clone()).await;
1148        let ttl = crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL;
1149        let (foreign, _) = other_store.acquire_scanner_publication_lease(0, ttl).await.expect("B token");
1150        let (released, _) = store.acquire_scanner_publication_lease(0, ttl).await.expect("A token");
1151        assert!(store.release_scanner_publication_lease(released).await);
1152        let (valid, _) = store.acquire_scanner_publication_lease(0, ttl).await.expect("new A token");
1153        for token in [Uuid::new_v4(), foreign, released] {
1154            assert!(
1155                store
1156                    .rename_local_data(
1157                        &disk.endpoint().to_string(),
1158                        ("target-bucket", "staged"),
1159                        &fi,
1160                        ("target-bucket", "destination"),
1161                        Some(token)
1162                    )
1163                    .await
1164                    .is_err()
1165            );
1166        }
1167        tokio::time::pause();
1168        tokio::time::advance(ttl + std::time::Duration::from_secs(1)).await;
1169        tokio::time::resume();
1170        assert!(
1171            store
1172                .rename_local_data(
1173                    &disk.endpoint().to_string(),
1174                    ("target-bucket", "staged"),
1175                    &fi,
1176                    ("target-bucket", "destination"),
1177                    Some(valid)
1178                )
1179                .await
1180                .is_err(),
1181            "expired real token"
1182        );
1183        let _ = other_store.release_scanner_publication_lease(foreign).await;
1184        assert_eq!(
1185            tokio::fs::read(root.path().join("target-bucket/staged/xl.meta"))
1186                .await
1187                .expect("source bytes"),
1188            before
1189        );
1190        assert!(!root.path().join("target-bucket/destination").exists());
1191        assert!(!ctx.namespace_commits_pending());
1192    }
1193
1194    #[tokio::test]
1195    async fn completed_namespace_commit_rejects_stale_scanner_target_admission() {
1196        use futures::FutureExt;
1197        use std::time::{Duration, Instant};
1198        use tokio::time::timeout;
1199
1200        let ttl = crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL;
1201        let ctx = Arc::new(InstanceContext::new());
1202        let root = tempfile::tempdir().expect("target root");
1203        let user_volume = "target-bucket";
1204        let metadata_volume = ".rustfs.sys/tmp";
1205        let setup = std::panic::AssertUnwindSafe(timeout(ttl / 2, async {
1206            let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
1207            let old_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed fixture modtime");
1208            let new_time = old_time + time::Duration::seconds(1);
1209            let mut user_old = target_file_info("destination", Uuid::new_v4(), b"user-before-scan");
1210            let mut user_new = target_file_info("destination", Uuid::new_v4(), b"user-after-scan");
1211            let mut metadata_old = target_file_info("destination", Uuid::new_v4(), b"metadata-before-stale-admission");
1212            let mut metadata_new = target_file_info("destination", Uuid::new_v4(), b"metadata-from-stale-scan");
1213            user_old.mod_time = Some(old_time);
1214            metadata_old.mod_time = Some(old_time);
1215            user_new.mod_time = Some(new_time);
1216            metadata_new.mod_time = Some(new_time);
1217            seed_target(&disk, user_volume, "destination", user_old).await;
1218            seed_target(&disk, user_volume, "staged", user_new.clone()).await;
1219            let metadata_before = seed_target(&disk, metadata_volume, "destination", metadata_old.clone()).await;
1220            seed_target(&disk, metadata_volume, "staged", metadata_new.clone()).await;
1221            (disk, user_new, metadata_old, metadata_new, metadata_before)
1222        }))
1223        .catch_unwind()
1224        .await;
1225        let (disk, user_new, metadata_old, metadata_new, metadata_before) = match setup {
1226            Ok(Ok(setup)) => setup,
1227            Ok(Err(error)) => {
1228                let retained = root.keep();
1229                panic!("fixture initialization must finish before lease acquisition: {error}; retained={retained:?}");
1230            }
1231            Err(panic) => {
1232                let retained = root.keep();
1233                eprintln!("fixture setup panicked; retained={retained:?}");
1234                std::panic::resume_unwind(panic);
1235            }
1236        };
1237        let store = super::super::tests::build_store_with_ctx(ctx.clone());
1238        let disk_ref = disk.endpoint().to_string();
1239        let metadata_path = root.path().join(metadata_volume).join("destination/xl.meta");
1240        let read_options = crate::disk::ReadOptions {
1241            read_data: true,
1242            ..Default::default()
1243        };
1244        let namespace_before = ctx.namespace_commit_generation();
1245        let namespace_completed = namespace_before.checked_add(2).expect("one begin and one physical drain");
1246        let movement_before = ctx.data_movement_generation();
1247        let operation_epoch_before = ctx.data_movement_operation_epoch();
1248        let mut acquired_token = None;
1249
1250        // Use real monotonic time: expiry must not supply a false rejection.
1251        let started = Instant::now();
1252        let observations = std::panic::AssertUnwindSafe(timeout(ttl / 2, async {
1253            assert!(!ctx.namespace_commits_pending(), "all fixture writes precede the baseline");
1254            let (token, generation) = store
1255                .acquire_scanner_publication_lease(movement_before, ttl)
1256                .await
1257                .expect("precondition: acquire a real current lease");
1258            acquired_token = Some(token);
1259            assert_eq!(generation, movement_before);
1260            store
1261                .validate_scanner_publication_lease(token, movement_before)
1262                .await
1263                .expect("precondition: validate succeeds before the ordinary commit");
1264            drop(
1265                store
1266                    .acquire_scanner_publication_lease_guard(token)
1267                    .await
1268                    .expect("precondition: actual target guard lookup accepts the live token"),
1269            );
1270            assert_eq!(ctx.namespace_commit_generation(), namespace_before);
1271            store
1272                .rename_local_data(&disk_ref, (user_volume, "staged"), &user_new, (user_volume, "destination"), None)
1273                .await
1274                .expect("precondition: complete a real ordinary rename after the successful lease checks");
1275            let ordinary = disk
1276                .read_version(user_volume, user_volume, "destination", "", &read_options)
1277                .await
1278                .expect("precondition: read latest ordinary committed body");
1279            assert_eq!(ordinary.data, user_new.data);
1280            assert_eq!(ordinary.version_id, user_new.version_id);
1281            while ctx.namespace_commits_pending() {
1282                tokio::task::yield_now().await;
1283            }
1284            assert_eq!(
1285                ctx.namespace_commit_generation(),
1286                namespace_completed,
1287                "ordinary physical owner must fully drain"
1288            );
1289            assert_eq!(ctx.data_movement_generation(), movement_before);
1290            assert_eq!(ctx.data_movement_operation_epoch(), operation_epoch_before);
1291
1292            // Collect both admissions and the physical result before asserting
1293            // rejection, so the first failure cannot hide the second entrypoint.
1294            let stale_validate = store.validate_scanner_publication_lease(token, movement_before).await;
1295            let target_rename = store
1296                .rename_local_data(
1297                    &disk_ref,
1298                    (metadata_volume, "staged"),
1299                    &metadata_new,
1300                    (metadata_volume, "destination"),
1301                    Some(token),
1302                )
1303                .await;
1304            let metadata_latest = disk
1305                .read_version(metadata_volume, metadata_volume, "destination", "", &read_options)
1306                .await;
1307            let metadata_after = tokio::fs::read(&metadata_path).await;
1308            let user_latest = disk
1309                .read_version(user_volume, user_volume, "destination", "", &read_options)
1310                .await;
1311            let final_state = (
1312                ctx.namespace_commits_pending(),
1313                ctx.namespace_commit_generation(),
1314                ctx.data_movement_generation(),
1315                ctx.data_movement_operation_epoch(),
1316            );
1317            (stale_validate, target_rename, metadata_latest, metadata_after, user_latest, final_state)
1318        }))
1319        .catch_unwind()
1320        .await;
1321        let observed_after = started.elapsed();
1322        // A table release does not drain an independently owned metadata call.
1323        // Collect every release outcome before checking either ownership chain.
1324        let cleanup = std::panic::AssertUnwindSafe(async {
1325            let mut releases = Vec::new();
1326            if let Some(token) = acquired_token {
1327                releases.push(
1328                    std::panic::AssertUnwindSafe(timeout(Duration::from_secs(5), store.release_scanner_publication_lease(token)))
1329                        .catch_unwind()
1330                        .await,
1331                );
1332            }
1333            let movement_guard = timeout(Duration::from_secs(5), ctx.data_movement_operation_gate().write_owned()).await;
1334            let namespace_drained = timeout(Duration::from_secs(5), async {
1335                while ctx.namespace_commits_pending() {
1336                    tokio::task::yield_now().await;
1337                }
1338            })
1339            .await;
1340            let drained = releases.iter().all(|release| matches!(release, Ok(Ok(_))))
1341                && movement_guard.is_ok()
1342                && namespace_drained.is_ok();
1343            #[cfg(not(windows))]
1344            let drained = {
1345                use crate::disk::os::prepared_publication_test_hooks as hooks;
1346
1347                let mut keys_drained = true;
1348                for volume in [user_volume, metadata_volume] {
1349                    // The physical lease uses the actual IO object directory,
1350                    // including descriptor-rooted aliases, not its xl.meta file.
1351                    let key_drained = match disk.get_object_path_for_io_if_local(volume, "destination") {
1352                        Some(Ok(path)) => timeout(Duration::from_secs(5), hooks::drain_namespace_key(&path))
1353                            .await
1354                            .is_ok(),
1355                        _ => false,
1356                    };
1357                    keys_drained &= key_drained;
1358                }
1359                drained && keys_drained
1360            };
1361            drop(movement_guard);
1362            if let Some(panic) = releases.into_iter().find_map(|release| release.err()) {
1363                std::panic::resume_unwind(panic);
1364            }
1365            drained
1366        })
1367        .catch_unwind()
1368        .await;
1369        // Generic cancelled IO cannot be proved drained by a namespace key.
1370        // Retain on any failed observation, even if best-effort cleanup succeeds.
1371        if !matches!(&observations, Ok(Ok((_, _, Ok(_), Ok(_), Ok(_), _)))) || !matches!(&cleanup, Ok(true)) {
1372            let retained = root.keep();
1373            eprintln!("fixture observations or physical cleanup incomplete; retained={retained:?}");
1374        }
1375        let observations = match observations {
1376            Ok(result) => result.expect("fixture timed out before complete observations; not a namespace rejection result"),
1377            Err(panic) => std::panic::resume_unwind(panic),
1378        };
1379        match cleanup {
1380            Ok(drained) => assert!(drained, "fixture physical cleanup must finish before deleting its root"),
1381            Err(panic) => std::panic::resume_unwind(panic),
1382        }
1383        assert!(
1384            observed_after < ttl,
1385            "fixture lease expired before observation: elapsed={observed_after:?}, ttl={ttl:?}"
1386        );
1387        assert!(
1388            observed_after < ttl / 2,
1389            "fixture exceeded its half-TTL observation budget: {observed_after:?}"
1390        );
1391        let (stale_validate, target_rename, metadata_latest, metadata_after, user_latest, final_state) = observations;
1392        let metadata_latest = metadata_latest.expect("observation: latest metadata must remain decodable");
1393        let metadata_after = metadata_after.expect("observation: read raw destination xl.meta");
1394        let user_latest = user_latest.expect("observation: latest ordinary body must remain decodable");
1395        let metadata_changed = metadata_after != metadata_before;
1396        let latest_is_staged = metadata_latest.version_id == metadata_new.version_id && metadata_latest.data == metadata_new.data;
1397        eprintln!(
1398            "namespace admission observation: stale_validate={stale_validate:?}, target_rename={:?}, metadata_changed={metadata_changed}, latest_is_staged={latest_is_staged}, latest_version={:?}, latest_body={:?}, final_state={final_state:?}, elapsed={observed_after:?}, ttl={ttl:?}",
1399            target_rename.as_ref().map(|_| ()),
1400            metadata_latest.version_id,
1401            metadata_latest.data,
1402        );
1403        assert_eq!(final_state, (false, namespace_completed, movement_before, operation_epoch_before));
1404        assert_eq!(user_latest.data, user_new.data);
1405        assert_eq!(user_latest.version_id, user_new.version_id);
1406        assert!(
1407            stale_validate.is_err() && target_rename.is_err(),
1408            "completed namespace commit must reject both stale admissions: validate={stale_validate:?}, target={:?}, metadata_changed={metadata_changed}",
1409            target_rename.as_ref().map(|_| ()),
1410        );
1411        assert_eq!(metadata_after, metadata_before, "stale target must not rewrite metadata");
1412        assert_eq!(metadata_latest.data, metadata_old.data, "latest metadata body must remain the old one");
1413        assert_eq!(
1414            metadata_latest.version_id, metadata_old.version_id,
1415            "latest metadata version must remain the old one"
1416        );
1417    }
1418
1419    #[tokio::test]
1420    async fn renewed_namespace_lease_allows_real_metadata_publication() {
1421        use futures::FutureExt;
1422        use std::time::Duration;
1423        use tokio::time::timeout;
1424
1425        let ctx = Arc::new(InstanceContext::new());
1426        let store = super::super::tests::build_store_with_ctx(ctx.clone());
1427        let root = tempfile::tempdir().expect("target root");
1428        let ttl = crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL;
1429        let user_volume = "target-bucket";
1430        let metadata_volume = ".rustfs.sys/tmp";
1431        let setup = std::panic::AssertUnwindSafe(timeout(ttl / 2, async {
1432            let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
1433            let user_new = target_file_info("destination", Uuid::new_v4(), b"completed-user-write");
1434            let mut metadata_old = target_file_info("destination", Uuid::new_v4(), b"old-metadata");
1435            let mut metadata_new = target_file_info("destination", Uuid::new_v4(), b"metadata-from-renewed-scan");
1436            metadata_old.mod_time = Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed old time"));
1437            metadata_new.mod_time = metadata_old.mod_time.map(|old| old + time::Duration::seconds(1));
1438            seed_target(&disk, user_volume, "staged", user_new.clone()).await;
1439            let metadata_before = seed_target(&disk, metadata_volume, "destination", metadata_old).await;
1440            seed_target(&disk, metadata_volume, "staged", metadata_new.clone()).await;
1441            (disk, user_new, metadata_new, metadata_before)
1442        }))
1443        .catch_unwind()
1444        .await;
1445        let (disk, user_new, metadata_new, metadata_before) = match setup {
1446            Ok(Ok(setup)) => setup,
1447            Ok(Err(error)) => {
1448                let retained = root.keep();
1449                panic!("fixture initialization must finish before lease acquisition: {error}; retained={retained:?}");
1450            }
1451            Err(panic) => {
1452                let retained = root.keep();
1453                eprintln!("fixture setup panicked; retained={retained:?}");
1454                std::panic::resume_unwind(panic);
1455            }
1456        };
1457        let disk_ref = disk.endpoint().to_string();
1458        let movement_generation = ctx.data_movement_generation();
1459        let namespace_generation = store.scanner_namespace_mutation_generation();
1460        let read_options = crate::disk::ReadOptions {
1461            read_data: true,
1462            ..Default::default()
1463        };
1464        let mut tokens = Vec::new();
1465        let result = std::panic::AssertUnwindSafe(timeout(ttl / 2, async {
1466            let (old_token, _) = store
1467                .acquire_scanner_publication_lease(movement_generation, ttl)
1468                .await
1469                .expect("original lease");
1470            tokens.push(old_token);
1471            store
1472                .rename_local_data(&disk_ref, (user_volume, "staged"), &user_new, (user_volume, "destination"), None)
1473                .await
1474                .expect("ordinary namespace write");
1475            while ctx.namespace_commits_pending() {
1476                tokio::task::yield_now().await;
1477            }
1478            assert_eq!(ctx.namespace_commit_generation(), 2);
1479            assert_eq!(ctx.data_movement_generation(), movement_generation);
1480            let user_latest = disk
1481                .read_version(user_volume, user_volume, "destination", "", &read_options)
1482                .await
1483                .expect("latest ordinary committed object");
1484            assert_eq!(user_latest.version_id, user_new.version_id);
1485            assert_eq!(user_latest.data, user_new.data);
1486            assert!(
1487                store
1488                    .validate_scanner_publication_lease(old_token, movement_generation)
1489                    .await
1490                    .is_err()
1491            );
1492            assert_eq!(
1493                ctx.scanner_publication_lease_generations(old_token).await,
1494                Some((movement_generation, namespace_generation)),
1495                "rejection must not refresh or discard the old lease"
1496            );
1497            let (fresh_token, fresh_generation) = store
1498                .acquire_scanner_publication_lease(movement_generation, ttl)
1499                .await
1500                .expect("a new scan can acquire a current lease after the completed write");
1501            tokens.push(fresh_token);
1502            store
1503                .validate_scanner_publication_lease(fresh_token, fresh_generation)
1504                .await
1505                .expect("fresh lease validates");
1506            store
1507                .rename_local_data(
1508                    &disk_ref,
1509                    (metadata_volume, "staged"),
1510                    &metadata_new,
1511                    (metadata_volume, "destination"),
1512                    Some(fresh_token),
1513                )
1514                .await
1515                .expect("fresh lease authorizes real internal metadata publication");
1516            let latest = disk
1517                .read_version(metadata_volume, metadata_volume, "destination", "", &read_options)
1518                .await
1519                .expect("latest published metadata");
1520            let raw = tokio::fs::read(root.path().join(metadata_volume).join("destination/xl.meta"))
1521                .await
1522                .expect("raw published metadata");
1523            assert_eq!(latest.version_id, metadata_new.version_id);
1524            assert_eq!(latest.data, metadata_new.data);
1525            assert_ne!(raw, metadata_before);
1526            assert!(!ctx.namespace_commits_pending());
1527            assert_eq!(
1528                ctx.namespace_commit_generation(),
1529                2,
1530                "internal metadata does not mutate the user namespace"
1531            );
1532            assert_eq!(ctx.data_movement_generation(), movement_generation);
1533        }))
1534        .catch_unwind()
1535        .await;
1536        // A table release does not drain an independently owned metadata call.
1537        // Collect every release outcome before checking either ownership chain.
1538        let cleanup = std::panic::AssertUnwindSafe(async {
1539            let mut releases = Vec::new();
1540            for token in tokens {
1541                releases.push(
1542                    std::panic::AssertUnwindSafe(timeout(Duration::from_secs(5), store.release_scanner_publication_lease(token)))
1543                        .catch_unwind()
1544                        .await,
1545                );
1546            }
1547            let movement_guard = timeout(Duration::from_secs(5), ctx.data_movement_operation_gate().write_owned()).await;
1548            let namespace_drained = timeout(Duration::from_secs(5), async {
1549                while ctx.namespace_commits_pending() {
1550                    tokio::task::yield_now().await;
1551                }
1552            })
1553            .await;
1554            let drained = releases.iter().all(|release| matches!(release, Ok(Ok(_))))
1555                && movement_guard.is_ok()
1556                && namespace_drained.is_ok();
1557            #[cfg(not(windows))]
1558            let drained = {
1559                use crate::disk::os::prepared_publication_test_hooks as hooks;
1560
1561                let mut keys_drained = true;
1562                for volume in [user_volume, metadata_volume] {
1563                    // The physical lease uses the actual IO object directory,
1564                    // including descriptor-rooted aliases, not its xl.meta file.
1565                    let key_drained = match disk.get_object_path_for_io_if_local(volume, "destination") {
1566                        Some(Ok(path)) => timeout(Duration::from_secs(5), hooks::drain_namespace_key(&path))
1567                            .await
1568                            .is_ok(),
1569                        _ => false,
1570                    };
1571                    keys_drained &= key_drained;
1572                }
1573                drained && keys_drained
1574            };
1575            drop(movement_guard);
1576            if let Some(panic) = releases.into_iter().find_map(|release| release.err()) {
1577                std::panic::resume_unwind(panic);
1578            }
1579            drained
1580        })
1581        .catch_unwind()
1582        .await;
1583        // Generic cancelled IO cannot be proved drained by a namespace key.
1584        // Retain on any failed observation, even if best-effort cleanup succeeds.
1585        if !matches!(&result, Ok(Ok(()))) || !matches!(&cleanup, Ok(true)) {
1586            let retained = root.keep();
1587            eprintln!("fixture observations or physical cleanup incomplete; retained={retained:?}");
1588        }
1589        match result {
1590            Ok(result) => result.expect("publication must finish before the original lease can expire"),
1591            Err(panic) => std::panic::resume_unwind(panic),
1592        }
1593        match cleanup {
1594            Ok(drained) => assert!(drained, "fixture physical cleanup must finish before deleting its root"),
1595            Err(panic) => std::panic::resume_unwind(panic),
1596        }
1597        assert!(ctx.data_movement_operation_gate().try_write_owned().is_ok());
1598    }
1599
1600    #[cfg(not(windows))]
1601    #[tokio::test]
1602    #[serial_test::serial]
1603    async fn target_ordinary_timeout_keeps_its_physical_namespace_owner() {
1604        use crate::disk::os::prepared_publication_test_hooks as hooks;
1605        temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("1"))], async {
1606            let ctx = Arc::new(InstanceContext::new());
1607            let store = super::super::tests::build_store_with_ctx(ctx.clone());
1608            let root = tempfile::tempdir().expect("root");
1609            let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
1610            let fi = target_file_info("destination", Uuid::new_v4(), b"timed-out-physical-commit");
1611            seed_target(&disk, "target-bucket", "staged", fi.clone()).await;
1612            let path = disk
1613                .get_object_path_for_io_if_local("target-bucket", "destination/xl.meta")
1614                .expect("local")
1615                .expect("destination IO path");
1616            let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
1617            let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
1618            let _hook = hooks::install(&path, move || {
1619                let _ = entered_tx.send(());
1620                let _ = release_rx.recv();
1621            });
1622            let disk_ref = disk.endpoint().to_string();
1623            let mut rename = Box::pin(store.rename_local_data(
1624                &disk_ref,
1625                ("target-bucket", "staged"),
1626                &fi,
1627                ("target-bucket", "destination"),
1628                None,
1629            ));
1630            tokio::time::timeout(std::time::Duration::from_secs(10), async {
1631                tokio::select! {
1632                    result = &mut rename => panic!("completed before physical pause: {result:?}"),
1633                    entered = entered_rx => entered.expect("physical entry"),
1634                }
1635            })
1636            .await
1637            .expect("bounded entry");
1638            tokio::time::pause();
1639            tokio::time::advance(std::time::Duration::from_secs(2)).await;
1640            tokio::time::resume();
1641            let result = tokio::time::timeout(std::time::Duration::from_secs(5), &mut rename)
1642                .await
1643                .expect("ordinary deadline remains enabled");
1644            assert!(matches!(result, Err(DiskError::Timeout)), "{result:?}");
1645            drop(rename);
1646            assert!(ctx.namespace_commits_pending(), "timeout is not a physical drain");
1647            drop(release_tx);
1648            tokio::time::timeout(std::time::Duration::from_secs(10), async {
1649                while ctx.namespace_commits_pending() {
1650                    tokio::task::yield_now().await;
1651                }
1652            })
1653            .await
1654            .expect("late physical owner drains");
1655            let read = disk
1656                .read_version(
1657                    "target-bucket",
1658                    "target-bucket",
1659                    "destination",
1660                    &fi.version_id.expect("version").to_string(),
1661                    &crate::disk::ReadOptions {
1662                        read_data: true,
1663                        ..Default::default()
1664                    },
1665                )
1666                .await
1667                .expect("read actual timeout tail");
1668            assert_eq!(read.data, fi.data);
1669        })
1670        .await;
1671    }
1672
1673    #[test]
1674    fn endpoint_rpc_authority_preserves_port_and_ipv6_brackets() {
1675        let endpoint = Endpoint::try_from("https://127.0.0.1:9001/d1").expect("URL endpoint");
1676        assert_eq!(endpoint_rpc_authority(&endpoint).as_deref(), Some("127.0.0.1:9001"));
1677
1678        let endpoint = Endpoint::try_from("https://[::1]:9002/d1").expect("IPv6 URL endpoint");
1679        assert_eq!(endpoint_rpc_authority(&endpoint).as_deref(), Some("[::1]:9002"));
1680    }
1681
1682    #[tokio::test]
1683    #[serial_test::serial]
1684    async fn init_local_peer_publishes_complete_rpc_authority() {
1685        let previous = rustfs_common::get_global_local_node_name().await;
1686        let mut endpoint = Endpoint::try_from("https://127.0.0.1:9001/d1").expect("URL endpoint");
1687        endpoint.is_local = true;
1688        let endpoint_pools = EndpointServerPools(vec![PoolEndpoints {
1689            legacy: false,
1690            set_count: 1,
1691            drives_per_set: 1,
1692            endpoints: Endpoints::from(vec![endpoint]),
1693            cmd_line: "rpc-authority-test".to_string(),
1694            platform: "test".to_string(),
1695        }]);
1696
1697        let host = String::new();
1698        let port = "9000".to_string();
1699        init_local_peer(&endpoint_pools, &host, &port).await;
1700        assert_eq!(rustfs_common::try_get_global_local_node_name().as_deref(), Some("127.0.0.1:9001"));
1701        rustfs_common::set_global_local_node_name(&previous).await;
1702    }
1703
1704    // Phase 5 follow-up (backlog#1052): registering local disks through the
1705    // ctx-explicit entry writes the passed context's registry only — the
1706    // process bootstrap context (and any other instance) stays clean, so a
1707    // future second server's disks cannot leak into the first one's registry.
1708    #[tokio::test]
1709    async fn init_local_disks_with_instance_ctx_isolates_disk_registry() {
1710        let temp_dir = tempfile::tempdir().expect("create temp disk dir");
1711        let endpoint_pools = single_local_disk_pools(temp_dir.path());
1712        let instance_ctx = Arc::new(InstanceContext::new());
1713
1714        init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools)
1715            .await
1716            .expect("local disks should register into the passed context");
1717
1718        let registered: Vec<String> = instance_ctx.local_disk_map().read().await.keys().cloned().collect();
1719        assert_eq!(registered.len(), 1, "the passed context must hold exactly the one local disk");
1720        assert_eq!(
1721            instance_ctx.local_disk_set_drives().read().await.len(),
1722            1,
1723            "the passed context must hold the pool/set/drive layout"
1724        );
1725
1726        let bootstrap = crate::runtime::instance::bootstrap_ctx();
1727        let bootstrap_map = bootstrap.local_disk_map();
1728        let bootstrap_map = bootstrap_map.read().await;
1729        let sibling = InstanceContext::new();
1730        for key in &registered {
1731            assert!(
1732                !bootstrap_map.contains_key(key),
1733                "bootstrap context must not absorb a disk registered into an explicit context"
1734            );
1735            assert!(
1736                !sibling.local_disk_map().read().await.contains_key(key),
1737                "a sibling context must not observe another instance's disks"
1738            );
1739        }
1740    }
1741
1742    #[tokio::test]
1743    async fn stale_local_disk_snapshot_cannot_repopulate_the_id_registry() {
1744        let temp_dir = tempfile::tempdir().expect("create temp disk dir");
1745        let endpoint_pools = single_local_disk_pools(temp_dir.path());
1746        let instance_ctx = Arc::new(InstanceContext::new());
1747        init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools)
1748            .await
1749            .expect("local disk should be registered");
1750        let disk = instance_ctx
1751            .local_disk_map()
1752            .read()
1753            .await
1754            .values()
1755            .find_map(|disk| disk.clone())
1756            .expect("registered local disk");
1757        let endpoint = disk.endpoint().to_string();
1758        let disk_id = Uuid::new_v4();
1759
1760        let local_disk_map = instance_ctx.local_disk_map();
1761        let mut quarantine = local_disk_map.write().await;
1762        let replacement = new_disk(
1763            &disk.endpoint(),
1764            &DiskOption {
1765                cleanup: false,
1766                health_check: false,
1767            },
1768        )
1769        .await
1770        .expect("replacement disk should initialize");
1771        assert!(!Arc::ptr_eq(&disk, &replacement));
1772        let task_ctx = instance_ctx.clone();
1773        let task_disk = disk.clone();
1774        let remember = tokio::spawn(async move { record_local_disk_id_if_active(&task_ctx, &task_disk, disk_id).await });
1775        tokio::task::yield_now().await;
1776        quarantine.insert(endpoint.clone(), Some(replacement.clone()));
1777        drop(quarantine);
1778
1779        assert!(!remember.await.expect("stale lookup task should complete"));
1780        assert!(!instance_ctx.local_disk_id_map().read().await.contains_key(&disk_id));
1781        let active = instance_ctx
1782            .local_disk_map()
1783            .read()
1784            .await
1785            .get(&endpoint)
1786            .cloned()
1787            .flatten()
1788            .expect("replacement disk should remain registered");
1789        assert!(Arc::ptr_eq(&active, &replacement));
1790        assert!(record_local_disk_id_if_active(&instance_ctx, &replacement, disk_id).await);
1791        assert_eq!(instance_ctx.local_disk_id_map().read().await.get(&disk_id), Some(&endpoint));
1792    }
1793}