Skip to main content

datui_lib/home/
home_app.rs

1//! The home screen's work: listing, probing remote places, measuring and
2//! classifying rows, previews, search, catalogs and the cloud sources, and the
3//! answers its workers send back.
4
5use crate::app::background::{CacheWrites, OwedAnswer};
6use crate::app::feedback::Confirm;
7use crate::cache::CacheManager;
8use crate::cli::FileFormat;
9use crate::loading::open_options::OpenOptions;
10#[cfg(feature = "cloud")]
11use crate::wait_on_runtime;
12use crate::{
13    APP_NAME, App, AppEvent, InputMode, cloud::source, config, home, home::catalog, home::discover,
14    loading, widgets,
15};
16use color_eyre::Result;
17use std::collections::HashMap;
18use std::path::{Path, PathBuf};
19use std::sync::{Arc, Mutex};
20
21/// The home screen's work in flight and what it keeps for the session: probes, listings,
22/// search, previews and schemas.
23#[derive(Default)]
24pub struct HomeApp {
25    /// Network roots being listed off-thread, so a probe is not started twice. Never
26    /// removed for a root that does not answer: its thread is lost, and a retry would
27    /// lose another.
28    pub(crate) probes_inflight: Vec<PathBuf>,
29    /// Each cloud listing's stop flag, by place: leaving the place stops it before its
30    /// next page.
31    pub(crate) listing_cancels: HashMap<PathBuf, Arc<std::sync::atomic::AtomicBool>>,
32    /// The listing a filter asked of a cut-short cloud directory: where, the name
33    /// prefix, and its stop flag.
34    pub(crate) narrowing: Option<(PathBuf, String, Arc<std::sync::atomic::AtomicBool>)>,
35    /// Cloud discovery has started. It costs a request per provider, so it runs once
36    /// per session.
37    #[cfg(feature = "cloud")]
38    pub(crate) cloud_discovery_started: bool,
39    /// A recursive search below the working directory is out. One at a time: a second
40    /// would only compete for the disk.
41    pub(crate) search_inflight: bool,
42    /// The home generation the walk started in. Its batches stay its own across
43    /// refreshes; the root decides whether they still apply.
44    pub(crate) search_generation: u64,
45    /// Why the last open failed, shown at home when the error is dismissed with nothing
46    /// to fall back to.
47    pub(crate) last_load_error: Option<String>,
48    /// Schema reads currently out, so the same one is not requested every frame.
49    pub(crate) schema_inflight: Vec<PathBuf>,
50    /// Invalidates listings and measurements from a request the user has moved past.
51    pub(crate) generation: u64,
52    /// Rows arrived for a listing still being read; it is listed again before the next
53    /// frame.
54    pub(crate) refresh_owed: bool,
55    /// Measurements arrived since the last frame; applied once before the next.
56    pub(crate) measured_owed: bool,
57    /// Schema previews, memoized for the session only (persisted, they would go stale).
58    pub(crate) schema_cache: HashMap<PathBuf, Option<discover::SchemaPreview>>,
59    /// The home screen's `ROWS` previews, and the dataset the newest one built.
60    pub previews: crate::home::home_preview::Previews,
61    /// The pre-0.4.0 Ctrl+D directories in the cache have been moved to `catalog.toml`.
62    pub(crate) remembered_moved: bool,
63    /// Which home workers panic before starting, for tests. The jobs' own is
64    /// [`Jobs::worker_dies`].
65    #[cfg(test)]
66    pub(crate) worker_dies: Option<crate::HomeWorkerDies>,
67    /// Whether a browser opened here appears in front of the user: `o` on a doc link
68    /// is offered only then (`link_open::local_desktop`).
69    pub local_desktop: bool,
70    /// The reads of data started this session, by kind.
71    pub reads: crate::home::home_preview::ReadCounts,
72}
73
74/// Rows measured per background pass: small, so a slow filesystem shows progress.
75const MEASURE_BATCH: usize = 12;
76
77/// Rows a probe measures while it is already reading a remote directory.
78const PROBE_MEASURE_LIMIT: usize = 24;
79
80/// Rows one classification pass looks into: a cap on work in flight, the next pass
81/// chosen from the viewport when this one lands. Small like [`MEASURE_BATCH`], so a
82/// screen fills in rather than going silent.
83pub(crate) const CLASSIFY_BATCH: usize = 16;
84
85/// Probes allowed at once: a probe of a gone share holds its thread until exit.
86pub(crate) const MAX_CONCURRENT_PROBES: usize = 4;
87
88/// A number per home search walk, so one walk's scorings are never taken for
89/// another's over the same place.
90fn next_search_epoch() -> u64 {
91    static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
92    NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
93}
94
95/// A source as a home-screen row, with the last run's buckets when they still apply.
96#[cfg(feature = "cloud")]
97pub(crate) fn home_cloud_source(
98    source: &crate::cloud::cloud_sources::Source,
99    cached: Option<&crate::cache::CloudListing>,
100    listing: bool,
101) -> home::CloudSource {
102    let mut details: Vec<(String, String)> = vec![
103        ("source".to_string(), source.id.clone()),
104        ("api".to_string(), source.kind.name().to_string()),
105    ];
106    if let Some(endpoint) = &source.s3.endpoint {
107        details.push(("endpoint".to_string(), endpoint.clone()));
108    }
109    if let Some(region) = &source.s3.region {
110        details.push(("region".to_string(), region.clone()));
111    }
112    if let Some(project) = &source.project {
113        details.push(("project".to_string(), project.clone()));
114    }
115    if let Some(profile) = &source.profile {
116        details.push(("profile".to_string(), profile.clone()));
117    }
118    if let Some(configuration) = &source.gcloud {
119        details.push(("configuration".to_string(), configuration.clone()));
120    }
121    if source.s3.virtual_hosted.is_some() {
122        let style = if source.s3.virtual_hosted_style() {
123            "virtual-hosted"
124        } else {
125            "path-style"
126        };
127        details.push(("addressing".to_string(), style.to_string()));
128    }
129    details.push(("login".to_string(), source.origin.clone()));
130
131    let short = source.problem.as_deref().map(|problem| {
132        if problem.starts_with("not signed in") {
133            "not signed in"
134        } else if problem.starts_with("unsupported login") {
135            "unsupported login"
136        } else {
137            "not configured"
138        }
139    });
140    // A source that cannot list says what to do where the row has room; the count
141    // carries the short problem.
142    let note = match (&source.problem, short) {
143        (Some(problem), Some(short)) => problem
144            .strip_prefix(short)
145            .map(|rest| rest.trim_start_matches([':', ' ']))
146            .filter(|rest| !rest.is_empty())
147            .unwrap_or(problem)
148            .to_string(),
149        _ => [source.detail(), Some(source.origin.clone())]
150            .into_iter()
151            .flatten()
152            .filter(|n| !n.is_empty())
153            .collect::<Vec<_>>()
154            .join(&format!(" {} ", crate::glyphs::get().middot)),
155    };
156    let mut names: Vec<String> = cached.map(|c| c.buckets.clone()).unwrap_or_default();
157    for bucket in &source.buckets {
158        if !names.contains(bucket) {
159            names.push(bucket.clone());
160        }
161    }
162    let status = match (&source.problem, short) {
163        (Some(problem), Some(short)) => home::CloudStatus::Failed {
164            short: short.to_string(),
165            detail: problem.clone(),
166        },
167        _ if cached.is_some() => home::CloudStatus::Listed,
168        _ if listing => home::CloudStatus::Listing,
169        _ => home::CloudStatus::Unlisted,
170    };
171    home::CloudSource {
172        id: source.id.clone(),
173        label: source.label.clone(),
174        api: source.kind,
175        note,
176        buckets: names
177            .iter()
178            .map(|b| PathBuf::from(source.bucket_url(b)))
179            .collect(),
180        refreshing: listing && cached.is_some() && source.problem.is_none(),
181        // A source that failed before any request has nothing to ask.
182        asked: listing || source.problem.is_some(),
183        listed_at: cached
184            .map(|c| std::time::UNIX_EPOCH + std::time::Duration::from_secs(c.listed_at)),
185        status,
186        details,
187        place_details: Default::default(),
188    }
189}
190
191/// A listing error as a word for the row and the full message for the details pane.
192#[cfg(feature = "cloud")]
193fn summarize_cloud_failure(error: &str) -> (String, String) {
194    let lower = error.to_lowercase();
195    // A missing tool is already as short as it gets: `needs the AWS CLI`.
196    if let Some(start) = lower.find("needs ") {
197        return (error[start..].to_string(), error.to_string());
198    }
199    let short = if lower.contains("403")
200        || lower.contains("forbidden")
201        || lower.contains("accessdenied")
202        || lower.contains("access denied")
203    {
204        "403"
205    } else if lower.contains("401")
206        || lower.contains("unauthorized")
207        || lower.contains("credential")
208        || lower.contains("invalidaccesskeyid")
209        || lower.contains("expired")
210        || lower.contains("sso")
211        || lower.contains("az login")
212    {
213        "not logged in"
214    } else if lower.contains("unsupported login") {
215        "unsupported login"
216    } else if lower.contains("no gcp project") {
217        "no project"
218    } else if lower.contains("is not set") {
219        "not configured"
220    } else if lower.contains("timed out")
221        || lower.contains("timeout")
222        || lower.contains("connection")
223        || lower.contains("dns")
224        || lower.contains("resolve")
225    {
226        "unavailable"
227    } else {
228        "error"
229    };
230    (short.to_string(), error.to_string())
231}
232
233impl App {
234    /// Schema for a home entry, from Parquet metadata, memoized for the session. `None`
235    /// means not knowable without a scan, and the UI says so.
236    pub fn home_schema(&mut self, entry: &discover::Entry) -> Option<discover::SchemaPreview> {
237        // Previews read local files only: nothing in an object store is read before it is
238        // opened.
239        if home::is_cloud_place(&entry.path) || home::is_object_store_url(&entry.path) {
240            return None;
241        }
242        if let Some(cached) = self.home_app.schema_cache.get(&entry.path) {
243            return cached.clone();
244        }
245        // Opening a file, so requested from a worker; the frame never waits.
246        self.request_home_schema(entry.clone());
247        None
248    }
249
250    /// What a home-screen worker owes in place of its answer if it panics.
251    fn owed_answer(&mut self, instead: AppEvent) -> OwedAnswer {
252        OwedAnswer {
253            tx: self.events.clone(),
254            #[cfg(test)]
255            dies: self
256                .home_app
257                .worker_dies
258                .as_mut()
259                .is_some_and(|dies| dies(&instead)),
260            instead: Some(instead),
261        }
262    }
263
264    /// List the directory the `~` prompt is typing, if not already listed: a URL from
265    /// what the screen knows, a local directory on a worker.
266    pub(crate) fn list_the_typed_directory(&mut self) {
267        if !self.home.path_input_active {
268            return;
269        }
270        let dir = home::typed_dir(&self.home.path_input).to_string();
271        if self
272            .home
273            .path_listing
274            .as_ref()
275            .is_some_and(|l| l.dir == dir)
276        {
277            return;
278        }
279        if home::typed_dir_is_url(&dir) {
280            self.home.path_listing = Some(home::names_under(&dir, self.home.known_urls()));
281            return;
282        }
283        // Off the UI thread: a typed path may name a dead mount.
284        let tx = self.events.clone();
285        let owed = self.owed_answer(AppEvent::HomePathListed {
286            listing: Box::new(home::PathListing {
287                dir: dir.clone(),
288                names: Vec::new(),
289                failed: true,
290            }),
291        });
292        std::thread::spawn(move || {
293            owed.run(|| {
294                let listing = home::list_typed_dir(&dir);
295                let _ = tx.send(AppEvent::HomePathListed {
296                    listing: Box::new(listing),
297                });
298            })
299        });
300    }
301
302    /// Complete the path being typed, on a worker.
303    pub(crate) fn request_path_completion(&mut self) {
304        let typed = self.home.path_input.clone();
305        if typed.is_empty() {
306            return;
307        }
308        let generation = self.home_app.generation;
309        let tx = self.events.clone();
310        std::thread::spawn(move || {
311            let (completed, candidates) = home::complete_path(&typed);
312            let _ = tx.send(AppEvent::HomePathCompleted {
313                generation,
314                typed,
315                completed,
316                candidates,
317            });
318        });
319    }
320
321    /// A home file's first rows for its `ROWS` preview, read on a worker as its open
322    /// reads them. `None` until they land or when not previewed. `screen_height` sizes
323    /// the page to the table's.
324    pub fn home_preview_rows(
325        &mut self,
326        entry: &discover::Entry,
327        screen_height: u16,
328    ) -> Option<Arc<crate::home::home_preview::PreviewRows>> {
329        let max = self.app_config.home.preview_max.bytes();
330        if !crate::home::home_preview::previewable(entry, max) {
331            return None;
332        }
333        let stamp = crate::home::home_preview::Stamp::of_entry(entry);
334        if let Some(known) = self.home_app.previews.rows(&entry.path, stamp) {
335            return known;
336        }
337        if self.home_app.previews.inflight.is_none() {
338            self.request_home_preview(entry.path.clone(), stamp, screen_height);
339        }
340        None
341    }
342
343    /// Whether `entry` is one the preview reads, before its rows are in.
344    pub fn home_preview_pending(&self, path: &Path) -> bool {
345        self.home_app.previews.reading(path)
346    }
347
348    /// Read a file's first page on a worker through the open's own scan and schema read,
349    /// so the open can install the result.
350    fn request_home_preview(
351        &mut self,
352        path: PathBuf,
353        stamp: crate::home::home_preview::Stamp,
354        screen_height: u16,
355    ) {
356        self.home_app.previews.inflight = Some(path.clone());
357        self.home_app.reads.previews += 1;
358        let tx = self.events.clone();
359        let cloud = self.app_config.cloud.clone();
360        let formats = self.formats.clone();
361        let runtime = self.runtime.clone();
362        let cache = self.cache.clone();
363        let writes = self.cache_writes.clone();
364        // The table's rows: the screen less the title, the header and the footer.
365        let visible = (screen_height as usize).saturating_sub(3).max(1);
366        let owed = self.owed_answer(AppEvent::HomePreviewReady {
367            path: path.clone(),
368            stamp,
369            read_at: None,
370            rows: None,
371            prepared: crate::home::home_preview::Handoff::default(),
372        });
373        self.runtime.spawn_blocking(move || {
374            owed.run(|| {
375                let began = std::time::Instant::now();
376                let read_at = crate::home::home_preview::Stamp::of_file(&path);
377                let read = Self::read_home_preview(
378                    &path, &cloud, &formats, &runtime, cache, writes, visible,
379                );
380                log::debug!(
381                    target: "datui",
382                    "home preview of {}: {:.1?}",
383                    path.display(),
384                    began.elapsed()
385                );
386                let (rows, prepared) = match read {
387                    Some((rows, prepared)) => (Some(Arc::new(rows)), Some(Box::new(prepared))),
388                    None => (None, None),
389                };
390                let _ = tx.send(AppEvent::HomePreviewReady {
391                    path,
392                    stamp,
393                    read_at,
394                    rows,
395                    prepared: Arc::new(Mutex::new(prepared)),
396                });
397            })
398        });
399    }
400
401    /// What opening `path` from home reads first: scan, schema and the page for
402    /// `visible` rows, built by the open's own steps and options so the dataset is the
403    /// one the open would build.
404    fn read_home_preview(
405        path: &Path,
406        cloud: &crate::config::CloudConfig,
407        formats: &crate::formats::Registry,
408        runtime: &tokio::runtime::Handle,
409        cache: CacheManager,
410        writes: CacheWrites,
411        visible: usize,
412    ) -> Option<(
413        crate::home::home_preview::PreviewRows,
414        crate::home::home_preview::Prepared,
415    )> {
416        let paths = [path.to_path_buf()];
417        let scanned = Self::scan_for_open(
418            cloud,
419            formats,
420            &paths,
421            OpenOptions::default(),
422            Some(path.to_path_buf()),
423        )
424        .ok()?;
425        let loading::LoadAnswer::Scanned { lf, path, options } = scanned else {
426            return None;
427        };
428        let progress = Arc::<crate::formats::schema_union::FooterProgress>::default();
429        let report = crate::loading::measurements::OpenReport {
430            progress: progress.clone(),
431            meter: Arc::new(crate::loading::measurements::Meter::default()),
432            remembered: Some(cache),
433            writes,
434        };
435        let read = Self::read_schema_for_open(
436            *lf,
437            path,
438            options,
439            cloud,
440            runtime,
441            &report,
442            loading::Made::default(),
443        )
444        .ok()?;
445        let loading::LoadAnswer::SchemaRead {
446            mut state,
447            options,
448            debug_label,
449            ..
450        } = read
451        else {
452            return None;
453        };
454        // Planned as the table plans its first page, so the page is the one it wants.
455        state.visible_rows = visible;
456        let began = std::time::Instant::now();
457        let request = state.prepare_async_collect(None)?;
458        let df =
459            crate::analysis::statistics::collect_lazy(request.lf, request.polars_streaming).ok()?;
460        let result = request.plan.fit(df);
461        let rows = crate::home::home_preview::PreviewRows::from_frame(result.rows());
462        state.measurements().read_page(began.elapsed(), Some(1));
463        state.apply_async_collect(result);
464        Some((
465            rows,
466            crate::home::home_preview::Prepared {
467                state,
468                options,
469                debug_label,
470                progress,
471            },
472        ))
473    }
474
475    /// Whether a schema read is currently out for this path.
476    pub fn home_schema_pending(&self, path: &Path) -> bool {
477        self.home_app.schema_inflight.iter().any(|p| p == path)
478    }
479
480    /// Read the selected dataset's schema on a worker.
481    fn request_home_schema(&mut self, entry: discover::Entry) {
482        if self.home_app.schema_inflight.contains(&entry.path) {
483            return;
484        }
485        self.home_app.schema_inflight.push(entry.path.clone());
486
487        let generation = self.home_app.generation;
488        let tx = self.events.clone();
489        // Remembered as none on failure, so it is not asked again.
490        let owed = self.owed_answer(AppEvent::HomeSchemaReady {
491            generation,
492            path: entry.path.clone(),
493            preview: None,
494        });
495        self.runtime.spawn_blocking(move || {
496            owed.run(|| {
497                let preview = discover::schema_preview(&entry);
498                let _ = tx.send(AppEvent::HomeSchemaReady {
499                    generation,
500                    path: entry.path,
501                    preview,
502                });
503            })
504        });
505    }
506
507    /// Start listing network roots that have not answered yet. Never waits: a gone
508    /// share (a `hard` NFS mount) blocks its thread uninterruptibly, so the task is
509    /// abandoned, not joined.
510    pub(crate) fn spawn_home_probes(&mut self) {
511        self.stop_listings_left_behind();
512        for root in self.home.pending_probes() {
513            if self.home_app.probes_inflight.contains(&root) {
514                // Left and returned to before its next page: it goes on.
515                if let Some(cancelled) = self.home_app.listing_cancels.get(&root) {
516                    cancelled.store(false, std::sync::atomic::Ordering::Relaxed);
517                }
518                continue;
519            }
520            // Each probe of an unreachable share costs a thread forever, so they are capped.
521            // The browsed directory is exempt: it is the whole screen, and the user bounds how
522            // many they open.
523            let browsed = self.home.browsing.as_ref() == Some(&root);
524            if !browsed && self.home_app.probes_inflight.len() >= MAX_CONCURRENT_PROBES {
525                continue;
526            }
527            self.home_app.probes_inflight.push(root.clone());
528            let tx = self.events.clone();
529            let cache = self.cache.clone();
530            let owed = self.owed_answer(AppEvent::HomeProbeFailed {
531                root: root.clone(),
532                message: "Could not read it; see the log".to_string(),
533            });
534            #[cfg(feature = "cloud")]
535            let cloud = self.app_config.cloud.clone();
536            #[cfg(feature = "cloud")]
537            let runtime = self.runtime.clone();
538            #[cfg(feature = "cloud")]
539            let cancelled = {
540                let flag = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
541                self.home_app
542                    .listing_cancels
543                    .insert(root.clone(), flag.clone());
544                flag
545            };
546            // A detached thread, not the runtime's blocking pool: a thread wedged on a dead
547            // mount never returns, and must not eat the pool that loads data.
548            std::thread::spawn(move || {
549                owed.run(|| {
550                    // A bucket or prefix: listed with an object-store listing, not `read_dir` (which
551                    // fails on `gs://`). Metadata only: names, sizes and times for one level; no
552                    // footers, schemas or counts, which would cost a paid ranged read per row.
553                    #[cfg(feature = "cloud")]
554                    if let Some((id, account)) = home::cloud_account(&root) {
555                        let listed = wait_on_runtime(&runtime, async move {
556                            crate::cloud::cloud_browse::list_account(&id, &account, &cloud).await
557                        });
558                        match listed {
559                            Some(Ok(rows)) => {
560                                let _ = tx.send(AppEvent::HomeProbeReady {
561                                    root,
562                                    rows: Some(rows),
563                                    cut_short: false,
564                                });
565                            }
566                            Some(Err(message)) => {
567                                log::warn!(
568                                    target: "datui::cloud",
569                                    "listing {} failed: {message}",
570                                    root.display()
571                                );
572                                let _ = tx.send(AppEvent::HomeProbeFailed { root, message });
573                            }
574                            None => {
575                                let _ = tx.send(AppEvent::HomeProbeReady {
576                                    root,
577                                    rows: None,
578                                    cut_short: false,
579                                });
580                            }
581                        }
582                        return;
583                    }
584                    #[cfg(feature = "cloud")]
585                    if crate::cloud::cloud_browse::split_bucket_url(&root.to_string_lossy())
586                        .is_some()
587                        || source::azure_parts(&root.to_string_lossy()).is_some()
588                    {
589                        let url = root.to_string_lossy().into_owned();
590                        // Each page's rows are drawn as they come; leaving the place stops the listing.
591                        let watch = crate::cloud::cloud_browse::Watch {
592                            progress: Some(std::sync::Arc::new({
593                                let (tx, root) = (tx.clone(), root.clone());
594                                move |page: &[crate::home::discover::Entry]| {
595                                    let _ = tx.send(AppEvent::HomeProbeProgress {
596                                        root: root.clone(),
597                                        rows: page.to_vec(),
598                                    });
599                                }
600                            })),
601                            cancelled,
602                            names_from: None,
603                        };
604                        let listed = wait_on_runtime(&runtime, async move {
605                            crate::cloud::cloud_browse::list_objects_watched(&url, &cloud, &watch)
606                                .await
607                        });
608                        // A refused listing says why, rather than reading as a place that stopped
609                        // answering.
610                        match listed {
611                            Some(Err(message)) => {
612                                log::warn!(
613                                    target: "datui::cloud",
614                                    "listing {} failed: {message}",
615                                    root.display()
616                                );
617                                let _ = tx.send(AppEvent::HomeProbeFailed { root, message });
618                            }
619                            Some(Ok(level)) if level.cancelled => {
620                                let _ = tx.send(AppEvent::HomeProbeCancelled { root });
621                            }
622                            other => {
623                                let (rows, cut_short) = match other {
624                                    Some(Ok(level)) => (Some(level.rows), level.truncated),
625                                    _ => (None, false),
626                                };
627                                let _ = tx.send(AppEvent::HomeProbeReady {
628                                    root,
629                                    rows,
630                                    cut_short,
631                                });
632                            }
633                        }
634                        return;
635                    }
636                    // No cloud feature: `read_dir` on a bucket URL would only say unavailable.
637                    #[cfg(not(feature = "cloud"))]
638                    if source::is_remote_url(&root) {
639                        let message = "cloud support not in this build".to_string();
640                        let _ = tx.send(AppEvent::HomeProbeFailed { root, message });
641                        return;
642                    }
643                    let mut cut_short = false;
644                    let rows = if std::fs::read_dir(&root).is_ok() {
645                        // What has been read shows while the rest is read (seconds for thousands of files).
646                        let scan = crate::home::discover::scan_dir_progressive(&root, |read| {
647                            let _ = tx.send(AppEvent::HomeProbeProgress {
648                                root: root.clone(),
649                                rows: read.to_vec(),
650                            });
651                        });
652                        cut_short = scan.truncated;
653                        let mut rows = scan.entries;
654                        // Measured here too: this thread is already the one allowed to block on the share.
655                        for row in rows.iter_mut().take(PROBE_MEASURE_LIMIT) {
656                            crate::home::discover::enrich(row);
657                        }
658                        // Remote datasets are measured nowhere else, so remember them here; otherwise a
659                        // remote row is blank on every run.
660                        let mounts = crate::home::locality::Mounts::current();
661                        for row in rows.iter_mut() {
662                            row.cost.source = Some(mounts.describe(&row.path).fstype);
663                        }
664                        let facts: Vec<_> = rows.iter().filter_map(home::facts_for).collect();
665                        cache.record_dataset_facts(&facts);
666                        Some(rows)
667                    } else {
668                        None
669                    };
670                    let _ = tx.send(AppEvent::HomeProbeReady {
671                        root,
672                        rows,
673                        cut_short,
674                    });
675                })
676            });
677        }
678    }
679
680    /// Stop cloud listings of places no longer on screen (the browsed one, or home's
681    /// roots). A place left and revisited is listed again.
682    fn stop_listings_left_behind(&mut self) {
683        let home = &self.home;
684        for (root, cancelled) in &self.home_app.listing_cancels {
685            let wanted = match &home.browsing {
686                Some(dir) => dir == root,
687                None => home
688                    .sections
689                    .iter()
690                    .any(|s| s.remote_root.as_ref() == Some(root)),
691            };
692            if !wanted {
693                cancelled.store(true, std::sync::atomic::Ordering::Relaxed);
694            }
695        }
696        if let Some((dir, _, cancelled)) = &self.home_app.narrowing
697            && home.browsing.as_ref() != Some(dir)
698        {
699            cancelled.store(true, std::sync::atomic::Ordering::Relaxed);
700            self.home_app.narrowing = None;
701        }
702        if self
703            .home
704            .narrowed
705            .as_ref()
706            .is_some_and(|n| self.home.browsing.as_ref() != Some(&n.dir))
707        {
708            self.home.narrowed = None;
709        }
710    }
711
712    /// In a cloud directory cut short at the cap, ask the server for names starting with
713    /// the filter, so names past the cap can be found. Nothing when the filter is empty
714    /// or what is held answers it.
715    #[cfg(feature = "cloud")]
716    pub(crate) fn narrow_cloud_listing(&mut self) {
717        let dir = self.home.browsing.clone();
718        let prefix = dir.as_ref().and_then(|dir| {
719            if !self.home.probes.cut_short(dir) {
720                return None;
721            }
722            let rows = self.home.probes.listed(dir)?;
723            let names: Vec<&str> = rows.iter().map(|row| row.name.as_str()).collect();
724            crate::cloud::cloud_browse::narrowing_prefix(&self.home.filter, &names)
725        });
726        let (Some(dir), Some(prefix)) = (dir, prefix) else {
727            if let Some((_, _, cancelled)) = self.home_app.narrowing.take() {
728                cancelled.store(true, std::sync::atomic::Ordering::Relaxed);
729            }
730            if self.home.narrowed.take().is_some() {
731                self.home_refresh();
732            }
733            return;
734        };
735        // Everything under a shorter prefix is everything under this one too.
736        if self.home.narrowed.as_ref().is_some_and(|n| {
737            n.dir == dir && (n.prefix == prefix || (!n.truncated && prefix.starts_with(&n.prefix)))
738        }) {
739            return;
740        }
741        if let Some((d, p, _)) = &self.home_app.narrowing
742            && *d == dir
743            && *p == prefix
744        {
745            return;
746        }
747        if let Some((_, _, cancelled)) = self.home_app.narrowing.take() {
748            cancelled.store(true, std::sync::atomic::Ordering::Relaxed);
749        }
750        let cancelled = Arc::new(std::sync::atomic::AtomicBool::new(false));
751        self.home_app.narrowing = Some((dir.clone(), prefix.clone(), cancelled.clone()));
752        let tx = self.events.clone();
753        let owed = self.owed_answer(AppEvent::HomeNarrowed {
754            dir: dir.clone(),
755            prefix: prefix.clone(),
756            listed: None,
757        });
758        let cloud = self.app_config.cloud.clone();
759        let runtime = self.runtime.clone();
760        std::thread::spawn(move || {
761            owed.run(|| {
762                let url = dir.to_string_lossy().into_owned();
763                let watch = crate::cloud::cloud_browse::Watch {
764                    progress: None,
765                    cancelled,
766                    names_from: Some(prefix.clone()),
767                };
768                let listed = wait_on_runtime(&runtime, async move {
769                    crate::cloud::cloud_browse::list_objects_watched(&url, &cloud, &watch).await
770                });
771                let listed = match listed {
772                    Some(Ok(level)) if !level.cancelled => Some((level.rows, level.truncated)),
773                    _ => None,
774                };
775                let _ = tx.send(AppEvent::HomeNarrowed {
776                    dir,
777                    prefix,
778                    listed,
779                });
780            })
781        });
782    }
783
784    /// Find the cloud sources this machine and the config describe, and list their
785    /// buckets when `[cloud] list_on_start` asks. Once per session; Ctrl+R asks again.
786    #[cfg(feature = "cloud")]
787    fn spawn_cloud_discovery(&mut self) {
788        if self.home_app.cloud_discovery_started {
789            return;
790        }
791        self.home_app.cloud_discovery_started = true;
792        let list = self.app_config.cloud.list_on_start;
793        self.list_cloud_sources(None, list);
794    }
795
796    /// List the browsed source if not asked this session; entering a source asks.
797    #[cfg(feature = "cloud")]
798    fn list_browsed_cloud_source(&mut self) {
799        let Some(id) = self
800            .home
801            .browsing
802            .as_deref()
803            .and_then(home::cloud_source_id)
804        else {
805            return;
806        };
807        let Some(source) = self.home.cloud.iter_mut().find(|s| s.id == id) else {
808            return;
809        };
810        if source.asked {
811            return;
812        }
813        source.begin_listing();
814        self.list_cloud_sources(Some(id), true);
815    }
816
817    /// Send every source's rows, or list the named one's buckets. Rows go first, filled
818    /// from the last run's listing when still valid. With `list`, sources are listed a
819    /// few at a time and each result sent on arrival; without, nothing leaves the
820    /// machine. On the runtime, not a detached thread: every call has a global timeout.
821    #[cfg(feature = "cloud")]
822    fn list_cloud_sources(&mut self, only: Option<String>, list: bool) {
823        let tx = self.events.clone();
824        let cloud = self.app_config.cloud.clone();
825        let cache = self.cache.clone();
826        self.runtime.spawn(async move {
827            let mut hidden = cache.load_hidden_cloud_sources();
828            hidden.extend(cloud.hide.iter().cloned());
829            let cached_for = |source: &crate::cloud::cloud_sources::Source| {
830                cache.cloud_listing(&source.id, &source.fingerprint())
831            };
832            // Looked for again, and kept for the opens and listings that follow.
833            let found = crate::cloud::cloud_sources::rediscover(&cloud).to_vec();
834            // A bucket under Recent opens with the login that listed it, shown or not, unless
835            // its source is hidden (perhaps for a dead login; the default opens it then).
836            // Only with the rows, so an old listing never overrides a newer one.
837            if only.is_none() {
838                for source in found.iter().filter(|s| !hidden.contains(&s.id)) {
839                    if let Some(cached) = cached_for(source) {
840                        crate::cloud::cloud_sources::remember_listed(source, &cached.buckets);
841                    }
842                }
843            }
844            let sources: Vec<crate::cloud::cloud_sources::Source> =
845                crate::cloud::cloud_sources::on_home(found, &cloud)
846                    .into_iter()
847                    .filter(|s| !hidden.contains(&s.id))
848                    .collect();
849
850            match &only {
851                None => {
852                    let rows = sources
853                        .iter()
854                        .map(|source| home_cloud_source(source, cached_for(source).as_ref(), list))
855                        .collect();
856                    let _ = tx.send(AppEvent::HomeCloudSources { sources: rows });
857                }
858                // Gone since its row was drawn (a profile removed, a source hidden): said, so the
859                // row does not wait forever.
860                Some(id) if !sources.iter().any(|s| &s.id == id) => {
861                    let _ = tx.send(AppEvent::HomeCloudListed {
862                        id: id.clone(),
863                        buckets: Vec::new(),
864                        details: Vec::new(),
865                        failure: Some((
866                            "not found".to_string(),
867                            format!("{id} is gone or hidden. Ctrl+R at the top looks again."),
868                        )),
869                        listed_at: std::time::SystemTime::now(),
870                    });
871                    return;
872                }
873                Some(_) => {}
874            }
875            if !list {
876                return;
877            }
878
879            // One slow endpoint does not delay the rest, and many sources do not storm.
880            const LISTING_AT_ONCE: usize = 4;
881            let permits = std::sync::Arc::new(tokio::sync::Semaphore::new(LISTING_AT_ONCE));
882            let mut listings = tokio::task::JoinSet::new();
883            for source in sources
884                .into_iter()
885                .filter(|s| only.as_ref().is_none_or(|id| &s.id == id))
886            {
887                let permits = permits.clone();
888                listings.spawn(async move {
889                    let _permit = permits.acquire_owned().await;
890                    let result = crate::cloud::cloud_browse::list_first_level(&source).await;
891                    (source, result)
892                });
893            }
894            while let Some(joined) = listings.join_next().await {
895                let Ok((source, result)) = joined else {
896                    continue;
897                };
898                let listed_at = std::time::SystemTime::now();
899                // Buckets named in the config show whether or not the login can list them.
900                let mut names = source.buckets.clone();
901                let mut details = Vec::new();
902                let failure = match result {
903                    Ok(listed) => {
904                        for item in listed {
905                            crate::cloud::cloud_sources::remember_bucket(&source, &item.name);
906                            if !item.details.is_empty() {
907                                details.push((item.place.clone(), item.details));
908                            }
909                            let name = item.name;
910                            if !names.contains(&name) {
911                                names.push(name);
912                            }
913                        }
914                        cache.save_cloud_listing(
915                            &source.id,
916                            crate::cache::CloudListing {
917                                fingerprint: source.fingerprint(),
918                                buckets: names.clone(),
919                                listed_at: listed_at
920                                    .duration_since(std::time::UNIX_EPOCH)
921                                    .map(|d| d.as_secs())
922                                    .unwrap_or(0),
923                            },
924                        );
925                        None
926                    }
927                    Err(e) => {
928                        log::warn!(target: "datui::cloud", "listing {} failed: {e}", source.id);
929                        Some(summarize_cloud_failure(&e))
930                    }
931                };
932                let _ = tx.send(AppEvent::HomeCloudListed {
933                    id: source.id.clone(),
934                    buckets: names
935                        .iter()
936                        .map(|b| PathBuf::from(source.bucket_url(b)))
937                        .collect(),
938                    details,
939                    failure,
940                    listed_at,
941                });
942            }
943        });
944    }
945
946    /// Ask again for what is on screen, bypassing the cache: the browsed source's
947    /// buckets, the browsed bucket or directory, or every source's buckets.
948    pub(crate) fn home_reload(&mut self) {
949        #[cfg(feature = "cloud")]
950        {
951            let browsing = self.home.browsing.clone();
952            match browsing.as_deref().and_then(home::cloud_source_id) {
953                Some(id) => {
954                    if let Some(source) = self.home.cloud.iter_mut().find(|s| s.id == id) {
955                        source.begin_listing();
956                    }
957                    self.list_cloud_sources(Some(id), true);
958                }
959                None if browsing.is_none() && !self.home.cloud.is_empty() => {
960                    for source in &mut self.home.cloud {
961                        source.begin_listing();
962                    }
963                    self.list_cloud_sources(None, true);
964                }
965                None => {}
966            }
967        }
968        if let Some(dir) = self.home.browsing.clone() {
969            // Its answer, not a listing still coming in.
970            if self.home.probes.settled(&dir) {
971                self.home.probes.forget(&dir);
972            }
973        }
974        // Ctrl+R retries failed peeks and missing web files too.
975        self.home.peek_failed.clear();
976        for path in std::mem::take(&mut self.home.web_gone).into_keys() {
977            self.home.sized.remove(&path);
978        }
979        self.home.status = None;
980        self.home_refresh();
981    }
982
983    /// Start the recursive search below the working directory, if wanted and not
984    /// running. Triggered by typing, not by opening home, so opening a recent costs no
985    /// walk.
986    pub(crate) fn spawn_home_search(&mut self) {
987        if self.home_app.search_inflight || self.home.search.done {
988            return;
989        }
990        let config = self.app_config.home.search.clone();
991        if !config.enabled {
992            return;
993        }
994        let Some(root) =
995            crate::home::search::search_root(self.home.browsing.as_ref(), self.home.network_check)
996        else {
997            return;
998        };
999
1000        self.home.search.reset();
1001        self.home.search.root = Some(root.clone());
1002        self.home.search.running = true;
1003        self.home.search.epoch = next_search_epoch();
1004        self.home.search_limit = config.max_results;
1005        self.home_app.search_inflight = true;
1006
1007        let generation = self.home_app.generation;
1008        self.home_app.search_generation = generation;
1009        let tx = self.events.clone();
1010        let formats = self.formats.clone();
1011        // Ended, with what the batches already found kept.
1012        let owed = self.owed_answer(AppEvent::HomeSearchDone {
1013            generation,
1014            root: root.clone(),
1015            scanned: 0,
1016            limited: Some(crate::glyphs::dotted("partial · failed")),
1017        });
1018        // A detached thread, as for probes: a filesystem stall must not stop drawing.
1019        std::thread::spawn(move || {
1020            owed.run(|| {
1021                let walk_root = root.clone();
1022                let batch_tx = tx.clone();
1023                let batch_gen = generation;
1024                let batch_root = root.clone();
1025                let outcome = crate::home::search::walk_with_specs(
1026                    &walk_root,
1027                    &config,
1028                    &formats,
1029                    move |found, outcome| {
1030                        // Sent even when empty: it carries progress, and a failed send tells the walk
1031                        // nobody listens.
1032                        batch_tx
1033                            .send(AppEvent::HomeSearchBatch {
1034                                generation: batch_gen,
1035                                root: batch_root.clone(),
1036                                found,
1037                                scanned: outcome.scanned,
1038                            })
1039                            // A closed channel means the app is gone; stop walking.
1040                            .is_ok()
1041                    },
1042                );
1043                let _ = tx.send(AppEvent::HomeSearchDone {
1044                    generation,
1045                    root,
1046                    scanned: outcome.scanned,
1047                    limited: outcome.note().map(crate::glyphs::dotted),
1048                });
1049            })
1050        });
1051    }
1052
1053    /// Score the filter against the search's files on a worker, when owed. Asked after
1054    /// every event; over tens of thousands of files scoring held back keystroke echo.
1055    /// One at a time; each answer asks for the next if the filter moved.
1056    pub(crate) fn home_score_search(&mut self) {
1057        if self.input_mode != InputMode::Home {
1058            return;
1059        }
1060        let Some(job) = self.home.score_job() else {
1061            return;
1062        };
1063        let epoch = job.epoch;
1064        let tx = self.events.clone();
1065        // A worker that dies answers with nothing.
1066        let owed = self.owed_answer(AppEvent::HomeSearchScored {
1067            epoch,
1068            matches: None,
1069        });
1070        self.runtime.spawn_blocking(move || {
1071            owed.run(move || {
1072                let matches = crate::home::search::score(
1073                    &job.results,
1074                    &job.query,
1075                    job.base.as_ref(),
1076                    job.limit,
1077                );
1078                let _ = tx.send(AppEvent::HomeSearchScored {
1079                    epoch,
1080                    matches: Some(Box::new(matches)),
1081                });
1082            })
1083        });
1084    }
1085
1086    /// Rebuild the home listing from the filesystem.
1087    pub(crate) fn home_refresh(&mut self) {
1088        self.home_app.refresh_owed = false;
1089        // Every way into a source comes through here: Enter, Backspace up from a bucket,
1090        // a jump, and rows arriving while it is open.
1091        #[cfg(feature = "cloud")]
1092        self.list_browsed_cloud_source();
1093        // Listings of where the user was are pages for nobody.
1094        self.stop_listings_left_behind();
1095        self.home_app.generation = self.home_app.generation.wrapping_add(1);
1096        let generation = self.home_app.generation;
1097
1098        self.move_remembered_places();
1099        let mut catalogs = home::catalogs(&self.app_config);
1100        // Hidden with Delete on its heading: only the bundled catalog, never a user's
1101        // `examples.toml`.
1102        if self.cache.examples_hidden() {
1103            catalogs.retain(|c| c.origin != crate::home::catalog::Origin::Bundled);
1104        }
1105        self.home.set_catalogs(catalogs);
1106        let mut request = home::ListingRequest {
1107            // Filled on the worker from the cache and the desktop's recents, so the first frame
1108            // waits on no file.
1109            recents: Vec::new(),
1110            desktop_dirs: Vec::new(),
1111            browsing: self.home.browsing.clone(),
1112            probes: self.home.probes.clone(),
1113            narrowed: self.home.narrowed.clone(),
1114            network_check: self.home.network_check,
1115            cloud: self.home.cloud.clone(),
1116            catalogs: self.home.catalogs.clone(),
1117            known: Default::default(),
1118            formats: self.formats.clone(),
1119        };
1120        let read_folds = std::mem::take(&mut self.home.folds_owed);
1121        let desktop = self.app_config.home.desktop_recents;
1122        let cache = self.cache.clone();
1123        let writes = self.cache_writes.clone();
1124
1125        self.home.listing_in_flight = true;
1126        let tx = self.events.clone();
1127        let owed = self.owed_answer(AppEvent::HomeListingFailed);
1128        self.runtime.spawn_blocking(move || {
1129            owed.run(move || {
1130                // After the dataset just left is in the recents with its shape.
1131                writes.settle();
1132                // Ranked by frecency; the cursor lands on the newest, one Enter from the last file.
1133                let (recents, visits) = cache.load_recents_with_visits();
1134                let newest = recents.first().cloned();
1135                request.recents = crate::cache::by_frecency(recents, &visits);
1136                request.known = cache.load_dataset_facts();
1137                if desktop {
1138                    request.desktop_dirs = home::desktop_recent_dirs();
1139                }
1140                let listing = home::build_listing(&request);
1141                let mut visits = visits;
1142                listing.alias_visits(&mut visits);
1143                // A record shown is a record used: the ones eviction keeps.
1144                let shown: Vec<PathBuf> = listing
1145                    .sections
1146                    .iter()
1147                    .flat_map(|s| s.rows.iter().chain(&s.door))
1148                    .flat_map(|row| [row.path.clone(), home::index_key(&row.path)])
1149                    .filter(|key| request.known.contains_key(key))
1150                    .collect();
1151                cache.touch_dataset_facts(shown.iter().map(PathBuf::as_path));
1152                let _ = tx.send(AppEvent::HomeListingReady {
1153                    generation,
1154                    listing: Box::new(listing),
1155                    known: request.known,
1156                    visits,
1157                    newest,
1158                    folds: read_folds.then(|| cache.load_folds()),
1159                });
1160            })
1161        });
1162    }
1163
1164    /// Ask the worker to measure on-screen rows not yet known. Opening a footer can
1165    /// block (a FIFO, a device, a wedged mount, a failing disk), so never on the draw
1166    /// thread.
1167    pub(crate) fn request_home_measurements(&mut self) {
1168        if self.home.measure_in_flight {
1169            return;
1170        }
1171        let wanted = self.home.unmeasured_visible(MEASURE_BATCH);
1172        if wanted.is_empty() {
1173            return;
1174        }
1175
1176        self.home.measure_in_flight = true;
1177        let tx = self.events.clone();
1178        let cache = self.cache.clone();
1179        self.runtime.spawn_blocking(move || {
1180            home::look_into_batch(wanted, &cache, |path, m| {
1181                let _ = tx.send(AppEvent::HomeMeasured {
1182                    measured: vec![(path, m)],
1183                    done: false,
1184                });
1185            });
1186            let _ = tx.send(AppEvent::HomeMeasured {
1187                measured: Vec::new(),
1188                done: true,
1189            });
1190        });
1191    }
1192
1193    /// Ask an HTTP(S) server the size of the file under the cursor, once a session,
1194    /// when unmeasured. The row shows the catalog's `~33 MB` until then; the answer is
1195    /// kept for the next listing.
1196    #[cfg(feature = "http")]
1197    pub(crate) fn size_selected_web_file(&mut self) {
1198        if !self.info.head_web_rows {
1199            return;
1200        }
1201        let Some(entry) = self.home.selected_entry().cloned() else {
1202            return;
1203        };
1204        if entry.size.is_some()
1205            || !matches!(
1206                source::input_source(&entry.path),
1207                source::InputSource::Http(_)
1208            )
1209            || !self.home.sized.insert(entry.path.clone())
1210        {
1211            return;
1212        }
1213        let tx = self.events.clone();
1214        let cache = self.cache.clone();
1215        self.runtime.spawn_blocking(move || {
1216            let size = match Self::fetch_remote_size_http(&entry.path.to_string_lossy()) {
1217                Ok(Some(size)) => size,
1218                Ok(None) => return,
1219                Err(gone) => {
1220                    let _ = tx.send(AppEvent::HomeWebGone {
1221                        path: entry.path,
1222                        gone,
1223                    });
1224                    return;
1225                }
1226            };
1227            let key = home::index_key(&entry.path);
1228            let mut facts = cache.dataset_facts(&key).unwrap_or_default();
1229            facts.size = size;
1230            cache.record_dataset_facts(&[(key, facts)]);
1231            let measured = home::Measured {
1232                rows: entry.rows,
1233                cols: entry.cols,
1234                cols_sampled: entry.cols_sampled,
1235                size: Some(size),
1236                columns: entry.columns.clone(),
1237                cost: entry.cost.clone(),
1238                kind: None,
1239                holds: entry.holds.clone(),
1240            };
1241            let _ = tx.send(AppEvent::HomeSized {
1242                path: entry.path,
1243                measured,
1244            });
1245        });
1246    }
1247
1248    /// Ask a worker what the rows on screen are. Classifying reads the named directory
1249    /// (a round trip on a share, forever on a wedged mount), so only on-screen rows,
1250    /// on a detached thread, one pass at a time. It also measures remote rows, which
1251    /// [`crate::home::HomeState::unmeasured_visible`] leaves alone since probes return them
1252    /// `Unknown`.
1253    pub(crate) fn request_home_classifications(&mut self) {
1254        if self.home.classify_in_flight {
1255            return;
1256        }
1257        let wanted = self.home.unclassified_visible(CLASSIFY_BATCH);
1258        if wanted.is_empty() {
1259            return;
1260        }
1261
1262        self.home.classify_in_flight = true;
1263        let tx = self.events.clone();
1264        let cache = self.cache.clone();
1265        std::thread::spawn(move || {
1266            home::look_into_batch(wanted, &cache, |path, m| {
1267                let _ = tx.send(AppEvent::HomeClassified {
1268                    measured: vec![(path, m)],
1269                    done: false,
1270                });
1271            });
1272            let _ = tx.send(AppEvent::HomeClassified {
1273                measured: Vec::new(),
1274                done: true,
1275            });
1276        });
1277    }
1278
1279    pub fn enter_home(&mut self) {
1280        self.counting.pause_indexing();
1281        if self.return_from_quality_evidence(false) {
1282            self.analysis_modal.close();
1283        }
1284        self.close_overlays();
1285        self.stop_find();
1286        // A count of the dataset being left is read for nobody.
1287        self.stop_value_count();
1288        self.export_modal.forget_counts();
1289        self.abandon_load();
1290        // Nobody is watching the file any more.
1291        if let Some(state) = self.data_table_state.as_mut() {
1292            state.stop_following();
1293        }
1294        self.home.status = None;
1295        // The search that found the dataset comes back selected: the next character starts
1296        // a new one, and `~` opens the path prompt.
1297        self.home.filter_selected = !self.home.filter.is_empty();
1298        self.home.folds_owed = true;
1299        self.home_refresh();
1300        if let Some(open_path) = self.path.clone() {
1301            let target =
1302                crate::canonical::canonicalize(&open_path).unwrap_or_else(|_| open_path.clone());
1303            if let Some(idx) = self.home.visible().iter().position(|row| match row {
1304                home::Row::Entry { entry, .. } => {
1305                    crate::canonical::canonicalize(&entry.path)
1306                        .unwrap_or_else(|_| entry.path.clone())
1307                        == target
1308                }
1309                // Not the door: its path is the directory's, and would take the cursor from the
1310                // file to the whole-directory row.
1311                home::Row::Header { .. }
1312                | home::Row::Place { .. }
1313                | home::Row::More { .. }
1314                | home::Row::Hidden { .. }
1315                | home::Row::Up { .. }
1316                | home::Row::Door { .. } => false,
1317            }) {
1318                self.home.selected = idx;
1319            }
1320        }
1321        self.input_mode = InputMode::Home;
1322    }
1323
1324    /// Esc backs out one layer: the filter, the directory descended into, then back to
1325    /// the open data. Nothing at the top level; Ctrl+C quits.
1326    pub(crate) fn home_escape(&mut self) -> Option<AppEvent> {
1327        if !self.home.filter.is_empty() {
1328            self.home.filter.clear();
1329            self.home.sync_search_section();
1330            // On a dataset, as at launch, not on the first section's header.
1331            self.home.select_first_entry();
1332            return None;
1333        }
1334        if self.home.browsing.is_some() {
1335            if self.home.below_browse_start() {
1336                self.home_ascend();
1337            } else {
1338                // Not past where the browse began: back to the listing it started from.
1339                self.home_leave_browsing(None);
1340            }
1341            return None;
1342        }
1343        if self.data_table_state.is_some() {
1344            self.show_table();
1345            // Said on arrival: one Esc too many lands here, and the next keys act on the table.
1346            let name = self
1347                .path
1348                .as_deref()
1349                .and_then(|p| p.file_name())
1350                .map(|n| n.to_string_lossy().into_owned());
1351            if let Some(name) = name {
1352                self.flash_note(format!("Back to {name}"));
1353            }
1354        }
1355        None
1356    }
1357
1358    /// Drop the highlighted dataset from the recents list. Only from Recent: elsewhere
1359    /// a row is a file on disk, and forgetting it would imply a deletion.
1360    pub(crate) fn home_forget_selected(&mut self) {
1361        // A catalog's heading: Delete hides the bundled one until the cache is cleared; a
1362        // user's catalog is hidden by id in the config.
1363        if let Some(catalog) = self.home.selected_catalog() {
1364            if catalog.origin == crate::home::catalog::Origin::Bundled {
1365                let message = format!(
1366                    "Hide {}? It comes back after datui cache clear.",
1367                    catalog.label
1368                );
1369                self.confirmation_modal
1370                    .show_destructive(message, "Hide", Confirm::HideExamples);
1371            } else {
1372                self.home.status = Some(format!(
1373                    "[home] hide = [\"{}\"] in config.toml hides it",
1374                    catalog.id
1375                ));
1376            }
1377            return;
1378        }
1379        // A place row stands for every recent under it, so forgetting them asks first, as
1380        // Shift+Delete does.
1381        if let Some(home::Row::Place { path, held, .. }) = self.home.selected_row() {
1382            let message = format!(
1383                "Forget {held} recently opened {} under {}?",
1384                if held == 1 { "dataset" } else { "datasets" },
1385                home::display_path(&path)
1386            );
1387            self.confirmation_modal
1388                .show(message, Confirm::ForgetPlace(path.clone()));
1389            return;
1390        }
1391        // A row in catalog.toml's own section is removed from the file, as Ctrl+D does; the
1392        // same place under Recent is only forgotten as a recent.
1393        let in_mine = self
1394            .home
1395            .selected_section()
1396            .and_then(|i| self.home.sections.get(i))
1397            .is_some_and(|s| s.origin == Some("catalog.toml"));
1398        if in_mine
1399            && let Some((location, _)) = self.home_row_for_catalog()
1400            && let Some((id, name)) = self.mine_entry_at(&location)
1401        {
1402            self.home_forget_from_catalog(&id, &name);
1403            return;
1404        }
1405        let section_title = self
1406            .home
1407            .selected_section()
1408            .and_then(|i| self.home.sections.get(i))
1409            .map(|s| s.title.clone())
1410            .unwrap_or_default();
1411        if self.home.browsing.is_none()
1412            && section_title == home::HomeState::CLOUD_SECTION
1413            && let Some(id) = self
1414                .home
1415                .selected_entry()
1416                .and_then(|e| home::cloud_source_id(&e.path))
1417        {
1418            self.cache.hide_cloud_source(&id);
1419            self.home.cloud.retain(|s| s.id != id);
1420            self.home_refresh();
1421            return;
1422        }
1423        let in_recents = section_title == "Recent";
1424        if !in_recents {
1425            self.home.status = Some("Only recents and catalog.toml rows can be forgotten".into());
1426            return;
1427        }
1428        let Some(entry) = self.home.selected_entry() else {
1429            return;
1430        };
1431        self.cache.forget_recent(&entry.path);
1432        // Nothing to say: the row going is the answer.
1433        self.home.status = None;
1434        self.home_refresh();
1435    }
1436
1437    /// The location and shown name Ctrl+D adds for the highlighted row; a heading stands
1438    /// for its section's directory. Tables inside files, cloud sources and non-place
1439    /// rows have none.
1440    fn home_row_for_catalog(&self) -> Option<(PathBuf, String)> {
1441        match self.home.selected_row()? {
1442            home::Row::Place { path, .. } => {
1443                let name = home::display_path(&path);
1444                Some((path, name))
1445            }
1446            home::Row::Door { entry, .. } => {
1447                // A local door's trailing slash goes; a URL keeps its `//`, which `components`
1448                // would fold.
1449                let path: PathBuf = if matches!(
1450                    source::input_source(&entry.path),
1451                    source::InputSource::Local(_)
1452                ) {
1453                    entry.path.components().collect()
1454                } else {
1455                    entry.path.clone()
1456                };
1457                Some((path.clone(), home::display_path(&path)))
1458            }
1459            home::Row::Entry { entry, .. } => (entry.table.is_none()
1460                && !home::is_cloud_place(&entry.path)
1461                && crate::formats::members::split(&entry.path).is_none())
1462            .then(|| (entry.path.clone(), entry.name.clone())),
1463            home::Row::Header { section, .. } => {
1464                let root = self.home.sections.get(section)?.root.clone()?;
1465                let name = home::display_path(&root);
1466                Some((root, name))
1467            }
1468            home::Row::More { .. } | home::Row::Hidden { .. } | home::Row::Up { .. } => None,
1469        }
1470    }
1471
1472    /// `catalog.toml`'s path: beside the config file read, else in the config directory.
1473    fn mine_catalog_file(&self) -> Option<PathBuf> {
1474        let dir = match &self.app_config.catalog_dir {
1475            Some(dir) => dir.clone(),
1476            None => config::ConfigManager::new(APP_NAME)
1477                .ok()?
1478                .config_dir()
1479                .to_path_buf(),
1480        };
1481        Some(dir.join(catalog::MINE_FILE))
1482    }
1483
1484    /// The id and name of the `catalog.toml` entry at `location`, when there is one.
1485    fn mine_entry_at(&self, location: &Path) -> Option<(String, String)> {
1486        self.app_config
1487            .read_catalogs
1488            .iter()
1489            .find(|c| c.origin == catalog::Origin::Mine)?
1490            .dataset_at(location)
1491            .map(|d| (d.id.clone(), d.name.clone()))
1492    }
1493
1494    /// Read the catalogs again after `catalog.toml` changed, and list again.
1495    fn reload_catalogs(&mut self) -> Result<(), String> {
1496        let dir = self
1497            .mine_catalog_file()
1498            .and_then(|f| f.parent().map(Path::to_path_buf));
1499        self.app_config
1500            .read_catalog_files(dir.as_deref())
1501            .map_err(|e| e.to_string())?;
1502        // The open dataset's notes and Documentation tab follow the file.
1503        self.info
1504            .follow_catalogs(&self.app_config, self.path.as_deref());
1505        self.open_info_documentation();
1506        self.home_refresh();
1507        Ok(())
1508    }
1509
1510    /// What Ctrl+D writes for `location` named `name`: a copy of another catalog's
1511    /// entry, or the place as is.
1512    fn new_catalog_dataset(&self, location: &Path, name: &str) -> catalog::NewDataset {
1513        if let Some((_, shown)) = self.home.catalog_dataset(location) {
1514            let entry = &shown.entry;
1515            return catalog::NewDataset {
1516                name: entry.name.clone(),
1517                path: entry.local_path().map(|p| home::display_path(&p)),
1518                url: entry.url.clone(),
1519                auth: entry.auth.clone(),
1520                connection: entry.connection.clone(),
1521                description: entry.description.clone(),
1522                size: entry.size,
1523            };
1524        }
1525        let mut new = catalog::NewDataset {
1526            name: name.to_string(),
1527            ..Default::default()
1528        };
1529        if matches!(
1530            source::input_source(location),
1531            source::InputSource::Local(_)
1532        ) {
1533            let absolute = if location.is_relative() {
1534                std::env::current_dir()
1535                    .map(|cwd| cwd.join(location))
1536                    .unwrap_or_else(|_| location.to_path_buf())
1537            } else {
1538                location.to_path_buf()
1539            };
1540            new.path = Some(home::display_path(&absolute));
1541            return new;
1542        }
1543        // A store reached through a configured source: the source becomes the connection
1544        // and leaves the URL.
1545        let text = location.to_string_lossy();
1546        let (id, plain) = source::split_source_id(&text);
1547        new.url = Some(plain.into_owned());
1548        if let Some(id) = id {
1549            new.connection = Some(id.to_string());
1550        }
1551        new
1552    }
1553
1554    /// Ctrl+D: add the row under the cursor to `catalog.toml`, or forget it from there.
1555    pub(crate) fn home_toggle_catalog(&mut self) {
1556        let Some((location, name)) = self.home_row_for_catalog() else {
1557            self.home.status = Some("Move to a dataset or directory to add it".into());
1558            return;
1559        };
1560        if let Some((id, name)) = self.mine_entry_at(&location) {
1561            self.home_forget_from_catalog(&id, &name);
1562            return;
1563        }
1564        let Some(file) = self.mine_catalog_file() else {
1565            self.home.status = Some("No config directory to keep catalog.toml in".into());
1566            return;
1567        };
1568        let new = self.new_catalog_dataset(&location, &name);
1569        // A source found on the machine but not configured cannot be named in a catalog:
1570        // the URL would be read elsewhere.
1571        if let Some(connection) = &new.connection
1572            && !self
1573                .app_config
1574                .cloud
1575                .connections
1576                .iter()
1577                .any(|c| c.name == *connection)
1578        {
1579            self.home.status = Some(format!(
1580                "Not added: {connection} is not a [[cloud.connections]] entry in the config"
1581            ));
1582            return;
1583        }
1584        if let Err(why) = new.check() {
1585            self.home.status = Some(format!("Not added: {why}"));
1586            return;
1587        }
1588        let label = self
1589            .app_config
1590            .read_catalogs
1591            .iter()
1592            .find(|c| c.origin == catalog::Origin::Mine)
1593            .map(|c| c.label.clone())
1594            .unwrap_or_else(|| catalog::MINE_LABEL.to_string());
1595        match catalog::add(&file, &new) {
1596            Ok(_) => match self.reload_catalogs() {
1597                Ok(()) => self.flash_note(format!("Added {} to {label}", new.name)),
1598                Err(e) => self.error_modal.show(e),
1599            },
1600            Err(e) => self.error_modal.show(e.to_string()),
1601        }
1602    }
1603
1604    /// Remove the entry `id` from `catalog.toml`.
1605    fn home_forget_from_catalog(&mut self, id: &str, name: &str) {
1606        let Some(file) = self.mine_catalog_file() else {
1607            return;
1608        };
1609        match catalog::forget(&file, id) {
1610            Ok(()) => match self.reload_catalogs() {
1611                Ok(()) => self.flash_note(format!("Forgot {name}")),
1612                Err(e) => self.error_modal.show(e),
1613            },
1614            Err(e) => self.error_modal.show(e.to_string()),
1615        }
1616    }
1617
1618    /// Move pre-0.4.0 Ctrl+D directories from the cache into `catalog.toml`, once.
1619    fn move_remembered_places(&mut self) {
1620        if std::mem::replace(&mut self.home_app.remembered_moved, true) {
1621            return;
1622        }
1623        let places = self.cache.load_remembered_places();
1624        if places.is_empty() {
1625            return;
1626        }
1627        let Some(file) = self.mine_catalog_file() else {
1628            return;
1629        };
1630        // The cache's list goes only once all are in catalog.toml; a failure retries next
1631        // run.
1632        match catalog::move_places(&file, &places) {
1633            Ok(_) => self.cache.clear_remembered_places(),
1634            Err(e) => {
1635                log::warn!(target: "datui", "moving remembered places into catalog.toml: {e:#}")
1636            }
1637        }
1638        let dir = file.parent().map(Path::to_path_buf);
1639        if let Err(e) = self.app_config.read_catalog_files(dir.as_deref()) {
1640            log::warn!(target: "datui", "reading catalog.toml: {e:#}");
1641        }
1642    }
1643
1644    /// Ctrl+E: the Documentation view of the row under the cursor.
1645    pub(crate) fn home_open_documentation(&mut self) {
1646        let Some((path, doc)) = self.home_documented_row() else {
1647            self.home.status =
1648                Some("Ctrl+E shows what a catalog or a format spec says of a row".into());
1649            return;
1650        };
1651        let measured = self
1652            .home
1653            .selected_entry()
1654            .filter(|e| {
1655                e.path == path
1656                    && doc
1657                        .catalog
1658                        .as_ref()
1659                        .is_some_and(|(_, entry)| entry.location() == path)
1660            })
1661            .and_then(|e| e.size);
1662        self.info.documentation.open(doc, measured);
1663        self.info.documentation.links_open = self.home_app.local_desktop;
1664    }
1665
1666    /// What Ctrl+E documents for the row under the cursor, with its path: the catalog
1667    /// dataset it is or is inside, and its format spec's docs, if any.
1668    pub(crate) fn home_documented_row(
1669        &self,
1670    ) -> Option<(PathBuf, widgets::documentation::Documented)> {
1671        let row = self.home.selected_row();
1672        let (path, file) = match &row {
1673            Some(home::Row::Entry { entry, .. }) | Some(home::Row::Door { entry, .. }) => {
1674                (Some(entry.path.clone()), Some(*entry))
1675            }
1676            Some(home::Row::Place { path, .. }) => (Some(path.clone()), None),
1677            Some(home::Row::Header { section, .. }) => (
1678                self.home
1679                    .sections
1680                    .get(*section)
1681                    .and_then(|s| s.root.clone()),
1682                None,
1683            ),
1684            _ => (None, None),
1685        };
1686        let path = path?;
1687        let catalog = home::catalog_entry_for(&self.home.catalogs, &path);
1688        let spec = file
1689            .filter(|e| e.kind == discover::EntryKind::File)
1690            .and_then(|e| e.format_spec.as_deref())
1691            .and_then(|name| self.home.formats.get(name))
1692            .and_then(|spec| spec.docs())
1693            .map(std::sync::Arc::new);
1694        // A record type's row (`day.ord/add`) documents its file.
1695        let name = file
1696            .map(|e| {
1697                let text = e.path.to_string_lossy();
1698                text.strip_suffix(e.name.as_str())
1699                    .filter(|_| e.table.is_some())
1700                    .map(|file| file.trim_end_matches(std::path::is_separator))
1701                    .and_then(|file| Path::new(file).file_name())
1702                    .map_or_else(|| e.name.clone(), |n| n.to_string_lossy().into_owned())
1703            })
1704            .unwrap_or_default();
1705        let doc = widgets::documentation::Documented::new(catalog, spec, name)?;
1706        Some((path, doc))
1707    }
1708
1709    /// Whether Delete on the selected row hides a catalog: the bundled one's heading.
1710    pub(crate) fn home_hides_catalog(&self) -> bool {
1711        self.home
1712            .selected_catalog()
1713            .is_some_and(|c| c.origin == crate::home::catalog::Origin::Bundled)
1714    }
1715
1716    /// What Ctrl+D does on the row under the cursor, as the footer names it: add to
1717    /// `catalog.toml` or forget from it; `None` where it cannot add.
1718    pub(crate) fn home_catalog_action(&self) -> Option<&'static str> {
1719        let (location, _) = self.home_row_for_catalog()?;
1720        Some(if self.mine_entry_at(&location).is_some() {
1721            "Forget"
1722        } else {
1723            "Add"
1724        })
1725    }
1726
1727    /// Fold or unfold the section under the cursor; folding moves the cursor to its
1728    /// header.
1729    pub(crate) fn home_toggle_fold(&mut self) {
1730        if let Some(section) = self.home.selected_section() {
1731            self.home.toggle_collapsed(section);
1732            self.home.clamp_selection();
1733            self.cache.save_folds(&self.home.folds);
1734        }
1735    }
1736
1737    pub(crate) fn home_collapse(&mut self, collapse: bool) {
1738        // The browsed listing is the whole screen: it never folds, and no fold is
1739        // remembered for its path (see `set_collapsed`).
1740        if self.home.browsing.is_some() {
1741            return;
1742        }
1743        let Some(section) = self.home.selected_section() else {
1744            return;
1745        };
1746        // → on a section's more row shows it whole; ← on a row its cut would hide cuts it
1747        // back. Elsewhere they fold.
1748        if collapse && self.home.cut_again(section) {
1749            return;
1750        }
1751        if !collapse && matches!(self.home.selected_row(), Some(home::Row::More { .. })) {
1752            self.home.show_all(section);
1753            return;
1754        }
1755        if collapse && !self.home.is_collapsed(section) {
1756            self.home.set_collapsed(section, true);
1757            if let Some(idx) = self
1758                .home
1759                .visible()
1760                .iter()
1761                .position(|row| row.section() == section)
1762            {
1763                self.home.selected = idx;
1764            }
1765        } else if !collapse {
1766            self.home.set_collapsed(section, false);
1767        }
1768        self.home.clamp_selection();
1769        self.cache.save_folds(&self.home.folds);
1770    }
1771
1772    /// Step out of a directory that was descended into.
1773    pub(crate) fn home_ascend(&mut self) {
1774        let Some(current) = self.home.browsing.clone() else {
1775            return;
1776        };
1777        let parent = self.home.parent_of(&current);
1778        self.home_leave_browsing(parent);
1779    }
1780
1781    /// Move the browse up to `to`, or back to the root listing when `None`.
1782    fn home_leave_browsing(&mut self, to: Option<PathBuf>) {
1783        // What the last place said of itself no longer applies.
1784        self.home.status = None;
1785        let from = std::mem::replace(&mut self.home.browsing, to);
1786        // Backspace can climb above the browse start; the start follows so Esc has a place
1787        // to stop.
1788        if !self.home.below_browse_start() {
1789            self.home.browse_start = self.home.browsing.clone();
1790        }
1791        // The filter, search and row left here come back; the cursor returns to that row
1792        // when the listing lands. A search of the old place no longer applies.
1793        self.home.come_back(from);
1794        self.home.sync_search_section();
1795        self.home.selected = 0;
1796        self.home_refresh();
1797        if !self.home.filter.is_empty() {
1798            self.spawn_home_search();
1799        }
1800    }
1801
1802    /// Whether a peek's answer changes what a row draws: its kind, or anything in
1803    /// `holds` (`truncated` turns `dir` into `dir+`). Kept answers rebuild the listing
1804    /// on the UI thread, so the rest become "a directory, nothing to say", still sent
1805    /// to clear `peeking`.
1806    #[cfg(feature = "cloud")]
1807    pub(crate) fn peek_tells_a_row_something(
1808        answer: &(discover::EntryKind, discover::Holds),
1809    ) -> bool {
1810        answer.0 != discover::EntryKind::Directory || !answer.1.is_empty()
1811    }
1812
1813    /// Peek inside cloud directories at or near the cursor so datasets show `hive` or
1814    /// `multi` and open as one: one small listing per directory, once a session, plus
1815    /// up to three few-KiB footer reads for a `multi` candidate. On-screen rows a batch
1816    /// at a time, the highlighted first.
1817    #[cfg(feature = "cloud")]
1818    pub(crate) fn peek_cloud_directories(&mut self) {
1819        const PEEKS_AT_ONCE: usize = 4;
1820        let directories = self.home.cloud_directories_to_peek(PEEKS_AT_ONCE);
1821        if directories.is_empty() {
1822            return;
1823        }
1824        // Marked out, not answered: a second pass must not ask again, and writing an
1825        // answer now would claim one before the request is made.
1826        for directory in &directories {
1827            self.home.peeking.insert(directory.clone());
1828        }
1829        let tx = self.events.clone();
1830        let cloud = self.app_config.cloud.clone();
1831        self.runtime.spawn(async move {
1832            let permits = Arc::new(tokio::sync::Semaphore::new(PEEKS_AT_ONCE));
1833            let mut peeks = tokio::task::JoinSet::new();
1834            // By task, so a panicking peek is sent back as failed rather than spinning
1835            // forever.
1836            let mut asked = std::collections::HashMap::new();
1837            for directory in directories {
1838                let (permits, cloud) = (permits.clone(), cloud.clone());
1839                let task_directory = directory.clone();
1840                let task = peeks.spawn(async move {
1841                    let directory = task_directory;
1842                    let _permit = permits.acquire_owned().await;
1843                    let kind =
1844                        crate::cloud::cloud_browse::peek_kind(&directory.to_string_lossy(), &cloud)
1845                            .await;
1846                    (directory, kind)
1847                });
1848                asked.insert(task.id(), directory);
1849            }
1850            // Sent a few at a time so labels fill in without a rebuild per directory. Every
1851            // directory asked is sent back, undecided and failed ones too: that clears
1852            // `peeking` and keeps one request per directory.
1853            let mut found = Vec::new();
1854            let mut failed = Vec::new();
1855            while let Some(joined) = peeks.join_next_with_id().await {
1856                match joined {
1857                    Ok((_, (directory, Ok(answer)))) => {
1858                        let answer = Some(answer)
1859                            .filter(Self::peek_tells_a_row_something)
1860                            .unwrap_or((discover::EntryKind::Directory, Default::default()));
1861                        found.push((directory, answer));
1862                    }
1863                    Ok((_, (directory, Err(_)))) => failed.push(directory),
1864                    Err(error) => failed.extend(asked.remove(&error.id())),
1865                }
1866                if found.len() + failed.len() >= PEEKS_AT_ONCE {
1867                    let _ = tx.send(AppEvent::HomeCloudKinds {
1868                        kinds: std::mem::take(&mut found),
1869                        failed: std::mem::take(&mut failed),
1870                    });
1871                }
1872            }
1873            if !found.is_empty() || !failed.is_empty() {
1874                let _ = tx.send(AppEvent::HomeCloudKinds {
1875                    kinds: found,
1876                    failed,
1877                });
1878            }
1879        });
1880    }
1881
1882    /// Browse into a directory or bucket, local or remote.
1883    pub(crate) fn home_browse_into(&mut self, path: PathBuf) {
1884        self.home.leave_mark();
1885        if self.home.browsing.is_none() {
1886            self.home.browse_start = Some(path.clone());
1887        } else if self.home.browse_start.is_none() {
1888            self.home.browse_start = self.home.browsing.clone();
1889        }
1890        self.home.browsing = Some(path);
1891        // What the last walk found and the status line describe a different place; a fresh
1892        // walk starts on the next keystroke. A caller with news of the new place says it
1893        // after this returns.
1894        self.home.status = None;
1895        self.home.search.reset();
1896        self.home.filter.clear();
1897        self.home.sync_search_section();
1898        self.home.selected = 0;
1899        self.home_refresh();
1900    }
1901
1902    /// Whether the highlighted row is the `(all files)` door, which opens the browsed
1903    /// directory and so is already inside it.
1904    fn selection_opens_the_whole_directory(&self) -> bool {
1905        self.home.selection_is_the_door()
1906    }
1907
1908    /// The highlighted row when → goes inside it: any directory, local or remote,
1909    /// whatever its label. Files, headers and the door are left out.
1910    pub(crate) fn selected_directory_to_enter(&self) -> Option<PathBuf> {
1911        // A place under `RECENT` has no entry, so it is answered first.
1912        if let Some(home::Row::Place { path, .. }) = self.home.selected_row() {
1913            return home::place_is_browsable(&path).then_some(path);
1914        }
1915        let entry = self.home.selected_entry()?;
1916        if self.selection_opens_the_whole_directory() || self.home.missing.contains(&entry.path) {
1917            return None;
1918        }
1919        // A SQLite database lists its tables, however many it has.
1920        if entry.cost.tables.is_some() {
1921            return Some(entry.path.clone());
1922        }
1923        (!matches!(
1924            entry.kind,
1925            discover::EntryKind::File | discover::EntryKind::Other
1926        ))
1927        .then_some(entry.path.clone())
1928    }
1929
1930    /// Why Enter on a bucket directory's `(all files)` row reads nothing, by Enter's
1931    /// own rule: a hive root or one-table directory reads through its files, and one
1932    /// with a reader for its contents reads with that.
1933    #[cfg(feature = "cloud")]
1934    pub(crate) fn why_a_door_reads_nothing(entry: &discover::Entry) -> Option<String> {
1935        if !home::is_object_store_url(&entry.path)
1936            || matches!(
1937                entry.kind,
1938                discover::EntryKind::Hive | discover::EntryKind::MultiFile
1939            )
1940            || Self::cloud_prefix_format(&entry.holds).is_some()
1941        {
1942            return None;
1943        }
1944        Self::why_a_cloud_prefix_cannot_be_read(&entry.holds)
1945    }
1946
1947    /// Why an object-store prefix cannot be read as one table, from its listed
1948    /// contents (no request). `None` when it may yet be Parquet: it holds Parquet,
1949    /// or no data files and perhaps data a level down.
1950    #[cfg(feature = "cloud")]
1951    pub(crate) fn why_a_cloud_prefix_cannot_be_read(holds: &discover::Holds) -> Option<String> {
1952        let reads_parquet =
1953            |name: &str| crate::FileFormat::from_name(name) == Some(crate::FileFormat::Parquet);
1954        if holds.formats.iter().any(|(name, _)| reads_parquet(name)) {
1955            return None;
1956        }
1957        match holds.formats.as_slice() {
1958            // Data files, none Parquet. `label()` says `mixed` for several formats, so the
1959            // line spells them out.
1960            [] => {
1961                // Nothing readable. A refusal only if nothing is below: sub-prefixes may hold
1962                // Parquet a level down.
1963                (holds.not_read > 0 && holds.directories == 0).then(|| {
1964                    "this prefix holds nothing datui can read — datui reads a directory in \
1965                     an object store as Parquet only."
1966                        .to_string()
1967                })
1968            }
1969            formats => {
1970                let held = formats
1971                    .iter()
1972                    .map(|(name, count)| format!("{count} {name}"))
1973                    .collect::<Vec<_>>()
1974                    .join(", ");
1975                Some(format!(
1976                    "this prefix holds {held} — datui reads a directory in an object store \
1977                     as Parquet only. Open one of the files below instead."
1978                ))
1979            }
1980        }
1981    }
1982
1983    /// The reader an object-store prefix calls for, from its listing's counts: the
1984    /// commonest format, ranked by `rank_formats` as on disk, Parquet included. `None`
1985    /// when nothing there has a multi-file reader, where the refusal belongs.
1986    #[cfg(feature = "cloud")]
1987    pub(crate) fn cloud_prefix_format(
1988        holds: &discover::Holds,
1989    ) -> Option<(FileFormat, Vec<(FileFormat, usize)>)> {
1990        // A saved DatasetDict: its splits are Arrow, read one at a time.
1991        if holds.dataset_dict {
1992            return Some((FileFormat::Arrow, Vec::new()));
1993        }
1994        // Model weights beside config and tokenizer JSON: the prefix is the model, and the
1995        // JSON is not data passed over.
1996        if let Some((name, _)) = holds.model_weights() {
1997            return FileFormat::from_name(name).map(|format| (format, Vec::new()));
1998        }
1999        let (name, _) = holds.formats.first()?;
2000        // A GPS log is read whole from disk; a bucket's logs open one at a time, as its
2001        // text files do.
2002        let format = FileFormat::from_name(name)
2003            .filter(|f| f.reads_many_files() && !f.reads_into() && !f.is_lines())?;
2004        // What taking the commonest passes over. Polars lists the prefix and never sees
2005        // other formats, so the note comes from the listing on screen.
2006        let left_out = holds
2007            .formats
2008            .iter()
2009            .skip(1)
2010            .filter_map(|(name, n)| FileFormat::from_name(name).map(|f| (f, *n)))
2011            .collect();
2012        Some((format, left_out))
2013    }
2014
2015    /// Open the highlighted entry: toggle a section, descend, or load a dataset.
2016    pub(crate) fn home_open_selected(&mut self) -> Option<AppEvent> {
2017        match self.home.selected_row() {
2018            // Into the directory or prefix the recents under it live in.
2019            Some(home::Row::Place { path, .. }) => {
2020                if home::place_is_browsable(&path) {
2021                    self.home_browse_into(path);
2022                } else {
2023                    self.home.status = Some(
2024                        "An HTTP server has no listing to browse. Open a file under it".into(),
2025                    );
2026                }
2027                return None;
2028            }
2029            // The rest of `RECENT`, or of a directory, for the session.
2030            Some(home::Row::More { section, .. }) => {
2031                self.home.show_all(section);
2032                return None;
2033            }
2034            // Up a level: as Backspace while browsing, and above a root at the listing.
2035            Some(home::Row::Up { section }) => {
2036                if self.home.browsing.is_some() {
2037                    self.home_ascend();
2038                } else if let Some(parent) = self
2039                    .home
2040                    .sections
2041                    .get(section)
2042                    .and_then(|s| s.root.as_deref())
2043                    .and_then(|root| self.home.parent_of(root))
2044                {
2045                    self.home_browse_into(parent);
2046                }
2047                return None;
2048            }
2049            // What Ctrl+A shows; the cursor goes to the first, where the hidden row stood.
2050            Some(home::Row::Hidden { .. }) => {
2051                self.home.hide_unreadable = false;
2052                if let Some(idx) = self.home.visible().iter().position(|row| {
2053                    matches!(row, home::Row::Entry { entry, .. }
2054                        if entry.hidden_by_default())
2055                }) {
2056                    self.home.selected = idx;
2057                }
2058                return None;
2059            }
2060            _ => {}
2061        }
2062        if self.home.selection_is_header() {
2063            self.home_toggle_fold();
2064            return None;
2065        }
2066        let entry = self.home.selected_entry()?.clone();
2067        // A collection's local dataset that is not there: said here, where it was named.
2068        if self.home.missing.contains(&entry.path) {
2069            self.home.status = Some(format!(
2070                "{} does not exist",
2071                home::display_path(&entry.path)
2072            ));
2073            return None;
2074        }
2075        // A place a collection suggests opens as one table on Enter; → still goes in.
2076        if entry.kind != discover::EntryKind::File && self.home.bookmark(&entry.path).is_some() {
2077            #[cfg(feature = "cloud")]
2078            let reader = if home::is_object_store_url(&entry.path)
2079                && !matches!(
2080                    entry.kind,
2081                    discover::EntryKind::Hive | discover::EntryKind::MultiFile
2082                ) {
2083                Self::cloud_prefix_format(&entry.holds)
2084            } else {
2085                None
2086            };
2087            #[cfg(not(feature = "cloud"))]
2088            let reader = None;
2089            let directory = home::directory_dataset_url(&entry.path);
2090            return Some(self.home_open_directory_as(directory, true, None, reader));
2091        }
2092        // The `(all files)` row opens its directory whatever the label: no label can lock
2093        // the user out. Sent straight to the open, since `open_what_it_is` would read the
2094        // label and step inside again.
2095        if self.selection_opens_the_whole_directory() {
2096            // A lake table read as Parquet counts tombstoned rows and every rewritten version,
2097            // so the read is labeled, not refused: a note in the panel and a footer chip say
2098            // so, and Enter one level up explains datui does not read the table itself yet.
2099            let lake = entry.kind.lake_name();
2100            // Pick an object-store prefix's reader from its listed contents (no request), so a
2101            // CSV prefix is not scanned as Parquet and blamed on credentials. A prefix the
2102            // listing already calls a dataset is left alone: a hive root reads through its
2103            // partitions despite a stray `manifest.csv`.
2104            #[cfg(feature = "cloud")]
2105            let reader = if home::is_object_store_url(&entry.path)
2106                && !matches!(
2107                    entry.kind,
2108                    discover::EntryKind::Hive | discover::EntryKind::MultiFile
2109                ) {
2110                let reader = Self::cloud_prefix_format(&entry.holds);
2111                // Nothing readable: the refusal names what is there rather than blaming the
2112                // connection.
2113                if reader.is_none()
2114                    && let Some(what) = Self::why_a_cloud_prefix_cannot_be_read(&entry.holds)
2115                {
2116                    self.home.status = Some(what);
2117                    return None;
2118                }
2119                reader
2120            } else {
2121                None
2122            };
2123            #[cfg(not(feature = "cloud"))]
2124            let reader = None;
2125            // `hive: true` reads it as one and carries partition columns through the hive
2126            // route. The cloud route returns before the dispatch.
2127            let directory = home::directory_dataset_url(&entry.path);
2128            return Some(self.home_open_directory_as(directory, true, lake, reader));
2129        }
2130        // A row nothing has looked at is looked at first: `Unknown` is offered as openable,
2131        // and a lake root would otherwise read as one table.
2132        let mut entry = entry;
2133        if entry.kind == discover::EntryKind::Unknown {
2134            if self.looking_could_block(&entry.path) {
2135                return Some(AppEvent::ClassifyThenOpen {
2136                    path: entry.path,
2137                    jump: false,
2138                });
2139            }
2140            if entry.path.is_dir() {
2141                entry.kind = discover::classify_directory(&entry.path);
2142            }
2143        }
2144        // A database of several tables lists them; one not yet measured opens and lands on
2145        // its tables the same way.
2146        if entry.enter_lists_tables() {
2147            self.home_browse_into(entry.path);
2148            return None;
2149        }
2150        self.open_what_it_is(entry.path, entry.kind, false)
2151    }
2152
2153    /// Browse into `path` as a jump: the browse starts here, so Esc returns to the
2154    /// listing rather than up through the path's parents.
2155    pub(crate) fn home_jump_into(&mut self, path: PathBuf) {
2156        // Only the listing's mark is still a way back.
2157        self.home.trail.retain(|mark| mark.place.is_none());
2158        if self.home.browsing.is_none() {
2159            self.home.leave_mark();
2160        }
2161        self.home.browse_start = Some(path.clone());
2162        self.home.browsing = Some(path);
2163        self.home.status = None;
2164        self.home.search.reset();
2165        self.home.filter.clear();
2166        self.home.sync_search_section();
2167        self.home.selected = 0;
2168        self.home_refresh();
2169    }
2170
2171    /// Load a path from the home screen. The `Open` handler records the recent.
2172    pub(crate) fn home_open_path(&mut self, path: PathBuf, hive: bool) -> AppEvent {
2173        self.home_open_directory(path, hive, None)
2174    }
2175
2176    /// As [`Self::home_open_path`], noting whether the directory is a lake table whose
2177    /// plain files are read, so the dataset can say so.
2178    fn home_open_directory(
2179        &mut self,
2180        path: PathBuf,
2181        hive: bool,
2182        lake: Option<&'static str>,
2183    ) -> AppEvent {
2184        self.home_open_directory_as(path, hive, lake, None)
2185    }
2186
2187    /// As [`Self::home_open_directory`], naming the reader. For an object-store prefix
2188    /// the cloud scan runs before the format dispatch, so the listing's format travels
2189    /// with the open or the scan falls back to Parquet.
2190    fn home_open_directory_as(
2191        &mut self,
2192        path: PathBuf,
2193        hive: bool,
2194        lake: Option<&'static str>,
2195        reader: Option<(FileFormat, Vec<(FileFormat, usize)>)>,
2196    ) -> AppEvent {
2197        let (format, left_out) = match reader {
2198            Some((format, left_out)) => (Some(format), left_out),
2199            None => (None, Vec::new()),
2200        };
2201        // Told, not stat'ed: the caller knows, and a stat on a gone share would freeze the
2202        // key thread (also why `Open` fills in the size).
2203        let options = OpenOptions {
2204            hive,
2205            read_as_plain_files_of: lake,
2206            format,
2207            left_out,
2208            ..self.open_defaults()
2209        };
2210        self.show_table();
2211        // Chosen here, so a failure is reported here.
2212        self.announce_open(true, "Scanning input".to_string(), 10);
2213        // The frame drawn before `Open` runs says which file; `Open` fills in the size a
2214        // frame later, off this thread.
2215        self.name_what_is_loading(path.clone());
2216        AppEvent::Open(vec![path], options)
2217    }
2218
2219    /// The home screen's worker answers.
2220    pub(crate) fn home_event(&mut self, event: AppEvent) -> Option<AppEvent> {
2221        match event {
2222            AppEvent::HomeListingReady {
2223                generation,
2224                listing,
2225                known,
2226                visits,
2227                newest,
2228                folds,
2229            } => {
2230                // Only the current listing's answer clears the flag; a stale one landing first
2231                // would say nothing is in flight.
2232                if generation == self.home_app.generation {
2233                    self.home.listing_in_flight = false;
2234                }
2235                // Read fresh from the cache, so true whichever listing carried them; only the
2236                // first listing after entering home carries the folds.
2237                self.home.known = known;
2238                self.home.set_visits(visits);
2239                self.home.newest_recent = newest;
2240                if let Some(folds) = folds {
2241                    self.home.folds = folds;
2242                }
2243                // A superseded listing describes somewhere the user has left.
2244                if generation != self.home_app.generation {
2245                    return None;
2246                }
2247                self.home.apply_listing(*listing);
2248                // Probes are chosen from the sections, so only once they exist.
2249                self.spawn_home_probes();
2250                #[cfg(feature = "cloud")]
2251                self.spawn_cloud_discovery();
2252                self.request_home_measurements();
2253                self.request_home_classifications();
2254                None
2255            }
2256            AppEvent::HomeListingFailed => {
2257                // The rows already listed stay. The panic is flashed as a raw worker's.
2258                self.home.listing_in_flight = false;
2259                None
2260            }
2261            AppEvent::HomeMeasured { measured, done } => {
2262                for (path, m) in measured {
2263                    self.home.enriched.insert(path, m);
2264                }
2265                // A batch answers one file per event; folding each into every row and
2266                // rebuilding the list per file made a big directory cost a list per file.
2267                self.home_app.measured_owed = true;
2268                if done {
2269                    self.apply_owed_measurements();
2270                    self.home.measure_in_flight = false;
2271                    self.request_home_measurements();
2272                }
2273                None
2274            }
2275            AppEvent::HomeSized { path, measured } => {
2276                // Only the size: a measurement that landed meanwhile keeps the rest.
2277                match self.home.enriched.get_mut(&path) {
2278                    Some(known) => known.size = measured.size,
2279                    None => {
2280                        self.home.enriched.insert(path, measured);
2281                    }
2282                }
2283                self.home.apply_measurements();
2284                None
2285            }
2286            AppEvent::HomeWebGone { path, gone } => {
2287                self.home.web_gone.insert(path, gone);
2288                None
2289            }
2290            AppEvent::HomeClassified { measured, done } => {
2291                // Kept even if the listing was rebuilt since: probes and peeks rebuild it often,
2292                // and dropping answers would leave rows unlabeled.
2293                for (path, m) in measured {
2294                    self.home.enriched.insert(path, m);
2295                }
2296                // Nothing re-sorts: kinds are written into rows in place, so the listing never
2297                // reshuffles under the cursor.
2298                self.home.apply_measurements();
2299                // The next batch comes from the viewport as it is now, not the rows scrolled past.
2300                if done {
2301                    self.home.classify_in_flight = false;
2302                    self.request_home_classifications();
2303                }
2304                None
2305            }
2306            AppEvent::HomePathListed { listing } => {
2307                // Kept only for the directory still being typed.
2308                if self.home.path_input_active
2309                    && home::typed_dir(&self.home.path_input) == listing.dir
2310                {
2311                    self.home.path_listing = Some(*listing);
2312                    if self.home.path_pick.is_none() {
2313                        self.home.pick_first_path();
2314                    }
2315                }
2316                None
2317            }
2318            AppEvent::HomePathCompleted {
2319                generation,
2320                typed,
2321                completed,
2322                candidates,
2323            } => {
2324                // Discard if the user typed since asking: completing would scramble their input.
2325                if generation != self.home_app.generation || self.home.path_input != typed {
2326                    return None;
2327                }
2328                if candidates == 0 {
2329                    self.home.status = Some("No such path".to_string());
2330                } else {
2331                    self.home.status = None;
2332                    if candidates > 1 {
2333                        self.flash_note(format!("{candidates} matches"));
2334                    }
2335                    self.home.path_input = completed;
2336                    self.home.pick_first_path();
2337                }
2338                None
2339            }
2340            AppEvent::HomePreviewReady {
2341                path,
2342                stamp,
2343                read_at,
2344                rows,
2345                prepared,
2346            } => {
2347                let prepared = prepared.lock().ok().and_then(|mut p| p.take());
2348                // The columns came with the rows: the pane lists them, for a CSV too.
2349                if let Some(prepared) = &prepared {
2350                    let schema = prepared
2351                        .state
2352                        .schema()
2353                        .iter()
2354                        .map(|(name, dtype)| (name.to_string(), dtype.clone()))
2355                        .collect();
2356                    self.home_app
2357                        .schema_cache
2358                        .insert(path.clone(), Some(schema));
2359                }
2360                let prepared = prepared.filter(|_| read_at.is_some());
2361                self.home_app.previews.landed(
2362                    path,
2363                    stamp,
2364                    read_at.unwrap_or(stamp),
2365                    rows,
2366                    prepared,
2367                );
2368                None
2369            }
2370            AppEvent::HomeSchemaReady {
2371                generation,
2372                path,
2373                preview,
2374            } => {
2375                self.home_app.schema_inflight.retain(|p| p != &path);
2376                // A preview's columns are not taken back by a metadata read that had none.
2377                let known = self
2378                    .home_app
2379                    .schema_cache
2380                    .get(&path)
2381                    .is_some_and(Option::is_some);
2382                if generation == self.home_app.generation && (preview.is_some() || !known) {
2383                    self.home_app.schema_cache.insert(path.clone(), preview);
2384                }
2385                None
2386            }
2387            AppEvent::HomeSearchBatch {
2388                generation,
2389                root,
2390                found,
2391                scanned,
2392            } => {
2393                // Batches from a walk a later navigation superseded describe a place left behind;
2394                // the walk is abandoned, not cancelled, so late batches are expected. A refresh of
2395                // the same place supersedes nothing.
2396                if generation == self.home_app.search_generation {
2397                    self.home.search_batch(&root, found, scanned);
2398                }
2399                None
2400            }
2401            AppEvent::HomeSearchScored { epoch, matches } => {
2402                // A scoring that died is not retried (it would die again); listed matches stand.
2403                if let Some(matches) = matches {
2404                    self.home.search_scored(epoch, *matches);
2405                }
2406                None
2407            }
2408            AppEvent::HomeSearchDone {
2409                generation,
2410                root,
2411                scanned,
2412                limited,
2413            } => {
2414                if generation == self.home_app.search_generation {
2415                    self.home.search_finished(&root, scanned, limited);
2416                }
2417                self.home_app.search_inflight = false;
2418                // A walk abandoned by a browse held up the one the filter now asks for.
2419                if !self.home.filter.is_empty() && self.home.search.root.is_none() {
2420                    self.spawn_home_search();
2421                }
2422                None
2423            }
2424            #[cfg(feature = "cloud")]
2425            AppEvent::HomeCloudSources { sources } => {
2426                self.home.cloud = sources;
2427                self.home_refresh();
2428                None
2429            }
2430            #[cfg(feature = "cloud")]
2431            AppEvent::HomeCloudListed {
2432                id,
2433                buckets,
2434                details,
2435                failure,
2436                listed_at,
2437            } => {
2438                if let Some(source) = self.home.cloud.iter_mut().find(|s| s.id == id) {
2439                    source.refreshing = false;
2440                    for (place, lines) in details {
2441                        source.place_details.insert(place, lines);
2442                    }
2443                    match failure {
2444                        // A failed refresh keeps the last buckets, and the row says it failed.
2445                        Some((short, detail)) => {
2446                            for bucket in buckets {
2447                                if !source.buckets.contains(&bucket) {
2448                                    source.buckets.push(bucket);
2449                                }
2450                            }
2451                            source.status = home::CloudStatus::Failed { short, detail };
2452                        }
2453                        None => {
2454                            source.buckets = buckets;
2455                            source.status = home::CloudStatus::Listed;
2456                            source.listed_at = Some(listed_at);
2457                        }
2458                    }
2459                }
2460                self.home_refresh();
2461                None
2462            }
2463            AppEvent::HomeNarrowed {
2464                dir,
2465                prefix,
2466                listed,
2467            } => {
2468                // Only the current request: a replaced one may answer late with a shorter prefix.
2469                let asked = self
2470                    .home_app
2471                    .narrowing
2472                    .as_ref()
2473                    .is_some_and(|(d, p, _)| *d == dir && *p == prefix);
2474                if asked {
2475                    self.home_app.narrowing = None;
2476                }
2477                // Only while it is still where the user is and what the filter asks.
2478                let wanted = asked
2479                    && self.home.browsing.as_ref() == Some(&dir)
2480                    && !self.home.filter.is_empty();
2481                if let (Some((rows, truncated)), true) = (listed, wanted) {
2482                    self.home.narrowed = Some(home::Narrowed {
2483                        dir,
2484                        prefix,
2485                        rows,
2486                        truncated,
2487                    });
2488                    self.home_refresh();
2489                }
2490                None
2491            }
2492            AppEvent::HomeProbeCancelled { root } => {
2493                self.home_app.probes_inflight.retain(|p| p != &root);
2494                self.home_app.listing_cancels.remove(&root);
2495                self.home.probes.stopped(&root);
2496                // Come back to after it had stopped: listed afresh.
2497                if self.home.browsing.as_ref() == Some(&root) {
2498                    self.home_refresh();
2499                }
2500                None
2501            }
2502            AppEvent::HomeProbeFailed { root, message } => {
2503                self.home_app.probes_inflight.retain(|p| p != &root);
2504                self.home_app.listing_cancels.remove(&root);
2505                self.home.probe_failed(root, Some(message));
2506                self.home_refresh();
2507                None
2508            }
2509            AppEvent::HomeProbeProgress { root, rows } => {
2510                // Only while that listing is out: a late batch must not paint over the answer.
2511                if self.home_app.probes_inflight.contains(&root) && !self.home.probes.settled(&root)
2512                {
2513                    self.home.probes.read(&root, &rows);
2514                    // Listed once a frame, however many batches came in it.
2515                    self.home_app.refresh_owed = true;
2516                }
2517                None
2518            }
2519            AppEvent::HomeProbeReady {
2520                root,
2521                rows,
2522                cut_short,
2523            } => {
2524                // Free the slot. The cap bounds threads wedged on dead mounts, which never send
2525                // this; without freeing, probing stops after MAX_CONCURRENT_PROBES roots.
2526                self.home_app.probes_inflight.retain(|p| p != &root);
2527                self.home_app.listing_cancels.remove(&root);
2528                let landed = rows.is_some();
2529                match rows {
2530                    Some(rows) => self.home.probe_ready(root.clone(), rows, cut_short),
2531                    None => self.home.probe_failed(root.clone(), None),
2532                }
2533                // A filter typed while it was listing asks the server too.
2534                #[cfg(feature = "cloud")]
2535                if cut_short && !self.home.filter.is_empty() {
2536                    self.narrow_cloud_listing();
2537                }
2538                // An account read with its keys (the sign-in has no data role) says so.
2539                #[cfg(feature = "cloud")]
2540                if let Some((account, _, _)) = source::azure_parts(&root.to_string_lossy())
2541                    && crate::cloud::azure::remembered_key(&account).is_some()
2542                {
2543                    for source in &mut self.home.cloud {
2544                        let place = source
2545                            .buckets
2546                            .iter()
2547                            .find(|b| home::cloud_account(b).is_some_and(|(_, a)| a == account))
2548                            .cloned();
2549                        if let Some(place) = place {
2550                            let lines = source.place_details.entry(place).or_default();
2551                            if !lines.iter().any(|(k, _)| k == "access") {
2552                                lines.push(("access".to_string(), "access key".to_string()));
2553                            }
2554                        }
2555                    }
2556                }
2557                // Rebuild so the listing picks up the result.
2558                self.home_refresh();
2559                // Then peek at its rows: after the rebuild, which writes the `visible()` the picker
2560                // reads. Here, because a listing landing under a still cursor may draw no frame to
2561                // notice.
2562                #[cfg(feature = "cloud")]
2563                if landed {
2564                    self.peek_cloud_directories();
2565                }
2566                #[cfg(not(feature = "cloud"))]
2567                let _ = landed;
2568                None
2569            }
2570            AppEvent::HomeCloudKinds { kinds, failed } => {
2571                for directory in failed {
2572                    self.home.peeking.remove(&directory);
2573                    self.home.peek_failed.insert(directory);
2574                }
2575                let roots: Vec<PathBuf> = self
2576                    .home
2577                    .probes
2578                    .answered()
2579                    .map(|(root, _)| root.clone())
2580                    .collect();
2581                for (directory, kind) in kinds {
2582                    // Answered: out of `peeking` into the set rows are labeled from; every directory
2583                    // comes back, so none is asked twice.
2584                    self.home.peeking.remove(&directory);
2585                    self.home.cloud_kinds.insert(directory, kind);
2586                }
2587                for root in roots {
2588                    self.home.apply_cloud_kinds(&root);
2589                }
2590                self.home_refresh();
2591                None
2592            }
2593            _ => unreachable!("not a home event"),
2594        }
2595    }
2596}