Skip to main content

datui_lib/home/
home_app.rs

1//! The home screen's work: listing, probing remote places, measuring and
2//! classifying rows, previews, search, catalogs and the cloud sources, and the
3//! answers its workers send back.
4
5use crate::app::background::{CacheWrites, OwedAnswer};
6use crate::app::feedback::Confirm;
7use crate::cache::CacheManager;
8use crate::cli::FileFormat;
9use crate::loading::open_options::OpenOptions;
10#[cfg(feature = "cloud")]
11use crate::wait_on_runtime;
12use crate::{
13    APP_NAME, App, AppEvent, InputMode, cloud::source, config, home, home::catalog, home::discover,
14    loading, widgets,
15};
16use color_eyre::Result;
17use std::collections::HashMap;
18use std::path::{Path, PathBuf};
19use std::sync::{Arc, Mutex};
20
21/// The home screen's work in flight and what it keeps for the session: probes, listings,
22/// search, previews and schemas.
23#[derive(Default)]
24pub struct HomeApp {
25    /// Network roots being listed off-thread, so a probe is not started twice. Never
26    /// removed for a root that does not answer: its thread is lost, and a retry would
27    /// lose another.
28    pub(crate) probes_inflight: Vec<PathBuf>,
29    /// Each cloud listing's stop flag, by place: leaving the place stops it before its
30    /// next page.
31    pub(crate) listing_cancels: HashMap<PathBuf, Arc<std::sync::atomic::AtomicBool>>,
32    /// The listing a filter asked of a cut-short cloud directory: where, the name
33    /// prefix, and its stop flag.
34    pub(crate) narrowing: Option<(PathBuf, String, Arc<std::sync::atomic::AtomicBool>)>,
35    /// Cloud discovery has started. It costs a request per provider, so it runs once
36    /// per session.
37    #[cfg(feature = "cloud")]
38    pub(crate) cloud_discovery_started: bool,
39    /// A recursive search below the working directory is out. One at a time: a second
40    /// would only compete for the disk.
41    pub(crate) search_inflight: bool,
42    /// The home generation the walk started in. Its batches stay its own across
43    /// refreshes; the root decides whether they still apply.
44    pub(crate) search_generation: u64,
45    /// Why the last open failed, shown at home when the error is dismissed with nothing
46    /// to fall back to.
47    pub(crate) last_load_error: Option<String>,
48    /// Schema reads currently out, so the same one is not requested every frame.
49    pub(crate) schema_inflight: Vec<PathBuf>,
50    /// Invalidates listings and measurements from a request the user has moved past.
51    pub(crate) generation: u64,
52    /// Rows arrived for a listing still being read; it is listed again before the next
53    /// frame.
54    pub(crate) refresh_owed: bool,
55    /// Schema previews, memoized for the session only (persisted, they would go stale).
56    pub(crate) schema_cache: HashMap<PathBuf, Option<discover::SchemaPreview>>,
57    /// The home screen's `ROWS` previews, and the dataset the newest one built.
58    pub previews: crate::home::home_preview::Previews,
59    /// The pre-0.4.0 Ctrl+D directories in the cache have been moved to `catalog.toml`.
60    pub(crate) remembered_moved: bool,
61    /// Which home workers panic before starting, for tests. The jobs' own is
62    /// [`Jobs::worker_dies`].
63    #[cfg(test)]
64    pub(crate) worker_dies: Option<crate::HomeWorkerDies>,
65    /// Whether a browser opened here appears in front of the user: `o` on a doc link
66    /// is offered only then (`link_open::local_desktop`).
67    pub local_desktop: bool,
68    /// The reads of data started this session, by kind.
69    pub reads: crate::home::home_preview::ReadCounts,
70}
71
72/// Rows measured per background pass: small, so a slow filesystem shows progress.
73const MEASURE_BATCH: usize = 12;
74
75/// Rows a probe measures while it is already reading a remote directory.
76const PROBE_MEASURE_LIMIT: usize = 24;
77
78/// Rows one classification pass looks into: a cap on work in flight, the next pass
79/// chosen from the viewport when this one lands. Small like [`MEASURE_BATCH`], so a
80/// screen fills in rather than going silent.
81pub(crate) const CLASSIFY_BATCH: usize = 16;
82
83/// Probes allowed at once: a probe of a gone share holds its thread until exit.
84pub(crate) const MAX_CONCURRENT_PROBES: usize = 4;
85
86/// A number per home search walk, so one walk's scorings are never taken for
87/// another's over the same place.
88fn next_search_epoch() -> u64 {
89    static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
90    NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
91}
92
93/// A source as a home-screen row, with the last run's buckets when they still apply.
94#[cfg(feature = "cloud")]
95pub(crate) fn home_cloud_source(
96    source: &crate::cloud::cloud_sources::Source,
97    cached: Option<&crate::cache::CloudListing>,
98    listing: bool,
99) -> home::CloudSource {
100    let mut details: Vec<(String, String)> = vec![
101        ("source".to_string(), source.id.clone()),
102        ("api".to_string(), source.kind.name().to_string()),
103    ];
104    if let Some(endpoint) = &source.s3.endpoint {
105        details.push(("endpoint".to_string(), endpoint.clone()));
106    }
107    if let Some(region) = &source.s3.region {
108        details.push(("region".to_string(), region.clone()));
109    }
110    if let Some(project) = &source.project {
111        details.push(("project".to_string(), project.clone()));
112    }
113    if let Some(profile) = &source.profile {
114        details.push(("profile".to_string(), profile.clone()));
115    }
116    if let Some(configuration) = &source.gcloud {
117        details.push(("configuration".to_string(), configuration.clone()));
118    }
119    if source.s3.virtual_hosted.is_some() {
120        let style = if source.s3.virtual_hosted_style() {
121            "virtual-hosted"
122        } else {
123            "path-style"
124        };
125        details.push(("addressing".to_string(), style.to_string()));
126    }
127    details.push(("login".to_string(), source.origin.clone()));
128
129    let short = source.problem.as_deref().map(|problem| {
130        if problem.starts_with("not signed in") {
131            "not signed in"
132        } else if problem.starts_with("unsupported login") {
133            "unsupported login"
134        } else {
135            "not configured"
136        }
137    });
138    // A source that cannot list says what to do where the row has room; the count
139    // carries the short problem.
140    let note = match (&source.problem, short) {
141        (Some(problem), Some(short)) => problem
142            .strip_prefix(short)
143            .map(|rest| rest.trim_start_matches([':', ' ']))
144            .filter(|rest| !rest.is_empty())
145            .unwrap_or(problem)
146            .to_string(),
147        _ => [source.detail(), Some(source.origin.clone())]
148            .into_iter()
149            .flatten()
150            .filter(|n| !n.is_empty())
151            .collect::<Vec<_>>()
152            .join(&format!(" {} ", crate::glyphs::get().middot)),
153    };
154    let mut names: Vec<String> = cached.map(|c| c.buckets.clone()).unwrap_or_default();
155    for bucket in &source.buckets {
156        if !names.contains(bucket) {
157            names.push(bucket.clone());
158        }
159    }
160    let status = match (&source.problem, short) {
161        (Some(problem), Some(short)) => home::CloudStatus::Failed {
162            short: short.to_string(),
163            detail: problem.clone(),
164        },
165        _ if cached.is_some() => home::CloudStatus::Listed,
166        _ if listing => home::CloudStatus::Listing,
167        _ => home::CloudStatus::Unlisted,
168    };
169    home::CloudSource {
170        id: source.id.clone(),
171        label: source.label.clone(),
172        api: source.kind,
173        note,
174        buckets: names
175            .iter()
176            .map(|b| PathBuf::from(source.bucket_url(b)))
177            .collect(),
178        refreshing: listing && cached.is_some() && source.problem.is_none(),
179        // A source that failed before any request has nothing to ask.
180        asked: listing || source.problem.is_some(),
181        listed_at: cached
182            .map(|c| std::time::UNIX_EPOCH + std::time::Duration::from_secs(c.listed_at)),
183        status,
184        details,
185        place_details: Default::default(),
186    }
187}
188
189/// A listing error as a word for the row and the full message for the details pane.
190#[cfg(feature = "cloud")]
191fn summarize_cloud_failure(error: &str) -> (String, String) {
192    let lower = error.to_lowercase();
193    // A missing tool is already as short as it gets: `needs the AWS CLI`.
194    if let Some(start) = lower.find("needs ") {
195        return (error[start..].to_string(), error.to_string());
196    }
197    let short = if lower.contains("403")
198        || lower.contains("forbidden")
199        || lower.contains("accessdenied")
200        || lower.contains("access denied")
201    {
202        "403"
203    } else if lower.contains("401")
204        || lower.contains("unauthorized")
205        || lower.contains("credential")
206        || lower.contains("invalidaccesskeyid")
207        || lower.contains("expired")
208        || lower.contains("sso")
209        || lower.contains("az login")
210    {
211        "not logged in"
212    } else if lower.contains("unsupported login") {
213        "unsupported login"
214    } else if lower.contains("no gcp project") {
215        "no project"
216    } else if lower.contains("is not set") {
217        "not configured"
218    } else if lower.contains("timed out")
219        || lower.contains("timeout")
220        || lower.contains("connection")
221        || lower.contains("dns")
222        || lower.contains("resolve")
223    {
224        "unavailable"
225    } else {
226        "error"
227    };
228    (short.to_string(), error.to_string())
229}
230
231impl App {
232    /// Schema for a home entry, from Parquet metadata, memoized for the session. `None`
233    /// means not knowable without a scan, and the UI says so.
234    pub fn home_schema(&mut self, entry: &discover::Entry) -> Option<discover::SchemaPreview> {
235        // Previews read local files only: nothing in an object store is read before it is
236        // opened.
237        if home::is_cloud_place(&entry.path) || home::is_object_store_url(&entry.path) {
238            return None;
239        }
240        if let Some(cached) = self.home_app.schema_cache.get(&entry.path) {
241            return cached.clone();
242        }
243        // Opening a file, so requested from a worker; the frame never waits.
244        self.request_home_schema(entry.clone());
245        None
246    }
247
248    /// What a home-screen worker owes in place of its answer if it panics.
249    fn owed_answer(&mut self, instead: AppEvent) -> OwedAnswer {
250        OwedAnswer {
251            tx: self.events.clone(),
252            #[cfg(test)]
253            dies: self
254                .home_app
255                .worker_dies
256                .as_mut()
257                .is_some_and(|dies| dies(&instead)),
258            instead: Some(instead),
259        }
260    }
261
262    /// List the directory the `~` prompt is typing, if not already listed: a URL from
263    /// what the screen knows, a local directory on a worker.
264    pub(crate) fn list_the_typed_directory(&mut self) {
265        if !self.home.path_input_active {
266            return;
267        }
268        let dir = home::typed_dir(&self.home.path_input).to_string();
269        if self
270            .home
271            .path_listing
272            .as_ref()
273            .is_some_and(|l| l.dir == dir)
274        {
275            return;
276        }
277        if home::typed_dir_is_url(&dir) {
278            self.home.path_listing = Some(home::names_under(&dir, self.home.known_urls()));
279            return;
280        }
281        // Off the UI thread: a typed path may name a dead mount.
282        let tx = self.events.clone();
283        let owed = self.owed_answer(AppEvent::HomePathListed {
284            listing: Box::new(home::PathListing {
285                dir: dir.clone(),
286                names: Vec::new(),
287                failed: true,
288            }),
289        });
290        std::thread::spawn(move || {
291            owed.run(|| {
292                let listing = home::list_typed_dir(&dir);
293                let _ = tx.send(AppEvent::HomePathListed {
294                    listing: Box::new(listing),
295                });
296            })
297        });
298    }
299
300    /// Complete the path being typed, on a worker.
301    pub(crate) fn request_path_completion(&mut self) {
302        let typed = self.home.path_input.clone();
303        if typed.is_empty() {
304            return;
305        }
306        let generation = self.home_app.generation;
307        let tx = self.events.clone();
308        std::thread::spawn(move || {
309            let (completed, candidates) = home::complete_path(&typed);
310            let _ = tx.send(AppEvent::HomePathCompleted {
311                generation,
312                typed,
313                completed,
314                candidates,
315            });
316        });
317    }
318
319    /// A home file's first rows for its `ROWS` preview, read on a worker as its open
320    /// reads them. `None` until they land or when not previewed. `screen_height` sizes
321    /// the page to the table's.
322    pub fn home_preview_rows(
323        &mut self,
324        entry: &discover::Entry,
325        screen_height: u16,
326    ) -> Option<Arc<crate::home::home_preview::PreviewRows>> {
327        let max = self.app_config.home.preview_max.bytes();
328        if !crate::home::home_preview::previewable(entry, max) {
329            return None;
330        }
331        let stamp = crate::home::home_preview::Stamp::of_entry(entry);
332        if let Some(known) = self.home_app.previews.rows(&entry.path, stamp) {
333            return known;
334        }
335        if self.home_app.previews.inflight.is_none() {
336            self.request_home_preview(entry.path.clone(), stamp, screen_height);
337        }
338        None
339    }
340
341    /// Whether `entry` is one the preview reads, before its rows are in.
342    pub fn home_preview_pending(&self, path: &Path) -> bool {
343        self.home_app.previews.reading(path)
344    }
345
346    /// Read a file's first page on a worker through the open's own scan and schema read,
347    /// so the open can install the result.
348    fn request_home_preview(
349        &mut self,
350        path: PathBuf,
351        stamp: crate::home::home_preview::Stamp,
352        screen_height: u16,
353    ) {
354        self.home_app.previews.inflight = Some(path.clone());
355        self.home_app.reads.previews += 1;
356        let tx = self.events.clone();
357        let cloud = self.app_config.cloud.clone();
358        let formats = self.formats.clone();
359        let runtime = self.runtime.clone();
360        let cache = self.cache.clone();
361        let writes = self.cache_writes.clone();
362        // The table's rows: the screen less the title, the header and the footer.
363        let visible = (screen_height as usize).saturating_sub(3).max(1);
364        let owed = self.owed_answer(AppEvent::HomePreviewReady {
365            path: path.clone(),
366            stamp,
367            read_at: None,
368            rows: None,
369            prepared: crate::home::home_preview::Handoff::default(),
370        });
371        self.runtime.spawn_blocking(move || {
372            owed.run(|| {
373                let began = std::time::Instant::now();
374                let read_at = crate::home::home_preview::Stamp::of_file(&path);
375                let read = Self::read_home_preview(
376                    &path, &cloud, &formats, &runtime, cache, writes, visible,
377                );
378                log::debug!(
379                    target: "datui",
380                    "home preview of {}: {:.1?}",
381                    path.display(),
382                    began.elapsed()
383                );
384                let (rows, prepared) = match read {
385                    Some((rows, prepared)) => (Some(Arc::new(rows)), Some(Box::new(prepared))),
386                    None => (None, None),
387                };
388                let _ = tx.send(AppEvent::HomePreviewReady {
389                    path,
390                    stamp,
391                    read_at,
392                    rows,
393                    prepared: Arc::new(Mutex::new(prepared)),
394                });
395            })
396        });
397    }
398
399    /// What opening `path` from home reads first: scan, schema and the page for
400    /// `visible` rows, built by the open's own steps and options so the dataset is the
401    /// one the open would build.
402    fn read_home_preview(
403        path: &Path,
404        cloud: &crate::config::CloudConfig,
405        formats: &crate::formats::Registry,
406        runtime: &tokio::runtime::Handle,
407        cache: CacheManager,
408        writes: CacheWrites,
409        visible: usize,
410    ) -> Option<(
411        crate::home::home_preview::PreviewRows,
412        crate::home::home_preview::Prepared,
413    )> {
414        let paths = [path.to_path_buf()];
415        let scanned = Self::scan_for_open(
416            cloud,
417            formats,
418            &paths,
419            OpenOptions::default(),
420            Some(path.to_path_buf()),
421        )
422        .ok()?;
423        let loading::LoadAnswer::Scanned { lf, path, options } = scanned else {
424            return None;
425        };
426        let progress = Arc::<crate::formats::schema_union::FooterProgress>::default();
427        let report = crate::loading::measurements::OpenReport {
428            progress: progress.clone(),
429            meter: Arc::new(crate::loading::measurements::Meter::default()),
430            remembered: Some(cache),
431            writes,
432        };
433        let read = Self::read_schema_for_open(
434            *lf,
435            path,
436            options,
437            cloud,
438            runtime,
439            &report,
440            loading::Made::default(),
441        )
442        .ok()?;
443        let loading::LoadAnswer::SchemaRead {
444            mut state,
445            options,
446            debug_label,
447            ..
448        } = read
449        else {
450            return None;
451        };
452        // Planned as the table plans its first page, so the page is the one it wants.
453        state.visible_rows = visible;
454        let began = std::time::Instant::now();
455        let request = state.prepare_async_collect(None)?;
456        let df =
457            crate::analysis::statistics::collect_lazy(request.lf, request.polars_streaming).ok()?;
458        let result = request.plan.fit(df);
459        let rows = crate::home::home_preview::PreviewRows::from_frame(result.rows());
460        state.measurements().read_page(began.elapsed(), Some(1));
461        state.apply_async_collect(result);
462        Some((
463            rows,
464            crate::home::home_preview::Prepared {
465                state,
466                options,
467                debug_label,
468                progress,
469            },
470        ))
471    }
472
473    /// Whether a schema read is currently out for this path.
474    pub fn home_schema_pending(&self, path: &Path) -> bool {
475        self.home_app.schema_inflight.iter().any(|p| p == path)
476    }
477
478    /// Read the selected dataset's schema on a worker.
479    fn request_home_schema(&mut self, entry: discover::Entry) {
480        if self.home_app.schema_inflight.contains(&entry.path) {
481            return;
482        }
483        self.home_app.schema_inflight.push(entry.path.clone());
484
485        let generation = self.home_app.generation;
486        let tx = self.events.clone();
487        // Remembered as none on failure, so it is not asked again.
488        let owed = self.owed_answer(AppEvent::HomeSchemaReady {
489            generation,
490            path: entry.path.clone(),
491            preview: None,
492        });
493        self.runtime.spawn_blocking(move || {
494            owed.run(|| {
495                let preview = discover::schema_preview(&entry);
496                let _ = tx.send(AppEvent::HomeSchemaReady {
497                    generation,
498                    path: entry.path,
499                    preview,
500                });
501            })
502        });
503    }
504
505    /// Start listing network roots that have not answered yet. Never waits: a gone
506    /// share (a `hard` NFS mount) blocks its thread uninterruptibly, so the task is
507    /// abandoned, not joined.
508    pub(crate) fn spawn_home_probes(&mut self) {
509        self.stop_listings_left_behind();
510        for root in self.home.pending_probes() {
511            if self.home_app.probes_inflight.contains(&root) {
512                // Left and returned to before its next page: it goes on.
513                if let Some(cancelled) = self.home_app.listing_cancels.get(&root) {
514                    cancelled.store(false, std::sync::atomic::Ordering::Relaxed);
515                }
516                continue;
517            }
518            // Each probe of an unreachable share costs a thread forever, so they are capped.
519            // The browsed directory is exempt: it is the whole screen, and the user bounds how
520            // many they open.
521            let browsed = self.home.browsing.as_ref() == Some(&root);
522            if !browsed && self.home_app.probes_inflight.len() >= MAX_CONCURRENT_PROBES {
523                continue;
524            }
525            self.home_app.probes_inflight.push(root.clone());
526            let tx = self.events.clone();
527            let cache = self.cache.clone();
528            let owed = self.owed_answer(AppEvent::HomeProbeFailed {
529                root: root.clone(),
530                message: "Could not read it; see the log".to_string(),
531            });
532            #[cfg(feature = "cloud")]
533            let cloud = self.app_config.cloud.clone();
534            #[cfg(feature = "cloud")]
535            let runtime = self.runtime.clone();
536            #[cfg(feature = "cloud")]
537            let cancelled = {
538                let flag = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
539                self.home_app
540                    .listing_cancels
541                    .insert(root.clone(), flag.clone());
542                flag
543            };
544            // A detached thread, not the runtime's blocking pool: a thread wedged on a dead
545            // mount never returns, and must not eat the pool that loads data.
546            std::thread::spawn(move || {
547                owed.run(|| {
548                    // A bucket or prefix: listed with an object-store listing, not `read_dir` (which
549                    // fails on `gs://`). Metadata only: names, sizes and times for one level; no
550                    // footers, schemas or counts, which would cost a paid ranged read per row.
551                    #[cfg(feature = "cloud")]
552                    if let Some((id, account)) = home::cloud_account(&root) {
553                        let listed = wait_on_runtime(&runtime, async move {
554                            crate::cloud::cloud_browse::list_account(&id, &account, &cloud).await
555                        });
556                        match listed {
557                            Some(Ok(rows)) => {
558                                let _ = tx.send(AppEvent::HomeProbeReady {
559                                    root,
560                                    rows: Some(rows),
561                                    cut_short: false,
562                                });
563                            }
564                            Some(Err(message)) => {
565                                log::warn!(
566                                    target: "datui::cloud",
567                                    "listing {} failed: {message}",
568                                    root.display()
569                                );
570                                let _ = tx.send(AppEvent::HomeProbeFailed { root, message });
571                            }
572                            None => {
573                                let _ = tx.send(AppEvent::HomeProbeReady {
574                                    root,
575                                    rows: None,
576                                    cut_short: false,
577                                });
578                            }
579                        }
580                        return;
581                    }
582                    #[cfg(feature = "cloud")]
583                    if crate::cloud::cloud_browse::split_bucket_url(&root.to_string_lossy())
584                        .is_some()
585                        || source::azure_parts(&root.to_string_lossy()).is_some()
586                    {
587                        let url = root.to_string_lossy().into_owned();
588                        // Each page's rows are drawn as they come; leaving the place stops the listing.
589                        let watch = crate::cloud::cloud_browse::Watch {
590                            progress: Some(std::sync::Arc::new({
591                                let (tx, root) = (tx.clone(), root.clone());
592                                move |page: &[crate::home::discover::Entry]| {
593                                    let _ = tx.send(AppEvent::HomeProbeProgress {
594                                        root: root.clone(),
595                                        rows: page.to_vec(),
596                                    });
597                                }
598                            })),
599                            cancelled,
600                            names_from: None,
601                        };
602                        let listed = wait_on_runtime(&runtime, async move {
603                            crate::cloud::cloud_browse::list_objects_watched(&url, &cloud, &watch)
604                                .await
605                        });
606                        // A refused listing says why, rather than reading as a place that stopped
607                        // answering.
608                        match listed {
609                            Some(Err(message)) => {
610                                log::warn!(
611                                    target: "datui::cloud",
612                                    "listing {} failed: {message}",
613                                    root.display()
614                                );
615                                let _ = tx.send(AppEvent::HomeProbeFailed { root, message });
616                            }
617                            Some(Ok(level)) if level.cancelled => {
618                                let _ = tx.send(AppEvent::HomeProbeCancelled { root });
619                            }
620                            other => {
621                                let (rows, cut_short) = match other {
622                                    Some(Ok(level)) => (Some(level.rows), level.truncated),
623                                    _ => (None, false),
624                                };
625                                let _ = tx.send(AppEvent::HomeProbeReady {
626                                    root,
627                                    rows,
628                                    cut_short,
629                                });
630                            }
631                        }
632                        return;
633                    }
634                    // No cloud feature: `read_dir` on a bucket URL would only say unavailable.
635                    #[cfg(not(feature = "cloud"))]
636                    if source::is_remote_url(&root) {
637                        let message = "cloud support not in this build".to_string();
638                        let _ = tx.send(AppEvent::HomeProbeFailed { root, message });
639                        return;
640                    }
641                    let mut cut_short = false;
642                    let rows = if std::fs::read_dir(&root).is_ok() {
643                        // What has been read shows while the rest is read (seconds for thousands of files).
644                        let scan = crate::home::discover::scan_dir_progressive(&root, |read| {
645                            let _ = tx.send(AppEvent::HomeProbeProgress {
646                                root: root.clone(),
647                                rows: read.to_vec(),
648                            });
649                        });
650                        cut_short = scan.truncated;
651                        let mut rows = scan.entries;
652                        // Measured here too: this thread is already the one allowed to block on the share.
653                        for row in rows.iter_mut().take(PROBE_MEASURE_LIMIT) {
654                            crate::home::discover::enrich(row);
655                        }
656                        // Remote datasets are measured nowhere else, so remember them here; otherwise a
657                        // remote row is blank on every run.
658                        let mounts = crate::home::locality::Mounts::current();
659                        for row in rows.iter_mut() {
660                            row.cost.source = Some(mounts.describe(&row.path).fstype);
661                        }
662                        let facts: Vec<_> = rows.iter().filter_map(home::facts_for).collect();
663                        cache.record_dataset_facts(&facts);
664                        Some(rows)
665                    } else {
666                        None
667                    };
668                    let _ = tx.send(AppEvent::HomeProbeReady {
669                        root,
670                        rows,
671                        cut_short,
672                    });
673                })
674            });
675        }
676    }
677
678    /// Stop cloud listings of places no longer on screen (the browsed one, or home's
679    /// roots). A place left and revisited is listed again.
680    fn stop_listings_left_behind(&mut self) {
681        let home = &self.home;
682        for (root, cancelled) in &self.home_app.listing_cancels {
683            let wanted = match &home.browsing {
684                Some(dir) => dir == root,
685                None => home
686                    .sections
687                    .iter()
688                    .any(|s| s.remote_root.as_ref() == Some(root)),
689            };
690            if !wanted {
691                cancelled.store(true, std::sync::atomic::Ordering::Relaxed);
692            }
693        }
694        if let Some((dir, _, cancelled)) = &self.home_app.narrowing
695            && home.browsing.as_ref() != Some(dir)
696        {
697            cancelled.store(true, std::sync::atomic::Ordering::Relaxed);
698            self.home_app.narrowing = None;
699        }
700        if self
701            .home
702            .narrowed
703            .as_ref()
704            .is_some_and(|n| self.home.browsing.as_ref() != Some(&n.dir))
705        {
706            self.home.narrowed = None;
707        }
708    }
709
710    /// In a cloud directory cut short at the cap, ask the server for names starting with
711    /// the filter, so names past the cap can be found. Nothing when the filter is empty
712    /// or what is held answers it.
713    #[cfg(feature = "cloud")]
714    pub(crate) fn narrow_cloud_listing(&mut self) {
715        let dir = self.home.browsing.clone();
716        let prefix = dir.as_ref().and_then(|dir| {
717            if !self.home.probes.cut_short(dir) {
718                return None;
719            }
720            let rows = self.home.probes.listed(dir)?;
721            let names: Vec<&str> = rows.iter().map(|row| row.name.as_str()).collect();
722            crate::cloud::cloud_browse::narrowing_prefix(&self.home.filter, &names)
723        });
724        let (Some(dir), Some(prefix)) = (dir, prefix) else {
725            if let Some((_, _, cancelled)) = self.home_app.narrowing.take() {
726                cancelled.store(true, std::sync::atomic::Ordering::Relaxed);
727            }
728            if self.home.narrowed.take().is_some() {
729                self.home_refresh();
730            }
731            return;
732        };
733        // Everything under a shorter prefix is everything under this one too.
734        if self.home.narrowed.as_ref().is_some_and(|n| {
735            n.dir == dir && (n.prefix == prefix || (!n.truncated && prefix.starts_with(&n.prefix)))
736        }) {
737            return;
738        }
739        if let Some((d, p, _)) = &self.home_app.narrowing
740            && *d == dir
741            && *p == prefix
742        {
743            return;
744        }
745        if let Some((_, _, cancelled)) = self.home_app.narrowing.take() {
746            cancelled.store(true, std::sync::atomic::Ordering::Relaxed);
747        }
748        let cancelled = Arc::new(std::sync::atomic::AtomicBool::new(false));
749        self.home_app.narrowing = Some((dir.clone(), prefix.clone(), cancelled.clone()));
750        let tx = self.events.clone();
751        let owed = self.owed_answer(AppEvent::HomeNarrowed {
752            dir: dir.clone(),
753            prefix: prefix.clone(),
754            listed: None,
755        });
756        let cloud = self.app_config.cloud.clone();
757        let runtime = self.runtime.clone();
758        std::thread::spawn(move || {
759            owed.run(|| {
760                let url = dir.to_string_lossy().into_owned();
761                let watch = crate::cloud::cloud_browse::Watch {
762                    progress: None,
763                    cancelled,
764                    names_from: Some(prefix.clone()),
765                };
766                let listed = wait_on_runtime(&runtime, async move {
767                    crate::cloud::cloud_browse::list_objects_watched(&url, &cloud, &watch).await
768                });
769                let listed = match listed {
770                    Some(Ok(level)) if !level.cancelled => Some((level.rows, level.truncated)),
771                    _ => None,
772                };
773                let _ = tx.send(AppEvent::HomeNarrowed {
774                    dir,
775                    prefix,
776                    listed,
777                });
778            })
779        });
780    }
781
782    /// Find the cloud sources this machine and the config describe, and list their
783    /// buckets when `[cloud] list_on_start` asks. Once per session; Ctrl+R asks again.
784    #[cfg(feature = "cloud")]
785    fn spawn_cloud_discovery(&mut self) {
786        if self.home_app.cloud_discovery_started {
787            return;
788        }
789        self.home_app.cloud_discovery_started = true;
790        let list = self.app_config.cloud.list_on_start;
791        self.list_cloud_sources(None, list);
792    }
793
794    /// List the browsed source if not asked this session; entering a source asks.
795    #[cfg(feature = "cloud")]
796    fn list_browsed_cloud_source(&mut self) {
797        let Some(id) = self
798            .home
799            .browsing
800            .as_deref()
801            .and_then(home::cloud_source_id)
802        else {
803            return;
804        };
805        let Some(source) = self.home.cloud.iter_mut().find(|s| s.id == id) else {
806            return;
807        };
808        if source.asked {
809            return;
810        }
811        source.begin_listing();
812        self.list_cloud_sources(Some(id), true);
813    }
814
815    /// Send every source's rows, or list the named one's buckets. Rows go first, filled
816    /// from the last run's listing when still valid. With `list`, sources are listed a
817    /// few at a time and each result sent on arrival; without, nothing leaves the
818    /// machine. On the runtime, not a detached thread: every call has a global timeout.
819    #[cfg(feature = "cloud")]
820    fn list_cloud_sources(&mut self, only: Option<String>, list: bool) {
821        let tx = self.events.clone();
822        let cloud = self.app_config.cloud.clone();
823        let cache = self.cache.clone();
824        self.runtime.spawn(async move {
825            let mut hidden = cache.load_hidden_cloud_sources();
826            hidden.extend(cloud.hide.iter().cloned());
827            let cached_for = |source: &crate::cloud::cloud_sources::Source| {
828                cache.cloud_listing(&source.id, &source.fingerprint())
829            };
830            // Looked for again, and kept for the opens and listings that follow.
831            let found = crate::cloud::cloud_sources::rediscover(&cloud).to_vec();
832            // A bucket under Recent opens with the login that listed it, shown or not, unless
833            // its source is hidden (perhaps for a dead login; the default opens it then).
834            // Only with the rows, so an old listing never overrides a newer one.
835            if only.is_none() {
836                for source in found.iter().filter(|s| !hidden.contains(&s.id)) {
837                    if let Some(cached) = cached_for(source) {
838                        crate::cloud::cloud_sources::remember_listed(source, &cached.buckets);
839                    }
840                }
841            }
842            let sources: Vec<crate::cloud::cloud_sources::Source> =
843                crate::cloud::cloud_sources::on_home(found, &cloud)
844                    .into_iter()
845                    .filter(|s| !hidden.contains(&s.id))
846                    .collect();
847
848            match &only {
849                None => {
850                    let rows = sources
851                        .iter()
852                        .map(|source| home_cloud_source(source, cached_for(source).as_ref(), list))
853                        .collect();
854                    let _ = tx.send(AppEvent::HomeCloudSources { sources: rows });
855                }
856                // Gone since its row was drawn (a profile removed, a source hidden): said, so the
857                // row does not wait forever.
858                Some(id) if !sources.iter().any(|s| &s.id == id) => {
859                    let _ = tx.send(AppEvent::HomeCloudListed {
860                        id: id.clone(),
861                        buckets: Vec::new(),
862                        details: Vec::new(),
863                        failure: Some((
864                            "not found".to_string(),
865                            format!("{id} is gone or hidden. Ctrl+R at the top looks again."),
866                        )),
867                        listed_at: std::time::SystemTime::now(),
868                    });
869                    return;
870                }
871                Some(_) => {}
872            }
873            if !list {
874                return;
875            }
876
877            // One slow endpoint does not delay the rest, and many sources do not storm.
878            const LISTING_AT_ONCE: usize = 4;
879            let permits = std::sync::Arc::new(tokio::sync::Semaphore::new(LISTING_AT_ONCE));
880            let mut listings = tokio::task::JoinSet::new();
881            for source in sources
882                .into_iter()
883                .filter(|s| only.as_ref().is_none_or(|id| &s.id == id))
884            {
885                let permits = permits.clone();
886                listings.spawn(async move {
887                    let _permit = permits.acquire_owned().await;
888                    let result = crate::cloud::cloud_browse::list_first_level(&source).await;
889                    (source, result)
890                });
891            }
892            while let Some(joined) = listings.join_next().await {
893                let Ok((source, result)) = joined else {
894                    continue;
895                };
896                let listed_at = std::time::SystemTime::now();
897                // Buckets named in the config show whether or not the login can list them.
898                let mut names = source.buckets.clone();
899                let mut details = Vec::new();
900                let failure = match result {
901                    Ok(listed) => {
902                        for item in listed {
903                            crate::cloud::cloud_sources::remember_bucket(&source, &item.name);
904                            if !item.details.is_empty() {
905                                details.push((item.place.clone(), item.details));
906                            }
907                            let name = item.name;
908                            if !names.contains(&name) {
909                                names.push(name);
910                            }
911                        }
912                        cache.save_cloud_listing(
913                            &source.id,
914                            crate::cache::CloudListing {
915                                fingerprint: source.fingerprint(),
916                                buckets: names.clone(),
917                                listed_at: listed_at
918                                    .duration_since(std::time::UNIX_EPOCH)
919                                    .map(|d| d.as_secs())
920                                    .unwrap_or(0),
921                            },
922                        );
923                        None
924                    }
925                    Err(e) => {
926                        log::warn!(target: "datui::cloud", "listing {} failed: {e}", source.id);
927                        Some(summarize_cloud_failure(&e))
928                    }
929                };
930                let _ = tx.send(AppEvent::HomeCloudListed {
931                    id: source.id.clone(),
932                    buckets: names
933                        .iter()
934                        .map(|b| PathBuf::from(source.bucket_url(b)))
935                        .collect(),
936                    details,
937                    failure,
938                    listed_at,
939                });
940            }
941        });
942    }
943
944    /// Ask again for what is on screen, bypassing the cache: the browsed source's
945    /// buckets, the browsed bucket or directory, or every source's buckets.
946    pub(crate) fn home_reload(&mut self) {
947        #[cfg(feature = "cloud")]
948        {
949            let browsing = self.home.browsing.clone();
950            match browsing.as_deref().and_then(home::cloud_source_id) {
951                Some(id) => {
952                    if let Some(source) = self.home.cloud.iter_mut().find(|s| s.id == id) {
953                        source.begin_listing();
954                    }
955                    self.list_cloud_sources(Some(id), true);
956                }
957                None if browsing.is_none() && !self.home.cloud.is_empty() => {
958                    for source in &mut self.home.cloud {
959                        source.begin_listing();
960                    }
961                    self.list_cloud_sources(None, true);
962                }
963                None => {}
964            }
965        }
966        if let Some(dir) = self.home.browsing.clone() {
967            // Its answer, not a listing still coming in.
968            if self.home.probes.settled(&dir) {
969                self.home.probes.forget(&dir);
970            }
971        }
972        // Ctrl+R retries failed peeks and missing web files too.
973        self.home.peek_failed.clear();
974        for path in std::mem::take(&mut self.home.web_gone).into_keys() {
975            self.home.sized.remove(&path);
976        }
977        self.home.status = None;
978        self.home_refresh();
979    }
980
981    /// Start the recursive search below the working directory, if wanted and not
982    /// running. Triggered by typing, not by opening home, so opening a recent costs no
983    /// walk.
984    pub(crate) fn spawn_home_search(&mut self) {
985        if self.home_app.search_inflight || self.home.search.done {
986            return;
987        }
988        let config = self.app_config.home.search.clone();
989        if !config.enabled {
990            return;
991        }
992        let Some(root) =
993            crate::home::search::search_root(self.home.browsing.as_ref(), self.home.network_check)
994        else {
995            return;
996        };
997
998        self.home.search.reset();
999        self.home.search.root = Some(root.clone());
1000        self.home.search.running = true;
1001        self.home.search.epoch = next_search_epoch();
1002        self.home.search_limit = config.max_results;
1003        self.home_app.search_inflight = true;
1004
1005        let generation = self.home_app.generation;
1006        self.home_app.search_generation = generation;
1007        let tx = self.events.clone();
1008        let formats = self.formats.clone();
1009        // Ended, with what the batches already found kept.
1010        let owed = self.owed_answer(AppEvent::HomeSearchDone {
1011            generation,
1012            root: root.clone(),
1013            scanned: 0,
1014            limited: Some(crate::glyphs::dotted("partial · failed")),
1015        });
1016        // A detached thread, as for probes: a filesystem stall must not stop drawing.
1017        std::thread::spawn(move || {
1018            owed.run(|| {
1019                let walk_root = root.clone();
1020                let batch_tx = tx.clone();
1021                let batch_gen = generation;
1022                let batch_root = root.clone();
1023                let outcome = crate::home::search::walk_with_specs(
1024                    &walk_root,
1025                    &config,
1026                    &formats,
1027                    move |found, outcome| {
1028                        // Sent even when empty: it carries progress, and a failed send tells the walk
1029                        // nobody listens.
1030                        batch_tx
1031                            .send(AppEvent::HomeSearchBatch {
1032                                generation: batch_gen,
1033                                root: batch_root.clone(),
1034                                found,
1035                                scanned: outcome.scanned,
1036                            })
1037                            // A closed channel means the app is gone; stop walking.
1038                            .is_ok()
1039                    },
1040                );
1041                let _ = tx.send(AppEvent::HomeSearchDone {
1042                    generation,
1043                    root,
1044                    scanned: outcome.scanned,
1045                    limited: outcome.note().map(crate::glyphs::dotted),
1046                });
1047            })
1048        });
1049    }
1050
1051    /// Score the filter against the search's files on a worker, when owed. Asked after
1052    /// every event; over tens of thousands of files scoring held back keystroke echo.
1053    /// One at a time; each answer asks for the next if the filter moved.
1054    pub(crate) fn home_score_search(&mut self) {
1055        if self.input_mode != InputMode::Home {
1056            return;
1057        }
1058        let Some(job) = self.home.score_job() else {
1059            return;
1060        };
1061        let epoch = job.epoch;
1062        let tx = self.events.clone();
1063        // A worker that dies answers with nothing.
1064        let owed = self.owed_answer(AppEvent::HomeSearchScored {
1065            epoch,
1066            matches: None,
1067        });
1068        self.runtime.spawn_blocking(move || {
1069            owed.run(move || {
1070                let matches = crate::home::search::score(
1071                    &job.results,
1072                    &job.query,
1073                    job.base.as_ref(),
1074                    job.limit,
1075                );
1076                let _ = tx.send(AppEvent::HomeSearchScored {
1077                    epoch,
1078                    matches: Some(Box::new(matches)),
1079                });
1080            })
1081        });
1082    }
1083
1084    /// Rebuild the home listing from the filesystem.
1085    pub(crate) fn home_refresh(&mut self) {
1086        self.home_app.refresh_owed = false;
1087        // Every way into a source comes through here: Enter, Backspace up from a bucket,
1088        // a jump, and rows arriving while it is open.
1089        #[cfg(feature = "cloud")]
1090        self.list_browsed_cloud_source();
1091        // Listings of where the user was are pages for nobody.
1092        self.stop_listings_left_behind();
1093        self.home_app.generation = self.home_app.generation.wrapping_add(1);
1094        let generation = self.home_app.generation;
1095
1096        self.move_remembered_places();
1097        let mut catalogs = home::catalogs(&self.app_config);
1098        // Hidden with Delete on its heading: only the bundled catalog, never a user's
1099        // `examples.toml`.
1100        if self.cache.examples_hidden() {
1101            catalogs.retain(|c| c.origin != crate::home::catalog::Origin::Bundled);
1102        }
1103        self.home.set_catalogs(catalogs);
1104        let mut request = home::ListingRequest {
1105            // Filled on the worker from the cache and the desktop's recents, so the first frame
1106            // waits on no file.
1107            recents: Vec::new(),
1108            desktop_dirs: Vec::new(),
1109            browsing: self.home.browsing.clone(),
1110            probes: self.home.probes.clone(),
1111            narrowed: self.home.narrowed.clone(),
1112            network_check: self.home.network_check,
1113            cloud: self.home.cloud.clone(),
1114            catalogs: self.home.catalogs.clone(),
1115            known: Default::default(),
1116            formats: self.formats.clone(),
1117        };
1118        let read_folds = std::mem::take(&mut self.home.folds_owed);
1119        let desktop = self.app_config.home.desktop_recents;
1120        let cache = self.cache.clone();
1121        let writes = self.cache_writes.clone();
1122
1123        self.home.listing_in_flight = true;
1124        let tx = self.events.clone();
1125        let owed = self.owed_answer(AppEvent::HomeListingFailed);
1126        self.runtime.spawn_blocking(move || {
1127            owed.run(move || {
1128                // After the dataset just left is in the recents with its shape.
1129                writes.settle();
1130                // Ranked by frecency; the cursor lands on the newest, one Enter from the last file.
1131                let (recents, visits) = cache.load_recents_with_visits();
1132                let newest = recents.first().cloned();
1133                request.recents = crate::cache::by_frecency(recents, &visits);
1134                request.known = cache.load_dataset_facts();
1135                if desktop {
1136                    request.desktop_dirs = home::desktop_recent_dirs();
1137                }
1138                let listing = home::build_listing(&request);
1139                let mut visits = visits;
1140                listing.alias_visits(&mut visits);
1141                // A record shown is a record used: the ones eviction keeps.
1142                let shown: Vec<PathBuf> = listing
1143                    .sections
1144                    .iter()
1145                    .flat_map(|s| s.rows.iter().chain(&s.door))
1146                    .flat_map(|row| [row.path.clone(), home::index_key(&row.path)])
1147                    .filter(|key| request.known.contains_key(key))
1148                    .collect();
1149                cache.touch_dataset_facts(shown.iter().map(PathBuf::as_path));
1150                let _ = tx.send(AppEvent::HomeListingReady {
1151                    generation,
1152                    listing: Box::new(listing),
1153                    known: request.known,
1154                    visits,
1155                    newest,
1156                    folds: read_folds.then(|| cache.load_folds()),
1157                });
1158            })
1159        });
1160    }
1161
1162    /// Ask the worker to measure on-screen rows not yet known. Opening a footer can
1163    /// block (a FIFO, a device, a wedged mount, a failing disk), so never on the draw
1164    /// thread.
1165    pub(crate) fn request_home_measurements(&mut self) {
1166        if self.home.measure_in_flight {
1167            return;
1168        }
1169        let wanted = self.home.unmeasured_visible(MEASURE_BATCH);
1170        if wanted.is_empty() {
1171            return;
1172        }
1173
1174        self.home.measure_in_flight = true;
1175        let tx = self.events.clone();
1176        let cache = self.cache.clone();
1177        self.runtime.spawn_blocking(move || {
1178            home::look_into_batch(wanted, &cache, |path, m| {
1179                let _ = tx.send(AppEvent::HomeMeasured {
1180                    measured: vec![(path, m)],
1181                    done: false,
1182                });
1183            });
1184            let _ = tx.send(AppEvent::HomeMeasured {
1185                measured: Vec::new(),
1186                done: true,
1187            });
1188        });
1189    }
1190
1191    /// Ask an HTTP(S) server the size of the file under the cursor, once a session,
1192    /// when unmeasured. The row shows the catalog's `~33 MB` until then; the answer is
1193    /// kept for the next listing.
1194    #[cfg(feature = "http")]
1195    pub(crate) fn size_selected_web_file(&mut self) {
1196        if !self.info.head_web_rows {
1197            return;
1198        }
1199        let Some(entry) = self.home.selected_entry().cloned() else {
1200            return;
1201        };
1202        if entry.size.is_some()
1203            || !matches!(
1204                source::input_source(&entry.path),
1205                source::InputSource::Http(_)
1206            )
1207            || !self.home.sized.insert(entry.path.clone())
1208        {
1209            return;
1210        }
1211        let tx = self.events.clone();
1212        let cache = self.cache.clone();
1213        self.runtime.spawn_blocking(move || {
1214            let size = match Self::fetch_remote_size_http(&entry.path.to_string_lossy()) {
1215                Ok(Some(size)) => size,
1216                Ok(None) => return,
1217                Err(gone) => {
1218                    let _ = tx.send(AppEvent::HomeWebGone {
1219                        path: entry.path,
1220                        gone,
1221                    });
1222                    return;
1223                }
1224            };
1225            let key = home::index_key(&entry.path);
1226            let mut facts = cache.dataset_facts(&key).unwrap_or_default();
1227            facts.size = size;
1228            cache.record_dataset_facts(&[(key, facts)]);
1229            let measured = home::Measured {
1230                rows: entry.rows,
1231                cols: entry.cols,
1232                cols_sampled: entry.cols_sampled,
1233                size: Some(size),
1234                columns: entry.columns.clone(),
1235                cost: entry.cost.clone(),
1236                kind: None,
1237                holds: entry.holds.clone(),
1238            };
1239            let _ = tx.send(AppEvent::HomeSized {
1240                path: entry.path,
1241                measured,
1242            });
1243        });
1244    }
1245
1246    /// Ask a worker what the rows on screen are. Classifying reads the named directory
1247    /// (a round trip on a share, forever on a wedged mount), so only on-screen rows,
1248    /// on a detached thread, one pass at a time. It also measures remote rows, which
1249    /// [`crate::home::HomeState::unmeasured_visible`] leaves alone since probes return them
1250    /// `Unknown`.
1251    pub(crate) fn request_home_classifications(&mut self) {
1252        if self.home.classify_in_flight {
1253            return;
1254        }
1255        let wanted = self.home.unclassified_visible(CLASSIFY_BATCH);
1256        if wanted.is_empty() {
1257            return;
1258        }
1259
1260        self.home.classify_in_flight = true;
1261        let tx = self.events.clone();
1262        let cache = self.cache.clone();
1263        std::thread::spawn(move || {
1264            home::look_into_batch(wanted, &cache, |path, m| {
1265                let _ = tx.send(AppEvent::HomeClassified {
1266                    measured: vec![(path, m)],
1267                    done: false,
1268                });
1269            });
1270            let _ = tx.send(AppEvent::HomeClassified {
1271                measured: Vec::new(),
1272                done: true,
1273            });
1274        });
1275    }
1276
1277    pub fn enter_home(&mut self) {
1278        self.counting.pause_indexing();
1279        if self.return_from_quality_evidence(false) {
1280            self.analysis_modal.close();
1281        }
1282        self.close_overlays();
1283        self.stop_find();
1284        // A count of the dataset being left is read for nobody.
1285        self.stop_value_count();
1286        self.export_modal.forget_counts();
1287        self.abandon_load();
1288        // Nobody is watching the file any more.
1289        if let Some(state) = self.data_table_state.as_mut() {
1290            state.stop_following();
1291        }
1292        self.home.status = None;
1293        // The search that found the dataset comes back selected: the next character starts
1294        // a new one, and `~` opens the path prompt.
1295        self.home.filter_selected = !self.home.filter.is_empty();
1296        self.home.folds_owed = true;
1297        self.home_refresh();
1298        if let Some(open_path) = self.path.clone() {
1299            let target =
1300                crate::canonical::canonicalize(&open_path).unwrap_or_else(|_| open_path.clone());
1301            if let Some(idx) = self.home.visible().iter().position(|row| match row {
1302                home::Row::Entry { entry, .. } => {
1303                    crate::canonical::canonicalize(&entry.path)
1304                        .unwrap_or_else(|_| entry.path.clone())
1305                        == target
1306                }
1307                // Not the door: its path is the directory's, and would take the cursor from the
1308                // file to the whole-directory row.
1309                home::Row::Header { .. }
1310                | home::Row::Place { .. }
1311                | home::Row::More { .. }
1312                | home::Row::Hidden { .. }
1313                | home::Row::Up { .. }
1314                | home::Row::Door { .. } => false,
1315            }) {
1316                self.home.selected = idx;
1317            }
1318        }
1319        self.input_mode = InputMode::Home;
1320    }
1321
1322    /// Esc backs out one layer: the filter, the directory descended into, then back to
1323    /// the open data. Nothing at the top level; Ctrl+C quits.
1324    pub(crate) fn home_escape(&mut self) -> Option<AppEvent> {
1325        if !self.home.filter.is_empty() {
1326            self.home.filter.clear();
1327            self.home.sync_search_section();
1328            // On a dataset, as at launch, not on the first section's header.
1329            self.home.select_first_entry();
1330            return None;
1331        }
1332        if self.home.browsing.is_some() {
1333            if self.home.below_browse_start() {
1334                self.home_ascend();
1335            } else {
1336                // Not past where the browse began: back to the listing it started from.
1337                self.home_leave_browsing(None);
1338            }
1339            return None;
1340        }
1341        if self.data_table_state.is_some() {
1342            self.show_table();
1343            // Said on arrival: one Esc too many lands here, and the next keys act on the table.
1344            let name = self
1345                .path
1346                .as_deref()
1347                .and_then(|p| p.file_name())
1348                .map(|n| n.to_string_lossy().into_owned());
1349            if let Some(name) = name {
1350                self.flash_note(format!("Back to {name}"));
1351            }
1352        }
1353        None
1354    }
1355
1356    /// Drop the highlighted dataset from the recents list. Only from Recent: elsewhere
1357    /// a row is a file on disk, and forgetting it would imply a deletion.
1358    pub(crate) fn home_forget_selected(&mut self) {
1359        // A catalog's heading: Delete hides the bundled one until the cache is cleared; a
1360        // user's catalog is hidden by id in the config.
1361        if let Some(catalog) = self.home.selected_catalog() {
1362            if catalog.origin == crate::home::catalog::Origin::Bundled {
1363                let message = format!(
1364                    "Hide {}? It comes back after datui cache clear.",
1365                    catalog.label
1366                );
1367                self.confirmation_modal
1368                    .show_destructive(message, "Hide", Confirm::HideExamples);
1369            } else {
1370                self.home.status = Some(format!(
1371                    "[home] hide = [\"{}\"] in config.toml hides it",
1372                    catalog.id
1373                ));
1374            }
1375            return;
1376        }
1377        // A place row stands for every recent under it, so forgetting them asks first, as
1378        // Shift+Delete does.
1379        if let Some(home::Row::Place { path, held, .. }) = self.home.selected_row() {
1380            let message = format!(
1381                "Forget {held} recently opened {} under {}?",
1382                if held == 1 { "dataset" } else { "datasets" },
1383                home::display_path(&path)
1384            );
1385            self.confirmation_modal
1386                .show(message, Confirm::ForgetPlace(path.clone()));
1387            return;
1388        }
1389        // A row in catalog.toml's own section is removed from the file, as Ctrl+D does; the
1390        // same place under Recent is only forgotten as a recent.
1391        let in_mine = self
1392            .home
1393            .selected_section()
1394            .and_then(|i| self.home.sections.get(i))
1395            .is_some_and(|s| s.origin == Some("catalog.toml"));
1396        if in_mine
1397            && let Some((location, _)) = self.home_row_for_catalog()
1398            && let Some((id, name)) = self.mine_entry_at(&location)
1399        {
1400            self.home_forget_from_catalog(&id, &name);
1401            return;
1402        }
1403        let section_title = self
1404            .home
1405            .selected_section()
1406            .and_then(|i| self.home.sections.get(i))
1407            .map(|s| s.title.clone())
1408            .unwrap_or_default();
1409        if self.home.browsing.is_none()
1410            && section_title == home::HomeState::CLOUD_SECTION
1411            && let Some(id) = self
1412                .home
1413                .selected_entry()
1414                .and_then(|e| home::cloud_source_id(&e.path))
1415        {
1416            self.cache.hide_cloud_source(&id);
1417            self.home.cloud.retain(|s| s.id != id);
1418            self.home_refresh();
1419            return;
1420        }
1421        let in_recents = section_title == "Recent";
1422        if !in_recents {
1423            self.home.status = Some("Only recents and catalog.toml rows can be forgotten".into());
1424            return;
1425        }
1426        let Some(entry) = self.home.selected_entry() else {
1427            return;
1428        };
1429        self.cache.forget_recent(&entry.path);
1430        // Nothing to say: the row going is the answer.
1431        self.home.status = None;
1432        self.home_refresh();
1433    }
1434
1435    /// The location and shown name Ctrl+D adds for the highlighted row; a heading stands
1436    /// for its section's directory. Tables inside files, cloud sources and non-place
1437    /// rows have none.
1438    fn home_row_for_catalog(&self) -> Option<(PathBuf, String)> {
1439        match self.home.selected_row()? {
1440            home::Row::Place { path, .. } => {
1441                let name = home::display_path(&path);
1442                Some((path, name))
1443            }
1444            home::Row::Door { entry, .. } => {
1445                // A local door's trailing slash goes; a URL keeps its `//`, which `components`
1446                // would fold.
1447                let path: PathBuf = if matches!(
1448                    source::input_source(&entry.path),
1449                    source::InputSource::Local(_)
1450                ) {
1451                    entry.path.components().collect()
1452                } else {
1453                    entry.path.clone()
1454                };
1455                Some((path.clone(), home::display_path(&path)))
1456            }
1457            home::Row::Entry { entry, .. } => (entry.table.is_none()
1458                && !home::is_cloud_place(&entry.path)
1459                && crate::formats::members::split(&entry.path).is_none())
1460            .then(|| (entry.path.clone(), entry.name.clone())),
1461            home::Row::Header { section, .. } => {
1462                let root = self.home.sections.get(section)?.root.clone()?;
1463                let name = home::display_path(&root);
1464                Some((root, name))
1465            }
1466            home::Row::More { .. } | home::Row::Hidden { .. } | home::Row::Up { .. } => None,
1467        }
1468    }
1469
1470    /// `catalog.toml`'s path: beside the config file read, else in the config directory.
1471    fn mine_catalog_file(&self) -> Option<PathBuf> {
1472        let dir = match &self.app_config.catalog_dir {
1473            Some(dir) => dir.clone(),
1474            None => config::ConfigManager::new(APP_NAME)
1475                .ok()?
1476                .config_dir()
1477                .to_path_buf(),
1478        };
1479        Some(dir.join(catalog::MINE_FILE))
1480    }
1481
1482    /// The id and name of the `catalog.toml` entry at `location`, when there is one.
1483    fn mine_entry_at(&self, location: &Path) -> Option<(String, String)> {
1484        self.app_config
1485            .read_catalogs
1486            .iter()
1487            .find(|c| c.origin == catalog::Origin::Mine)?
1488            .dataset_at(location)
1489            .map(|d| (d.id.clone(), d.name.clone()))
1490    }
1491
1492    /// Read the catalogs again after `catalog.toml` changed, and list again.
1493    fn reload_catalogs(&mut self) -> Result<(), String> {
1494        let dir = self
1495            .mine_catalog_file()
1496            .and_then(|f| f.parent().map(Path::to_path_buf));
1497        self.app_config
1498            .read_catalog_files(dir.as_deref())
1499            .map_err(|e| e.to_string())?;
1500        // The open dataset's notes and Documentation tab follow the file.
1501        self.info
1502            .follow_catalogs(&self.app_config, self.path.as_deref());
1503        self.open_info_documentation();
1504        self.home_refresh();
1505        Ok(())
1506    }
1507
1508    /// What Ctrl+D writes for `location` named `name`: a copy of another catalog's
1509    /// entry, or the place as is.
1510    fn new_catalog_dataset(&self, location: &Path, name: &str) -> catalog::NewDataset {
1511        if let Some((_, shown)) = self.home.catalog_dataset(location) {
1512            let entry = &shown.entry;
1513            return catalog::NewDataset {
1514                name: entry.name.clone(),
1515                path: entry.local_path().map(|p| home::display_path(&p)),
1516                url: entry.url.clone(),
1517                auth: entry.auth.clone(),
1518                connection: entry.connection.clone(),
1519                description: entry.description.clone(),
1520                size: entry.size,
1521            };
1522        }
1523        let mut new = catalog::NewDataset {
1524            name: name.to_string(),
1525            ..Default::default()
1526        };
1527        if matches!(
1528            source::input_source(location),
1529            source::InputSource::Local(_)
1530        ) {
1531            let absolute = if location.is_relative() {
1532                std::env::current_dir()
1533                    .map(|cwd| cwd.join(location))
1534                    .unwrap_or_else(|_| location.to_path_buf())
1535            } else {
1536                location.to_path_buf()
1537            };
1538            new.path = Some(home::display_path(&absolute));
1539            return new;
1540        }
1541        // A store reached through a configured source: the source becomes the connection
1542        // and leaves the URL.
1543        let text = location.to_string_lossy();
1544        let (id, plain) = source::split_source_id(&text);
1545        new.url = Some(plain.into_owned());
1546        if let Some(id) = id {
1547            new.connection = Some(id.to_string());
1548        }
1549        new
1550    }
1551
1552    /// Ctrl+D: add the row under the cursor to `catalog.toml`, or forget it from there.
1553    pub(crate) fn home_toggle_catalog(&mut self) {
1554        let Some((location, name)) = self.home_row_for_catalog() else {
1555            self.home.status = Some("Move to a dataset or directory to add it".into());
1556            return;
1557        };
1558        if let Some((id, name)) = self.mine_entry_at(&location) {
1559            self.home_forget_from_catalog(&id, &name);
1560            return;
1561        }
1562        let Some(file) = self.mine_catalog_file() else {
1563            self.home.status = Some("No config directory to keep catalog.toml in".into());
1564            return;
1565        };
1566        let new = self.new_catalog_dataset(&location, &name);
1567        // A source found on the machine but not configured cannot be named in a catalog:
1568        // the URL would be read elsewhere.
1569        if let Some(connection) = &new.connection
1570            && !self
1571                .app_config
1572                .cloud
1573                .connections
1574                .iter()
1575                .any(|c| c.name == *connection)
1576        {
1577            self.home.status = Some(format!(
1578                "Not added: {connection} is not a [[cloud.connections]] entry in the config"
1579            ));
1580            return;
1581        }
1582        if let Err(why) = new.check() {
1583            self.home.status = Some(format!("Not added: {why}"));
1584            return;
1585        }
1586        let label = self
1587            .app_config
1588            .read_catalogs
1589            .iter()
1590            .find(|c| c.origin == catalog::Origin::Mine)
1591            .map(|c| c.label.clone())
1592            .unwrap_or_else(|| catalog::MINE_LABEL.to_string());
1593        match catalog::add(&file, &new) {
1594            Ok(_) => match self.reload_catalogs() {
1595                Ok(()) => self.flash_note(format!("Added {} to {label}", new.name)),
1596                Err(e) => self.error_modal.show(e),
1597            },
1598            Err(e) => self.error_modal.show(e.to_string()),
1599        }
1600    }
1601
1602    /// Remove the entry `id` from `catalog.toml`.
1603    fn home_forget_from_catalog(&mut self, id: &str, name: &str) {
1604        let Some(file) = self.mine_catalog_file() else {
1605            return;
1606        };
1607        match catalog::forget(&file, id) {
1608            Ok(()) => match self.reload_catalogs() {
1609                Ok(()) => self.flash_note(format!("Forgot {name}")),
1610                Err(e) => self.error_modal.show(e),
1611            },
1612            Err(e) => self.error_modal.show(e.to_string()),
1613        }
1614    }
1615
1616    /// Move pre-0.4.0 Ctrl+D directories from the cache into `catalog.toml`, once.
1617    fn move_remembered_places(&mut self) {
1618        if std::mem::replace(&mut self.home_app.remembered_moved, true) {
1619            return;
1620        }
1621        let places = self.cache.load_remembered_places();
1622        if places.is_empty() {
1623            return;
1624        }
1625        let Some(file) = self.mine_catalog_file() else {
1626            return;
1627        };
1628        // The cache's list goes only once all are in catalog.toml; a failure retries next
1629        // run.
1630        match catalog::move_places(&file, &places) {
1631            Ok(_) => self.cache.clear_remembered_places(),
1632            Err(e) => {
1633                log::warn!(target: "datui", "moving remembered places into catalog.toml: {e:#}")
1634            }
1635        }
1636        let dir = file.parent().map(Path::to_path_buf);
1637        if let Err(e) = self.app_config.read_catalog_files(dir.as_deref()) {
1638            log::warn!(target: "datui", "reading catalog.toml: {e:#}");
1639        }
1640    }
1641
1642    /// Ctrl+E: the Documentation view of the row under the cursor.
1643    pub(crate) fn home_open_documentation(&mut self) {
1644        let Some((path, doc)) = self.home_documented_row() else {
1645            self.home.status =
1646                Some("Ctrl+E shows what a catalog or a format spec says of a row".into());
1647            return;
1648        };
1649        let measured = self
1650            .home
1651            .selected_entry()
1652            .filter(|e| {
1653                e.path == path
1654                    && doc
1655                        .catalog
1656                        .as_ref()
1657                        .is_some_and(|(_, entry)| entry.location() == path)
1658            })
1659            .and_then(|e| e.size);
1660        self.info.documentation.open(doc, measured);
1661        self.info.documentation.links_open = self.home_app.local_desktop;
1662    }
1663
1664    /// What Ctrl+E documents for the row under the cursor, with its path: the catalog
1665    /// dataset it is or is inside, and its format spec's docs, if any.
1666    pub(crate) fn home_documented_row(
1667        &self,
1668    ) -> Option<(PathBuf, widgets::documentation::Documented)> {
1669        let row = self.home.selected_row();
1670        let (path, file) = match &row {
1671            Some(home::Row::Entry { entry, .. }) | Some(home::Row::Door { entry, .. }) => {
1672                (Some(entry.path.clone()), Some(*entry))
1673            }
1674            Some(home::Row::Place { path, .. }) => (Some(path.clone()), None),
1675            Some(home::Row::Header { section, .. }) => (
1676                self.home
1677                    .sections
1678                    .get(*section)
1679                    .and_then(|s| s.root.clone()),
1680                None,
1681            ),
1682            _ => (None, None),
1683        };
1684        let path = path?;
1685        let catalog = home::catalog_entry_for(&self.home.catalogs, &path);
1686        let spec = file
1687            .filter(|e| e.kind == discover::EntryKind::File)
1688            .and_then(|e| e.format_spec.as_deref())
1689            .and_then(|name| self.home.formats.get(name))
1690            .and_then(|spec| spec.docs())
1691            .map(std::sync::Arc::new);
1692        // A record type's row (`day.ord/add`) documents its file.
1693        let name = file
1694            .map(|e| {
1695                let text = e.path.to_string_lossy();
1696                text.strip_suffix(e.name.as_str())
1697                    .filter(|_| e.table.is_some())
1698                    .map(|file| file.trim_end_matches(std::path::is_separator))
1699                    .and_then(|file| Path::new(file).file_name())
1700                    .map_or_else(|| e.name.clone(), |n| n.to_string_lossy().into_owned())
1701            })
1702            .unwrap_or_default();
1703        let doc = widgets::documentation::Documented::new(catalog, spec, name)?;
1704        Some((path, doc))
1705    }
1706
1707    /// Whether Delete on the selected row hides a catalog: the bundled one's heading.
1708    pub(crate) fn home_hides_catalog(&self) -> bool {
1709        self.home
1710            .selected_catalog()
1711            .is_some_and(|c| c.origin == crate::home::catalog::Origin::Bundled)
1712    }
1713
1714    /// What Ctrl+D does on the row under the cursor, as the footer names it: add to
1715    /// `catalog.toml` or forget from it; `None` where it cannot add.
1716    pub(crate) fn home_catalog_action(&self) -> Option<&'static str> {
1717        let (location, _) = self.home_row_for_catalog()?;
1718        Some(if self.mine_entry_at(&location).is_some() {
1719            "Forget"
1720        } else {
1721            "Add"
1722        })
1723    }
1724
1725    /// Fold or unfold the section under the cursor; folding moves the cursor to its
1726    /// header.
1727    pub(crate) fn home_toggle_fold(&mut self) {
1728        if let Some(section) = self.home.selected_section() {
1729            self.home.toggle_collapsed(section);
1730            self.home.clamp_selection();
1731            self.cache.save_folds(&self.home.folds);
1732        }
1733    }
1734
1735    pub(crate) fn home_collapse(&mut self, collapse: bool) {
1736        // The browsed listing is the whole screen: it never folds, and no fold is
1737        // remembered for its path (see `set_collapsed`).
1738        if self.home.browsing.is_some() {
1739            return;
1740        }
1741        let Some(section) = self.home.selected_section() else {
1742            return;
1743        };
1744        // → on a section's more row shows it whole; ← on a row its cut would hide cuts it
1745        // back. Elsewhere they fold.
1746        if collapse && self.home.cut_again(section) {
1747            return;
1748        }
1749        if !collapse && matches!(self.home.selected_row(), Some(home::Row::More { .. })) {
1750            self.home.show_all(section);
1751            return;
1752        }
1753        if collapse && !self.home.is_collapsed(section) {
1754            self.home.set_collapsed(section, true);
1755            if let Some(idx) = self
1756                .home
1757                .visible()
1758                .iter()
1759                .position(|row| row.section() == section)
1760            {
1761                self.home.selected = idx;
1762            }
1763        } else if !collapse {
1764            self.home.set_collapsed(section, false);
1765        }
1766        self.home.clamp_selection();
1767        self.cache.save_folds(&self.home.folds);
1768    }
1769
1770    /// Step out of a directory that was descended into.
1771    pub(crate) fn home_ascend(&mut self) {
1772        let Some(current) = self.home.browsing.clone() else {
1773            return;
1774        };
1775        let parent = self.home.parent_of(&current);
1776        self.home_leave_browsing(parent);
1777    }
1778
1779    /// Move the browse up to `to`, or back to the root listing when `None`.
1780    fn home_leave_browsing(&mut self, to: Option<PathBuf>) {
1781        // What the last place said of itself no longer applies.
1782        self.home.status = None;
1783        let from = std::mem::replace(&mut self.home.browsing, to);
1784        // Backspace can climb above the browse start; the start follows so Esc has a place
1785        // to stop.
1786        if !self.home.below_browse_start() {
1787            self.home.browse_start = self.home.browsing.clone();
1788        }
1789        // The filter, search and row left here come back; the cursor returns to that row
1790        // when the listing lands. A search of the old place no longer applies.
1791        self.home.come_back(from);
1792        self.home.sync_search_section();
1793        self.home.selected = 0;
1794        self.home_refresh();
1795        if !self.home.filter.is_empty() {
1796            self.spawn_home_search();
1797        }
1798    }
1799
1800    /// Whether a peek's answer changes what a row draws: its kind, or anything in
1801    /// `holds` (`truncated` turns `dir` into `dir+`). Kept answers rebuild the listing
1802    /// on the UI thread, so the rest become "a directory, nothing to say", still sent
1803    /// to clear `peeking`.
1804    #[cfg(feature = "cloud")]
1805    pub(crate) fn peek_tells_a_row_something(
1806        answer: &(discover::EntryKind, discover::Holds),
1807    ) -> bool {
1808        answer.0 != discover::EntryKind::Directory || !answer.1.is_empty()
1809    }
1810
1811    /// Peek inside cloud directories at or near the cursor so datasets show `hive` or
1812    /// `multi` and open as one: one small listing per directory, once a session, plus
1813    /// up to three few-KiB footer reads for a `multi` candidate. On-screen rows a batch
1814    /// at a time, the highlighted first.
1815    #[cfg(feature = "cloud")]
1816    pub(crate) fn peek_cloud_directories(&mut self) {
1817        const PEEKS_AT_ONCE: usize = 4;
1818        let directories = self.home.cloud_directories_to_peek(PEEKS_AT_ONCE);
1819        if directories.is_empty() {
1820            return;
1821        }
1822        // Marked out, not answered: a second pass must not ask again, and writing an
1823        // answer now would claim one before the request is made.
1824        for directory in &directories {
1825            self.home.peeking.insert(directory.clone());
1826        }
1827        let tx = self.events.clone();
1828        let cloud = self.app_config.cloud.clone();
1829        self.runtime.spawn(async move {
1830            let permits = Arc::new(tokio::sync::Semaphore::new(PEEKS_AT_ONCE));
1831            let mut peeks = tokio::task::JoinSet::new();
1832            // By task, so a panicking peek is sent back as failed rather than spinning
1833            // forever.
1834            let mut asked = std::collections::HashMap::new();
1835            for directory in directories {
1836                let (permits, cloud) = (permits.clone(), cloud.clone());
1837                let task_directory = directory.clone();
1838                let task = peeks.spawn(async move {
1839                    let directory = task_directory;
1840                    let _permit = permits.acquire_owned().await;
1841                    let kind =
1842                        crate::cloud::cloud_browse::peek_kind(&directory.to_string_lossy(), &cloud)
1843                            .await;
1844                    (directory, kind)
1845                });
1846                asked.insert(task.id(), directory);
1847            }
1848            // Sent a few at a time so labels fill in without a rebuild per directory. Every
1849            // directory asked is sent back, undecided and failed ones too: that clears
1850            // `peeking` and keeps one request per directory.
1851            let mut found = Vec::new();
1852            let mut failed = Vec::new();
1853            while let Some(joined) = peeks.join_next_with_id().await {
1854                match joined {
1855                    Ok((_, (directory, Ok(answer)))) => {
1856                        let answer = Some(answer)
1857                            .filter(Self::peek_tells_a_row_something)
1858                            .unwrap_or((discover::EntryKind::Directory, Default::default()));
1859                        found.push((directory, answer));
1860                    }
1861                    Ok((_, (directory, Err(_)))) => failed.push(directory),
1862                    Err(error) => failed.extend(asked.remove(&error.id())),
1863                }
1864                if found.len() + failed.len() >= PEEKS_AT_ONCE {
1865                    let _ = tx.send(AppEvent::HomeCloudKinds {
1866                        kinds: std::mem::take(&mut found),
1867                        failed: std::mem::take(&mut failed),
1868                    });
1869                }
1870            }
1871            if !found.is_empty() || !failed.is_empty() {
1872                let _ = tx.send(AppEvent::HomeCloudKinds {
1873                    kinds: found,
1874                    failed,
1875                });
1876            }
1877        });
1878    }
1879
1880    /// Browse into a directory or bucket, local or remote.
1881    pub(crate) fn home_browse_into(&mut self, path: PathBuf) {
1882        self.home.leave_mark();
1883        if self.home.browsing.is_none() {
1884            self.home.browse_start = Some(path.clone());
1885        } else if self.home.browse_start.is_none() {
1886            self.home.browse_start = self.home.browsing.clone();
1887        }
1888        self.home.browsing = Some(path);
1889        // What the last walk found and the status line describe a different place; a fresh
1890        // walk starts on the next keystroke. A caller with news of the new place says it
1891        // after this returns.
1892        self.home.status = None;
1893        self.home.search.reset();
1894        self.home.filter.clear();
1895        self.home.sync_search_section();
1896        self.home.selected = 0;
1897        self.home_refresh();
1898    }
1899
1900    /// Whether the highlighted row is the `(all files)` door, which opens the browsed
1901    /// directory and so is already inside it.
1902    fn selection_opens_the_whole_directory(&self) -> bool {
1903        self.home.selection_is_the_door()
1904    }
1905
1906    /// The highlighted row when → goes inside it: any directory, local or remote,
1907    /// whatever its label. Files, headers and the door are left out.
1908    pub(crate) fn selected_directory_to_enter(&self) -> Option<PathBuf> {
1909        // A place under `RECENT` has no entry, so it is answered first.
1910        if let Some(home::Row::Place { path, .. }) = self.home.selected_row() {
1911            return home::place_is_browsable(&path).then_some(path);
1912        }
1913        let entry = self.home.selected_entry()?;
1914        if self.selection_opens_the_whole_directory() || self.home.missing.contains(&entry.path) {
1915            return None;
1916        }
1917        // A SQLite database lists its tables, however many it has.
1918        if entry.cost.tables.is_some() {
1919            return Some(entry.path.clone());
1920        }
1921        (!matches!(
1922            entry.kind,
1923            discover::EntryKind::File | discover::EntryKind::Other
1924        ))
1925        .then_some(entry.path.clone())
1926    }
1927
1928    /// Why Enter on a bucket directory's `(all files)` row reads nothing, by Enter's
1929    /// own rule: a hive root or one-table directory reads through its files, and one
1930    /// with a reader for its contents reads with that.
1931    #[cfg(feature = "cloud")]
1932    pub(crate) fn why_a_door_reads_nothing(entry: &discover::Entry) -> Option<String> {
1933        if !home::is_object_store_url(&entry.path)
1934            || matches!(
1935                entry.kind,
1936                discover::EntryKind::Hive | discover::EntryKind::MultiFile
1937            )
1938            || Self::cloud_prefix_format(&entry.holds).is_some()
1939        {
1940            return None;
1941        }
1942        Self::why_a_cloud_prefix_cannot_be_read(&entry.holds)
1943    }
1944
1945    /// Why an object-store prefix cannot be read as one table, from its listed
1946    /// contents (no request). `None` when it may yet be Parquet: it holds Parquet,
1947    /// or no data files and perhaps data a level down.
1948    #[cfg(feature = "cloud")]
1949    pub(crate) fn why_a_cloud_prefix_cannot_be_read(holds: &discover::Holds) -> Option<String> {
1950        let reads_parquet =
1951            |name: &str| crate::FileFormat::from_name(name) == Some(crate::FileFormat::Parquet);
1952        if holds.formats.iter().any(|(name, _)| reads_parquet(name)) {
1953            return None;
1954        }
1955        match holds.formats.as_slice() {
1956            // Data files, none Parquet. `label()` says `mixed` for several formats, so the
1957            // line spells them out.
1958            [] => {
1959                // Nothing readable. A refusal only if nothing is below: sub-prefixes may hold
1960                // Parquet a level down.
1961                (holds.not_read > 0 && holds.directories == 0).then(|| {
1962                    "this prefix holds nothing datui can read — datui reads a directory in \
1963                     an object store as Parquet only."
1964                        .to_string()
1965                })
1966            }
1967            formats => {
1968                let held = formats
1969                    .iter()
1970                    .map(|(name, count)| format!("{count} {name}"))
1971                    .collect::<Vec<_>>()
1972                    .join(", ");
1973                Some(format!(
1974                    "this prefix holds {held} — datui reads a directory in an object store \
1975                     as Parquet only. Open one of the files below instead."
1976                ))
1977            }
1978        }
1979    }
1980
1981    /// The reader an object-store prefix calls for, from its listing's counts: the
1982    /// commonest format, ranked by `rank_formats` as on disk, Parquet included. `None`
1983    /// when nothing there has a multi-file reader, where the refusal belongs.
1984    #[cfg(feature = "cloud")]
1985    pub(crate) fn cloud_prefix_format(
1986        holds: &discover::Holds,
1987    ) -> Option<(FileFormat, Vec<(FileFormat, usize)>)> {
1988        // A saved DatasetDict: its splits are Arrow, read one at a time.
1989        if holds.dataset_dict {
1990            return Some((FileFormat::Arrow, Vec::new()));
1991        }
1992        // Model weights beside config and tokenizer JSON: the prefix is the model, and the
1993        // JSON is not data passed over.
1994        if let Some((name, _)) = holds.model_weights() {
1995            return FileFormat::from_name(name).map(|format| (format, Vec::new()));
1996        }
1997        let (name, _) = holds.formats.first()?;
1998        // A GPS log is read whole from disk; a bucket's logs open one at a time, as its
1999        // text files do.
2000        let format = FileFormat::from_name(name)
2001            .filter(|f| f.reads_many_files() && !f.reads_into() && !f.is_lines())?;
2002        // What taking the commonest passes over. Polars lists the prefix and never sees
2003        // other formats, so the note comes from the listing on screen.
2004        let left_out = holds
2005            .formats
2006            .iter()
2007            .skip(1)
2008            .filter_map(|(name, n)| FileFormat::from_name(name).map(|f| (f, *n)))
2009            .collect();
2010        Some((format, left_out))
2011    }
2012
2013    /// Open the highlighted entry: toggle a section, descend, or load a dataset.
2014    pub(crate) fn home_open_selected(&mut self) -> Option<AppEvent> {
2015        match self.home.selected_row() {
2016            // Into the directory or prefix the recents under it live in.
2017            Some(home::Row::Place { path, .. }) => {
2018                if home::place_is_browsable(&path) {
2019                    self.home_browse_into(path);
2020                } else {
2021                    self.home.status = Some(
2022                        "An HTTP server has no listing to browse. Open a file under it".into(),
2023                    );
2024                }
2025                return None;
2026            }
2027            // The rest of `RECENT`, or of a directory, for the session.
2028            Some(home::Row::More { section, .. }) => {
2029                self.home.show_all(section);
2030                return None;
2031            }
2032            // Up a level: as Backspace while browsing, and above a root at the listing.
2033            Some(home::Row::Up { section }) => {
2034                if self.home.browsing.is_some() {
2035                    self.home_ascend();
2036                } else if let Some(parent) = self
2037                    .home
2038                    .sections
2039                    .get(section)
2040                    .and_then(|s| s.root.as_deref())
2041                    .and_then(|root| self.home.parent_of(root))
2042                {
2043                    self.home_browse_into(parent);
2044                }
2045                return None;
2046            }
2047            // What Ctrl+A shows; the cursor goes to the first, where the hidden row stood.
2048            Some(home::Row::Hidden { .. }) => {
2049                self.home.hide_unreadable = false;
2050                if let Some(idx) = self.home.visible().iter().position(|row| {
2051                    matches!(row, home::Row::Entry { entry, .. }
2052                        if entry.hidden_by_default())
2053                }) {
2054                    self.home.selected = idx;
2055                }
2056                return None;
2057            }
2058            _ => {}
2059        }
2060        if self.home.selection_is_header() {
2061            self.home_toggle_fold();
2062            return None;
2063        }
2064        let entry = self.home.selected_entry()?.clone();
2065        // A collection's local dataset that is not there: said here, where it was named.
2066        if self.home.missing.contains(&entry.path) {
2067            self.home.status = Some(format!(
2068                "{} does not exist",
2069                home::display_path(&entry.path)
2070            ));
2071            return None;
2072        }
2073        // A place a collection suggests opens as one table on Enter; → still goes in.
2074        if entry.kind != discover::EntryKind::File && self.home.bookmark(&entry.path).is_some() {
2075            #[cfg(feature = "cloud")]
2076            let reader = if home::is_object_store_url(&entry.path)
2077                && !matches!(
2078                    entry.kind,
2079                    discover::EntryKind::Hive | discover::EntryKind::MultiFile
2080                ) {
2081                Self::cloud_prefix_format(&entry.holds)
2082            } else {
2083                None
2084            };
2085            #[cfg(not(feature = "cloud"))]
2086            let reader = None;
2087            let directory = home::directory_dataset_url(&entry.path);
2088            return Some(self.home_open_directory_as(directory, true, None, reader));
2089        }
2090        // The `(all files)` row opens its directory whatever the label: no label can lock
2091        // the user out. Sent straight to the open, since `open_what_it_is` would read the
2092        // label and step inside again.
2093        if self.selection_opens_the_whole_directory() {
2094            // A lake table read as Parquet counts tombstoned rows and every rewritten version,
2095            // so the read is labeled, not refused: a note in the panel and a footer chip say
2096            // so, and Enter one level up explains datui does not read the table itself yet.
2097            let lake = entry.kind.lake_name();
2098            // Pick an object-store prefix's reader from its listed contents (no request), so a
2099            // CSV prefix is not scanned as Parquet and blamed on credentials. A prefix the
2100            // listing already calls a dataset is left alone: a hive root reads through its
2101            // partitions despite a stray `manifest.csv`.
2102            #[cfg(feature = "cloud")]
2103            let reader = if home::is_object_store_url(&entry.path)
2104                && !matches!(
2105                    entry.kind,
2106                    discover::EntryKind::Hive | discover::EntryKind::MultiFile
2107                ) {
2108                let reader = Self::cloud_prefix_format(&entry.holds);
2109                // Nothing readable: the refusal names what is there rather than blaming the
2110                // connection.
2111                if reader.is_none()
2112                    && let Some(what) = Self::why_a_cloud_prefix_cannot_be_read(&entry.holds)
2113                {
2114                    self.home.status = Some(what);
2115                    return None;
2116                }
2117                reader
2118            } else {
2119                None
2120            };
2121            #[cfg(not(feature = "cloud"))]
2122            let reader = None;
2123            // `hive: true` reads it as one and carries partition columns through the hive
2124            // route. The cloud route returns before the dispatch.
2125            let directory = home::directory_dataset_url(&entry.path);
2126            return Some(self.home_open_directory_as(directory, true, lake, reader));
2127        }
2128        // A row nothing has looked at is looked at first: `Unknown` is offered as openable,
2129        // and a lake root would otherwise read as one table.
2130        let mut entry = entry;
2131        if entry.kind == discover::EntryKind::Unknown {
2132            if self.looking_could_block(&entry.path) {
2133                return Some(AppEvent::ClassifyThenOpen {
2134                    path: entry.path,
2135                    jump: false,
2136                });
2137            }
2138            if entry.path.is_dir() {
2139                entry.kind = discover::classify_directory(&entry.path);
2140            }
2141        }
2142        // A database of several tables lists them; one not yet measured opens and lands on
2143        // its tables the same way.
2144        if entry.enter_lists_tables() {
2145            self.home_browse_into(entry.path);
2146            return None;
2147        }
2148        self.open_what_it_is(entry.path, entry.kind, false)
2149    }
2150
2151    /// Browse into `path` as a jump: the browse starts here, so Esc returns to the
2152    /// listing rather than up through the path's parents.
2153    pub(crate) fn home_jump_into(&mut self, path: PathBuf) {
2154        // Only the listing's mark is still a way back.
2155        self.home.trail.retain(|mark| mark.place.is_none());
2156        if self.home.browsing.is_none() {
2157            self.home.leave_mark();
2158        }
2159        self.home.browse_start = Some(path.clone());
2160        self.home.browsing = Some(path);
2161        self.home.status = None;
2162        self.home.search.reset();
2163        self.home.filter.clear();
2164        self.home.sync_search_section();
2165        self.home.selected = 0;
2166        self.home_refresh();
2167    }
2168
2169    /// Load a path from the home screen. The `Open` handler records the recent.
2170    pub(crate) fn home_open_path(&mut self, path: PathBuf, hive: bool) -> AppEvent {
2171        self.home_open_directory(path, hive, None)
2172    }
2173
2174    /// As [`Self::home_open_path`], noting whether the directory is a lake table whose
2175    /// plain files are read, so the dataset can say so.
2176    fn home_open_directory(
2177        &mut self,
2178        path: PathBuf,
2179        hive: bool,
2180        lake: Option<&'static str>,
2181    ) -> AppEvent {
2182        self.home_open_directory_as(path, hive, lake, None)
2183    }
2184
2185    /// As [`Self::home_open_directory`], naming the reader. For an object-store prefix
2186    /// the cloud scan runs before the format dispatch, so the listing's format travels
2187    /// with the open or the scan falls back to Parquet.
2188    fn home_open_directory_as(
2189        &mut self,
2190        path: PathBuf,
2191        hive: bool,
2192        lake: Option<&'static str>,
2193        reader: Option<(FileFormat, Vec<(FileFormat, usize)>)>,
2194    ) -> AppEvent {
2195        let (format, left_out) = match reader {
2196            Some((format, left_out)) => (Some(format), left_out),
2197            None => (None, Vec::new()),
2198        };
2199        // Told, not stat'ed: the caller knows, and a stat on a gone share would freeze the
2200        // key thread (also why `Open` fills in the size).
2201        let options = OpenOptions {
2202            hive,
2203            read_as_plain_files_of: lake,
2204            format,
2205            left_out,
2206            ..self.open_defaults()
2207        };
2208        self.show_table();
2209        // Chosen here, so a failure is reported here.
2210        self.announce_open(true, "Scanning input".to_string(), 10);
2211        // The frame drawn before `Open` runs says which file; `Open` fills in the size a
2212        // frame later, off this thread.
2213        self.name_what_is_loading(path.clone());
2214        AppEvent::Open(vec![path], options)
2215    }
2216
2217    /// The home screen's worker answers.
2218    pub(crate) fn home_event(&mut self, event: AppEvent) -> Option<AppEvent> {
2219        match event {
2220            AppEvent::HomeListingReady {
2221                generation,
2222                listing,
2223                known,
2224                visits,
2225                newest,
2226                folds,
2227            } => {
2228                // Only the current listing's answer clears the flag; a stale one landing first
2229                // would say nothing is in flight.
2230                if generation == self.home_app.generation {
2231                    self.home.listing_in_flight = false;
2232                }
2233                // Read fresh from the cache, so true whichever listing carried them; only the
2234                // first listing after entering home carries the folds.
2235                self.home.known = known;
2236                self.home.set_visits(visits);
2237                self.home.newest_recent = newest;
2238                if let Some(folds) = folds {
2239                    self.home.folds = folds;
2240                }
2241                // A superseded listing describes somewhere the user has left.
2242                if generation != self.home_app.generation {
2243                    return None;
2244                }
2245                self.home.apply_listing(*listing);
2246                // Probes are chosen from the sections, so only once they exist.
2247                self.spawn_home_probes();
2248                #[cfg(feature = "cloud")]
2249                self.spawn_cloud_discovery();
2250                self.request_home_measurements();
2251                self.request_home_classifications();
2252                None
2253            }
2254            AppEvent::HomeListingFailed => {
2255                // The rows already listed stay. The panic is flashed as a raw worker's.
2256                self.home.listing_in_flight = false;
2257                None
2258            }
2259            AppEvent::HomeMeasured { measured, done } => {
2260                for (path, m) in measured {
2261                    self.home.record_measurement(path, m);
2262                }
2263                // A batch answers one file per event; they are folded in at the next frame
2264                // (`begin_frame`), not one list per file.
2265                if done {
2266                    self.home.measure_in_flight = false;
2267                    self.request_home_measurements();
2268                }
2269                None
2270            }
2271            AppEvent::HomeSized { path, measured } => {
2272                self.home.record_size(path, measured);
2273                self.home.apply_new_measurements();
2274                None
2275            }
2276            AppEvent::HomeWebGone { path, gone } => {
2277                self.home.web_gone.insert(path, gone);
2278                None
2279            }
2280            AppEvent::HomeClassified { measured, done } => {
2281                // Kept even if the listing was rebuilt since: probes and peeks rebuild it often,
2282                // and dropping answers would leave rows unlabeled.
2283                for (path, m) in measured {
2284                    self.home.record_measurement(path, m);
2285                }
2286                // Nothing re-sorts: kinds are written into rows in place, so the listing never
2287                // reshuffles under the cursor. Folded in at the next frame, as measurements.
2288                // The next batch comes from the viewport as it is now, not the rows scrolled past.
2289                if done {
2290                    self.home.classify_in_flight = false;
2291                    self.request_home_classifications();
2292                }
2293                None
2294            }
2295            AppEvent::HomePathListed { listing } => {
2296                // Kept only for the directory still being typed.
2297                if self.home.path_input_active
2298                    && home::typed_dir(&self.home.path_input) == listing.dir
2299                {
2300                    self.home.path_listing = Some(*listing);
2301                    if self.home.path_pick.is_none() {
2302                        self.home.pick_first_path();
2303                    }
2304                }
2305                None
2306            }
2307            AppEvent::HomePathCompleted {
2308                generation,
2309                typed,
2310                completed,
2311                candidates,
2312            } => {
2313                // Discard if the user typed since asking: completing would scramble their input.
2314                if generation != self.home_app.generation || self.home.path_input != typed {
2315                    return None;
2316                }
2317                if candidates == 0 {
2318                    self.home.status = Some("No such path".to_string());
2319                } else {
2320                    self.home.status = None;
2321                    if candidates > 1 {
2322                        self.flash_note(format!("{candidates} matches"));
2323                    }
2324                    self.home.path_input = completed;
2325                    self.home.pick_first_path();
2326                }
2327                None
2328            }
2329            AppEvent::HomePreviewReady {
2330                path,
2331                stamp,
2332                read_at,
2333                rows,
2334                prepared,
2335            } => {
2336                let prepared = prepared.lock().ok().and_then(|mut p| p.take());
2337                // The columns came with the rows: the pane lists them, for a CSV too.
2338                if let Some(prepared) = &prepared {
2339                    let schema = prepared
2340                        .state
2341                        .schema()
2342                        .iter()
2343                        .map(|(name, dtype)| (name.to_string(), dtype.clone()))
2344                        .collect();
2345                    self.home_app
2346                        .schema_cache
2347                        .insert(path.clone(), Some(schema));
2348                }
2349                let prepared = prepared.filter(|_| read_at.is_some());
2350                self.home_app.previews.landed(
2351                    path,
2352                    stamp,
2353                    read_at.unwrap_or(stamp),
2354                    rows,
2355                    prepared,
2356                );
2357                None
2358            }
2359            AppEvent::HomeSchemaReady {
2360                generation,
2361                path,
2362                preview,
2363            } => {
2364                self.home_app.schema_inflight.retain(|p| p != &path);
2365                // A preview's columns are not taken back by a metadata read that had none.
2366                let known = self
2367                    .home_app
2368                    .schema_cache
2369                    .get(&path)
2370                    .is_some_and(Option::is_some);
2371                if generation == self.home_app.generation && (preview.is_some() || !known) {
2372                    self.home_app.schema_cache.insert(path.clone(), preview);
2373                }
2374                None
2375            }
2376            AppEvent::HomeSearchBatch {
2377                generation,
2378                root,
2379                found,
2380                scanned,
2381            } => {
2382                // Batches from a walk a later navigation superseded describe a place left behind;
2383                // the walk is abandoned, not cancelled, so late batches are expected. A refresh of
2384                // the same place supersedes nothing.
2385                if generation == self.home_app.search_generation {
2386                    self.home.search_batch(&root, found, scanned);
2387                }
2388                None
2389            }
2390            AppEvent::HomeSearchScored { epoch, matches } => {
2391                // A scoring that died is not retried (it would die again); listed matches stand.
2392                if let Some(matches) = matches {
2393                    self.home.search_scored(epoch, *matches);
2394                }
2395                None
2396            }
2397            AppEvent::HomeSearchDone {
2398                generation,
2399                root,
2400                scanned,
2401                limited,
2402            } => {
2403                if generation == self.home_app.search_generation {
2404                    self.home.search_finished(&root, scanned, limited);
2405                }
2406                self.home_app.search_inflight = false;
2407                // A walk abandoned by a browse held up the one the filter now asks for.
2408                if !self.home.filter.is_empty() && self.home.search.root.is_none() {
2409                    self.spawn_home_search();
2410                }
2411                None
2412            }
2413            #[cfg(feature = "cloud")]
2414            AppEvent::HomeCloudSources { sources } => {
2415                self.home.cloud = sources;
2416                self.home_refresh();
2417                None
2418            }
2419            #[cfg(feature = "cloud")]
2420            AppEvent::HomeCloudListed {
2421                id,
2422                buckets,
2423                details,
2424                failure,
2425                listed_at,
2426            } => {
2427                if let Some(source) = self.home.cloud.iter_mut().find(|s| s.id == id) {
2428                    source.refreshing = false;
2429                    for (place, lines) in details {
2430                        source.place_details.insert(place, lines);
2431                    }
2432                    match failure {
2433                        // A failed refresh keeps the last buckets, and the row says it failed.
2434                        Some((short, detail)) => {
2435                            for bucket in buckets {
2436                                if !source.buckets.contains(&bucket) {
2437                                    source.buckets.push(bucket);
2438                                }
2439                            }
2440                            source.status = home::CloudStatus::Failed { short, detail };
2441                        }
2442                        None => {
2443                            source.buckets = buckets;
2444                            source.status = home::CloudStatus::Listed;
2445                            source.listed_at = Some(listed_at);
2446                        }
2447                    }
2448                }
2449                self.home_refresh();
2450                None
2451            }
2452            AppEvent::HomeNarrowed {
2453                dir,
2454                prefix,
2455                listed,
2456            } => {
2457                // Only the current request: a replaced one may answer late with a shorter prefix.
2458                let asked = self
2459                    .home_app
2460                    .narrowing
2461                    .as_ref()
2462                    .is_some_and(|(d, p, _)| *d == dir && *p == prefix);
2463                if asked {
2464                    self.home_app.narrowing = None;
2465                }
2466                // Only while it is still where the user is and what the filter asks.
2467                let wanted = asked
2468                    && self.home.browsing.as_ref() == Some(&dir)
2469                    && !self.home.filter.is_empty();
2470                if let (Some((rows, truncated)), true) = (listed, wanted) {
2471                    self.home.narrowed = Some(home::Narrowed {
2472                        dir,
2473                        prefix,
2474                        rows,
2475                        truncated,
2476                    });
2477                    self.home_refresh();
2478                }
2479                None
2480            }
2481            AppEvent::HomeProbeCancelled { root } => {
2482                self.home_app.probes_inflight.retain(|p| p != &root);
2483                self.home_app.listing_cancels.remove(&root);
2484                self.home.probes.stopped(&root);
2485                // Come back to after it had stopped: listed afresh.
2486                if self.home.browsing.as_ref() == Some(&root) {
2487                    self.home_refresh();
2488                }
2489                None
2490            }
2491            AppEvent::HomeProbeFailed { root, message } => {
2492                self.home_app.probes_inflight.retain(|p| p != &root);
2493                self.home_app.listing_cancels.remove(&root);
2494                self.home.probe_failed(root, Some(message));
2495                self.home_refresh();
2496                None
2497            }
2498            AppEvent::HomeProbeProgress { root, rows } => {
2499                // Only while that listing is out: a late batch must not paint over the answer.
2500                if self.home_app.probes_inflight.contains(&root) && !self.home.probes.settled(&root)
2501                {
2502                    self.home.probes.read(&root, &rows);
2503                    // Listed once a frame, however many batches came in it.
2504                    self.home_app.refresh_owed = true;
2505                }
2506                None
2507            }
2508            AppEvent::HomeProbeReady {
2509                root,
2510                rows,
2511                cut_short,
2512            } => {
2513                // Free the slot. The cap bounds threads wedged on dead mounts, which never send
2514                // this; without freeing, probing stops after MAX_CONCURRENT_PROBES roots.
2515                self.home_app.probes_inflight.retain(|p| p != &root);
2516                self.home_app.listing_cancels.remove(&root);
2517                let landed = rows.is_some();
2518                match rows {
2519                    Some(rows) => self.home.probe_ready(root.clone(), rows, cut_short),
2520                    None => self.home.probe_failed(root.clone(), None),
2521                }
2522                // A filter typed while it was listing asks the server too.
2523                #[cfg(feature = "cloud")]
2524                if cut_short && !self.home.filter.is_empty() {
2525                    self.narrow_cloud_listing();
2526                }
2527                // An account read with its keys (the sign-in has no data role) says so.
2528                #[cfg(feature = "cloud")]
2529                if let Some((account, _, _)) = source::azure_parts(&root.to_string_lossy())
2530                    && crate::cloud::azure::remembered_key(&account).is_some()
2531                {
2532                    for source in &mut self.home.cloud {
2533                        let place = source
2534                            .buckets
2535                            .iter()
2536                            .find(|b| home::cloud_account(b).is_some_and(|(_, a)| a == account))
2537                            .cloned();
2538                        if let Some(place) = place {
2539                            let lines = source.place_details.entry(place).or_default();
2540                            if !lines.iter().any(|(k, _)| k == "access") {
2541                                lines.push(("access".to_string(), "access key".to_string()));
2542                            }
2543                        }
2544                    }
2545                }
2546                // Rebuild so the listing picks up the result.
2547                self.home_refresh();
2548                // Then peek at its rows: after the rebuild, which writes the `visible()` the picker
2549                // reads. Here, because a listing landing under a still cursor may draw no frame to
2550                // notice.
2551                #[cfg(feature = "cloud")]
2552                if landed {
2553                    self.peek_cloud_directories();
2554                }
2555                #[cfg(not(feature = "cloud"))]
2556                let _ = landed;
2557                None
2558            }
2559            AppEvent::HomeCloudKinds { kinds, failed } => {
2560                for directory in failed {
2561                    self.home.peeking.remove(&directory);
2562                    self.home.peek_failed.insert(directory);
2563                }
2564                let roots: Vec<PathBuf> = self
2565                    .home
2566                    .probes
2567                    .answered()
2568                    .map(|(root, _)| root.clone())
2569                    .collect();
2570                for (directory, kind) in kinds {
2571                    // Answered: out of `peeking` into the set rows are labeled from; every directory
2572                    // comes back, so none is asked twice.
2573                    self.home.peeking.remove(&directory);
2574                    self.home.cloud_kinds.insert(directory, kind);
2575                }
2576                for root in roots {
2577                    self.home.apply_cloud_kinds(&root);
2578                }
2579                self.home_refresh();
2580                None
2581            }
2582            _ => unreachable!("not a home event"),
2583        }
2584    }
2585}