d-major 0.0.0

Traverse directory trees in parallel, using relative entries to minimize allocation and maximize parallelism.
Documentation
/*
 * Description: Crawl the filesystem in parallel.
 *
 * Copyright (C) 2025 d@nny mc² <dmc2@hypnicjerk.ai>
 * SPDX-License-Identifier: LGPL-3.0-or-later
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published
 * by the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */

//! Crawl the filesystem in parallel.


pub mod traits {
  use crate::vfs::traits;

  pub trait RecvErr<Err> {
    fn signal(&self, err: Err);
  }

  pub trait RecvStat<'vfs, VFS, Err>: RecvErr<Err>
  where
    VFS: traits::VFS,
    Err: From<VFS::Err>,
  {
    fn stat<'s>(&self, ctx: VFS::Ctx<'vfs>, rel: VFS::PathRef<'s>) -> Result<VFS::Stat, VFS::Err>;
  }

  #[derive(Clone, Debug)]
  pub struct HydratedLink<P> {
    pub relpath: P,
    pub target: P,
  }

  #[derive(Copy, Clone, Default, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
  pub enum BrokenLinkBehavior {
    #[default]
    IgnoreBroken,
    ErrorOnBroken,
  }

  pub trait RecvLink<'vfs, VFS, Err>: RecvStat<'vfs, VFS, Err>
  where
    VFS: traits::VFS,
    Err: From<VFS::Err>,
  {
    fn read_link<'s>(
      &self,
      ctx: VFS::Ctx<'vfs>,
      rel: VFS::PathRef<'s>,
    ) -> Result<VFS::OwnedPath, VFS::Err>;
    fn send_link(
      &self,
      ctx: VFS::Ctx<'vfs>,
      link: HydratedLink<VFS::OwnedPath>,
      opts: BrokenLinkBehavior,
    );
  }

  pub trait RecvFile<'vfs, VFS, Err>: RecvStat<'vfs, VFS, Err>
  where
    VFS: traits::VFS,
    Err: From<VFS::Err>,
  {
    fn send_file(
      &self,
      ctx: VFS::Ctx<'vfs>,
      rel: VFS::OwnedPath,
      stat: Option<VFS::Stat>,
      opts: VFS::FileOptions,
    );
  }

  pub trait RecvDir<'vfs, VFS, Err>: RecvStat<'vfs, VFS, Err>
  where
    VFS: traits::VFS,
    Err: From<VFS::Err>,
  {
    fn send_dir(&self, ctx: VFS::Ctx<'vfs>, rel: VFS::OwnedPath, stat: Option<VFS::Stat>);
  }

  pub trait RecvDirEntry<'vfs, VFS, Err>: RecvStat<'vfs, VFS, Err>
  where
    VFS: traits::VFS,
    Err: From<VFS::Err>,
  {
    fn owned_name<'dir, 's>(
      name: <<VFS::Dir<'vfs> as traits::DirectoryStream>::DirEntry<'dir> as traits::DirectoryEntry>::PathRef<'s>,
    ) -> <<VFS::Dir<'vfs> as traits::DirectoryStream>::DirEntry<'dir> as traits::DirectoryEntry>::OwnedPath;
    fn send_dir_entry(
      &self,
      ctx: VFS::Ctx<'vfs>,
      file_type: traits::FileType,
      name: <<VFS::Dir<'vfs> as traits::DirectoryStream>::DirEntry<'static> as traits::DirectoryEntry>::OwnedPath,
      stat: Option<VFS::Stat>,
    );
  }
}


pub mod channels {
  use std::sync::mpsc;

  use super::traits;

  #[derive(Debug, Clone)]
  pub struct RecvErr<Err> {
    sender: mpsc::SyncSender<Err>,
  }

  impl<Err> traits::RecvErr<Err> for RecvErr<Err> {
    fn signal(&self, err: Err) { self.sender.send(err).unwrap(); }
  }

  #[derive(Debug, Clone)]
  pub struct RecvStat<'vfs, VFS, Err> {
    vfs: &'vfs VFS,
    err: RecvErr<Err>,
  }

  impl<'vfs, VFS, Err> traits::RecvErr<Err> for RecvStat<'vfs, VFS, Err> {
    fn signal(&self, err: Err) { self.err.signal(err); }
  }

  impl<'vfs, VFS, Err> traits::RecvStat<'vfs, VFS, Err> for RecvStat<'vfs, VFS, Err>
  where
    VFS: crate::vfs::traits::VFS,
    Err: From<VFS::Err>,
  {
    fn stat<'s>(&self, ctx: VFS::Ctx<'vfs>, rel: VFS::PathRef<'s>) -> Result<VFS::Stat, VFS::Err> {
      self.vfs.stat(ctx, rel)
    }
  }

  #[derive(Debug, Clone)]
  pub struct RecvLink<'vfs, VFS, Err, Link> {
    stat: RecvStat<'vfs, VFS, Err>,
    sender: mpsc::Sender<Link>,
  }

  impl<'vfs, VFS, Err, Link> traits::RecvErr<Err> for RecvLink<'vfs, VFS, Err, Link> {
    fn signal(&self, err: Err) { self.stat.signal(err); }
  }

  impl<'vfs, VFS, Err, Link> traits::RecvStat<'vfs, VFS, Err> for RecvLink<'vfs, VFS, Err, Link>
  where
    VFS: crate::vfs::traits::VFS,
    Err: From<VFS::Err>,
  {
    fn stat<'s>(&self, ctx: VFS::Ctx<'vfs>, rel: VFS::PathRef<'s>) -> Result<VFS::Stat, VFS::Err> {
      self.stat.stat(ctx, rel)
    }
  }

  impl<'vfs, VFS, Err> traits::RecvLink<'vfs, VFS, Err>
    for RecvLink<
      'vfs,
      VFS,
      Err,
      (
        VFS::Ctx<'vfs>,
        traits::HydratedLink<VFS::OwnedPath>,
        traits::BrokenLinkBehavior,
      ),
    >
  where
    VFS: crate::vfs::traits::VFS,
    Err: From<VFS::Err>,
  {
    fn read_link<'s>(
      &self,
      ctx: VFS::Ctx<'vfs>,
      rel: VFS::PathRef<'s>,
    ) -> Result<VFS::OwnedPath, VFS::Err> {
      self.stat.vfs.read_link(ctx, rel)
    }
    fn send_link(
      &self,
      ctx: VFS::Ctx<'vfs>,
      link: traits::HydratedLink<VFS::OwnedPath>,
      opts: traits::BrokenLinkBehavior,
    ) {
      self.sender.send((ctx, link, opts)).unwrap();
    }
  }

  #[derive(Debug, Clone)]
  pub struct RecvFile<'vfs, VFS, Err, File> {
    stat: RecvStat<'vfs, VFS, Err>,
    sender: mpsc::Sender<File>,
  }

  impl<'vfs, VFS, Err, File> traits::RecvErr<Err> for RecvFile<'vfs, VFS, Err, File> {
    fn signal(&self, err: Err) { self.stat.signal(err); }
  }

  impl<'vfs, VFS, Err, File> traits::RecvStat<'vfs, VFS, Err> for RecvFile<'vfs, VFS, Err, File>
  where
    VFS: crate::vfs::traits::VFS,
    Err: From<VFS::Err>,
  {
    fn stat<'s>(&self, ctx: VFS::Ctx<'vfs>, rel: VFS::PathRef<'s>) -> Result<VFS::Stat, VFS::Err> {
      self.stat.stat(ctx, rel)
    }
  }

  impl<'vfs, VFS, Err> traits::RecvFile<'vfs, VFS, Err>
    for RecvFile<
      'vfs,
      VFS,
      Err,
      (
        VFS::Ctx<'vfs>,
        VFS::OwnedPath,
        Option<VFS::Stat>,
        VFS::FileOptions,
      ),
    >
  where
    VFS: crate::vfs::traits::VFS,
    Err: From<VFS::Err>,
  {
    fn send_file(
      &self,
      ctx: VFS::Ctx<'vfs>,
      rel: VFS::OwnedPath,
      stat: Option<VFS::Stat>,
      opts: VFS::FileOptions,
    ) {
      self.sender.send((ctx, rel, stat, opts)).unwrap();
    }
  }

  #[derive(Debug, Clone)]
  pub struct RecvDir<'vfs, VFS, Err, Dir> {
    stat: RecvStat<'vfs, VFS, Err>,
    sender: mpsc::Sender<Dir>,
  }

  impl<'vfs, VFS, Err, Dir> traits::RecvErr<Err> for RecvDir<'vfs, VFS, Err, Dir> {
    fn signal(&self, err: Err) { self.stat.signal(err); }
  }

  impl<'vfs, VFS, Err, Dir> traits::RecvStat<'vfs, VFS, Err> for RecvDir<'vfs, VFS, Err, Dir>
  where
    VFS: crate::vfs::traits::VFS,
    Err: From<VFS::Err>,
  {
    fn stat<'s>(&self, ctx: VFS::Ctx<'vfs>, rel: VFS::PathRef<'s>) -> Result<VFS::Stat, VFS::Err> {
      self.stat.stat(ctx, rel)
    }
  }

  impl<'vfs, VFS, Err> traits::RecvDir<'vfs, VFS, Err>
    for RecvDir<'vfs, VFS, Err, (VFS::Ctx<'vfs>, VFS::OwnedPath, Option<VFS::Stat>)>
  where
    VFS: crate::vfs::traits::VFS,
    Err: From<VFS::Err>,
  {
    fn send_dir(&self, ctx: VFS::Ctx<'vfs>, rel: VFS::OwnedPath, stat: Option<VFS::Stat>) {
      self.sender.send((ctx, rel, stat)).unwrap();
    }
  }

  #[derive(Debug, Clone)]
  pub struct RecvDirEntry<'vfs, VFS, Err, DirEntry> {
    stat: RecvStat<'vfs, VFS, Err>,
    sender: mpsc::Sender<DirEntry>,
  }

  impl<'vfs, VFS, Err, DirEntry> traits::RecvErr<Err> for RecvDirEntry<'vfs, VFS, Err, DirEntry> {
    fn signal(&self, err: Err) { self.stat.signal(err); }
  }

  impl<'vfs, VFS, Err, DirEntry> traits::RecvStat<'vfs, VFS, Err>
    for RecvDirEntry<'vfs, VFS, Err, DirEntry>
  where
    VFS: crate::vfs::traits::VFS,
    Err: From<VFS::Err>,
  {
    fn stat<'s>(&self, ctx: VFS::Ctx<'vfs>, rel: VFS::PathRef<'s>) -> Result<VFS::Stat, VFS::Err> {
      self.stat.stat(ctx, rel)
    }
  }

  impl<'vfs, VFS, Err> traits::RecvDirEntry<'vfs, VFS, Err>
    for RecvDirEntry<
      'vfs,
      VFS,
      Err,
      (
        VFS::Ctx<'vfs>,
        crate::vfs::traits::FileType,
        <<VFS::Dir<'vfs> as crate::vfs::traits::DirectoryStream>::DirEntry<'static> as crate::vfs::traits::DirectoryEntry>::OwnedPath,
        Option<VFS::Stat>,
      ),
    >
  where
    VFS: crate::vfs::traits::VFS,
    Err: From<VFS::Err>,
  {
    fn owned_name<'dir, 's>(
      name: <<VFS::Dir<'vfs> as crate::vfs::traits::DirectoryStream>::DirEntry<'dir> as crate::vfs::traits::DirectoryEntry>::PathRef<'s>,
    ) -> <<VFS::Dir<'vfs> as crate::vfs::traits::DirectoryStream>::DirEntry<'dir> as crate::vfs::traits::DirectoryEntry>::OwnedPath {
      <<VFS::Dir<'vfs> as crate::vfs::traits::DirectoryStream>::DirEntry<'dir> as crate::vfs::traits::DirectoryEntry>::owned_name(name)
    }
    fn send_dir_entry(
      &self,
      ctx: VFS::Ctx<'vfs>,
      file_type: crate::vfs::traits::FileType,
      name: <<VFS::Dir<'vfs> as crate::vfs::traits::DirectoryStream>::DirEntry<'static> as crate::vfs::traits::DirectoryEntry>::OwnedPath,
      stat: Option<VFS::Stat>,
    ) {
      self.sender.send((ctx, file_type, name, stat)).unwrap();
    }
  }
}


pub mod dir_handles {
  use std::{
    os::fd,
    sync::atomic::{AtomicUsize, Ordering},
  };

  use indexmap::IndexMap;
  use parking_lot::RwLock;

  use crate::handles::DirFd;


  #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
  pub struct DirToken(fd::RawFd);

  pub struct DirState<R, Ctx> {
    dir: DirFd<R>,
    num_entries: AtomicUsize,
    ctx: Ctx,
  }

  impl<R, Ctx> DirState<R, Ctx> {
    pub const fn new(dir: DirFd<R>, ctx: Ctx) -> Self {
      Self {
        dir,
        num_entries: AtomicUsize::new(0),
        ctx,
      }
    }
  }

  pub struct OpenDirHandles<R, Ctx> {
    handles: RwLock<IndexMap<DirToken, DirState<R, Ctx>>>,
  }

  pub struct DirEntry<Info> {
    token: DirToken,
    info: Info,
  }
}