link_r/source/
mod.rs

1// Author: support@lituus.io
2// Copyright: lituus-io, all rights reserved
3//
4//! Sources: discover the pages to index from a parent.
5//!
6//! A [`Source`] streams already-fetched [`Page`]s (the crawler must fetch a page
7//! to find its links, so discovery and content-fetch are fused — no double fetch).
8//! The primary source is the recursive [`CrawlSource`]; a
9//! local-directory [`FsSource`] serves offline/test use.
10
11#[cfg(feature = "crawl")]
12pub mod crawl;
13#[cfg(feature = "fs")]
14pub mod fs;
15#[cfg(feature = "github")]
16pub mod github;
17
18#[cfg(feature = "crawl")]
19pub use crawl::{CrawlConfig, CrawlScope, CrawlSource};
20#[cfg(feature = "fs")]
21pub use fs::FsSource;
22#[cfg(feature = "github")]
23pub use github::{GitHubSource, GithubAuth, GithubSpec};
24
25use crate::error::Result;
26use crate::resource::{Page, SourceRef};
27use futures::Stream;
28
29/// Discovers and yields the pages to index from a parent reference.
30pub trait Source: Send + Sync {
31    /// The stream of discovered pages.
32    type Pages<'a>: Stream<Item = Result<Page<'a>>> + Send + 'a
33    where
34        Self: 'a;
35
36    /// A stable label for this source kind (`"crawl"`, `"fs"`).
37    fn kind(&self) -> &'static str;
38
39    /// Begin discovery from `root`.
40    fn discover<'a>(&'a self, root: &'a SourceRef) -> Self::Pages<'a>;
41}
link_r/source/mod.rs

link_r/source/
mod.rs
