Struct Engine 

pub struct Engine<S: Spider> { /* private fields */ }

Implementations

impl<S> Engine<S>

where S: Spider + 'static, S::Item: ScrapedItem,

pub async fn start_crawl(self) -> Result<(), SpiderError>

Examples found in repository

examples/quotes.rs (line 117)

async fn main() -> Result<(), anyhow::Error> {
    let filter = EnvFilter::new("info,spider_lib=debug");
    fmt().with_env_filter(filter).init();

    info!("Starting RustScraper example...");

    let engine = EngineBuilder::new(MySpider)
        .add_pipeline(Box::new(JsonlWriterPipeline::<QuoteItem>::new(
            "result.jsonl",
        )?))
        .build()
        .await?;

    engine.start_crawl().await?;

    info!("RustScraper example finished.");

    Ok(())
}

examples/books.rs (line 131)

async fn main() -> Result<(), anyhow::Error> {
    let filter = EnvFilter::new("info,spider_lib=debug");
    fmt().with_env_filter(filter).init();

    info!("Starting RustScraper example...");

    let engine = EngineBuilder::new(BooksSpider)
        .add_pipeline(Box::new(CsvExporterPipeline::new("books.csv")))
        .max_concurrent_downloads(10)
        .build()
        .await?;

    engine.start_crawl().await?;

    info!("RustScraper example finished.");

    Ok(())
}