Struct EngineBuilder

Source

pub struct EngineBuilder<S: Spider> { /* private fields */ }

Implementations§

Source §

impl<S: Spider> EngineBuilder<S>

Source

pub fn new(spider: S) -> Self

Examples found in repository ?

examples/quotes.rs (line 110)

104async fn main() -> Result<(), anyhow::Error> {
105    let filter = EnvFilter::new("info,spider_lib=debug");
106    fmt().with_env_filter(filter).init();
107
108    info!("Starting RustScraper example...");
109
110    let engine = EngineBuilder::new(MySpider)
111        .add_pipeline(Box::new(JsonlWriterPipeline::<QuoteItem>::new(
112            "result.jsonl",
113        )?))
114        .build()
115        .await?;
116
117    engine.start_crawl().await?;
118
119    info!("RustScraper example finished.");
120
121    Ok(())
122}

More examples

Hide additional examples

examples/books.rs (line 125)

119async fn main() -> Result<(), anyhow::Error> {
120    let filter = EnvFilter::new("info,spider_lib=debug");
121    fmt().with_env_filter(filter).init();
122
123    info!("Starting RustScraper example...");
124
125    let engine = EngineBuilder::new(BooksSpider)
126        .add_pipeline(Box::new(CsvExporterPipeline::new("books.csv")))
127        .max_concurrent_downloads(10)
128        .build()
129        .await?;
130
131    engine.start_crawl().await?;
132
133    info!("RustScraper example finished.");
134
135    Ok(())
136}

Source

pub fn max_concurrent_downloads(self, limit: usize) -> Self

Examples found in repository ?

examples/books.rs (line 127)

119async fn main() -> Result<(), anyhow::Error> {
120    let filter = EnvFilter::new("info,spider_lib=debug");
121    fmt().with_env_filter(filter).init();
122
123    info!("Starting RustScraper example...");
124
125    let engine = EngineBuilder::new(BooksSpider)
126        .add_pipeline(Box::new(CsvExporterPipeline::new("books.csv")))
127        .max_concurrent_downloads(10)
128        .build()
129        .await?;
130
131    engine.start_crawl().await?;
132
133    info!("RustScraper example finished.");
134
135    Ok(())
136}

Source

pub fn add_pipeline(self, pipeline: Box<dyn ItemPipeline<S::Item>>) -> Self

Examples found in repository ?

examples/quotes.rs (lines 111-113)

104async fn main() -> Result<(), anyhow::Error> {
105    let filter = EnvFilter::new("info,spider_lib=debug");
106    fmt().with_env_filter(filter).init();
107
108    info!("Starting RustScraper example...");
109
110    let engine = EngineBuilder::new(MySpider)
111        .add_pipeline(Box::new(JsonlWriterPipeline::<QuoteItem>::new(
112            "result.jsonl",
113        )?))
114        .build()
115        .await?;
116
117    engine.start_crawl().await?;
118
119    info!("RustScraper example finished.");
120
121    Ok(())
122}

More examples

Hide additional examples

examples/books.rs (line 126)

119async fn main() -> Result<(), anyhow::Error> {
120    let filter = EnvFilter::new("info,spider_lib=debug");
121    fmt().with_env_filter(filter).init();
122
123    info!("Starting RustScraper example...");
124
125    let engine = EngineBuilder::new(BooksSpider)
126        .add_pipeline(Box::new(CsvExporterPipeline::new("books.csv")))
127        .max_concurrent_downloads(10)
128        .build()
129        .await?;
130
131    engine.start_crawl().await?;
132
133    info!("RustScraper example finished.");
134
135    Ok(())
136}

Source

pub async fn build(self) -> Result<Engine<S>, SpiderError>

Examples found in repository ?

examples/quotes.rs (line 114)

104async fn main() -> Result<(), anyhow::Error> {
105    let filter = EnvFilter::new("info,spider_lib=debug");
106    fmt().with_env_filter(filter).init();
107
108    info!("Starting RustScraper example...");
109
110    let engine = EngineBuilder::new(MySpider)
111        .add_pipeline(Box::new(JsonlWriterPipeline::<QuoteItem>::new(
112            "result.jsonl",
113        )?))
114        .build()
115        .await?;
116
117    engine.start_crawl().await?;
118
119    info!("RustScraper example finished.");
120
121    Ok(())
122}

More examples

Hide additional examples

examples/books.rs (line 128)

119async fn main() -> Result<(), anyhow::Error> {
120    let filter = EnvFilter::new("info,spider_lib=debug");
121    fmt().with_env_filter(filter).init();
122
123    info!("Starting RustScraper example...");
124
125    let engine = EngineBuilder::new(BooksSpider)
126        .add_pipeline(Box::new(CsvExporterPipeline::new("books.csv")))
127        .max_concurrent_downloads(10)
128        .build()
129        .await?;
130
131    engine.start_crawl().await?;
132
133    info!("RustScraper example finished.");
134
135    Ok(())
136}