kreuzcrawl 0.2.0

High-performance web crawling engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

//! Research planner that decides the next step in the research loop.
#![allow(dead_code)]

use super::types::{Finding, StepAction};
use crate::error::CrawlError;

/// Plans the next research action based on current state.
///
/// Currently uses a simple round-robin over seed URLs, then synthesizes.
/// Future iterations will use LLM to plan intelligently based on findings.
pub(crate) struct ResearchPlanner {}

impl ResearchPlanner {
    pub(crate) fn new() -> Self {
        Self {}
    }

    /// Decide the next action given the current research state.
    ///
    /// Returns [`StepAction::Crawl`] while there are unvisited seed URLs and
    /// we have not reached `max_steps - 1`. Returns [`StepAction::Synthesize`]
    /// otherwise.
    pub(crate) async fn plan_next_step(
        &self,
        _query: &str,
        seed_urls: &[String],
        _findings: &[Finding],
        step_number: usize,
        max_steps: usize,
    ) -> Result<StepAction, CrawlError> {
        if step_number < seed_urls.len() && step_number < max_steps {
            Ok(StepAction::Crawl {
                url: seed_urls[step_number].clone(),
                depth: 1,
            })
        } else {
            Ok(StepAction::Synthesize)
        }
    }
}