rover-fetch 0.2.0

An MCP server for fetching and prepping web content for LLM agents.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

//! End-to-end lifecycle test: real DB, real extractive backend, params
//! variation, cache invalidation when `backend_name` changes.

#![cfg(feature = "test-loopback")]

use std::sync::Arc;

use rover::storage::Db;
use rover::summarizer::backend::{CompactMode, CompactOpts, PreserveSection, Style};
use rover::summarizer::extractive::ExtractiveBackend;
use rover::summarizer::registry::SummarizerRegistry;
use rover::summarizer::{SummarizerService, SummaryCacheStatus};
use rover::tokenizer::Tokenizer;

async fn make_service() -> (SummarizerService, tempfile::TempDir) {
    let tmp = tempfile::tempdir().unwrap();
    let db = Db::open(tmp.path().join("r.db")).await.unwrap();
    let mut map: std::collections::HashMap<
        String,
        Arc<dyn rover::summarizer::backend::SummarizerBackend>,
    > = Default::default();
    map.insert(
        "default".into(),
        Arc::new(ExtractiveBackend::new("default", Tokenizer::O200k)),
    );
    let reg = Arc::new(SummarizerRegistry::__test_construct(
        map,
        "default".into(),
        Some("default".into()),
    ));
    (SummarizerService::new(db, reg, true), tmp)
}

fn opts() -> CompactOpts {
    CompactOpts {
        mode: CompactMode::Extractive,
        style: Style::Prose,
        target_tokens: None,
        focus: None,
        preserve: vec![],
        backend_name: "default".into(),
    }
}

#[tokio::test]
async fn second_call_with_same_params_hits_cache() {
    let (svc, _tmp) = make_service().await;
    let r1 = svc
        .compact("h1", "First sentence here. Second sentence here.", &opts())
        .await
        .unwrap();
    let r2 = svc
        .compact("h1", "First sentence here. Second sentence here.", &opts())
        .await
        .unwrap();
    assert!(matches!(r1.cache_status, SummaryCacheStatus::Miss));
    assert!(matches!(r2.cache_status, SummaryCacheStatus::Hit));
    assert_eq!(r1.summary_md, r2.summary_md);
}

#[tokio::test]
async fn changing_target_tokens_creates_independent_cache_row() {
    let (svc, _tmp) = make_service().await;
    let mut o = opts();
    o.target_tokens = Some(10);
    let r1 = svc
        .compact("h1", "First sentence. Second sentence. Third one.", &o)
        .await
        .unwrap();
    o.target_tokens = Some(100);
    let r2 = svc
        .compact("h1", "First sentence. Second sentence. Third one.", &o)
        .await
        .unwrap();
    assert!(matches!(r1.cache_status, SummaryCacheStatus::Miss));
    assert!(matches!(r2.cache_status, SummaryCacheStatus::Miss));
}

#[tokio::test]
async fn changing_preserve_list_order_does_not_re_summarize() {
    let (svc, _tmp) = make_service().await;
    let mut o = opts();
    o.preserve = vec![PreserveSection::Code, PreserveSection::Tables];
    let _r1 = svc.compact("h2", "Sentence here.", &o).await.unwrap();
    o.preserve = vec![PreserveSection::Tables, PreserveSection::Code];
    let r2 = svc.compact("h2", "Sentence here.", &o).await.unwrap();
    assert!(matches!(r2.cache_status, SummaryCacheStatus::Hit));
}