webfind 0.5.0

Self-hosted native search engine for AI agents — pure CLI, zero cost
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128

use std::path::PathBuf;
use std::process::Command;
use std::sync::Once;

mod common;

static BUILD: Once = Once::new();

fn webfind_binary() -> PathBuf {
    BUILD.call_once(|| {
        let status = Command::new(env!("CARGO"))
            .args(["build", "--bin", "webfind", "--quiet"])
            .status()
            .expect("cargo build should start");
        assert!(status.success(), "cargo build --bin webfind failed");
    });
    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("target/debug/webfind")
}

async fn run_webfind(
    args: Vec<String>,
    env_vars: Vec<(&'static str, String)>,
) -> std::process::Output {
    let binary = webfind_binary();
    tokio::task::spawn_blocking(move || {
        let mut cmd = Command::new(binary);
        cmd.args(args);
        for (k, v) in env_vars {
            cmd.env(k, v);
        }
        cmd.output().expect("webfind command should run")
    })
    .await
    .expect("spawn_blocking should succeed")
}

#[tokio::test]
async fn test_cli_bulk_crawl_persists_to_turso_and_graph_traverses() {
    let (_server, base_url) = common::start_test_server().await;
    let tmp = tempfile::tempdir().unwrap();
    let data_dir = tmp.path();
    let turso_path = data_dir.join("graph.db");
    let turso_path_arg = turso_path.to_str().unwrap().to_string();

    // Bulk crawl with the Turso graph store.
    let crawl = run_webfind(
        vec![
            "crawl".to_string(),
            "--bulk".to_string(),
            "--graph-store".to_string(),
            "turso".to_string(),
            "--turso-path".to_string(),
            turso_path_arg.clone(),
            "--seed".to_string(),
            base_url.clone(),
            "--max-pages".to_string(),
            "3".to_string(),
            "--delay".to_string(),
            "50".to_string(),
            "--rps".to_string(),
            "100".to_string(),
        ],
        vec![
            ("WEBFIND_DATA_DIR", data_dir.to_str().unwrap().to_string()),
            ("RUST_LOG", "info".to_string()),
        ],
    )
    .await;
    let crawl_stdout = String::from_utf8_lossy(&crawl.stdout);
    let crawl_stderr = String::from_utf8_lossy(&crawl.stderr);
    println!("CRAWL STDOUT:\n{}", crawl_stdout);
    println!("CRAWL STDERR:\n{}", crawl_stderr);
    assert!(
        crawl.status.success(),
        "crawl failed:\nstdout: {}\nstderr: {}",
        crawl_stdout,
        crawl_stderr
    );
    assert!(
        crawl_stdout.contains("Bulk crawl complete"),
        "crawl did not report completion:\n{}",
        crawl_stdout
    );

    // Graph traversal should discover linked pages.
    let graph = run_webfind(
        vec![
            "graph".to_string(),
            base_url.clone(),
            "--direction".to_string(),
            "outbound".to_string(),
            "--depth".to_string(),
            "2".to_string(),
            "--graph-store".to_string(),
            "turso".to_string(),
            "--turso-path".to_string(),
            turso_path_arg,
        ],
        vec![],
    )
    .await;
    let graph_stdout = String::from_utf8_lossy(&graph.stdout);
    let graph_stderr = String::from_utf8_lossy(&graph.stderr);
    println!("GRAPH STDOUT:\n{}", graph_stdout);
    println!("GRAPH STDERR:\n{}", graph_stderr);
    assert!(
        graph.status.success(),
        "graph failed:\nstdout: {}\nstderr: {}",
        graph_stdout,
        graph_stderr
    );

    assert!(
        graph_stdout.contains("/page1"),
        "graph output missing /page1:\n{}",
        graph_stdout
    );
    assert!(
        graph_stdout.contains("/page2"),
        "graph output missing /page2:\n{}",
        graph_stdout
    );
    assert!(
        graph_stdout.contains("Discovered"),
        "graph output missing summary line:\n{}",
        graph_stdout
    );
}