webfind 0.5.0

Self-hosted native search engine for AI agents — pure CLI, zero cost
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98

use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main};
use webfind::engine::proxy_pool::{ProxyEndpoint, ProxyPool, RotationStrategy};

fn create_test_proxies(n: usize) -> Vec<String> {
    (0..n)
        .map(|i| format!("http://proxy{}.example.com:8080", i))
        .collect()
}

fn bench_proxy_selection(c: &mut Criterion) {
    let mut group = c.benchmark_group("proxy_selection");

    for pool_size in [10, 100, 1000, 10000].iter() {
        let proxies = create_test_proxies(*pool_size);
        let pool = ProxyPool::from_list(&proxies).unwrap();

        group.throughput(Throughput::Elements(*pool_size as u64));
        group.bench_with_input(BenchmarkId::from_parameter(pool_size), &pool, |b, pool| {
            b.iter(|| {
                let proxy = pool.select(None).unwrap();
                black_box(proxy);
            });
        });
    }

    group.finish();
}

fn bench_proxy_selection_with_strategies(c: &mut Criterion) {
    let mut group = c.benchmark_group("proxy_selection_strategies");

    let proxies = create_test_proxies(1000);

    let strategies = vec![
        ("random", RotationStrategy::Random),
        ("round_robin", RotationStrategy::RoundRobin),
        ("weighted", RotationStrategy::Weighted),
        ("sticky", RotationStrategy::Sticky),
    ];

    for (name, strategy) in strategies {
        let pool = ProxyPool::from_list(&proxies)
            .unwrap()
            .with_strategy(strategy);

        group.bench_with_input(BenchmarkId::from_parameter(name), &pool, |b, pool| {
            b.iter(|| {
                let proxy = pool.select(None).unwrap();
                black_box(proxy);
            });
        });
    }

    group.finish();
}

fn bench_proxy_pool_len(c: &mut Criterion) {
    let mut group = c.benchmark_group("proxy_pool_len");

    let proxies = create_test_proxies(10000);
    let pool = ProxyPool::from_list(&proxies).unwrap();

    group.bench_function("len_10000", |b| {
        b.iter(|| {
            let len = pool.len().unwrap();
            black_box(len);
        });
    });

    group.finish();
}

fn bench_proxy_add(c: &mut Criterion) {
    let mut group = c.benchmark_group("proxy_add");

    let pool = ProxyPool::new();

    group.bench_function("add_1000", |b| {
        b.iter(|| {
            for i in 0..1000 {
                let ep = ProxyEndpoint::from_url(&format!("http://proxy{}.example.com:8080", i))
                    .unwrap();
                pool.add(ep).unwrap();
            }
        });
    });

    group.finish();
}

criterion_group!(
    benches,
    bench_proxy_selection,
    bench_proxy_selection_with_strategies,
    bench_proxy_pool_len,
    bench_proxy_add
);
criterion_main!(benches);