webshift 0.2.13

Denoised web search library — fetch, clean, and rerank web content for AI agents.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

[package]

name = "webshift"

version.workspace = true

edition.workspace = true

license.workspace = true

authors.workspace = true

repository.workspace = true

description = "Denoised web search library — fetch, clean, and rerank web content for AI agents."

readme = "../../README.md"

keywords = ["web-search", "html-to-text", "mcp", "llm", "web-scraping"]

categories = ["text-processing", "web-programming"]



[features]

default = ["backends"]

backends = []

llm = []

text-map = ["lol_html"]



[dependencies]

# Async runtime

tokio = { version = "1", features = ["rt-multi-thread", "macros", "time"] }



# HTTP client (streaming support critical for size cap)

reqwest = { version = "0.12", default-features = false, features = ["stream", "gzip", "brotli", "json", "rustls-tls"] }

futures = "0.3"



# HTML parsing — pure Rust, no system deps

scraper = "0.20"



# Serialization

serde = { version = "1", features = ["derive"] }

serde_json = "1"

toml = "0.8"



# Text processing

regex = "1"

rand = "0.8"

url = "2"



# Directories (home dir for config file search)

dirs-next = "2"



# HTML streaming rewriter (text-map feature)

lol_html = { version = "2", optional = true }



# Error handling

thiserror = "2"



# Logging

tracing = "0.1"



# Async trait (for SearchBackend dyn dispatch)

async-trait = "0.1"



[package.metadata.docs.rs]

all-features = true

rustdoc-args = ["--cfg", "docsrs"]



[dev-dependencies]

tokio = { version = "1", features = ["rt-multi-thread", "macros", "test-util"] }

wiremock = "0.6"