webfind 0.5.0

[workspace]
members = [
    ".",
]
resolver = "2"

[package]
name = "webfind"
readme = "README.md"
version = "0.5.0"
edition = "2024"
rust-version = "1.98.0"
description = "Self-hosted native search engine for AI agents — pure CLI, zero cost"
license = "MIT"
keywords = ["search", "crawler", "agent", "turso", "offline"]
categories = ["command-line-utilities", "web-programming"]
repository = "https://github.com/Gaurav-Wankhede/webfind"

[[bin]]
name = "webfind"
path = "src/main.rs"

[dependencies]
# CLI/Server
clap = { version = "4", features = ["derive", "env"] }
tokio = { version = "1", features = ["full"] }
tokio-util = "0.7"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
toml = "0.8"
uuid = { version = "1", features = ["v4"] }
chrono = { version = "0.4", features = ["serde"] }
tracing = "0.1"
dashmap = "6"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
anyhow = "1"
thiserror = "2"

# Crawler
# NOTE: spider's default features pull `disk` -> sqlx -> libsqlite3-sys/bundled,
# which links a 2nd static sqlite3 alongside libsql-ffi's bundled fork
# ("ld: duplicate symbol _sqlite3_*"). We don't use spider's disk persistence
# (WebFind has its own storage), so default-features = false and re-enable
# everything except the disk-* features.
spider = { version = "2.52", default-features = false, features = [
    "sync",
    "cookies",
    "ua_generator",
    "encoding",
    "string_interner_buffer_backend",
    "balance",
    "real_browser",
    "time",
    "adaptive_concurrency",
    "priority_frontier",
    "dns_cache",
    "rate_limit",
    "request_coalesce",
    "auto_throttle",
    "etag_cache",
    "warc",
    "reqwest_native_tls_native_roots",
] }
reqwest = { version = "0.13", features = ["rustls", "gzip", "http2", "stream", "cookies", "json"] }
reqwest-middleware = "0.5"
reqwest-retry = "0.6"
robotstxt = "0.3"
sitemap = "0.4"
governor = "0.10.4"
rand = "0.10.2"
flate2 = "1"

# Fetcher (content extraction + normalization)
regex = "1"
scraper = "0.27"
html2text = "0.17"
readability-rust = "0.1"
unicode-normalization = "0.1"
unicode-segmentation = "1.13"
whatlang = "0.18"
dateparser = "0.3"
keyword_extraction = { version = "1.5.0", features = ["all", "co_occurrence"] }
stop-words = "0.8"
textstat = "0.1"
simdutf8 = "0.1"
ahash = "0.8"

# Tokenization: LLM-aligned BPE token counts (context budgeting).
# The linguistic tokenizer (words/sentences) is unicode-segmentation below.
tiktoken = "4.1.2"

# Index
tantivy = "0.26"
fastembed = "4"

# Real BM25 lexical scoring. Emits SPARSE embeddings (token-index -> weight)
# that persist in the same Turso store as the dense vectors, so lexical and
# semantic retrieval fuse in one place with no second on-disk index.
# `parallelism` enables rayon batch-fitting; `language_detection` adds whichlang.
bm25 = { version = "2.3.2", features = ["parallelism", "language_detection"] }

# Storage
blake3 = "1.8.5"
bytes = "1"
url = "2"
base64 = "0.22"
bytemuck = { version = "1.25", features = ["derive"] }
parking_lot = "0.12"

# Vector index (DiskANN approximate nearest neighbor search, FR-1)
anndists = "0.1"
diskann-rs = "0.5"

# Turso/libSQL (embedded storage backend, replaces SurrealDB)
# The `encryption` feature enables SQLCipher encryption-at-rest support.
libsql = { version = "0.6", features = ["core", "encryption"] }

# HTTP API
mimalloc = { version = "0.1", features = ["secure"] }
schemars = "1.0"
axum = "0.8.9"
tower = "0.5"
tower_governor = "0.7"
tower-http = { version = "0.6", features = ["fs", "limit", "cors", "trace"] }
futures = "0.3.33"
socket2 = "0.6.5"
async-trait = "0.1"

# Code Mode sandbox (FR-10)
rquickjs = { version = "0.11", features = ["full-async"] }

# Templating / GUI
askama = "0.13"
tokio-stream = { version = "0.1", features = ["sync"] }
tree-sitter = "0.27.0"
tree-sitter-language-pack = { version = "1.21.0", features = ["download"] }
petgraph = "0.8.3"

[features]
# `dynamic` (headless-Chromium rendering) is OPT-IN. It used to be a default
# feature, which pulled chromiumoxide — and therefore a full Chromium/CDP
# dependency chain — into every build, including the common case where nobody
# passes `--dynamic`. Enable it explicitly when SPA rendering is needed:
#   cargo build --release --features dynamic
default = []
dynamic = ["dep:chromiumoxide"]

[dependencies.chromiumoxide]
version = "0.9.1"
optional = true

[dev-dependencies]
tempfile = "3"
tokio-test = "0.4"
criterion = { version = "0.5", features = ["html_reports"] }
insta = { version = "1", features = ["json"] }
proptest = "1"
# Benchmark-only SHA-256 to compare against BLAKE3 (FR-8 acceptance: URL ID
# generation must be >=3x faster with BLAKE3). Kept out of [dependencies] so the
# production binary stays free of sha2.
sha2 = "0.10"

[profile.release]
opt-level = 3
lto = true
codegen-units = 1
strip = true

[lints.rust]
dead_code = "deny"
unused_variables = "deny"
unused_imports = "deny"
unreachable_code = "deny"

[lints.clippy]
all = "warn"
correctness = { level = "deny", priority = -1 }
suspicious = { level = "deny", priority = -1 }
perf = { level = "warn", priority = -1 }

[[bench]]
name = "score_scalar"
harness = false

[[bench]]
name = "fingerprint_fast"
harness = false

[[bench]]
name = "proxy_select"
harness = false

[[bench]]
name = "content_deserialize"
harness = false

[[bench]]
name = "url_id"
harness = false