link_r/lib.rs
1// Author: support@lituus.io
2// Copyright: lituus-io, all rights reserved
3//
4//! `link-r` — a tiny, embeddable crawl-and-resolve link index.
5//!
6//! Give it a parent HTTPS link. It recursively crawls sublinks to a configurable
7//! depth, distills keywords/topics from each page, and builds a small, mmap-able
8//! hybrid index (dense embeddings + BM25 + structured metadata). A query resolves
9//! to ranked **remote URLs** — document bodies stay at the URL and are fetched
10//! lazily by the caller. Each link is unique in the index (canonical-URL keyed),
11//! so re-crawling never duplicates a link.
12//!
13//! # Design tenets
14//!
15//! - **Zero-copy**: the on-disk index *is* the file; search runs against mapped
16//! pages with no deserialization step.
17//! - **Static dispatch**: async traits use GATs (no `Box<dyn Future>`); runtime
18//! variant selection uses enum-dispatch, not `dyn`.
19//! - **Self-contained**: no server, no sidecar database; the production embedder
20//! (`OnnxEmbedder`, feature `onnx`) and a deterministic
21//! [`HashEmbedder`] fallback both run in-process.
22//!
23//! # Dead-simple API
24//!
25//! ```ignore
26//! use link_r::prelude::*;
27//!
28//! let mut idx = LinkIndex::open_or_create("kb.lnkr")?;
29//! idx.update("https://example.com/docs/").depth(2).run().await?;
30//! idx.save()?;
31//!
32//! for hit in idx.search("how does auth work", 10).await? {
33//! println!("{:.3} {}", hit.score, hit.url);
34//! }
35//!
36//! // Maintain the knowledge base: re-validate links older than a day, evict dead ones.
37//! idx.refresh().ttl(std::time::Duration::from_secs(86_400)).run().await?;
38//! ```
39// GAT TAIT for the async Source/Fetcher/Auth traits. Only the impls need it --
40// source/crawl.rs, source/fs.rs, fetch/http.rs and auth/oauth.rs -- so the
41// attribute is conditional: declared unconditionally it trips `unused_features`
42// under -D warnings in a build where none of them is compiled. All four are
43// listed rather than relying on `crawl`/`oauth` implying `http`, so a later
44// change to the feature graph cannot silently drop the declaration.
45#![cfg_attr(
46 any(feature = "crawl", feature = "fs", feature = "http", feature = "oauth"),
47 feature(impl_trait_in_assoc_type)
48)]
49#![forbid(unsafe_code)] // the mmap island lives in graph-r
50#![deny(
51 missing_docs,
52 missing_debug_implementations,
53 rust_2018_idioms,
54 unreachable_pub
55)]
56#![warn(clippy::all, clippy::pedantic, clippy::cargo)]
57#![allow(
58 clippy::module_name_repetitions,
59 clippy::missing_errors_doc,
60 clippy::missing_panics_doc,
61 clippy::multiple_crate_versions,
62 // This crate is a byte-format + scoring engine: document counts, dimensions,
63 // and lengths are bounded by `u32`/`usize` by construction, and float scores
64 // are intentionally lossy. These casts are deliberate, so the pedantic cast
65 // family would only add noise.
66 clippy::cast_possible_truncation,
67 clippy::cast_precision_loss,
68 clippy::cast_sign_loss,
69 clippy::cast_possible_wrap
70)]
71
72pub mod auth;
73#[cfg(feature = "graph")]
74pub mod bridge;
75pub mod embed;
76pub mod error;
77pub mod extract;
78#[cfg(feature = "crawl")]
79pub mod facade;
80pub mod fetch;
81pub mod index;
82#[cfg(feature = "kb")]
83pub mod kb;
84pub mod metric;
85pub mod payload;
86pub mod query;
87#[cfg(feature = "remote")]
88pub mod remote;
89pub mod resource;
90pub mod source;
91pub mod text;
92pub mod url_key;
93
94pub use auth::{AnonymousAuth, AuthProvider, Credential, DynAuthProvider, StaticTokenAuth};
95pub use embed::{Embedder, EmbedderId, HashEmbedder};
96pub use error::{Error, Result};
97pub use extract::{AutoExtractor, Descriptor, Extractor};
98pub use fetch::{FetchMeta, FetchOptions, Fetched, Fetcher};
99pub use index::{DocMeta, Document, Filter, Index, IndexBuilder, UpsertOutcome};
100pub use metric::Metric;
101pub use payload::{ByteStream, DocPayload, MmapView};
102pub use query::{Fusion, Hit, Hits, PreparedQuery, RankParams};
103pub use resource::{DocId, Page, Resource, ResourceKind, SourceRef};
104pub use source::Source;
105pub use url_key::{canonicalize, UrlKey, UrlKeyExt};
106// `url::Url` appears in this crate's own public signatures (crawl scopes,
107// `UpdateBuilder::known_edges`), so consumers need the exact same version --
108// re-exporting it keeps one source of truth instead of a version-sync hazard.
109pub use url::Url;
110
111#[cfg(all(feature = "crawl", feature = "github"))]
112pub use facade::GithubUpdateBuilder;
113#[cfg(feature = "crawl")]
114pub use facade::{LinkIndex, SearchResult, UpdateReport};
115#[cfg(feature = "http")]
116pub use fetch::HttpFetcher;
117#[cfg(feature = "fs")]
118pub use source::FsSource;
119#[cfg(feature = "crawl")]
120pub use source::{CrawlConfig, CrawlScope, CrawlSource};
121#[cfg(feature = "github")]
122pub use source::{GitHubSource, GithubAuth, GithubSpec};
123
124/// Everything you need for the dead-simple API, in one glob import.
125pub mod prelude {
126 pub use crate::error::{Error, Result};
127 pub use crate::index::{Document, Filter, Index, IndexBuilder, UpsertOutcome};
128 pub use crate::query::{Hit, Hits, PreparedQuery, RankParams};
129 pub use crate::resource::{Resource, ResourceKind, SourceRef};
130 pub use crate::url_key::{UrlKey, UrlKeyExt};
131
132 #[cfg(feature = "crawl")]
133 pub use crate::facade::{LinkIndex, SearchResult, UpdateReport};
134 #[cfg(feature = "crawl")]
135 pub use crate::source::CrawlScope;
136}