use std::sync::OnceLock;
use pyo3::create_exception;
use pyo3::exceptions::{PyException, PyValueError};
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyList};
use crate::chunk::ChunkConfig;
use crate::denoise::{DenoiseConfig, DenoiseStats};
use crate::error::Error;
use crate::http::{FetchConfig, Impersonate, Page};
use crate::parse::{
Article, ArticleKind, ExtractOptions, IndexMode, Link, Meta, RenderOptions, Unit,
};
use crate::pipeline::{Pipeline, Research};
use crate::rank::{Context, SlimConfig};
use crate::search::{Provider, SearchConfig, SearchResult};
create_exception!(rustai, RustaiError, PyException, "Base class for every rustai error.");
create_exception!(rustai, NetworkError, RustaiError, "The request failed at the transport level.");
create_exception!(
rustai,
HttpStatusError,
RustaiError,
"The server answered with a failure status."
);
create_exception!(rustai, RobotsError, RustaiError, "robots.txt disallows this URL.");
create_exception!(rustai, ExtractError, RustaiError, "The document could not be parsed.");
create_exception!(rustai, ProviderError, RustaiError, "A search provider failed.");
create_exception!(rustai, BrowserError, RustaiError, "The headless fallback was unavailable.");
impl From<Error> for PyErr {
fn from(e: Error) -> PyErr {
let msg = e.to_string();
match e {
Error::Network { .. } => NetworkError::new_err(msg),
Error::Status { .. } => HttpStatusError::new_err(msg),
Error::BodyTooLarge { .. } => NetworkError::new_err(msg),
Error::RobotsDenied(_) => RobotsError::new_err(msg),
Error::InvalidUrl(_) | Error::Config(_) => PyValueError::new_err(msg),
Error::Parse(_) => ExtractError::new_err(msg),
Error::Provider { .. } => ProviderError::new_err(msg),
Error::Browser(_) => BrowserError::new_err(msg),
}
}
}
fn runtime() -> PyResult<&'static tokio::runtime::Runtime> {
static RUNTIME: OnceLock<std::io::Result<tokio::runtime::Runtime>> = OnceLock::new();
RUNTIME
.get_or_init(|| tokio::runtime::Builder::new_multi_thread().enable_all().build())
.as_ref()
.map_err(|e| RustaiError::new_err(format!("could not start the async runtime: {e}")))
}
fn json_to_py<'py>(py: Python<'py>, v: &serde_json::Value) -> PyResult<Bound<'py, PyAny>> {
use serde_json::Value;
Ok(match v {
Value::Null => py.None().into_bound(py),
Value::Bool(b) => b.into_pyobject(py)?.to_owned().into_any(),
Value::Number(n) => match (n.as_i64(), n.as_f64()) {
(Some(i), _) => i.into_pyobject(py)?.into_any(),
(_, Some(f)) => f.into_pyobject(py)?.into_any(),
_ => py.None().into_bound(py),
},
Value::String(s) => s.into_pyobject(py)?.into_any(),
Value::Array(items) => {
let list = PyList::empty(py);
for item in items {
list.append(json_to_py(py, item)?)?;
}
list.into_any()
}
Value::Object(map) => {
let dict = PyDict::new(py);
for (k, val) in map {
dict.set_item(k, json_to_py(py, val)?)?;
}
dict.into_any()
}
})
}
fn to_dict<T: serde::Serialize>(py: Python<'_>, value: &T) -> PyResult<Py<PyAny>> {
let json = serde_json::to_value(value)
.map_err(|e| RustaiError::new_err(format!("could not serialise: {e}")))?;
Ok(json_to_py(py, &json)?.unbind())
}
#[pyclass(name = "Meta", frozen, module = "rustai", skip_from_py_object)]
#[derive(Clone)]
pub struct PyMeta {
inner: Meta,
}
#[pymethods]
impl PyMeta {
#[getter]
fn title(&self) -> Option<&str> {
self.inner.title.as_deref()
}
#[getter]
fn description(&self) -> Option<&str> {
self.inner.description.as_deref()
}
#[getter]
fn byline(&self) -> Option<&str> {
self.inner.byline.as_deref()
}
#[getter]
fn published(&self) -> Option<&str> {
self.inner.published.as_deref()
}
#[getter]
fn language(&self) -> Option<&str> {
self.inner.language.as_deref()
}
#[getter]
fn canonical(&self) -> Option<&str> {
self.inner.canonical.as_deref()
}
#[getter]
fn site_name(&self) -> Option<&str> {
self.inner.site_name.as_deref()
}
fn to_dict(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
to_dict(py, &self.inner)
}
fn __repr__(&self) -> String {
format!("Meta(title={:?}, language={:?})", self.inner.title, self.inner.language)
}
}
#[pyclass(name = "Unit", frozen, module = "rustai", skip_from_py_object)]
#[derive(Clone)]
pub struct PyUnit {
inner: Unit,
}
#[pymethods]
impl PyUnit {
#[getter]
fn kind(&self) -> String {
serde_json::to_value(self.inner.kind)
.ok()
.and_then(|v| v.as_str().map(str::to_string))
.unwrap_or_default()
}
#[getter]
fn level(&self) -> u8 {
self.inner.level
}
#[getter]
fn text(&self) -> &str {
&self.inner.text
}
#[getter]
fn markdown(&self) -> &str {
&self.inner.markdown
}
#[getter]
fn heading_path(&self) -> Vec<String> {
self.inner.heading_path.clone()
}
#[getter]
fn position(&self) -> usize {
self.inner.position
}
#[getter]
fn tokens(&self) -> usize {
self.inner.tokens
}
fn to_dict(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
to_dict(py, &self.inner)
}
fn __repr__(&self) -> String {
let preview: String = self.inner.text.chars().take(40).collect();
format!("Unit(kind={}, tokens={}, text={preview:?})", self.kind(), self.inner.tokens)
}
}
#[pyclass(name = "DenoiseStats", frozen, module = "rustai", skip_from_py_object)]
#[derive(Clone)]
pub struct PyStats {
inner: DenoiseStats,
}
#[pymethods]
impl PyStats {
#[getter]
fn nodes_visited(&self) -> usize {
self.inner.nodes_visited
}
#[getter]
fn nodes_dropped(&self) -> usize {
self.inner.nodes_dropped
}
#[getter]
fn html_bytes(&self) -> usize {
self.inner.html_bytes
}
#[getter]
fn markdown_bytes(&self) -> usize {
self.inner.markdown_bytes
}
#[getter]
fn compression(&self) -> f32 {
self.inner.compression()
}
fn to_dict(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
to_dict(py, &self.inner)
}
fn __repr__(&self) -> String {
format!(
"DenoiseStats(html_bytes={}, markdown_bytes={}, compression={:.2})",
self.inner.html_bytes,
self.inner.markdown_bytes,
self.inner.compression()
)
}
}
#[pyclass(name = "Link", frozen, module = "rustai", skip_from_py_object)]
#[derive(Clone)]
pub struct PyLink {
inner: Link,
}
#[pymethods]
impl PyLink {
#[getter]
fn text(&self) -> &str {
&self.inner.text
}
#[getter]
fn url(&self) -> &str {
&self.inner.url
}
#[getter]
fn snippet(&self) -> &str {
&self.inner.snippet
}
#[getter]
fn heading_path(&self) -> Vec<String> {
self.inner.heading_path.clone()
}
fn to_dict(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
to_dict(py, &self.inner)
}
fn __repr__(&self) -> String {
format!("Link(text={:?}, url={:?})", self.inner.text, self.inner.url)
}
}
#[pyclass(name = "Article", frozen, module = "rustai", from_py_object)]
#[derive(Clone)]
pub struct PyArticle {
pub(crate) inner: Article,
}
#[pymethods]
impl PyArticle {
#[pyo3(signature = (*, target_tokens = 512, overlap_tokens = 64, min_tokens = 24))]
fn chunks(
&self,
py: Python<'_>,
target_tokens: usize,
overlap_tokens: usize,
min_tokens: usize,
) -> PyResult<Py<PyAny>> {
if target_tokens == 0 {
return Err(PyValueError::new_err("target_tokens must be greater than 0"));
}
if overlap_tokens >= target_tokens {
return Err(PyValueError::new_err(
"overlap_tokens must be smaller than target_tokens, or every \
chunk repeats the last one and chunking never advances",
));
}
let cfg = ChunkConfig { target_tokens, overlap_tokens, min_tokens };
let out = py.detach(|| crate::chunk::chunk(&self.inner, &cfg));
to_dict(py, &out)
}
#[getter]
fn url(&self) -> Option<&str> {
self.inner.url.as_deref()
}
#[getter]
fn title(&self) -> Option<&str> {
self.inner.title()
}
#[getter]
fn markdown(&self) -> &str {
&self.inner.markdown
}
#[getter]
fn text(&self) -> &str {
&self.inner.text
}
#[getter]
fn kind(&self) -> &'static str {
match self.inner.kind {
ArticleKind::Article => "article",
ArticleKind::Index => "index",
}
}
#[getter]
fn links(&self) -> Vec<PyLink> {
self.inner.links.iter().cloned().map(|inner| PyLink { inner }).collect()
}
#[getter]
fn meta(&self) -> PyMeta {
PyMeta { inner: self.inner.meta.clone() }
}
#[getter]
fn units(&self) -> Vec<PyUnit> {
self.inner.units.iter().cloned().map(|inner| PyUnit { inner }).collect()
}
#[getter]
fn stats(&self) -> PyStats {
PyStats { inner: self.inner.stats.clone() }
}
#[getter]
fn tokens(&self) -> usize {
self.inner.tokens()
}
fn to_dict(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
to_dict(py, &self.inner)
}
fn __len__(&self) -> usize {
self.inner.units.len()
}
fn __repr__(&self) -> String {
format!(
"Article(kind={}, title={:?}, units={}, tokens={})",
self.kind(),
self.inner.title(),
self.inner.units.len(),
self.inner.tokens()
)
}
}
#[pyclass(name = "Page", frozen, module = "rustai", skip_from_py_object)]
#[derive(Clone)]
pub struct PyPage {
inner: Page,
}
#[pymethods]
impl PyPage {
#[getter]
fn url(&self) -> &str {
&self.inner.url
}
#[getter]
fn final_url(&self) -> &str {
&self.inner.final_url
}
#[getter]
fn status(&self) -> u16 {
self.inner.status
}
#[getter]
fn content_type(&self) -> Option<&str> {
self.inner.content_type.as_deref()
}
#[getter]
fn body(&self) -> &str {
&self.inner.body
}
#[getter]
fn elapsed_ms(&self) -> u64 {
self.inner.elapsed_ms
}
#[getter]
fn rendered(&self) -> bool {
self.inner.rendered
}
fn extract(&self) -> PyResult<PyArticle> {
let inner = crate::parse::extract(&self.inner.body, Some(&self.inner.final_url))?;
Ok(PyArticle { inner })
}
fn to_dict(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
to_dict(py, &self.inner)
}
fn __repr__(&self) -> String {
format!(
"Page(url={:?}, status={}, bytes={}, elapsed_ms={})",
self.inner.url,
self.inner.status,
self.inner.body.len(),
self.inner.elapsed_ms
)
}
}
#[pyclass(name = "SearchResult", frozen, module = "rustai", skip_from_py_object)]
#[derive(Clone)]
pub struct PySearchResult {
inner: SearchResult,
}
#[pymethods]
impl PySearchResult {
#[getter]
fn title(&self) -> &str {
&self.inner.title
}
#[getter]
fn url(&self) -> &str {
&self.inner.url
}
#[getter]
fn snippet(&self) -> &str {
&self.inner.snippet
}
#[getter]
fn providers(&self) -> Vec<String> {
self.inner.providers.clone()
}
#[getter]
fn score(&self) -> f32 {
self.inner.score
}
#[getter]
fn rank(&self) -> usize {
self.inner.rank
}
fn to_dict(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
to_dict(py, &self.inner)
}
fn __repr__(&self) -> String {
format!(
"SearchResult(title={:?}, url={:?}, providers={:?})",
self.inner.title, self.inner.url, self.inner.providers
)
}
}
#[pyclass(name = "Context", frozen, module = "rustai", skip_from_py_object)]
#[derive(Clone)]
pub struct PyContext {
inner: Context,
}
#[pymethods]
impl PyContext {
#[getter]
fn query(&self) -> &str {
&self.inner.query
}
#[getter]
fn markdown(&self) -> &str {
&self.inner.markdown
}
#[getter]
fn tokens(&self) -> usize {
self.inner.tokens
}
#[getter]
fn sources(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
to_dict(py, &self.inner.sources)
}
#[getter]
fn selected(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
to_dict(py, &self.inner.selected)
}
#[getter]
fn units_considered(&self) -> usize {
self.inner.units_considered
}
fn to_dict(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
to_dict(py, &self.inner)
}
fn __str__(&self) -> &str {
&self.inner.markdown
}
fn __repr__(&self) -> String {
format!(
"Context(tokens={}, sources={}, selected={}/{})",
self.inner.tokens,
self.inner.sources.len(),
self.inner.selected.len(),
self.inner.units_considered
)
}
}
#[pyclass(name = "Research", frozen, module = "rustai", skip_from_py_object)]
#[derive(Clone)]
pub struct PyResearch {
inner: Research,
}
#[pymethods]
impl PyResearch {
#[getter]
fn query(&self) -> &str {
&self.inner.query
}
#[getter]
fn context(&self) -> PyContext {
PyContext { inner: self.inner.context.clone() }
}
#[getter]
fn markdown(&self) -> &str {
&self.inner.context.markdown
}
#[getter]
fn results(&self) -> Vec<PySearchResult> {
self.inner.results.iter().cloned().map(|inner| PySearchResult { inner }).collect()
}
#[getter]
fn articles(&self) -> Vec<PyArticle> {
self.inner.articles.iter().cloned().map(|inner| PyArticle { inner }).collect()
}
#[getter]
fn failures(&self) -> Vec<(String, String)> {
self.inner.failures.clone()
}
fn to_dict(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
to_dict(py, &self.inner)
}
fn __str__(&self) -> &str {
&self.inner.context.markdown
}
fn __repr__(&self) -> String {
format!(
"Research(query={:?}, articles={}, tokens={}, failures={})",
self.inner.query,
self.inner.articles.len(),
self.inner.context.tokens,
self.inner.failures.len()
)
}
}
#[pyclass(name = "Client", frozen, module = "rustai")]
pub struct PyClient {
pipeline: Pipeline,
summary: String,
}
#[pymethods]
impl PyClient {
#[new]
#[pyo3(signature = (
*,
providers = None,
concurrency = 16,
timeout = 20.0,
impersonate = "chrome",
respect_robots = true,
respect_crawl_delay = true,
per_host_delay = 0.25,
retries = 2,
max_body_bytes = 8 * 1024 * 1024,
accept_language = "en-US,en;q=0.9",
browser_fallback = false,
contact_email = None,
proxies = None,
max_retry_after = 60.0,
cookie_file = None,
allow_private_hosts = false,
max_tokens = 2048,
max_tokens_per_source = None,
diversity = 0.35,
include_links = true,
include_images = false,
include_tables = true,
index_mode = "auto",
max_results = None,
limit = None,
))]
#[allow(clippy::too_many_arguments)]
fn new(
providers: Option<Vec<String>>,
concurrency: usize,
timeout: f64,
impersonate: &str,
respect_robots: bool,
respect_crawl_delay: bool,
per_host_delay: f64,
retries: u32,
max_body_bytes: usize,
accept_language: &str,
browser_fallback: bool,
contact_email: Option<String>,
proxies: Option<Vec<String>>,
max_retry_after: f64,
cookie_file: Option<String>,
allow_private_hosts: bool,
max_tokens: usize,
max_tokens_per_source: Option<usize>,
diversity: f32,
include_links: bool,
include_images: bool,
include_tables: bool,
index_mode: &str,
max_results: Option<usize>,
limit: Option<usize>,
) -> PyResult<Self> {
if !timeout.is_finite() || timeout <= 0.0 {
return Err(PyValueError::new_err("timeout must be a positive number of seconds"));
}
if !(0.0..=1.0).contains(&diversity) {
return Err(PyValueError::new_err("diversity must be between 0.0 and 1.0"));
}
let max_results = match (max_results, limit) {
(Some(a), Some(b)) if a != b => {
return Err(PyValueError::new_err(format!(
"max_results={a} and limit={b} disagree; limit is the old name \
for max_results, so pass only max_results"
)));
}
(Some(n), _) | (None, Some(n)) => n,
(None, None) => 10,
};
let providers = match providers {
Some(specs) => {
specs.iter().map(|s| Provider::parse(s)).collect::<Result<Vec<_>, _>>()?
}
None => SearchConfig::default().providers,
};
if providers.is_empty() {
return Err(PyValueError::new_err("at least one provider is required"));
}
let fetch = FetchConfig {
concurrency,
timeout: std::time::Duration::from_secs_f64(timeout),
max_body_bytes,
retries,
per_host_delay: std::time::Duration::from_secs_f64(per_host_delay.max(0.0)),
respect_robots,
respect_crawl_delay,
impersonate: Impersonate::parse(impersonate)?,
accept_language: accept_language.to_string(),
browser_fallback,
proxies: proxies.unwrap_or_default(),
allow_private_hosts,
max_retry_after: std::time::Duration::from_secs_f64(max_retry_after.max(0.0)),
cookie_file: cookie_file.map(std::path::PathBuf::from),
..Default::default()
};
let summary = format!(
"Client(providers={:?}, concurrency={concurrency}, impersonate={impersonate:?}, \
respect_robots={respect_robots}, proxies={}, max_tokens={max_tokens})",
providers.iter().map(Provider::name).collect::<Vec<_>>(),
fetch.proxies.len()
);
let pipeline = Pipeline::builder()
.fetch(fetch)
.search(SearchConfig {
providers,
limit: max_results,
per_provider: max_results.max(10),
contact_email,
})
.extract(ExtractOptions {
render: RenderOptions { include_links, include_images, include_tables },
denoise: DenoiseConfig { keep_tables: include_tables, ..Default::default() },
index_mode: parse_index_mode(index_mode)?,
..ExtractOptions::new()
})
.slim(SlimConfig { max_tokens, max_tokens_per_source, diversity, ..Default::default() })
.build()?;
Ok(PyClient { pipeline, summary })
}
#[pyo3(signature = (query, *, strict = false))]
fn search(&self, py: Python<'_>, query: &str, strict: bool) -> PyResult<Vec<PySearchResult>> {
let outcome = py.detach(|| runtime().map(|rt| rt.block_on(self.pipeline.search(query))))?;
if strict && outcome.results.is_empty() && !outcome.failures.is_empty() {
let detail = outcome
.failures
.iter()
.map(|(p, m)| format!("{p}: {m}"))
.collect::<Vec<_>>()
.join("; ");
return Err(ProviderError::new_err(format!("every provider failed ({detail})")));
}
Ok(outcome.results.into_iter().map(|inner| PySearchResult { inner }).collect())
}
#[pyo3(signature = (urls, *, raise_on_error = false))]
fn fetch(
&self,
py: Python<'_>,
urls: Vec<String>,
raise_on_error: bool,
) -> PyResult<Vec<PyPage>> {
let results = py.detach(|| runtime().map(|rt| rt.block_on(self.pipeline.fetch(&urls))))?;
collect(results, raise_on_error)
.map(|pages| pages.into_iter().map(|inner| PyPage { inner }).collect())
}
#[pyo3(signature = (urls, *, raise_on_error = false))]
fn read(
&self,
py: Python<'_>,
urls: Vec<String>,
raise_on_error: bool,
) -> PyResult<Vec<PyArticle>> {
let results = py.detach(|| runtime().map(|rt| rt.block_on(self.pipeline.read(&urls))))?;
collect(results, raise_on_error)
.map(|arts| arts.into_iter().map(|inner| PyArticle { inner }).collect())
}
#[pyo3(signature = (query, *, max_sources = 5))]
fn research(&self, py: Python<'_>, query: &str, max_sources: usize) -> PyResult<PyResearch> {
let inner = py.detach(|| {
runtime().map(|rt| rt.block_on(self.pipeline.research(query, max_sources)))
})?;
Ok(PyResearch { inner })
}
fn save_cookies(&self) -> PyResult<usize> {
Ok(self.pipeline.fetcher().save_cookies()?)
}
fn __repr__(&self) -> String {
self.summary.clone()
}
}
fn collect<T>(results: Vec<crate::error::Result<T>>, raise: bool) -> PyResult<Vec<T>> {
let mut out = Vec::with_capacity(results.len());
for r in results {
match r {
Ok(v) => out.push(v),
Err(e) if raise => return Err(e.into()),
Err(_) => {}
}
}
Ok(out)
}
fn parse_index_mode(mode: &str) -> PyResult<IndexMode> {
match mode.trim().to_ascii_lowercase().as_str() {
"auto" => Ok(IndexMode::Auto),
"never" | "off" => Ok(IndexMode::Never),
"always" | "on" => Ok(IndexMode::Always),
other => Err(PyValueError::new_err(format!(
"index_mode must be `auto`, `never` or `always`, not {other:?}"
))),
}
}
fn reject_url_as_html(html: &str) -> PyResult<()> {
let trimmed = html.trim();
let looks_like_url = trimmed.len() < 2048
&& !trimmed.contains('<')
&& !trimmed.contains(char::is_whitespace)
&& (trimmed.starts_with("http://") || trimmed.starts_with("https://"));
if looks_like_url {
return Err(PyValueError::new_err(format!(
"expected HTML, got what looks like a URL ({trimmed:?}).\n\
`extract` takes markup you already have and does not fetch. To \
read a URL:\n rustai.Client().read([{trimmed:?}])[0]\n\
or, for a whole question:\n rustai.research(\"your question\")"
)));
}
Ok(())
}
#[pyfunction]
#[pyo3(signature = (html, url = None, *, include_links = true, include_images = false, include_tables = true, index_mode = "auto"))]
fn extract(
py: Python<'_>,
html: &str,
url: Option<&str>,
include_links: bool,
include_images: bool,
include_tables: bool,
index_mode: &str,
) -> PyResult<PyArticle> {
reject_url_as_html(html)?;
let opts = ExtractOptions {
render: RenderOptions { include_links, include_images, include_tables },
denoise: DenoiseConfig { keep_tables: include_tables, ..Default::default() },
index_mode: parse_index_mode(index_mode)?,
..ExtractOptions::new()
};
let inner = py.detach(|| crate::parse::extract_with(html, url, &opts))?;
Ok(PyArticle { inner })
}
#[pyfunction]
#[pyo3(signature = (articles, *, target_tokens = 512, overlap_tokens = 64, min_tokens = 24))]
fn chunk_many(
py: Python<'_>,
articles: Vec<PyRef<'_, PyArticle>>,
target_tokens: usize,
overlap_tokens: usize,
min_tokens: usize,
) -> PyResult<Py<PyAny>> {
if target_tokens == 0 {
return Err(PyValueError::new_err("target_tokens must be greater than 0"));
}
if overlap_tokens >= target_tokens {
return Err(PyValueError::new_err("overlap_tokens must be smaller than target_tokens"));
}
let cfg = ChunkConfig { target_tokens, overlap_tokens, min_tokens };
let owned: Vec<Article> = articles.iter().map(|a| a.inner.clone()).collect();
let out = py.detach(|| {
use rayon::prelude::*;
owned.par_iter().flat_map(|a| crate::chunk::chunk(a, &cfg)).collect::<Vec<_>>()
});
to_dict(py, &out)
}
#[pyfunction]
#[pyo3(signature = (documents, urls = None, *, include_links = true, include_images = false, include_tables = true, index_mode = "auto"))]
fn extract_many(
py: Python<'_>,
documents: Vec<String>,
urls: Option<Vec<Option<String>>>,
include_links: bool,
include_images: bool,
include_tables: bool,
index_mode: &str,
) -> PyResult<Vec<PyArticle>> {
for doc in &documents {
reject_url_as_html(doc)?;
}
if let Some(urls) = &urls
&& urls.len() != documents.len()
{
return Err(PyValueError::new_err(format!(
"urls has {} entries but documents has {}",
urls.len(),
documents.len()
)));
}
let opts = ExtractOptions {
render: RenderOptions { include_links, include_images, include_tables },
denoise: DenoiseConfig { keep_tables: include_tables, ..Default::default() },
index_mode: parse_index_mode(index_mode)?,
..ExtractOptions::new()
};
let results = py.detach(|| {
let pairs: Vec<(&str, Option<&str>)> = documents
.iter()
.enumerate()
.map(|(i, html)| {
let url = urls.as_ref().and_then(|u| u[i].as_deref());
(html.as_str(), url)
})
.collect();
crate::parse::extract_many(pairs, &opts)
});
results.into_iter().map(|r| r.map(|inner| PyArticle { inner }).map_err(PyErr::from)).collect()
}
#[pyfunction]
#[pyo3(signature = (query, articles, *, max_tokens = 2048, diversity = 0.35,
max_tokens_per_source = None, include_breadcrumbs = true,
query_vector = None, unit_vectors = None, semantic_weight = 0.5))]
#[allow(clippy::too_many_arguments)]
fn slim(
py: Python<'_>,
query: &str,
articles: Vec<PyRef<'_, PyArticle>>,
max_tokens: usize,
diversity: f32,
max_tokens_per_source: Option<usize>,
include_breadcrumbs: bool,
query_vector: Option<Vec<f32>>,
unit_vectors: Option<Vec<Vec<f32>>>,
semantic_weight: f32,
) -> PyResult<PyContext> {
if !(0.0..=1.0).contains(&diversity) {
return Err(PyValueError::new_err("diversity must be between 0.0 and 1.0"));
}
if !(0.0..=1.0).contains(&semantic_weight) {
return Err(PyValueError::new_err("semantic_weight must be between 0.0 and 1.0"));
}
let owned: Vec<Article> = articles.into_iter().map(|a| a.inner.clone()).collect();
let vectors = match (&query_vector, &unit_vectors) {
(Some(_), None) | (None, Some(_)) => {
return Err(PyValueError::new_err(
"query_vector and unit_vectors go together; pass both or neither",
));
}
(None, None) => None,
(Some(q), Some(units)) => {
let expected: usize = owned.iter().map(|a| a.units.len()).sum();
if units.len() != expected {
return Err(PyValueError::new_err(format!(
"unit_vectors has {} entries but the articles hold {expected} units. \
One vector per unit, flattened in order: \
[u.text for a in articles for u in a.units]",
units.len()
)));
}
if let Some((i, bad)) = units.iter().enumerate().find(|(_, u)| u.len() != q.len()) {
return Err(PyValueError::new_err(format!(
"unit_vectors[{i}] has {} dimensions and query_vector has {}. \
Embed the query with the same model as the units",
bad.len(),
q.len()
)));
}
Some((q.as_slice(), units.as_slice()))
}
};
let cfg = SlimConfig {
max_tokens,
diversity,
max_tokens_per_source,
include_breadcrumbs,
semantic_weight,
..Default::default()
};
let inner = py.detach(|| match vectors {
Some((q, units)) => crate::rank::slim_with_vectors(
query,
&owned,
&cfg,
Some(crate::rank::Vectors { query: q, units }),
),
None => crate::rank::slim(query, &owned, &cfg),
});
Ok(PyContext { inner })
}
#[pyfunction]
#[pyo3(signature = (query, *, max_sources = 5, max_tokens = 2048, max_tokens_per_source = None,
providers = None, impersonate = "chrome", respect_robots = true, contact_email = None))]
#[allow(clippy::too_many_arguments)]
fn research(
py: Python<'_>,
query: &str,
max_sources: usize,
max_tokens: usize,
max_tokens_per_source: Option<usize>,
providers: Option<Vec<String>>,
impersonate: &str,
respect_robots: bool,
contact_email: Option<String>,
) -> PyResult<PyResearch> {
let client = PyClient::new(
providers,
16,
20.0,
impersonate,
respect_robots,
true,
0.25,
2,
8 * 1024 * 1024,
"en-US,en;q=0.9",
false,
contact_email,
None,
60.0,
None,
false,
max_tokens,
max_tokens_per_source,
0.35,
true,
false,
true,
"auto",
None,
None,
)?;
client.research(py, query, max_sources)
}
#[pyfunction]
fn count_tokens(text: &str) -> usize {
crate::text::estimate_tokens(text)
}
#[pyfunction]
fn tokenize(text: &str) -> Vec<String> {
crate::text::tokenize(text)
}
#[pyfunction]
fn density(text: &str) -> f32 {
crate::rank::density(text).score
}
#[pyfunction]
fn canonical_url(url: &str) -> Option<String> {
crate::search::canonical_key(url)
}
#[pyfunction]
fn parse_feed(py: Python<'_>, xml: &str) -> PyResult<Py<PyAny>> {
to_dict(py, &crate::search::parse_rss(xml)?)
}
#[pyfunction]
fn parse_sitemap(py: Python<'_>, xml: &str) -> PyResult<Py<PyAny>> {
to_dict(py, &crate::search::parse_sitemap(xml)?)
}
#[pymodule]
#[pyo3(name = "_rustai")]
pub fn rustai_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add("__version__", env!("CARGO_PKG_VERSION"))?;
m.add_class::<PyClient>()?;
m.add_class::<PyArticle>()?;
m.add_class::<PyPage>()?;
m.add_class::<PyUnit>()?;
m.add_class::<PyLink>()?;
m.add_class::<PyMeta>()?;
m.add_class::<PyStats>()?;
m.add_class::<PySearchResult>()?;
m.add_class::<PyContext>()?;
m.add_class::<PyResearch>()?;
m.add_function(wrap_pyfunction!(extract, m)?)?;
m.add_function(wrap_pyfunction!(extract_many, m)?)?;
m.add_function(wrap_pyfunction!(chunk_many, m)?)?;
m.add_function(wrap_pyfunction!(slim, m)?)?;
m.add_function(wrap_pyfunction!(research, m)?)?;
m.add_function(wrap_pyfunction!(count_tokens, m)?)?;
m.add_function(wrap_pyfunction!(tokenize, m)?)?;
m.add_function(wrap_pyfunction!(density, m)?)?;
m.add_function(wrap_pyfunction!(canonical_url, m)?)?;
m.add_function(wrap_pyfunction!(parse_feed, m)?)?;
m.add_function(wrap_pyfunction!(parse_sitemap, m)?)?;
m.add("RustaiError", m.py().get_type::<RustaiError>())?;
m.add("NetworkError", m.py().get_type::<NetworkError>())?;
m.add("HttpStatusError", m.py().get_type::<HttpStatusError>())?;
m.add("RobotsError", m.py().get_type::<RobotsError>())?;
m.add("ExtractError", m.py().get_type::<ExtractError>())?;
m.add("ProviderError", m.py().get_type::<ProviderError>())?;
m.add("BrowserError", m.py().get_type::<BrowserError>())?;
Ok(())
}
#[cfg(test)]
mod guard_tests {
use super::reject_url_as_html;
#[test]
fn rejects_a_bare_url() {
assert!(reject_url_as_html("https://example.com/x").is_err());
assert!(reject_url_as_html(" http://example.com ").is_err());
}
#[test]
fn passes_anything_that_could_be_a_document() {
assert!(reject_url_as_html("<p>hi</p>").is_ok());
assert!(reject_url_as_html("").is_ok());
assert!(reject_url_as_html("just some prose").is_ok());
assert!(reject_url_as_html("see https://example.com for more").is_ok());
assert!(reject_url_as_html("<a href=\"https://example.com\">x</a>").is_ok());
assert!(reject_url_as_html("ftp://example.com/file").is_ok());
}
}