use crate::privacy::{PrivacyReport, discover_subresources};
use crate::reader::{ReaderArticle, amp_canonical_redirect, extract_article};
use crate::storage::Profile;
use anyhow::Context;
use reqwest::blocking::Client;
use reqwest::header::{HeaderMap, HeaderName, HeaderValue, LOCATION, USER_AGENT};
use reqwest::redirect::Policy;
use std::io::Read;
use std::time::Duration;
use url::Url;
pub const DEFAULT_MAX_NAVIGATION_STACK: usize = 100;
pub const DEFAULT_MAX_PAGE_BODY_BYTES: usize = 5 * 1024 * 1024;
const MAX_HTTP_REDIRECTS: usize = 8;
#[derive(Debug, Clone)]
pub struct BrowserConfig {
pub user_agent: String,
pub timeout: Duration,
pub max_de_amp_hops: usize,
pub max_navigation_stack: usize,
pub max_page_body_bytes: usize,
}
impl Default for BrowserConfig {
fn default() -> Self {
Self {
user_agent: "Cephas/0.1 privacy-first Rust browser".to_string(),
timeout: Duration::from_secs(30),
max_de_amp_hops: 2,
max_navigation_stack: DEFAULT_MAX_NAVIGATION_STACK,
max_page_body_bytes: DEFAULT_MAX_PAGE_BODY_BYTES,
}
}
}
#[derive(Debug, Clone)]
pub struct LoadedPage {
pub url: Url,
pub status: u16,
pub content_type: Option<String>,
pub body: String,
pub body_truncated: bool,
pub article: ReaderArticle,
pub privacy: PrivacyReport,
}
#[derive(Debug, Clone)]
struct FetchResult {
status: u16,
content_type: Option<String>,
body: String,
body_truncated: bool,
response_url: Option<Url>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BrowserDiagnostics {
pub back_stack_entries: usize,
pub forward_stack_entries: usize,
pub max_navigation_stack: usize,
pub history_entries: usize,
pub max_history_entries: usize,
pub bookmark_entries: usize,
pub max_bookmark_entries: usize,
pub current_body_bytes: usize,
pub max_page_body_bytes: usize,
}
pub struct Browser {
client: Client,
pub profile: Profile,
pub current: Option<LoadedPage>,
back_stack: Vec<Url>,
forward_stack: Vec<Url>,
config: BrowserConfig,
}
impl Browser {
pub fn new(profile: Profile, config: BrowserConfig) -> anyhow::Result<Self> {
let client = Client::builder()
.timeout(config.timeout)
.redirect(Policy::none())
.user_agent(config.user_agent.clone())
.build()
.context("failed to build HTTP client")?;
Ok(Self {
client,
profile,
current: None,
back_stack: Vec::new(),
forward_stack: Vec::new(),
config,
})
}
pub fn open(&mut self, input: &str) -> anyhow::Result<&LoadedPage> {
let (url, report) = self
.profile
.shields
.prepare_navigation_with_search(input, self.profile.search_provider)?;
self.load(url, report, true)
}
pub fn reload(&mut self) -> anyhow::Result<&LoadedPage> {
let url = self
.current
.as_ref()
.map(|page| page.url.clone())
.unwrap_or_else(|| self.profile.home.clone());
let mut report = PrivacyReport {
original_url: url.to_string(),
..PrivacyReport::default()
};
report.final_url = url.to_string();
self.load(url, report, false)
}
pub fn back(&mut self) -> anyhow::Result<Option<&LoadedPage>> {
let Some(url) = self.back_stack.pop() else {
return Ok(None);
};
if let Some(current) = &self.current {
push_bounded(
&mut self.forward_stack,
current.url.clone(),
self.config.max_navigation_stack,
);
}
let mut report = PrivacyReport {
original_url: url.to_string(),
..PrivacyReport::default()
};
report.final_url = url.to_string();
self.load(url, report, false).map(Some)
}
pub fn forward(&mut self) -> anyhow::Result<Option<&LoadedPage>> {
let Some(url) = self.forward_stack.pop() else {
return Ok(None);
};
if let Some(current) = &self.current {
push_bounded(
&mut self.back_stack,
current.url.clone(),
self.config.max_navigation_stack,
);
}
let mut report = PrivacyReport {
original_url: url.to_string(),
..PrivacyReport::default()
};
report.final_url = url.to_string();
self.load(url, report, false).map(Some)
}
pub fn bookmark_current(&mut self) -> anyhow::Result<bool> {
let page = self.current.as_ref().context("no page is currently open")?;
Ok(self
.profile
.add_bookmark(page.article.title.clone(), page.url.clone()))
}
pub fn forget_current_site(&mut self) -> anyhow::Result<usize> {
let page = self.current.as_ref().context("no page is currently open")?;
let host = page
.url
.host_str()
.context("current page has no host")?
.to_string();
Ok(self.profile.forget_history_for_host(&host))
}
pub fn set_shields_for_current_site(&mut self, enabled: bool) -> anyhow::Result<()> {
let page = self.current.as_ref().context("no page is currently open")?;
if enabled {
self.profile.shields.protect_site(&page.url);
} else {
self.profile.shields.allow_site(&page.url);
}
Ok(())
}
pub fn can_go_back(&self) -> bool {
!self.back_stack.is_empty()
}
pub fn can_go_forward(&self) -> bool {
!self.forward_stack.is_empty()
}
pub fn diagnostics(&self) -> BrowserDiagnostics {
BrowserDiagnostics {
back_stack_entries: self.back_stack.len(),
forward_stack_entries: self.forward_stack.len(),
max_navigation_stack: self.config.max_navigation_stack.max(1),
history_entries: self.profile.history.len(),
max_history_entries: self.profile.max_history.max(1),
bookmark_entries: self.profile.bookmarks.len(),
max_bookmark_entries: self.profile.max_bookmarks.max(1),
current_body_bytes: self
.current
.as_ref()
.map(|page| page.body.len())
.unwrap_or(0),
max_page_body_bytes: self.config.max_page_body_bytes,
}
}
fn load(
&mut self,
url: Url,
mut report: PrivacyReport,
update_history_stack: bool,
) -> anyhow::Result<&LoadedPage> {
let page = self.fetch_with_de_amp(url, &mut report, 0)?;
if update_history_stack && let Some(current) = &self.current {
push_bounded(
&mut self.back_stack,
current.url.clone(),
self.config.max_navigation_stack,
);
self.forward_stack.clear();
}
self.profile
.record_visit(page.article.title.clone(), page.url.clone());
self.current = Some(page);
Ok(self.current.as_ref().expect("page set"))
}
fn fetch_with_de_amp(
&self,
url: Url,
report: &mut PrivacyReport,
hop: usize,
) -> anyhow::Result<LoadedPage> {
let FetchResult {
status,
content_type,
body,
body_truncated,
response_url,
} = self.fetch(&url)?;
let effective_url = response_url.unwrap_or(url);
if self.profile.shields.de_amp
&& hop < self.config.max_de_amp_hops
&& let Some(canonical) = amp_canonical_redirect(&body, &effective_url)
&& canonical != effective_url
&& de_amp_canonical_allowed(&effective_url, &canonical)
{
let (canonical, _) = self
.profile
.shields
.prepare_navigation_with_search(canonical.as_str(), self.profile.search_provider)?;
if !is_fetchable_web_url(&canonical) || canonical == effective_url {
report.final_url = effective_url.to_string();
return Ok(self.loaded_page(
status,
content_type,
body,
body_truncated,
effective_url,
report,
));
}
report.de_amped_to = Some(canonical.to_string());
report.final_url = canonical.to_string();
return self.fetch_with_de_amp(canonical, report, hop + 1);
}
Ok(self.loaded_page(
status,
content_type,
body,
body_truncated,
effective_url,
report,
))
}
fn loaded_page(
&self,
status: u16,
content_type: Option<String>,
body: String,
body_truncated: bool,
effective_url: Url,
report: &mut PrivacyReport,
) -> LoadedPage {
let resources = discover_subresources(&body, &effective_url);
report.blocked_requests = resources
.iter()
.filter(|resource| {
self.profile
.shields
.should_block_request(&effective_url, resource)
})
.map(ToString::to_string)
.collect();
report.final_url = effective_url.to_string();
let article = extract_article(&body, &effective_url);
LoadedPage {
url: effective_url,
status,
content_type,
body,
body_truncated,
article,
privacy: report.clone(),
}
}
fn fetch(&self, url: &Url) -> anyhow::Result<FetchResult> {
let mut current = url.clone();
for _ in 0..=MAX_HTTP_REDIRECTS {
let headers = self.headers_for(¤t)?;
let mut response = self
.client
.get(current.clone())
.headers(headers)
.send()
.with_context(|| format!("failed to load {current}"))?;
if response.status().is_redirection()
&& let Some(next) = self.redirect_target(response.url(), &response)?
{
current = next;
continue;
}
let status = response.status().as_u16();
let response_url = response.url().clone();
let content_type = response
.headers()
.get(reqwest::header::CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.map(ToString::to_string);
let mut body = Vec::new();
response
.by_ref()
.take(self.config.max_page_body_bytes.saturating_add(1) as u64)
.read_to_end(&mut body)
.with_context(|| format!("failed to read response body from {response_url}"))?;
let (body, body_truncated) =
decode_limited_body_bytes(body, self.config.max_page_body_bytes);
return Ok(FetchResult {
status,
content_type,
body,
body_truncated,
response_url: Some(response_url),
});
}
anyhow::bail!("too many redirects while loading {url}")
}
fn redirect_target(
&self,
base: &Url,
response: &reqwest::blocking::Response,
) -> anyhow::Result<Option<Url>> {
let Some(location) = response.headers().get(LOCATION) else {
return Ok(None);
};
let location = location
.to_str()
.context("redirect location is not valid UTF-8")?;
let target = base
.join(location)
.with_context(|| format!("invalid redirect location {location}"))?;
let (target, _) = self
.profile
.shields
.prepare_navigation_with_search(target.as_str(), self.profile.search_provider)?;
if !is_fetchable_web_url(&target) {
anyhow::bail!(
"redirect target uses unsupported scheme: {}",
target.scheme()
);
}
Ok(Some(target))
}
fn headers_for(&self, url: &Url) -> anyhow::Result<HeaderMap> {
let mut headers = HeaderMap::new();
headers.insert(USER_AGENT, HeaderValue::from_str(&self.config.user_agent)?);
for (name, value) in self.profile.shields.request_headers(url) {
let name = HeaderName::from_bytes(name.as_bytes())?;
headers.insert(name, HeaderValue::from_static(value));
}
Ok(headers)
}
}
#[cfg(test)]
fn truncate_body_to_cap(mut body: String, max_bytes: usize) -> (String, bool) {
if body.len() <= max_bytes {
return (body, false);
}
let mut boundary = max_bytes;
while boundary > 0 && !body.is_char_boundary(boundary) {
boundary -= 1;
}
body.truncate(boundary);
(body, true)
}
pub(crate) fn decode_limited_body_bytes(mut body: Vec<u8>, max_bytes: usize) -> (String, bool) {
let truncated = body.len() > max_bytes;
if truncated {
body.truncate(max_bytes);
}
(String::from_utf8_lossy(&body).into_owned(), truncated)
}
fn push_bounded<T>(stack: &mut Vec<T>, value: T, max_len: usize) {
let max_len = max_len.max(1);
if stack.len() >= max_len {
let overflow = stack.len() + 1 - max_len;
stack.drain(0..overflow);
}
stack.push(value);
}
fn is_fetchable_web_url(url: &Url) -> bool {
matches!(url.scheme(), "http" | "https")
}
fn de_amp_canonical_allowed(source: &Url, canonical: &Url) -> bool {
is_fetchable_web_url(canonical) && source.host_str() == canonical.host_str()
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
use std::net::TcpListener;
use std::sync::{Arc, Mutex};
use std::thread;
use std::time::{Duration, Instant};
#[test]
fn push_bounded_evicts_oldest_entries() {
let mut stack = Vec::new();
for value in 0..5 {
push_bounded(&mut stack, value, 3);
}
assert_eq!(stack, vec![2, 3, 4]);
}
#[test]
fn de_amp_canonical_requires_fetchable_same_host_url() {
let source = Url::parse("https://news.example/amp/story").unwrap();
assert!(de_amp_canonical_allowed(
&source,
&Url::parse("https://news.example/story").unwrap()
));
assert!(!de_amp_canonical_allowed(
&source,
&Url::parse("https://other.example/story").unwrap()
));
assert!(!de_amp_canonical_allowed(
&source,
&Url::parse("cephas://agent").unwrap()
));
}
#[test]
fn truncate_body_to_cap_reports_source_truncation() {
assert_eq!(
truncate_body_to_cap("abc".to_string(), 3),
("abc".to_string(), false)
);
assert_eq!(
truncate_body_to_cap("abcdef".to_string(), 3),
("abc".to_string(), true)
);
assert_eq!(
truncate_body_to_cap("éclair".to_string(), 1),
("".to_string(), true)
);
}
#[test]
fn decode_limited_body_bytes_handles_binary_and_split_utf8() {
assert_eq!(
decode_limited_body_bytes(vec![0xff, b'a', b'b'], 8),
("\u{fffd}ab".to_string(), false)
);
assert_eq!(
decode_limited_body_bytes("éclair".as_bytes().to_vec(), 1),
("\u{fffd}".to_string(), true)
);
}
#[test]
fn follows_redirects_through_navigation_sanitation() {
let (base_url, seen_paths, handle) = spawn_http_server(2, |path, base_url| {
if path == "/start" {
http_response(
"302 Found",
&[(
"Location",
format!("{base_url}/final?utm_source=news&id=42"),
)],
"",
)
} else if path == "/final?id=42" {
http_response(
"200 OK",
&[("Content-Type", "text/html; charset=utf-8".to_string())],
"<html><head><title>Redirected</title></head><body><article>Done</article></body></html>",
)
} else {
http_response("404 Not Found", &[], "not found")
}
});
let mut browser = Browser::new(
Profile::default(),
BrowserConfig {
timeout: Duration::from_secs(2),
..BrowserConfig::default()
},
)
.unwrap();
let page = browser.open(&format!("{base_url}/start")).unwrap();
assert_eq!(page.url.as_str(), format!("{base_url}/final?id=42"));
assert!(page.body.contains("Done"));
handle.join().unwrap();
let paths = seen_paths.lock().unwrap().clone();
assert_eq!(paths, vec!["/start", "/final?id=42"]);
}
#[test]
fn rejects_redirects_to_non_web_targets() {
let (base_url, seen_paths, handle) = spawn_http_server(1, |_, _| {
http_response(
"302 Found",
&[("Location", "cephas://agent".to_string())],
"",
)
});
let mut browser = Browser::new(
Profile::default(),
BrowserConfig {
timeout: Duration::from_secs(2),
..BrowserConfig::default()
},
)
.unwrap();
let err = browser.open(&format!("{base_url}/start")).unwrap_err();
assert!(err.to_string().contains("unsupported scheme"));
handle.join().unwrap();
assert_eq!(seen_paths.lock().unwrap().as_slice(), ["/start"]);
}
fn spawn_http_server<F>(
expected_requests: usize,
handler: F,
) -> (String, Arc<Mutex<Vec<String>>>, thread::JoinHandle<()>)
where
F: Fn(&str, &str) -> String + Send + 'static,
{
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
listener.set_nonblocking(true).unwrap();
let base_url = format!("http://{}", listener.local_addr().unwrap());
let server_base_url = base_url.clone();
let seen_paths = Arc::new(Mutex::new(Vec::new()));
let seen_paths_for_thread = Arc::clone(&seen_paths);
let handle = thread::spawn(move || {
let deadline = Instant::now() + Duration::from_secs(2);
while seen_paths_for_thread.lock().unwrap().len() < expected_requests
&& Instant::now() < deadline
{
match listener.accept() {
Ok((mut stream, _)) => {
let mut request = [0_u8; 2048];
let size = stream.read(&mut request).unwrap_or(0);
let request = String::from_utf8_lossy(&request[..size]);
let path = request
.lines()
.next()
.and_then(|line| line.split_whitespace().nth(1))
.unwrap_or("/")
.to_string();
seen_paths_for_thread.lock().unwrap().push(path.clone());
let response = handler(&path, &server_base_url);
stream.write_all(response.as_bytes()).unwrap();
}
Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {
thread::sleep(Duration::from_millis(10));
}
Err(err) => panic!("test server accept failed: {err}"),
}
}
});
(base_url, seen_paths, handle)
}
fn http_response(status: &str, headers: &[(&str, String)], body: &str) -> String {
let mut response = format!(
"HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n",
body.len()
);
for (name, value) in headers {
response.push_str(name);
response.push_str(": ");
response.push_str(value);
response.push_str("\r\n");
}
response.push_str("\r\n");
response.push_str(body);
response
}
}