use super::*;
use std::borrow::Cow;
use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr};
use std::process::Stdio;
use std::sync::{Arc, RwLock};
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::{TcpListener, TcpStream};
use tokio::sync::watch;
use crate::execution::{ExecutionCancellation, ExecutionDeadlines};
use crate::rendering::RenderedSite;
use super::watching::{Watcher, cancelled, clean_cancellation};
use crate::reporting::{Event, Reporting};
pub async fn serve_with(
options: ServeOptions,
deadlines: ExecutionDeadlines,
cancellation: ExecutionCancellation<'_>,
) -> Result<(), CommandError> {
CommandRunner::default()
.serve_with(options, deadlines, cancellation)
.await
}
pub(super) async fn serve_reported(
options: ServeOptions,
deadlines: ExecutionDeadlines,
cancellation: ExecutionCancellation<'_>,
reporting: &Reporting,
) -> Result<(), CommandError> {
let (stop, receiver) = watch::channel(false);
let worker = run(options, deadlines, receiver, reporting);
tokio::pin!(worker);
tokio::select! {
result = &mut worker => result,
_ = cancellation => {
let _ = stop.send(true);
worker.await
}
}
}
async fn run(
options: ServeOptions,
deadlines: ExecutionDeadlines,
stop: watch::Receiver<bool>,
reporting: &Reporting,
) -> Result<(), CommandError> {
let setup = reporting.stage("Starting preview", None);
let listener = TcpListener::bind((options.host, options.port)).await?;
let address = listener.local_addr()?;
let build = BuildOptions {
config: options.config.clone(),
output: options.output.clone(),
};
let mut watcher = Watcher::new(&options.config, &options.output, reporting)?;
setup.finish();
let command = reporting.command("build");
let mut attempt = pipeline::AttemptState::watching();
let initial = pipeline::build_attempt_observed(
&build,
deadlines,
Box::pin(cancelled(stop.clone())),
reporting,
&mut attempt,
)
.await;
watcher.acknowledge(attempt);
let initial = match initial {
Ok(site) => site,
Err(error) if *stop.borrow() && clean_cancellation(&error) => return Ok(()),
Err(error) => return Err(error),
};
if *stop.borrow() {
return Ok(());
}
command.finish(Some(&build.output), Some(initial.page_count()));
let current = Arc::new(RwLock::new(Arc::new(ServedGeneration {
site: initial,
revision: getrandom::u64().map_err(std::io::Error::other)?,
})));
let http = tokio::spawn(http(
listener,
current.clone(),
options.live_reload,
stop.clone(),
));
let mut http = AbortOnDrop(http);
let url = preview_url(address);
reporting.emit(Event::Serving { url: &url });
reporting.emit(Event::Watching {
retained_previous: false,
});
let browser_reporting = reporting.clone();
let _browser = options.open.then(|| {
AbortOnDrop(tokio::spawn(async move {
if let Err(error) = open_browser(&url).await {
browser_reporting.emit(Event::Warning(&format!(
"could not open browser for {url}: {error}"
)));
}
}))
});
let mut http_finished = false;
let result = loop {
tokio::select! {
_ = cancelled(stop.clone()) => break Ok(()),
result = &mut http.0 => {
http_finished = true;
break match result {
Ok(result) => result.map_err(CommandError::from),
Err(error) => Err(std::io::Error::other(error).into()),
};
}
_ = watcher.next_change() => {
let command = reporting.command("rebuild");
let mut attempt = pipeline::AttemptState::watching();
let result = pipeline::build_attempt_observed(&build, deadlines, Box::pin(cancelled(stop.clone())), reporting, &mut attempt).await;
watcher.acknowledge(attempt);
match result {
Ok(site) => {
let pages = site.page_count();
{
let mut current = current.write().expect("preview generation lock");
*current = Arc::new(ServedGeneration {
site,
revision: current.revision.wrapping_add(1),
});
}
command.finish(Some(&build.output), Some(pages));
reporting.emit(Event::Rebuilt);
reporting.emit(Event::Watching { retained_previous: false });
}
Err(error) => {
drop(command);
if *stop.borrow() && clean_cancellation(&error) { break Ok(()); }
reporting.error(&error);
if *stop.borrow() { break Err(error); }
reporting.emit(Event::Watching { retained_previous: true });
}
}
}
}
};
if !http_finished {
http.0.abort();
let _ = (&mut http.0).await;
}
result
}
fn preview_url(mut address: SocketAddr) -> String {
if address.ip().is_unspecified() {
address.set_ip(match address {
SocketAddr::V4(_) => Ipv4Addr::LOCALHOST.into(),
SocketAddr::V6(_) => Ipv6Addr::LOCALHOST.into(),
});
}
format!("http://{address}/")
}
async fn open_browser(url: &str) -> std::io::Result<()> {
#[cfg(target_os = "macos")]
let mut command = tokio::process::Command::new("open");
#[cfg(target_os = "windows")]
let mut command = {
let mut command = tokio::process::Command::new("rundll32.exe");
command.arg("url.dll,FileProtocolHandler");
command
};
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
let mut command = tokio::process::Command::new("xdg-open");
let status = command
.arg(url)
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.await?;
if status.success() {
Ok(())
} else {
Err(std::io::Error::other(format!(
"browser launcher exited with {status}"
)))
}
}
struct AbortOnDrop<T>(tokio::task::JoinHandle<T>);
struct ServedGeneration {
site: RenderedSite,
revision: u64,
}
impl<T> Drop for AbortOnDrop<T> {
fn drop(&mut self) {
self.0.abort();
}
}
async fn http(
listener: TcpListener,
site: Arc<RwLock<Arc<ServedGeneration>>>,
live_reload: bool,
stop: watch::Receiver<bool>,
) -> std::io::Result<()> {
let mut requests = tokio::task::JoinSet::new();
loop {
tokio::select! {
_ = cancelled(stop.clone()) => return Ok(()),
accepted = listener.accept() => {
let (socket, _) = accepted?;
let site = site.read().expect("preview generation lock").clone();
requests.spawn(async move { let _ = tokio::time::timeout(Duration::from_secs(5), response(socket, &site, live_reload)).await; });
}
_ = requests.join_next(), if !requests.is_empty() => {}
}
}
}
async fn response(
mut socket: TcpStream,
generation: &ServedGeneration,
live_reload: bool,
) -> std::io::Result<()> {
let mut bytes = Vec::new();
let mut buffer = [0; 2048];
while !bytes.windows(4).any(|w| w == b"\r\n\r\n") {
let n = socket.read(&mut buffer).await?;
if n == 0 || bytes.len() + n > 16384 {
return Ok(());
}
bytes.extend_from_slice(&buffer[..n]);
}
let line = std::str::from_utf8(&bytes)
.ok()
.and_then(|s| s.lines().next())
.unwrap_or("");
let mut parts = line.split_ascii_whitespace();
let method = parts.next().unwrap_or("");
let target = parts.next().unwrap_or("");
let request_path = target.split('?').next().unwrap_or("");
let decoded = percent_encoding::percent_decode_str(request_path)
.decode_utf8()
.ok();
if matches!(method, "GET" | "HEAD")
&& let Some(destination) = generation.site.redirects().get(request_path)
{
let query = target.split_once('?').map_or("", |(_, query)| query);
let location = if query.is_empty() {
destination.clone()
} else {
format!("{destination}?{query}")
};
socket.write_all(format!("HTTP/1.1 301 Moved Permanently\r\nLocation: {location}\r\nContent-Length: 0\r\nCache-Control: no-cache\r\nConnection: close\r\n\r\n").as_bytes()).await?;
return socket.shutdown().await;
}
let path = decoded
.as_deref()
.and_then(|p| p.strip_prefix('/'))
.map(|p| {
if p.is_empty() || p.ends_with('/') {
format!("{p}index.html")
} else {
p.into()
}
});
let path = path.map(|path| {
let clean = format!("{path}.html");
if !generation.site.files().contains_key(&path)
&& !path.ends_with(".html")
&& generation.site.files().contains_key(&clean)
{
clean
} else {
path
}
});
let accept = std::str::from_utf8(&bytes)
.unwrap_or("")
.lines()
.skip(1)
.filter_map(|line| line.split_once(':'))
.filter(|(name, _)| name.eq_ignore_ascii_case("accept"))
.map(|(_, value)| value)
.collect::<Vec<_>>()
.join(",");
let markdown = accepts_markdown(&accept);
let markdown_path = path
.as_deref()
.and_then(|p| p.strip_suffix(".html"))
.map(|p| format!("{p}.md"));
let file = path
.as_ref()
.filter(|p| crate::diagnostics::DiagnosticPath::try_from(p.as_str()).is_ok())
.and_then(|p| generation.site.files().get(p));
let negotiable = file.is_some_and(|file| file.media_type.starts_with("text/html"))
&& markdown_path
.as_ref()
.is_some_and(|p| generation.site.files().contains_key(p));
let file = if negotiable && markdown {
markdown_path
.as_ref()
.and_then(|p| generation.site.files().get(p))
} else {
file
};
let vary = if negotiable { "Vary: Accept\r\n" } else { "" };
let (status, media, content) = if !matches!(method, "GET" | "HEAD") {
(
"405 Method Not Allowed",
"text/plain; charset=utf-8",
Cow::Borrowed(b"Method not allowed\n".as_slice()),
)
} else if live_reload && path.as_deref() == Some("__diplodocus/revision") {
(
"200 OK",
"text/plain; charset=utf-8",
Cow::Owned(generation.revision.to_string().into_bytes()),
)
} else if let Some(file) = file {
let content = if live_reload && file.media_type.starts_with("text/html") {
Cow::Owned(with_live_reload(&file.bytes, generation.revision))
} else {
Cow::Borrowed(file.bytes.as_slice())
};
("200 OK", file.media_type.as_str(), content)
} else {
(
"404 Not Found",
"text/plain; charset=utf-8",
Cow::Borrowed(b"Not found\n".as_slice()),
)
};
socket.write_all(format!("HTTP/1.1 {status}\r\nContent-Type: {media}\r\n{vary}Content-Length: {}\r\nCache-Control: no-cache\r\nX-Content-Type-Options: nosniff\r\nConnection: close\r\n\r\n", content.len()).as_bytes()).await?;
if method != "HEAD" {
socket.write_all(&content).await?;
}
socket.shutdown().await
}
fn accepts_markdown(accept: &str) -> bool {
let mut markdown = 0.0_f32;
let mut html = 0.0_f32;
for range in accept.split(',') {
let mut parts = range.trim().split(';');
let media = parts.next().unwrap_or("").trim();
let quality = parts
.filter_map(|part| part.trim().split_once('='))
.find(|(key, _)| key.eq_ignore_ascii_case("q"))
.map_or(1.0, |(_, value)| value.parse::<f32>().unwrap_or(0.0));
if !(0.0..=1.0).contains(&quality) {
continue;
}
if media.eq_ignore_ascii_case("text/markdown") {
markdown = markdown.max(quality);
}
if media.eq_ignore_ascii_case("text/html") {
html = html.max(quality);
}
}
markdown > 0.0 && markdown >= html
}
fn with_live_reload(html: &[u8], revision: u64) -> Vec<u8> {
let marker = b"</body>";
let Some(position) = html
.windows(marker.len())
.rposition(|window| window == marker)
else {
return html.to_vec();
};
let script = LIVE_RELOAD.replace("__REVISION__", &revision.to_string());
let mut content = Vec::with_capacity(html.len() + script.len());
content.extend_from_slice(&html[..position]);
content.extend_from_slice(script.as_bytes());
content.extend_from_slice(&html[position..]);
content
}
const LIVE_RELOAD: &str = r#"<script>(()=>{const served='__REVISION__';async function check(){try{const response=await fetch('/__diplodocus/revision',{cache:'no-store'});if(response.ok&&await response.text()!==served){location.reload();return}}catch{}setTimeout(check,1000)}setTimeout(check,1000)})()</script>"#;
#[cfg(test)]
mod tests {
use super::*;
use crate::assembly::assemble_workspace;
use std::fs;
#[test]
fn browser_urls_use_bound_ports_loopback_and_ipv6_brackets() {
for (address, expected) in [
("127.0.0.1:12345", "http://127.0.0.1:12345/"),
("0.0.0.0:45678", "http://127.0.0.1:45678/"),
("[::]:12345", "http://[::1]:12345/"),
("[::1]:45678", "http://[::1]:45678/"),
("192.0.2.1:12345", "http://192.0.2.1:12345/"),
] {
assert_eq!(preview_url(address.parse().unwrap()), expected);
}
}
#[tokio::test]
async fn http_negotiates_markdown_and_preserves_head_and_assets() {
let root = tempfile::tempdir().unwrap();
let config = root.path().join("diplodocus.yaml");
fs::write(&config, "project:\n name: \"Negotiation\"\nrepositories:\n - id: \"repo\"\n path: \".\"\ncontent:\n - id: \"guide\"\n owner: \"project\"\n repository: \"repo\"\n path: \"docs\"\n mount: \"guide\"\n format: \"gfm\"\n").unwrap();
fs::create_dir(root.path().join("docs")).unwrap();
fs::write(
root.path().join("docs/index.md"),
"# Guide\n\nAgent content.\n",
)
.unwrap();
fs::write(
root.path().join("docs/start.md"),
"# Start\n\nAgent content.\n",
)
.unwrap();
let sources = assemble_workspace(&config).unwrap();
let snapshot = crate::snapshots::Snapshot::from_sources(
&sources,
&resolve_workspace(&sources).unwrap(),
)
.unwrap();
let site = crate::site::Site::new(&snapshot).unwrap();
let generation = ServedGeneration {
site: crate::rendering::render_site(&site).unwrap(),
revision: 7,
};
for (method, target, accept, media) in [
("GET", "/guide/", "text/markdown", "text/markdown"),
(
"GET",
"/guide/index.html?query=1",
"TEXT/MARKDOWN",
"text/markdown",
),
("HEAD", "/guide/", "text/markdown", "text/markdown"),
("GET", "/guide/start", "text/markdown", "text/markdown"),
("HEAD", "/guide/start", "text/markdown", "text/markdown"),
("GET", "/guide/start", "text/html", "text/html"),
("GET", "/guide/", "*/*", "text/html"),
("GET", "/guide/", "text/markdown;q=0", "text/html"),
(
"GET",
"/assets/search.json",
"text/markdown",
"application/json",
),
] {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let mut client = TcpStream::connect(listener.local_addr().unwrap())
.await
.unwrap();
let (socket, _) = listener.accept().await.unwrap();
let request = format!(
"{method} {target} HTTP/1.1\r\nHost: localhost\r\nAccept: {accept}\r\n\r\n"
);
let server = response(socket, &generation, true);
let client = async {
client.write_all(request.as_bytes()).await.unwrap();
let mut bytes = Vec::new();
client.read_to_end(&mut bytes).await.unwrap();
String::from_utf8(bytes).unwrap()
};
let (result, response) = tokio::join!(server, client);
result.unwrap();
let (headers, body) = response.split_once("\r\n\r\n").unwrap();
assert!(headers.contains("200 OK"), "{response}");
assert!(
headers.contains(&format!("Content-Type: {media}")),
"{headers}"
);
if target.starts_with("/guide/") {
assert!(headers.contains("Vary: Accept"));
}
if method == "HEAD" {
assert!(body.is_empty());
} else if media == "text/markdown" {
assert!(body.contains("Agent content."));
assert!(!body.contains("<script>"));
}
}
}
#[test]
fn markdown_accept_quality_and_browser_defaults() {
for accept in [
"text/markdown",
"TEXT/MARKDOWN; charset=utf-8",
"text/html, text/markdown",
"text/markdown;q=0.8, */*;q=0.1",
] {
assert!(accepts_markdown(accept), "{accept}");
}
for accept in [
"",
"*/*",
"text/html",
"text/markdown;q=0",
"text/markdown;q=0.2,text/html;q=0.9",
"text/markdown;q=invalid",
"text/markdown;q=NaN",
] {
assert!(!accepts_markdown(accept), "{accept}");
}
}
#[tokio::test]
async fn http_redirects_use_local_canonical_urls_and_preserve_queries() {
let root = tempfile::tempdir().unwrap();
let config = root.path().join("diplodocus.yaml");
fs::write(&config, "project:\n name: Redirects\npresentation:\n site-url: https://example.test/docs/\n canonical-url-style: clean\n redirects:\n /old: guide/start.html\n /old.html: guide/start.html\n /café%20old/: guide/start.html\n /old%2dpath: guide/start.html\n /old&path: guide/start.html\nrepositories:\n - id: repo\n path: .\ncontent:\n - id: guide\n owner: project\n repository: repo\n path: docs\n mount: guide\n format: gfm\n").unwrap();
fs::create_dir(root.path().join("docs")).unwrap();
fs::write(root.path().join("docs/start.md"), "# Start\n").unwrap();
let sources = assemble_workspace(config).unwrap();
let snapshot = crate::snapshots::Snapshot::from_sources(
&sources,
&resolve_workspace(&sources).unwrap(),
)
.unwrap();
let generation = ServedGeneration {
site: crate::rendering::render_site(&crate::site::Site::new(&snapshot).unwrap())
.unwrap(),
revision: 1,
};
for (method, target, expected_status, destination) in [
("GET", "/old", "301", Some("/guide/start")),
("HEAD", "/old.html", "301", Some("/guide/start")),
(
"GET",
"/old?from=old&q=a%20b",
"301",
Some("/guide/start?from=old&q=a%20b"),
),
("GET", "/caf%C3%A9%20old/", "301", Some("/guide/start")),
("GET", "/old%2dpath", "301", Some("/guide/start")),
("GET", "/old&path", "301", Some("/guide/start")),
("GET", "/%6Fld", "404", None),
("GET", "/old-path", "404", None),
("GET", "/old/", "404", None),
("GET", "/guide/start", "200", None),
("GET", "/missing", "404", None),
("POST", "/old", "405", None),
] {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let mut client = TcpStream::connect(listener.local_addr().unwrap())
.await
.unwrap();
let (socket, _) = listener.accept().await.unwrap();
let server = response(socket, &generation, true);
let client = async {
client.write_all(format!("{method} {target} HTTP/1.1\r\nHost: localhost\r\nAccept: text/markdown\r\n\r\n").as_bytes()).await.unwrap();
let mut bytes = Vec::new();
client.read_to_end(&mut bytes).await.unwrap();
String::from_utf8(bytes).unwrap()
};
let (result, response) = tokio::join!(server, client);
result.unwrap();
let (headers, body) = response.split_once("\r\n\r\n").unwrap();
assert!(
headers.starts_with(&format!("HTTP/1.1 {expected_status}")),
"{method} {target}: {response}"
);
if let Some(destination) = destination {
assert!(
headers.contains(&format!("\r\nLocation: {destination}\r\n")),
"{headers}"
);
assert!(body.is_empty());
} else {
assert!(!headers.contains("Location:"));
}
}
}
}