pub mod commerce;
pub mod dns_aid;
pub(crate) mod documents;
mod entities;
pub mod markdown;
pub(crate) mod negotiate;
pub mod robots;
pub mod web_bot_auth;
pub mod webmcp;
use std::sync::Arc;
use axum::body::Body;
use axum::http::{HeaderValue, Method, Request};
use axum::response::{IntoResponse as _, Response};
use serde::Deserialize;
pub use documents::{AgentSkill, AgentSkillError};
pub use robots::{AI_SEARCH_CRAWLERS, AI_TRAINING_CRAWLERS, AI_USER_FETCHERS, BotPolicy};
#[derive(Debug, Clone, Deserialize)]
#[non_exhaustive]
#[allow(clippy::struct_excessive_bools)] pub struct AeoConfig {
#[serde(default = "default_true")]
pub enabled: bool,
#[serde(default)]
pub content_signals: ContentSignalsConfig,
#[serde(default)]
pub ai_crawlers: AiCrawlersConfig,
#[serde(default)]
pub site_name: Option<String>,
#[serde(default)]
pub site_description: Option<String>,
#[serde(default = "default_true")]
pub markdown: bool,
#[serde(default = "default_markdown_max_bytes")]
pub markdown_max_bytes: usize,
#[serde(default = "default_true")]
pub link_headers: bool,
#[serde(default = "default_true")]
pub llms_txt: bool,
#[serde(default = "default_true")]
pub site_guide_skill: bool,
#[serde(default = "default_true")]
pub publish_tools: bool,
#[serde(default)]
pub auth_md: AuthMdConfig,
#[serde(default)]
pub oauth: OAuthConfig,
#[serde(default)]
pub web_bot_auth: web_bot_auth::WebBotAuthConfig,
#[serde(default)]
pub x402: commerce::X402Config,
#[serde(default)]
pub paid_routes: Vec<commerce::PaidRoute>,
}
impl AeoConfig {
pub fn validate(&self) -> Result<(), String> {
if !self.enabled {
return Ok(());
}
for route in self.paid_routes.iter().filter(|r| r.is_x402()) {
if let Some(problem) = commerce::unmatchable_route(route) {
return Err(format!("[[aeo.paid_routes]] {problem}"));
}
}
if let Some(problem) = commerce::mixed_protocol_route(&self.paid_routes) {
return Err(format!("[[aeo.paid_routes]] {problem}"));
}
Ok(())
}
}
impl Default for AeoConfig {
fn default() -> Self {
Self {
enabled: true,
content_signals: ContentSignalsConfig::default(),
ai_crawlers: AiCrawlersConfig::default(),
site_name: None,
site_description: None,
markdown: true,
markdown_max_bytes: default_markdown_max_bytes(),
link_headers: true,
llms_txt: true,
site_guide_skill: true,
publish_tools: true,
auth_md: AuthMdConfig::default(),
oauth: OAuthConfig::default(),
web_bot_auth: web_bot_auth::WebBotAuthConfig::default(),
x402: commerce::X402Config::default(),
paid_routes: Vec::new(),
}
}
}
#[derive(Debug, Clone, Deserialize)]
#[non_exhaustive]
pub struct AuthMdConfig {
#[serde(default = "default_true")]
pub enabled: bool,
#[serde(default)]
pub registration_url: Option<String>,
#[serde(default)]
pub instructions: Option<String>,
}
impl Default for AuthMdConfig {
fn default() -> Self {
Self {
enabled: true,
registration_url: None,
instructions: None,
}
}
}
#[derive(Debug, Clone, Default, Deserialize)]
#[non_exhaustive]
pub struct OAuthConfig {
#[serde(default)]
pub authorization_servers: Vec<String>,
#[serde(default)]
pub scopes_supported: Vec<String>,
#[serde(default)]
pub authorization_server: AuthorizationServerConfig,
}
#[derive(Debug, Clone, Default, Deserialize)]
#[non_exhaustive]
pub struct AuthorizationServerConfig {
#[serde(default)]
pub issuer: Option<String>,
#[serde(default)]
pub authorization_endpoint: Option<String>,
#[serde(default)]
pub token_endpoint: Option<String>,
#[serde(default)]
pub jwks_uri: Option<String>,
#[serde(default)]
pub registration_endpoint: Option<String>,
#[serde(default)]
pub scopes_supported: Vec<String>,
#[serde(default)]
pub response_types_supported: Vec<String>,
#[serde(default)]
pub grant_types_supported: Vec<String>,
#[serde(default)]
pub code_challenge_methods_supported: Vec<String>,
#[serde(default)]
pub agent_identity_endpoint: Option<String>,
#[serde(default)]
pub agent_claim_endpoint: Option<String>,
#[serde(default)]
pub agent_identity_types: Vec<String>,
#[serde(default)]
pub agent_credential_types: Vec<String>,
#[serde(default)]
pub agent_assertion_types: Vec<String>,
#[serde(default)]
pub agent_revocation_endpoint: Option<String>,
#[serde(default)]
pub agent_events_endpoint: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[non_exhaustive]
#[allow(clippy::struct_excessive_bools)] pub struct ContentSignalsConfig {
#[serde(default = "default_true")]
pub enabled: bool,
#[serde(default = "default_true")]
pub search: bool,
#[serde(default = "default_true")]
pub ai_input: bool,
#[serde(default)]
pub ai_train: bool,
}
impl Default for ContentSignalsConfig {
fn default() -> Self {
Self {
enabled: true,
search: true,
ai_input: true,
ai_train: false,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[non_exhaustive]
pub struct AiCrawlersConfig {
#[serde(default)]
pub search: CrawlerAccess,
#[serde(default)]
pub user_fetch: CrawlerAccess,
#[serde(default)]
pub training: CrawlerAccess,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "lowercase")]
#[non_exhaustive]
pub enum CrawlerAccess {
#[default]
Allow,
Disallow,
}
#[derive(Debug, Clone, Default)]
pub(crate) struct RegisteredAgentSkills(pub Vec<AgentSkill>);
#[derive(Debug, Clone, Default)]
pub(crate) struct RegisteredCommerceDocs {
pub(crate) ucp: Option<commerce::UcpProfile>,
pub(crate) acp: Option<commerce::AcpDiscovery>,
}
#[derive(Debug, Clone, Default)]
pub(crate) struct AeoSite {
pub(crate) enabled: bool,
pub(crate) base_url: Option<String>,
pub(crate) facts: documents::SiteFacts,
pub(crate) negotiate: negotiate::NegotiateConfig,
}
impl AeoSite {
#[must_use]
pub(crate) fn new(
config: &crate::config::AutumnConfig,
mut facts: documents::SiteFacts,
) -> Self {
let aeo = &config.aeo;
if aeo.site_name.is_some() {
facts.name.clone_from(&aeo.site_name);
}
if aeo.site_description.is_some() {
facts.description.clone_from(&aeo.site_description);
}
facts.site_guide_skill = aeo.site_guide_skill;
facts.llms_txt = aeo.llms_txt;
facts.markdown = aeo.markdown;
facts.auth_md = aeo.auth_md.clone();
facts.oauth = aeo.oauth.clone();
facts.prefixed = !base_path(config).is_empty();
facts
.csrf_header
.clone_from(&config.security.csrf.token_header);
facts.csrf_required = facts
.mcp
.as_ref()
.is_some_and(|mcp| csrf_checks_post(config, &mcp.path));
if !aeo.publish_tools
&& let Some(mcp) = facts.mcp.as_mut()
{
mcp.public_tools = false;
mcp.tools.clear();
}
facts.robots_txt = aeo.enabled.then(|| default_robots_txt(config));
facts.sitemap = aeo.enabled;
warn_on_config(config);
let wba_key = aeo
.enabled
.then(|| {
web_bot_auth::WebBotAuthKey::from_config(&aeo.web_bot_auth, &crate::config::OsEnv)
})
.flatten();
facts.web_bot_auth = match wba_key {
Some(Ok(key)) => Some(key),
Some(Err(err)) => {
tracing::warn!(error = %err, "aeo: Web Bot Auth key not loaded");
None
}
None => None,
};
let base_path = base_path(config);
let home_links: Vec<(String, HeaderValue)> = if aeo.link_headers && !facts.no_home {
let homes = if facts.home_paths.is_empty() {
vec!["/".to_owned()]
} else {
facts.home_paths.clone()
};
homes
.into_iter()
.filter_map(|home| {
let link = documents::homepage_link_header(&facts, &home, &base_path);
HeaderValue::from_str(&link).ok().map(|v| (home, v))
})
.collect()
} else {
Vec::new()
};
let content_signal = robots::wildcard_signal(&config.seo.robots.additional_rules)
.map(ToOwned::to_owned)
.or_else(|| {
aeo.content_signals
.enabled
.then(|| robots::content_signal_value(aeo.content_signals))
})
.and_then(|value| HeaderValue::from_str(&value).ok());
let configured = documents::Origin::resolve(config.seo.base_url.as_deref(), None);
let prefixed = facts.prefixed;
let resource_metadata =
(!aeo.oauth.authorization_servers.is_empty() && configured.configured && !prefixed)
.then(|| {
HeaderValue::from_str(&format!(
"Bearer resource_metadata=\"{}\"",
configured.url(documents::OAUTH_RESOURCE_PATH)
))
.ok()
})
.flatten();
Self {
enabled: aeo.enabled,
base_url: config.seo.base_url.clone(),
facts,
negotiate: negotiate::NegotiateConfig {
markdown: aeo.markdown,
max_bytes: aeo.markdown_max_bytes,
home_links,
content_signal,
resource_metadata_from_host: !aeo.oauth.authorization_servers.is_empty()
&& !prefixed
&& resource_metadata.is_none(),
resource_metadata,
},
}
}
#[must_use]
pub(crate) fn web_bot_auth_signer(
&self,
config: &crate::config::AutumnConfig,
) -> Option<web_bot_auth::WebBotAuthSigner> {
if !self.enabled {
return None;
}
let key = self.facts.web_bot_auth.clone()?;
let wba = &config.aeo.web_bot_auth;
let fallback = wba.signature_agent.is_none();
let agent = wba
.signature_agent
.clone()
.or_else(|| self.base_url.clone())?;
if fallback && url::Url::parse(agent.trim()).is_ok_and(|u| !matches!(u.path(), "" | "/")) {
tracing::warn!(
base_url = %agent,
"aeo: [seo] base_url has a path, so verifiers cannot find the key \
directory at its origin; set [aeo.web_bot_auth] signature_agent. \
Requests are not signed"
);
return None;
}
if !url::Url::parse(agent.trim()).is_ok_and(|u| u.scheme() == "https" && u.has_host()) {
tracing::warn!(
signature_agent = %agent,
"aeo: Signature-Agent is not an https URL; requests are not signed"
);
return None;
}
let signer = web_bot_auth::WebBotAuthSigner::new(key, agent.trim());
Some(match wba.expires_secs {
Some(secs) => signer.expires_secs(secs),
None => signer,
})
}
#[must_use]
pub(crate) fn layer(&self) -> Option<negotiate::NegotiateLayer> {
self.enabled
.then(|| negotiate::NegotiateLayer::new(self.negotiate.clone()))
}
}
#[cfg(test)]
pub(crate) async fn fallback(site: Option<Arc<AeoSite>>, req: Request<Body>) -> Response {
match serve_document(site, &req) {
Some(res) => res,
None => {
crate::middleware::error_page_filter::fallback_404_handler(
req.method().clone(),
req.uri().clone(),
)
.await
}
}
}
pub(crate) fn serve_document(site: Option<Arc<AeoSite>>, req: &Request<Body>) -> Option<Response> {
let method = req.method().clone();
let uri = req.uri().clone();
let readable = method == Method::GET || method == Method::HEAD;
if let Some(site) = site.filter(|s| s.enabled)
&& (readable || method == Method::OPTIONS)
&& documents::is_document_path(uri.path())
{
let (origin, host) = documents::Origin::for_request(site.base_url.as_deref(), req);
let host = host.as_deref();
if let Some(mut doc) = documents::render(&site.facts, &origin, uri.path()) {
if uri.path() == web_bot_auth::DIRECTORY_PATH && !origin.is_request_authority(host) {
doc.headers
.retain(|(name, _)| !matches!(*name, "signature" | "signature-input"));
}
return Some(document_response(doc, &method, req.headers()));
}
}
None
}
#[derive(Debug, Clone, Copy)]
pub(crate) struct AeoDocument;
#[cfg_attr(not(feature = "i18n"), allow(dead_code))]
pub(crate) fn localize_pages(
pages: Vec<documents::PageFacts>,
locales: &[String],
exclude_prefixes: &[String],
exclude_exact: &[String],
) -> Vec<documents::PageFacts> {
if locales.is_empty() {
return pages;
}
let mut out = Vec::with_capacity(pages.len() * locales.len());
for page in pages {
if exclude_exact.contains(&page.path)
|| crate::seo::matches_locale_exclude_prefix(&page.path, exclude_prefixes)
{
out.push(page);
continue;
}
for locale in locales {
let path = if page.path == "/" {
format!("/{locale}")
} else {
format!("/{locale}{}", page.path)
};
out.push(documents::PageFacts {
path,
..page.clone()
});
}
}
out
}
fn document_response(
doc: documents::Document,
method: &Method,
request_headers: &axum::http::HeaderMap,
) -> Response {
use axum::http::{StatusCode, header};
let etag = format!(
"\"{}\"",
&documents::sha256_digest(doc.body.as_bytes())["sha256:".len()..][..32]
);
let not_modified = request_headers
.get_all(header::IF_NONE_MATCH)
.iter()
.filter_map(|v| v.to_str().ok())
.flat_map(|v| v.split(','))
.map(|t| t.trim().trim_start_matches("W/"))
.any(|t| t == etag || t == "*");
let mut res = if *method == Method::OPTIONS {
if !doc
.headers
.iter()
.any(|(k, _)| *k == "access-control-allow-origin")
{
return StatusCode::NOT_FOUND.into_response();
}
StatusCode::NO_CONTENT.into_response()
} else if not_modified {
StatusCode::NOT_MODIFIED.into_response()
} else {
let len = doc.body.len();
let mut res = if *method == Method::HEAD {
axum::body::Body::empty().into_response()
} else {
doc.body.into_response()
};
res.headers_mut().insert(
header::CONTENT_TYPE,
HeaderValue::from_static(doc.content_type),
);
res.headers_mut()
.insert(header::CONTENT_LENGTH, HeaderValue::from(len));
res
};
let headers = res.headers_mut();
if let Ok(v) = HeaderValue::from_str(&etag) {
headers.insert(header::ETAG, v);
}
for (name, value) in doc.headers {
if let Ok(value) = HeaderValue::from_str(&value) {
headers.append(name, value);
}
}
res.extensions_mut().insert(AeoDocument);
res
}
fn default_robots_txt(config: &crate::config::AutumnConfig) -> String {
robots::robots_txt_with_policy(robots_profile(config), None, &[], &robots_policy(config))
}
#[must_use]
pub(crate) fn robots_profile(config: &crate::config::AutumnConfig) -> &'static str {
match config.profile.as_deref() {
Some("dev" | "test") | None => "dev",
Some(_) => "prod",
}
}
fn warn_on_config(config: &crate::config::AutumnConfig) {
let aeo = &config.aeo;
if !aeo.enabled {
return;
}
let prod = matches!(config.profile.as_deref(), Some("prod" | "production"));
if let Some(base) = config.seo.base_url.as_deref()
&& !documents::Origin::resolve(Some(base), None).configured
{
tracing::warn!(
base_url = base,
"aeo: [seo] base_url is not an http(s) URL without credentials, a query or a fragment; the \
agent documents use the request Host instead, and a static build writes none"
);
}
if prod && config.seo.base_url.is_none() {
tracing::warn!(
"aeo: set [seo] base_url in production; without it the agent documents use \
the request Host and are not cached"
);
}
for problem in commerce::paid_route_problems(&aeo.paid_routes) {
tracing::warn!(
"aeo: [[aeo.paid_routes]] {problem}; an x402 route answers 503, and the route \
is not in the OpenAPI payment info"
);
}
if !aeo.oauth.authorization_servers.is_empty() && !base_path(config).is_empty() {
tracing::warn!(
"aeo: [aeo.oauth] authorization_servers needs an app served at the host root: \
[seo] base_url has a path, and RFC 9728 discovery asks the root; Autumn does \
not publish the protected resource metadata"
);
}
if let Some(issuer) = aeo.oauth.authorization_server.issuer.as_deref() {
let origin =
documents::Origin::resolve(config.seo.base_url.as_deref().or(Some(issuer)), None);
let facts = documents::SiteFacts {
oauth: aeo.oauth.clone(),
..documents::SiteFacts::default()
};
if !base_path(config).is_empty() {
tracing::warn!(
issuer,
"aeo: [aeo.oauth.authorization_server] needs an app served at the host \
root: [seo] base_url has a path, and RFC 8414 discovery asks the root; \
Autumn does not publish its metadata"
);
} else if documents::render(&facts, &origin, documents::OAUTH_SERVER_PATH).is_none() {
tracing::warn!(
issuer,
"aeo: [aeo.oauth.authorization_server] issuer must be this site's origin \
with no path; Autumn does not publish its metadata"
);
}
}
}
pub const AI_CATALOG_PATH: &str = "/.well-known/ai-catalog.json";
const STATIC_DOCUMENT_PATHS: &[&str] = &[
documents::LLMS_TXT_PATH,
documents::SKILLS_INDEX_PATH,
AI_CATALOG_PATH,
documents::ARD_PATH,
documents::API_CATALOG_PATH,
documents::SERVER_CARD_PATH,
documents::OAUTH_RESOURCE_PATH,
documents::OAUTH_SERVER_PATH,
documents::AUTH_MD_PATH,
commerce::UCP_PATH,
commerce::ACP_PATH,
];
pub(crate) async fn write_static_documents(
router: axum::Router,
base_url: Option<&str>,
home_links: &[(String, HeaderValue)],
dist: &std::path::Path,
) -> std::io::Result<Vec<String>> {
use tower::ServiceExt as _;
let configured = documents::Origin::resolve(base_url, None).configured;
let mut queue: Vec<String> = if configured {
STATIC_DOCUMENT_PATHS
.iter()
.map(|p| (*p).to_owned())
.collect()
} else {
Vec::new()
};
queue.push(webmcp::WEBMCP_JS_PATH.to_owned());
let mut written = Vec::new();
let base = documents::Origin::resolve(base_url, None).base;
let mut index = None;
let mut host_headers: Vec<(String, Vec<(String, String)>)> = Vec::new();
while let Some(path) = queue.pop() {
let Ok(req) = Request::get(path.as_str())
.extension(crate::static_gen::RenderDeadlineExempt)
.body(Body::empty())
else {
continue;
};
let Ok(res) = router.clone().oneshot(req).await;
if res.status() != axum::http::StatusCode::OK
|| res.extensions().get::<AeoDocument>().is_none()
{
continue;
}
let headers = static_host_headers(res.headers());
let Ok(body) = axum::body::to_bytes(res.into_body(), usize::MAX).await else {
continue;
};
if path == documents::SKILLS_INDEX_PATH {
queue.extend(skill_paths(&body, &base));
index = Some((body, headers));
continue;
}
let file = dist.join(path.trim_start_matches('/'));
if tokio::fs::try_exists(&file).await.unwrap_or(false) {
continue;
}
if let Some(parent) = file.parent() {
tokio::fs::create_dir_all(parent).await?;
}
tokio::fs::write(&file, &body).await?;
if path != webmcp::WEBMCP_JS_PATH {
host_headers.push((path.clone(), headers));
}
written.push(path);
}
if let Some((body, headers)) = index {
let file = dist.join(documents::SKILLS_INDEX_PATH.trim_start_matches('/'));
if !tokio::fs::try_exists(&file).await.unwrap_or(false) {
let body = index_with_disk_digests(&body, &base, dist).await;
if let Some(parent) = file.parent() {
tokio::fs::create_dir_all(parent).await?;
}
tokio::fs::write(&file, &body).await?;
host_headers.push((documents::SKILLS_INDEX_PATH.to_owned(), headers));
written.push(documents::SKILLS_INDEX_PATH.to_owned());
}
}
if configured {
for (home, link) in home_links {
let Some(link) = link.to_str().ok().and_then(static_home_link) else {
continue;
};
if static_page_exists(dist, home).await {
host_headers.push((home.clone(), vec![("link".to_owned(), link)]));
}
}
}
if !host_headers.is_empty() {
let file = dist.join(STATIC_HEADERS_FILE);
if tokio::fs::try_exists(&file).await.unwrap_or(false) {
tracing::warn!(
"aeo: dist/{STATIC_HEADERS_FILE} exists; the agent documents' headers are not added to it"
);
} else {
tokio::fs::write(&file, headers_file(&host_headers)).await?;
written.push(format!("/{STATIC_HEADERS_FILE}"));
}
}
written.sort();
Ok(written)
}
async fn index_with_disk_digests(index: &[u8], base: &str, dist: &std::path::Path) -> Vec<u8> {
let Ok(mut value) = serde_json::from_slice::<serde_json::Value>(index) else {
return index.to_vec();
};
let mut changed = false;
if let Some(skills) = value.get_mut("skills").and_then(|s| s.as_array_mut()) {
for skill in skills {
let Some(path) = skill
.get("url")
.and_then(|u| u.as_str())
.and_then(|u| skill_path(u, base))
else {
continue;
};
let Ok(bytes) = tokio::fs::read(dist.join(path.trim_start_matches('/'))).await else {
continue;
};
let digest = serde_json::Value::String(documents::sha256_digest(&bytes));
if skill.get("digest") != Some(&digest) {
skill["digest"] = digest;
changed = true;
}
}
}
if changed {
serde_json::to_vec_pretty(&value).unwrap_or_else(|_| index.to_vec())
} else {
index.to_vec()
}
}
#[must_use]
pub fn base_path(config: &crate::config::AutumnConfig) -> String {
documents::Origin::resolve(config.seo.base_url.as_deref(), None)
.configured
.then_some(config.seo.base_url.as_deref())
.flatten()
.and_then(|base| url::Url::parse(base.trim()).ok())
.map(|url| url.path().trim_end_matches('/').to_owned())
.unwrap_or_default()
}
const STATIC_HEADERS_FILE: &str = "_headers";
fn static_host_headers(headers: &axum::http::HeaderMap) -> Vec<(String, String)> {
headers
.iter()
.filter(|(name, _)| {
let name = name.as_str();
name == "content-type"
|| name == "cache-control"
|| name == "link"
|| name.starts_with("access-control-")
})
.filter_map(|(name, value)| {
Some((name.as_str().to_owned(), value.to_str().ok()?.to_owned()))
})
.collect()
}
fn static_home_link(link: &str) -> Option<String> {
let kept: Vec<String> = link
.split(", <")
.enumerate()
.map(|(i, entry)| {
if i == 0 {
entry.to_owned()
} else {
format!("<{entry}")
}
})
.filter(|entry| !entry.contains("type=\"text/markdown\""))
.collect();
(!kept.is_empty()).then(|| kept.join(", "))
}
async fn static_page_exists(dist: &std::path::Path, path: &str) -> bool {
let page = dist.join(path.trim_matches('/')).join("index.html");
tokio::fs::try_exists(page).await.unwrap_or(false)
}
fn headers_file(files: &[(String, Vec<(String, String)>)]) -> String {
use std::fmt::Write as _;
let mut out =
String::from("# Written by `autumn build`: the agent documents' response headers.\n");
for (path, headers) in files {
if headers.is_empty() {
continue;
}
let _ = writeln!(out, "\n{path}");
for (name, value) in headers {
let _ = writeln!(out, " {name}: {value}");
}
}
out
}
fn skill_paths(index: &[u8], base: &str) -> Vec<String> {
serde_json::from_slice::<serde_json::Value>(index)
.ok()
.and_then(|v| v.get("skills")?.as_array().cloned())
.unwrap_or_default()
.iter()
.filter_map(|s| skill_path(s.get("url")?.as_str()?, base))
.map(str::to_owned)
.collect()
}
fn skill_path<'a>(url: &'a str, base: &str) -> Option<&'a str> {
let path = url.strip_prefix(base)?;
(path.starts_with('/') && !path.starts_with("//") && !path.contains("..")).then_some(path)
}
#[must_use]
pub fn robots_policy(config: &crate::config::AutumnConfig) -> BotPolicy {
let agentmap = config
.aeo
.enabled
.then_some(config.seo.base_url.as_deref())
.flatten()
.map(|base| documents::Origin::resolve(Some(base), None))
.filter(|origin| origin.configured)
.map(|origin| origin.url(AI_CATALOG_PATH));
BotPolicy::from_config(&config.aeo).agentmap(agentmap)
}
fn csrf_checks_post(config: &crate::config::AutumnConfig, path: &str) -> bool {
let csrf = &config.security.csrf;
let exempt: Vec<String> = csrf
.exempt_paths
.iter()
.cloned()
.chain(
config
.security
.webhooks
.endpoints
.iter()
.map(|e| e.path.clone()),
)
.collect();
csrf.enabled
&& !csrf
.safe_methods
.iter()
.any(|m| m.eq_ignore_ascii_case("POST"))
&& !crate::security::path::is_exempt_path(path, &exempt)
}
const fn default_markdown_max_bytes() -> usize {
2 * 1024 * 1024
}
const fn default_true() -> bool {
true
}
#[cfg(test)]
mod tests {
use super::*;
fn site_router(config: &crate::config::AutumnConfig) -> axum::Router {
let site = Arc::new(AeoSite::new(config, documents::SiteFacts::default()));
axum::Router::new().fallback(move |req| fallback(Some(Arc::clone(&site)), req))
}
#[test]
fn a_prefixed_app_publishes_no_oauth_metadata() {
let mut config = crate::config::AutumnConfig::default();
config.aeo.oauth.authorization_server.issuer = Some("https://example.com".to_owned());
config.aeo.oauth.authorization_server.token_endpoint =
Some("https://example.com/token".to_owned());
let published = |config: &crate::config::AutumnConfig| {
let site = AeoSite::new(config, documents::SiteFacts::default());
let origin = documents::Origin::resolve(config.seo.base_url.as_deref(), None);
documents::render(&site.facts, &origin, documents::OAUTH_SERVER_PATH).is_some()
};
config.seo.base_url = Some("https://example.com".to_owned());
assert!(published(&config));
config.seo.base_url = Some("https://example.com/shop".to_owned());
assert!(!published(&config));
config.aeo.oauth.authorization_servers = vec!["https://auth.example".to_owned()];
let site = AeoSite::new(&config, documents::SiteFacts::default());
let origin = documents::Origin::resolve(config.seo.base_url.as_deref(), None);
assert!(documents::render(&site.facts, &origin, documents::OAUTH_RESOURCE_PATH).is_none());
assert!(site.negotiate.resource_metadata.is_none());
assert!(!site.negotiate.resource_metadata_from_host);
config.seo.base_url = Some("https://example.com".to_owned());
let site = AeoSite::new(&config, documents::SiteFacts::default());
let origin = documents::Origin::resolve(config.seo.base_url.as_deref(), None);
assert!(documents::render(&site.facts, &origin, documents::OAUTH_RESOURCE_PATH).is_some());
assert!(site.negotiate.resource_metadata.is_some());
}
#[tokio::test]
async fn a_static_build_writes_the_agent_documents() {
let mut config = crate::config::AutumnConfig::default();
config.seo.base_url = Some("https://example.com".to_owned());
let dist = tempfile::tempdir().unwrap();
std::fs::write(dist.path().join("llms.txt"), "mine").unwrap();
let written = write_static_documents(
site_router(&config),
config.seo.base_url.as_deref(),
&[],
dist.path(),
)
.await
.unwrap();
let read = |p: &str| std::fs::read_to_string(dist.path().join(p)).unwrap();
assert_eq!(read("llms.txt"), "mine", "a file in dist stays");
assert!(!written.iter().any(|p| p == "/llms.txt"), "{written:?}");
assert!(read(".well-known/ai-catalog.json").contains("https://example.com"));
let index: serde_json::Value =
serde_json::from_str(&read(".well-known/agent-skills/index.json")).unwrap();
let skill = index["skills"][0]["url"].as_str().unwrap();
let skill = skill
.strip_prefix("https://example.com/")
.expect("absolute");
assert!(read(skill).starts_with("---\n"));
assert!(!dist.path().join(".well-known/api-catalog").exists());
assert!(written.iter().any(|p| p == "/_headers"), "{written:?}");
let headers = read("_headers");
let index_entry = headers
.split("\n\n")
.find(|e| e.starts_with("/.well-known/agent-skills/index.json\n"))
.unwrap_or_else(|| panic!("{headers}"));
for want in [
" content-type: application/json",
" access-control-allow-origin: *",
" access-control-expose-headers: ETag",
] {
assert!(index_entry.contains(want), "{want}:\n{headers}");
}
assert!(!headers.contains("/_autumn/webmcp.js"), "{headers}");
}
#[tokio::test]
async fn a_static_home_page_keeps_its_link_header() {
let mut config = crate::config::AutumnConfig::default();
config.seo.base_url = Some("https://example.com".to_owned());
let dist = tempfile::tempdir().unwrap();
std::fs::write(dist.path().join("index.html"), "<h1>Home</h1>").unwrap();
let link = HeaderValue::from_static(
"</llms.txt>; rel=\"describedby\"; type=\"text/plain\", \
</>; rel=\"alternate\"; type=\"text/markdown\"",
);
let homes = [
("/".to_owned(), link.clone()),
("/fr".to_owned(), link),
];
write_static_documents(
site_router(&config),
config.seo.base_url.as_deref(),
&homes,
dist.path(),
)
.await
.unwrap();
let headers = std::fs::read_to_string(dist.path().join("_headers")).unwrap();
assert!(
headers
.contains("\n/\n link: </llms.txt>; rel=\"describedby\"; type=\"text/plain\"\n"),
"{headers}"
);
assert!(!headers.contains("rel=\"alternate\""), "{headers}");
assert!(!headers.contains("\n/fr\n"), "{headers}");
}
#[test]
fn the_markdown_signal_follows_the_robots_wildcard_signal() {
let mut config = crate::config::AutumnConfig::default();
let site = AeoSite::new(&config, documents::SiteFacts::default());
assert_eq!(
site.negotiate.content_signal.as_ref().unwrap(),
robots::content_signal_value(config.aeo.content_signals).as_str()
);
config.seo.robots.additional_rules =
vec!["Content-Signal: search=yes, ai-input=no, ai-train=yes".to_owned()];
let site = AeoSite::new(&config, documents::SiteFacts::default());
assert_eq!(
site.negotiate.content_signal.as_ref().unwrap(),
"search=yes, ai-input=no, ai-train=yes"
);
}
#[tokio::test]
async fn a_static_headers_file_matches_the_files_in_dist() {
let mut config = crate::config::AutumnConfig::default();
config.seo.base_url = Some("https://example.com/shop".to_owned());
let dist = tempfile::tempdir().unwrap();
write_static_documents(
site_router(&config),
config.seo.base_url.as_deref(),
&[],
dist.path(),
)
.await
.unwrap();
let headers = std::fs::read_to_string(dist.path().join("_headers")).unwrap();
let entry = "/.well-known/agent-skills/index.json";
assert!(headers.contains(&format!("\n{entry}\n")), "{headers}");
assert!(
dist.path().join(entry.trim_start_matches('/')).exists(),
"{headers}"
);
assert!(!headers.contains("/shop/"), "{headers}");
}
#[tokio::test]
async fn a_static_build_keeps_the_apps_own_headers_file() {
let mut config = crate::config::AutumnConfig::default();
config.seo.base_url = Some("https://example.com".to_owned());
let dist = tempfile::tempdir().unwrap();
std::fs::write(dist.path().join("_headers"), "/*\n X-Mine: 1\n").unwrap();
let written = write_static_documents(
site_router(&config),
config.seo.base_url.as_deref(),
&[],
dist.path(),
)
.await
.unwrap();
assert!(!written.iter().any(|p| p == "/_headers"), "{written:?}");
assert_eq!(
std::fs::read_to_string(dist.path().join("_headers")).unwrap(),
"/*\n X-Mine: 1\n"
);
}
#[tokio::test]
async fn a_kept_skill_file_sets_its_index_digest() {
let mut config = crate::config::AutumnConfig::default();
config.seo.base_url = Some("https://example.com".to_owned());
let dist = tempfile::tempdir().unwrap();
let kept = dist
.path()
.join(".well-known/agent-skills/site-guide/SKILL.md");
std::fs::create_dir_all(kept.parent().unwrap()).unwrap();
std::fs::write(&kept, "---\nname: site-guide\ndescription: mine\n---\n").unwrap();
write_static_documents(
site_router(&config),
config.seo.base_url.as_deref(),
&[],
dist.path(),
)
.await
.unwrap();
let index: serde_json::Value = serde_json::from_slice(
&std::fs::read(dist.path().join(".well-known/agent-skills/index.json")).unwrap(),
)
.unwrap();
let entry = index["skills"]
.as_array()
.unwrap()
.iter()
.find(|s| s["name"] == "site-guide")
.unwrap();
assert_eq!(
entry["digest"],
documents::sha256_digest(&std::fs::read(&kept).unwrap()),
"the index names the bytes served"
);
}
#[tokio::test]
async fn a_static_build_without_a_usable_base_url_writes_only_the_script() {
let config = crate::config::AutumnConfig::default();
let dist = tempfile::tempdir().unwrap();
for base in [
None,
Some("https://example.com?tenant=a"),
Some("ftp://example.com"),
] {
let written = write_static_documents(site_router(&config), base, &[], dist.path())
.await
.unwrap();
assert!(
written.iter().all(|p| p == webmcp::WEBMCP_JS_PATH),
"{base:?}: {written:?}"
);
}
assert!(!dist.path().join("llms.txt").exists());
assert!(dist.path().join("_autumn/webmcp.js").exists());
}
#[test]
fn every_profile_but_dev_and_test_is_public() {
let mut config = crate::config::AutumnConfig::default();
for (profile, want) in [
(None, "dev"),
(Some("dev"), "dev"),
(Some("test"), "dev"),
(Some("prod"), "prod"),
(Some("live"), "prod"),
] {
config.profile = profile.map(str::to_owned);
assert_eq!(robots_profile(&config), want, "{profile:?}");
}
}
#[test]
fn csrf_checks_a_post_to_mcp_unless_it_is_exempt() {
let mut config = crate::config::AutumnConfig::default();
config.security.csrf.enabled = false;
assert!(!csrf_checks_post(&config, "/mcp"));
config.security.csrf.enabled = true;
assert!(csrf_checks_post(&config, "/mcp"));
config.security.csrf.exempt_paths = vec!["/mcp".to_owned()];
assert!(!csrf_checks_post(&config, "/mcp"));
config.security.csrf.exempt_paths.clear();
config.security.csrf.safe_methods.push("post".to_owned());
assert!(!csrf_checks_post(&config, "/mcp"));
}
#[test]
fn localized_pages_get_one_entry_per_locale() {
let page = |path: &str| documents::PageFacts {
path: path.to_owned(),
title: "T".to_owned(),
description: None,
};
let locales = ["en".to_owned(), "fr".to_owned()];
let out = localize_pages(
vec![page("/"), page("/about"), page("/api/docs"), page("/legal")],
&locales,
&["/api".to_owned()],
&["/legal".to_owned()],
);
let paths: Vec<&str> = out.iter().map(|p| p.path.as_str()).collect();
assert_eq!(
paths,
[
"/en",
"/fr",
"/en/about",
"/fr/about",
"/api/docs",
"/legal"
]
);
assert_eq!(localize_pages(vec![page("/x")], &[], &[], &[]).len(), 1);
}
#[tokio::test]
async fn a_static_build_needs_a_base_url() {
let config = crate::config::AutumnConfig::default();
let dist = tempfile::tempdir().unwrap();
let written = write_static_documents(site_router(&config), None, &[], dist.path())
.await
.unwrap();
assert_eq!(written, [webmcp::WEBMCP_JS_PATH], "{written:?}");
}
}