use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, AtomicI64, Ordering};
use std::sync::Arc;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use secrecy::ExposeSecret;
use crate::cloud::{CloudState, CredentialProvider};
use crate::config::PolicyConfig;
use crate::core::error::{
ERR_BUNDLE_FETCH_FAILED, ERR_BUNDLE_INVALID, ERR_BUNDLE_REJECTED, ERR_BUNDLE_STALE,
};
use crate::core::policy::{store, PolicyHandle, ResidentBundle};
use crate::generated::types::PolicyBundle;
pub const SUPPORTED_SCHEMA_VERSION: i64 = 1;
const BUNDLE_ENDPOINT: &str = "/api/v1/policy/bundle";
const AGENT_ID_HEADER: &str = "X-OpenLatch-Agent-Id";
const MAX_RETRY_AFTER_SECS: u64 = 3_600;
const MIN_POLL_INTERVAL_SECS: u64 = 30;
#[derive(Debug, Clone, PartialEq, Eq)]
enum PollOutcome {
Activated { revision: i64 },
NotModified,
Rejected(&'static str),
NoBundle,
AuthFailed,
RateLimited(Option<Duration>),
Failed,
Skipped(&'static str),
}
#[allow(clippy::too_many_arguments)]
pub async fn run_policy_poller(
handle: PolicyHandle,
last_fetch_ok: Arc<AtomicBool>,
last_poll_ok_at: Arc<AtomicI64>,
cloud_state: CloudState,
credentials: Arc<dyn CredentialProvider>,
api_url: String,
policy_config: PolicyConfig,
base_dir: PathBuf,
http_client: reqwest::Client,
agent_id: Option<String>,
) {
let mut poller = PolicyPoller::new(
handle,
last_fetch_ok,
last_poll_ok_at,
cloud_state,
credentials,
api_url,
policy_config,
base_dir,
http_client,
agent_id,
);
poller.seed_poll_clock();
let mut outcome = poller.poll_once().await;
poller.check_staleness();
loop {
let mut delay = jittered(poller.config.poll_interval_secs.max(MIN_POLL_INTERVAL_SECS));
if let PollOutcome::RateLimited(Some(retry_after)) = outcome {
if retry_after > delay {
delay = retry_after;
}
}
tokio::time::sleep(delay).await;
outcome = poller.poll_once().await;
poller.check_staleness();
}
}
fn parse_retry_after(raw: &str) -> Option<u64> {
let value = raw.trim();
if let Ok(secs) = value.parse::<u64>() {
return Some(secs);
}
let when = chrono::DateTime::parse_from_rfc2822(value).ok()?;
let delta = when.timestamp() - chrono::Utc::now().timestamp();
Some(delta.max(0) as u64)
}
fn jittered(base: u64) -> Duration {
let b = uuid::Uuid::new_v4().as_bytes()[0] as u64; let span = base.saturating_mul(20) / 100; let offset = (b * span) / 255;
Duration::from_secs(base.saturating_sub(span / 2).saturating_add(offset))
}
fn parse_etag(raw: &str) -> Option<&str> {
let t = raw.trim();
let t = t.strip_prefix("W/").unwrap_or(t).trim();
t.strip_prefix('"')?.strip_suffix('"')
}
struct PolicyPoller {
handle: PolicyHandle,
last_fetch_ok: Arc<AtomicBool>,
last_poll_ok_at: Arc<AtomicI64>,
cloud_state: CloudState,
credentials: Arc<dyn CredentialProvider>,
url: String,
config: PolicyConfig,
base_dir: PathBuf,
http: reqwest::Client,
agent_id: Option<String>,
failed_credential: Option<Vec<u8>>,
}
impl PolicyPoller {
#[allow(clippy::too_many_arguments)]
fn new(
handle: PolicyHandle,
last_fetch_ok: Arc<AtomicBool>,
last_poll_ok_at: Arc<AtomicI64>,
cloud_state: CloudState,
credentials: Arc<dyn CredentialProvider>,
api_url: String,
config: PolicyConfig,
base_dir: PathBuf,
http: reqwest::Client,
agent_id: Option<String>,
) -> Self {
let agent_id = agent_id.filter(|id| {
reqwest::header::HeaderValue::from_str(id)
.inspect_err(|_| {
tracing::warn!(
target: "policy",
agent_id = ?id,
"configured agent_id is not a usable HTTP header value; polling without it. The platform serves the org bundle with no agent_context, so every scoped rule matches nothing"
);
})
.is_ok()
});
Self {
handle,
last_fetch_ok,
last_poll_ok_at,
cloud_state,
credentials,
url: format!("{}{}", api_url.trim_end_matches('/'), BUNDLE_ENDPOINT),
config,
base_dir,
http,
agent_id,
failed_credential: None,
}
}
fn seed_poll_clock(&self) {
let Ok(Some(meta)) = store::read_meta(&self.base_dir) else {
return;
};
if let Some(secs) = meta.last_poll_ok_at.as_deref().and_then(parse_unix_secs) {
self.last_poll_ok_at.store(secs, Ordering::Relaxed);
}
self.last_fetch_ok
.store(meta.last_fetch_ok, Ordering::Relaxed);
}
async fn poll_once(&mut self) -> PollOutcome {
if self.cloud_state.is_auth_error() {
tracing::debug!(
target: "policy",
"cloud auth error is latched; skipping the policy poll (the resident bundle keeps enforcing)"
);
return PollOutcome::Skipped("auth_error_latched");
}
let Some(token) = self.credentials.retrieve() else {
tracing::debug!(
target: "policy",
"no credential available; skipping the policy poll"
);
return PollOutcome::Skipped("no_credential");
};
let credential = token.expose_secret().as_bytes().to_vec();
if self.failed_credential.as_deref() == Some(credential.as_slice()) {
tracing::debug!(
target: "policy",
"credential unchanged since the last 401/403; skipping the policy poll"
);
return PollOutcome::Skipped("credential_unchanged_after_auth_failure");
}
let meta = match store::read_meta(&self.base_dir) {
Ok(meta) => meta,
Err(e) => {
tracing::warn!(
target: "policy",
code = ERR_BUNDLE_FETCH_FAILED,
error = %e,
"could not read bundle.meta.json; polling without an If-None-Match validator"
);
None
}
};
let mut req = self.http.get(&self.url).bearer_auth(token.expose_secret());
let validator = meta
.as_ref()
.filter(|m| m.agent_id.as_deref() == self.agent_id.as_deref())
.and_then(|m| m.etag.as_deref());
if let Some(etag) = validator {
req = req.header(reqwest::header::IF_NONE_MATCH, etag);
}
if let Some(agent_id) = self.agent_id.as_deref() {
req = req.header(AGENT_ID_HEADER, agent_id);
}
let resp = match req.send().await {
Ok(resp) => resp,
Err(e) => {
return self.transport_failure(&format!("policy bundle request failed: {e}"));
}
};
let status = resp.status();
if status.as_u16() != 401 && status.as_u16() != 403 {
self.failed_credential = None;
}
match status.as_u16() {
200 => self.handle_ok(resp, meta).await,
304 => self.handle_not_modified(meta),
401 | 403 => {
self.failed_credential = Some(credential);
tracing::error!(
target: "policy",
code = ERR_BUNDLE_FETCH_FAILED,
status = status.as_u16(),
"policy bundle fetch rejected the credential; pausing bundle refresh until the credential changes. The resident bundle keeps enforcing"
);
self.record_poll_failure(&format!(
"{ERR_BUNDLE_FETCH_FAILED} auth rejected ({})",
status.as_u16()
));
PollOutcome::AuthFailed
}
404 => {
let resident = self.handle.load().is_some();
tracing::warn!(
target: "policy",
code = ERR_BUNDLE_FETCH_FAILED,
resident_bundle = resident,
"policy bundle endpoint returned 404; keeping the last-known-good bundle enforcing"
);
self.record_poll_failure(&format!("{ERR_BUNDLE_FETCH_FAILED} 404 no bundle"));
PollOutcome::NoBundle
}
429 => {
let retry_after = resp
.headers()
.get(reqwest::header::RETRY_AFTER)
.and_then(|v| v.to_str().ok())
.and_then(parse_retry_after)
.map(|s| Duration::from_secs(s.min(MAX_RETRY_AFTER_SECS)));
tracing::warn!(
target: "policy",
code = ERR_BUNDLE_FETCH_FAILED,
retry_after_secs = retry_after.map(|d| d.as_secs()),
"policy bundle fetch rate limited (429); backing off"
);
self.record_poll_failure(&format!("{ERR_BUNDLE_FETCH_FAILED} rate limited"));
PollOutcome::RateLimited(retry_after)
}
code => self.transport_failure(&format!(
"policy bundle fetch returned an unusable status {code}"
)),
}
}
async fn handle_ok(
&mut self,
resp: reqwest::Response,
meta: Option<store::BundleMeta>,
) -> PollOutcome {
let raw_etag = resp
.headers()
.get(reqwest::header::ETAG)
.and_then(|v| v.to_str().ok())
.map(|s| s.to_string());
let body = match resp.bytes().await {
Ok(b) => b,
Err(e) => {
return self.transport_failure(&format!("policy bundle body was not readable: {e}"))
}
};
let Some(raw_etag) = raw_etag else {
return self.reject(
ERR_BUNDLE_REJECTED,
"200 response carried no ETag header, so the body's digest cannot be verified",
);
};
let Some(tag) = parse_etag(&raw_etag) else {
return self.reject(
ERR_BUNDLE_REJECTED,
"ETag header is not a well-formed entity-tag",
);
};
if let Err(e) = store::verify_digest(&body, tag) {
return self.reject(ERR_BUNDLE_REJECTED, &e.to_string());
}
let value: serde_json::Value = match serde_json::from_slice(&body) {
Ok(v) => v,
Err(e) => {
return self.reject(ERR_BUNDLE_INVALID, &format!("body is not valid JSON: {e}"))
}
};
match value.get("schema_version").and_then(|v| v.as_i64()) {
Some(SUPPORTED_SCHEMA_VERSION) => {}
other => {
return self.reject(
ERR_BUNDLE_INVALID,
&format!("unsupported schema_version {other:?}"),
)
}
}
let bundle: PolicyBundle = match crate::core::policy::parse_bundle_tolerant(value) {
Ok(b) => b,
Err(e) => {
return self.reject(
ERR_BUNDLE_INVALID,
&format!("body is not a valid policy bundle: {e}"),
)
}
};
let expected_org: Option<String> = meta
.as_ref()
.map(|m| m.organization_id.clone())
.or_else(|| {
self.handle
.load()
.as_ref()
.as_ref()
.map(|b| b.organization_id.clone())
});
if let Some(expected) = expected_org {
if bundle.organization_id != expected {
return self.reject(
ERR_BUNDLE_REJECTED,
&format!(
"organization_id {} does not match this host's {expected}",
bundle.organization_id
),
);
}
}
if bundle.signature.is_some() {
return self.reject(
ERR_BUNDLE_INVALID,
"bundle carries a signature this client cannot verify (D32 lands in v1.1)",
);
}
let digest = tag.to_string();
let mut new_meta = store::BundleMeta::activated(&bundle, digest.clone(), Some(raw_etag));
new_meta.agent_id = self.agent_id.clone();
if let Err(e) = store::store(&self.base_dir, &body, &new_meta) {
tracing::warn!(
target: "policy",
code = ERR_BUNDLE_FETCH_FAILED,
error = %e,
"could not persist the policy bundle; activating it in memory anyway"
);
}
let revision = bundle.revision;
let resident = ResidentBundle::from_bundle(&bundle);
let rule_count = resident.command_rules.len();
let request_rule_count = resident.request_rules.len();
let enforcement_enabled = resident.enforcement_enabled;
self.handle.store(Arc::new(Some(resident)));
self.last_fetch_ok.store(true, Ordering::Relaxed);
self.last_poll_ok_at
.store(now_unix_secs(), Ordering::Relaxed);
tracing::info!(
target: "policy",
revision,
digest = %digest,
rules = rule_count,
request_rules = request_rule_count,
enforcement_enabled,
"policy bundle activated"
);
PollOutcome::Activated { revision }
}
fn handle_not_modified(&self, meta: Option<store::BundleMeta>) -> PollOutcome {
self.last_fetch_ok.store(true, Ordering::Relaxed);
self.last_poll_ok_at
.store(now_unix_secs(), Ordering::Relaxed);
if let Some(mut meta) = meta {
meta.last_poll_ok_at = Some(store::now_rfc3339());
meta.last_fetch_ok = true;
meta.last_error = None;
if let Err(e) = store::write_meta(&self.base_dir, &meta) {
tracing::warn!(
target: "policy",
code = ERR_BUNDLE_FETCH_FAILED,
error = %e,
"could not update the policy poll clock on disk"
);
}
}
tracing::debug!(
target: "policy",
"policy bundle unchanged (304); the resident bundle stays active"
);
PollOutcome::NotModified
}
fn reject(&self, code: &'static str, detail: &str) -> PollOutcome {
tracing::warn!(
target: "policy",
code,
detail,
"policy bundle rejected; keeping the last-known-good bundle enforcing"
);
self.record_poll_failure(&format!("{code} {detail}"));
PollOutcome::Rejected(code)
}
fn transport_failure(&self, detail: &str) -> PollOutcome {
tracing::warn!(
target: "policy",
code = ERR_BUNDLE_FETCH_FAILED,
detail,
"policy bundle poll failed; keeping the last-known-good bundle enforcing"
);
self.record_poll_failure(&format!("{ERR_BUNDLE_FETCH_FAILED} {detail}"));
PollOutcome::Failed
}
fn record_poll_failure(&self, message: &str) {
self.last_fetch_ok.store(false, Ordering::Relaxed);
let Ok(Some(mut meta)) = store::read_meta(&self.base_dir) else {
return;
};
meta.last_fetch_ok = false;
meta.last_error = Some(message.to_string());
if let Err(e) = store::write_meta(&self.base_dir, &meta) {
tracing::debug!(
target: "policy",
error = %e,
"could not record the policy poll failure on disk"
);
}
}
fn check_staleness(&self) -> bool {
let last = self.last_poll_ok_at.load(Ordering::Relaxed);
if last <= 0 {
return false;
}
let age = now_unix_secs().saturating_sub(last).max(0) as u64;
if age <= self.config.stale_warn_after_secs {
return false;
}
tracing::warn!(
target: "policy",
code = ERR_BUNDLE_STALE,
stale_seconds = age,
threshold_seconds = self.config.stale_warn_after_secs,
"no successful policy bundle poll within the staleness threshold; the resident bundle keeps enforcing"
);
true
}
}
fn now_unix_secs() -> i64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs() as i64)
.unwrap_or(0)
}
fn parse_unix_secs(raw: &str) -> Option<i64> {
chrono::DateTime::parse_from_rfc3339(raw)
.ok()
.map(|dt| dt.timestamp())
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::AtomicUsize;
use std::sync::Mutex;
use secrecy::SecretString;
use crate::core::policy::test_support::wire_rule;
use crate::core::policy::{evaluate_command, new_handle};
use crate::generated::types::{PolicyRule, PolicyRuleMode, PolicyRuleSeverity};
const ORG: &str = "0192f8a1-4c3b-7e2a-9f10-5d8c3b1a7e42";
const OTHER_ORG: &str = "0192f8a1-0000-0000-0000-000000000000";
struct TestCredentialProvider {
key: Mutex<Option<String>>,
calls: AtomicUsize,
}
impl TestCredentialProvider {
fn with_key(key: &str) -> Arc<Self> {
Arc::new(Self {
key: Mutex::new(Some(key.to_string())),
calls: AtomicUsize::new(0),
})
}
fn set_key(&self, key: &str) {
*self.key.lock().expect("lock") = Some(key.to_string());
}
}
impl CredentialProvider for TestCredentialProvider {
fn retrieve(&self) -> Option<SecretString> {
self.calls.fetch_add(1, Ordering::Relaxed);
self.key
.lock()
.ok()
.and_then(|g| g.as_ref().map(|k| SecretString::from(k.clone())))
}
}
fn rule(rule_id: &str, pattern: &str, mode: PolicyRuleMode) -> PolicyRule {
wire_rule(rule_id, pattern, mode, PolicyRuleSeverity::High)
}
fn bundle_json(revision: i64, org: &str, rules: Vec<PolicyRule>) -> serde_json::Value {
serde_json::json!({
"schema_version": 1,
"organization_id": org,
"revision": revision,
"built_at": "2026-07-21T09:00:00Z",
"enforcement_enabled": true,
"rules": rules,
"signature": serde_json::Value::Null,
})
}
fn body(revision: i64) -> Vec<u8> {
serde_json::to_vec(&bundle_json(
revision,
ORG,
vec![rule("OL-CMD-001", "*rm -rf*", PolicyRuleMode::Enforce)],
))
.expect("serialise fixture")
}
fn etag_for(body: &[u8]) -> String {
format!("\"{}\"", store::digest_of(body))
}
struct Harness {
poller: PolicyPoller,
dir: tempfile::TempDir,
credentials: Arc<TestCredentialProvider>,
cloud_state: CloudState,
handle: PolicyHandle,
last_fetch_ok: Arc<AtomicBool>,
last_poll_ok_at: Arc<AtomicI64>,
}
const AGENT_ID: &str = "0192f8a1-4c3b-7e2a-9f10-5d8c3b1a7e42";
impl Harness {
fn new(api_url: String) -> Self {
Self::with_agent_id(api_url, Some(AGENT_ID.to_string()))
}
fn with_agent_id(api_url: String, agent_id: Option<String>) -> Self {
let dir = tempfile::tempdir().expect("tempdir");
let handle = new_handle(None);
let last_fetch_ok = Arc::new(AtomicBool::new(false));
let last_poll_ok_at = Arc::new(AtomicI64::new(0));
let cloud_state = CloudState::new();
let credentials = TestCredentialProvider::with_key("ol_org_test");
let poller = PolicyPoller::new(
handle.clone(),
last_fetch_ok.clone(),
last_poll_ok_at.clone(),
cloud_state.clone(),
credentials.clone(),
api_url,
PolicyConfig {
enabled: true,
poll_interval_secs: 300,
stale_warn_after_secs: 86_400,
},
dir.path().to_path_buf(),
reqwest::Client::new(),
agent_id,
);
Self {
poller,
dir,
credentials,
cloud_state,
handle,
last_fetch_ok,
last_poll_ok_at,
}
}
fn still_enforcing(&self) -> bool {
let loaded = self.handle.load();
let Some(bundle) = loaded.as_ref().as_ref() else {
return false;
};
evaluate_command(bundle, "rm -rf /tmp").is_some_and(|m| !m.shadow)
}
fn resident_revision(&self) -> Option<i64> {
self.handle.load().as_ref().as_ref().map(|b| b.revision)
}
fn meta(&self) -> Option<store::BundleMeta> {
store::read_meta(self.dir.path()).expect("meta readable")
}
}
#[test]
fn parse_etag_strips_quotes_and_the_weak_prefix() {
assert_eq!(parse_etag("\"sha256:abc\""), Some("sha256:abc"));
assert_eq!(parse_etag("W/\"sha256:abc\""), Some("sha256:abc"));
assert_eq!(parse_etag(" W/ \"sha256:abc\" "), Some("sha256:abc"));
assert_ne!(parse_etag("W/\"sha256:abc\""), Some("\"sha256:abc\""));
assert_eq!(parse_etag("sha256:abc"), None);
assert_eq!(parse_etag(""), None);
}
#[test]
fn jitter_stays_within_ten_percent_and_varies() {
let base = 300u64;
let intervals: Vec<u64> = (0..20).map(|_| jittered(base).as_secs()).collect();
for secs in &intervals {
assert!(
(270..=330).contains(secs),
"interval {secs}s outside ±10% of {base}s"
);
}
assert!(
intervals.windows(2).any(|w| w[0] != w[1]),
"at least one consecutive pair must differ: {intervals:?}"
);
}
#[test]
fn jitter_does_not_underflow_on_a_tiny_interval() {
for _ in 0..20 {
let _ = jittered(1);
let _ = jittered(0);
}
}
#[test]
fn a_zero_poll_interval_cannot_produce_a_tight_loop() {
let configured: u64 = 0;
assert_eq!(
jittered(configured),
Duration::ZERO,
"precondition: 0 really is degenerate"
);
for _ in 0..50 {
let delay = jittered(configured.max(MIN_POLL_INTERVAL_SECS));
assert!(
delay >= Duration::from_secs(MIN_POLL_INTERVAL_SECS * 9 / 10),
"clamped interval fell below the floor: {delay:?}"
);
}
}
#[test]
fn retry_after_parses_delta_seconds() {
assert_eq!(parse_retry_after("120"), Some(120));
assert_eq!(parse_retry_after(" 120 "), Some(120));
}
#[test]
fn retry_after_parses_the_http_date_form() {
let future = chrono::Utc::now() + chrono::Duration::seconds(600);
let header = future.format("%a, %d %b %Y %H:%M:%S GMT").to_string();
let parsed = parse_retry_after(&header).expect("http-date must parse");
assert!(
(susp_range()).contains(&parsed),
"expected ~600s from an http-date, got {parsed}"
);
}
fn susp_range() -> std::ops::RangeInclusive<u64> {
590..=600
}
#[test]
fn retry_after_in_the_past_is_zero_not_a_wrapped_negative() {
let past = chrono::Utc::now() - chrono::Duration::seconds(600);
let header = past.format("%a, %d %b %Y %H:%M:%S GMT").to_string();
assert_eq!(parse_retry_after(&header), Some(0));
}
#[test]
fn retry_after_garbage_is_none_not_a_panic() {
assert_eq!(parse_retry_after("soon"), None);
assert_eq!(parse_retry_after(""), None);
assert_eq!(parse_retry_after("-5"), None);
}
#[tokio::test]
async fn ok_activates_and_stores_the_activation_etag() {
let mut server = mockito::Server::new_async().await;
let body = body(42);
let etag = etag_for(&body);
let mock = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(200)
.with_header("ETag", &etag)
.with_body(&body)
.create_async()
.await;
let mut h = Harness::new(server.url());
assert_eq!(
h.poller.poll_once().await,
PollOutcome::Activated { revision: 42 }
);
assert!(h.still_enforcing(), "the new bundle must be enforcing");
assert_eq!(h.resident_revision(), Some(42));
let meta = h.meta().expect("meta written");
assert_eq!(meta.etag.as_deref(), Some(etag.as_str()));
assert_eq!(meta.digest, store::digest_of(&body));
assert_eq!(meta.organization_id, ORG);
assert!(meta.last_activated_at.is_some());
assert!(h.last_fetch_ok.load(Ordering::Relaxed));
assert!(h.last_poll_ok_at.load(Ordering::Relaxed) > 0);
assert_eq!(
std::fs::read(store::bundle_path(h.dir.path())).expect("body on disk"),
body
);
mock.assert_async().await;
}
#[tokio::test]
async fn weak_etag_is_accepted() {
let mut server = mockito::Server::new_async().await;
let body = body(7);
let mock = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(200)
.with_header("ETag", &format!("W/\"{}\"", store::digest_of(&body)))
.with_body(&body)
.create_async()
.await;
let mut h = Harness::new(server.url());
assert_eq!(
h.poller.poll_once().await,
PollOutcome::Activated { revision: 7 }
);
assert!(h.still_enforcing());
mock.assert_async().await;
}
#[tokio::test]
async fn empty_rule_set_activates_and_keeps_a_revision() {
let mut server = mockito::Server::new_async().await;
let body = serde_json::to_vec(&bundle_json(9, ORG, vec![])).expect("serialise");
let mock = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(200)
.with_header("ETag", &etag_for(&body))
.with_body(&body)
.create_async()
.await;
let mut h = Harness::new(server.url());
assert_eq!(
h.poller.poll_once().await,
PollOutcome::Activated { revision: 9 }
);
assert!(!h.still_enforcing());
assert_eq!(h.resident_revision(), Some(9));
mock.assert_async().await;
}
#[tokio::test]
async fn a_provisioned_install_sends_its_agent_id_on_every_poll() {
let mut server = mockito::Server::new_async().await;
let body = body(11);
let mock = server
.mock("GET", BUNDLE_ENDPOINT)
.match_header("x-openlatch-agent-id", AGENT_ID)
.with_status(200)
.with_header("ETag", &etag_for(&body))
.with_body(&body)
.expect(1)
.create_async()
.await;
let mut h = Harness::new(server.url());
assert_eq!(
h.poller.poll_once().await,
PollOutcome::Activated { revision: 11 }
);
mock.assert_async().await;
}
#[tokio::test]
async fn the_agent_id_header_rides_beside_the_validator() {
let mut server = mockito::Server::new_async().await;
let body = body(12);
let etag = etag_for(&body);
let first = server
.mock("GET", BUNDLE_ENDPOINT)
.match_header("x-openlatch-agent-id", AGENT_ID)
.with_status(200)
.with_header("ETag", &etag)
.with_body(&body)
.expect(1)
.create_async()
.await;
let mut h = Harness::new(server.url());
assert!(matches!(
h.poller.poll_once().await,
PollOutcome::Activated { .. }
));
first.assert_async().await;
let revalidate = server
.mock("GET", BUNDLE_ENDPOINT)
.match_header("if-none-match", etag.as_str())
.match_header("x-openlatch-agent-id", AGENT_ID)
.with_status(304)
.expect(1)
.create_async()
.await;
assert_eq!(h.poller.poll_once().await, PollOutcome::NotModified);
revalidate.assert_async().await;
}
#[tokio::test]
async fn an_unprovisioned_install_omits_the_agent_id_header() {
let mut server = mockito::Server::new_async().await;
let body = body(13);
let mock = server
.mock("GET", BUNDLE_ENDPOINT)
.match_header("x-openlatch-agent-id", mockito::Matcher::Missing)
.with_status(200)
.with_header("ETag", &etag_for(&body))
.with_body(&body)
.expect(1)
.create_async()
.await;
let mut h = Harness::with_agent_id(server.url(), None);
assert_eq!(
h.poller.poll_once().await,
PollOutcome::Activated { revision: 13 }
);
mock.assert_async().await;
}
#[tokio::test]
async fn a_validator_fetched_under_another_identity_is_not_revalidated() {
let mut server = mockito::Server::new_async().await;
let body = body(14);
let etag = etag_for(&body);
let mut h = Harness::new(server.url());
let mut meta = store::BundleMeta::activated(
&serde_json::from_slice(&body).expect("fixture parses"),
store::digest_of(&body),
Some(etag.clone()),
);
meta.agent_id = None;
let mut raw = serde_json::to_value(&meta).expect("meta serialises");
raw.as_object_mut().expect("object").remove("agent_id");
store::write_body(h.dir.path(), &body).expect("write body");
std::fs::write(
store::meta_path(h.dir.path()),
serde_json::to_vec(&raw).expect("serialise"),
)
.expect("write legacy meta");
assert!(
h.meta().expect("meta readable").agent_id.is_none(),
"precondition: the legacy file carries no identity"
);
let full = server
.mock("GET", BUNDLE_ENDPOINT)
.match_header("if-none-match", mockito::Matcher::Missing)
.match_header("x-openlatch-agent-id", AGENT_ID)
.with_status(200)
.with_header("ETag", &etag)
.with_body(&body)
.expect(1)
.create_async()
.await;
assert_eq!(
h.poller.poll_once().await,
PollOutcome::Activated { revision: 14 }
);
full.assert_async().await;
assert_eq!(h.meta().expect("meta").agent_id.as_deref(), Some(AGENT_ID));
let revalidate = server
.mock("GET", BUNDLE_ENDPOINT)
.match_header("if-none-match", etag.as_str())
.match_header("x-openlatch-agent-id", AGENT_ID)
.with_status(304)
.expect(1)
.create_async()
.await;
assert_eq!(h.poller.poll_once().await, PollOutcome::NotModified);
revalidate.assert_async().await;
}
#[tokio::test]
async fn losing_the_agent_id_also_drops_the_validator() {
let mut server = mockito::Server::new_async().await;
let body = body(15);
let etag = etag_for(&body);
let seed = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(200)
.with_header("ETag", &etag)
.with_body(&body)
.expect(1)
.create_async()
.await;
let mut h = Harness::new(server.url());
assert!(matches!(
h.poller.poll_once().await,
PollOutcome::Activated { .. }
));
seed.assert_async().await;
h.poller.agent_id = None;
let full = server
.mock("GET", BUNDLE_ENDPOINT)
.match_header("if-none-match", mockito::Matcher::Missing)
.match_header("x-openlatch-agent-id", mockito::Matcher::Missing)
.with_status(200)
.with_header("ETag", &etag)
.with_body(&body)
.expect(1)
.create_async()
.await;
assert!(matches!(
h.poller.poll_once().await,
PollOutcome::Activated { .. }
));
full.assert_async().await;
}
#[tokio::test]
async fn an_agent_id_that_is_not_a_header_value_is_dropped_not_sent() {
let mut server = mockito::Server::new_async().await;
let body = body(16);
let mock = server
.mock("GET", BUNDLE_ENDPOINT)
.match_header("x-openlatch-agent-id", mockito::Matcher::Missing)
.with_status(200)
.with_header("ETag", &etag_for(&body))
.with_body(&body)
.expect(1)
.create_async()
.await;
let mut h = Harness::with_agent_id(
server.url(),
Some(
"bad
id"
.to_string(),
),
);
assert!(h.poller.agent_id.is_none(), "dropped at construction");
assert_eq!(
h.poller.poll_once().await,
PollOutcome::Activated { revision: 16 }
);
mock.assert_async().await;
}
async fn assert_rejected(
builder: impl FnOnce(mockito::Mock) -> mockito::Mock,
code: &'static str,
) {
let mut server = mockito::Server::new_async().await;
let mock = builder(server.mock("GET", BUNDLE_ENDPOINT))
.create_async()
.await;
let mut h = Harness::new(server.url());
assert_eq!(h.poller.poll_once().await, PollOutcome::Rejected(code));
assert!(
h.handle.load().is_none(),
"a rejected bundle must never activate"
);
assert!(!h.last_fetch_ok.load(Ordering::Relaxed));
assert_eq!(
h.last_poll_ok_at.load(Ordering::Relaxed),
0,
"a rejection is not a successful poll"
);
mock.assert_async().await;
}
#[tokio::test]
async fn missing_etag_on_200_is_rejected() {
let body = body(1);
assert_rejected(
move |m| m.with_status(200).with_body(body),
ERR_BUNDLE_REJECTED,
)
.await;
}
#[tokio::test]
async fn malformed_etag_on_200_is_rejected() {
let body = body(1);
assert_rejected(
move |m| {
m.with_status(200)
.with_header("ETag", "sha256:unquoted")
.with_body(body)
},
ERR_BUNDLE_REJECTED,
)
.await;
}
#[tokio::test]
async fn digest_mismatch_is_rejected() {
let served = body(1);
let other = etag_for(&body(2));
assert_rejected(
move |m| {
m.with_status(200)
.with_header("ETag", &other)
.with_body(served)
},
ERR_BUNDLE_REJECTED,
)
.await;
}
#[tokio::test]
async fn org_mismatch_is_rejected() {
let mut server = mockito::Server::new_async().await;
let first = body(1);
let good = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(200)
.with_header("ETag", &etag_for(&first))
.with_body(&first)
.expect(1)
.create_async()
.await;
let mut h = Harness::new(server.url());
assert!(matches!(
h.poller.poll_once().await,
PollOutcome::Activated { .. }
));
good.assert_async().await;
let intruder = serde_json::to_vec(&bundle_json(
2,
OTHER_ORG,
vec![rule("OL-CMD-999", "*", PolicyRuleMode::Enforce)],
))
.expect("serialise");
let bad = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(200)
.with_header("ETag", &etag_for(&intruder))
.with_body(&intruder)
.create_async()
.await;
assert_eq!(
h.poller.poll_once().await,
PollOutcome::Rejected(ERR_BUNDLE_REJECTED)
);
assert_eq!(h.resident_revision(), Some(1), "never evaluate another org");
bad.assert_async().await;
}
#[tokio::test]
async fn unknown_schema_version_is_rejected() {
let mut doc = bundle_json(1, ORG, vec![]);
doc["schema_version"] = serde_json::json!(2);
let body = serde_json::to_vec(&doc).expect("serialise");
let etag = etag_for(&body);
assert_rejected(
move |m| {
m.with_status(200)
.with_header("ETag", &etag)
.with_body(body)
},
ERR_BUNDLE_INVALID,
)
.await;
}
#[tokio::test]
async fn malformed_json_is_rejected() {
let body = b"{ not json".to_vec();
let etag = etag_for(&body);
assert_rejected(
move |m| {
m.with_status(200)
.with_header("ETag", &etag)
.with_body(body)
},
ERR_BUNDLE_INVALID,
)
.await;
}
#[tokio::test]
async fn non_null_signature_is_rejected() {
let mut doc = bundle_json(1, ORG, vec![]);
doc["signature"] = serde_json::json!("ed25519:deadbeef");
let body = serde_json::to_vec(&doc).expect("serialise");
let etag = etag_for(&body);
assert_rejected(
move |m| {
m.with_status(200)
.with_header("ETag", &etag)
.with_body(body)
},
ERR_BUNDLE_INVALID,
)
.await;
}
#[tokio::test]
async fn activation_failure_rewinds_the_etag() {
let mut server = mockito::Server::new_async().await;
let good = body(1);
let good_etag = etag_for(&good);
let first = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(200)
.with_header("ETag", &good_etag)
.with_body(&good)
.expect(1)
.create_async()
.await;
let mut h = Harness::new(server.url());
assert!(matches!(
h.poller.poll_once().await,
PollOutcome::Activated { .. }
));
first.assert_async().await;
let corrupt = body(2);
let second = server
.mock("GET", BUNDLE_ENDPOINT)
.match_header("if-none-match", good_etag.as_str())
.with_status(200)
.with_header("ETag", &etag_for(b"a different document entirely"))
.with_body(&corrupt)
.expect(1)
.create_async()
.await;
assert_eq!(
h.poller.poll_once().await,
PollOutcome::Rejected(ERR_BUNDLE_REJECTED)
);
second.assert_async().await;
assert_eq!(h.resident_revision(), Some(1), "revision 1 keeps enforcing");
assert!(h.still_enforcing());
assert_eq!(
h.meta().expect("meta").etag.as_deref(),
Some(good_etag.as_str()),
"the stored validator must still be revision 1's"
);
let fixed = body(3);
let third = server
.mock("GET", BUNDLE_ENDPOINT)
.match_header("if-none-match", good_etag.as_str())
.with_status(200)
.with_header("ETag", &etag_for(&fixed))
.with_body(&fixed)
.expect(1)
.create_async()
.await;
assert_eq!(
h.poller.poll_once().await,
PollOutcome::Activated { revision: 3 }
);
third.assert_async().await;
}
#[tokio::test]
async fn bare_304s_do_not_clear_the_validator() {
let mut server = mockito::Server::new_async().await;
let good = body(11);
let etag = etag_for(&good);
let body_on_disk = good.clone();
let download = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(200)
.with_header("ETag", &etag)
.with_body(&good)
.expect(1) .create_async()
.await;
let mut h = Harness::new(server.url());
assert!(matches!(
h.poller.poll_once().await,
PollOutcome::Activated { .. }
));
download.assert_async().await;
let revalidate = server
.mock("GET", BUNDLE_ENDPOINT)
.match_header("if-none-match", etag.as_str())
.with_status(304)
.expect(3)
.create_async()
.await;
for _ in 0..3 {
assert_eq!(h.poller.poll_once().await, PollOutcome::NotModified);
}
revalidate.assert_async().await;
download.assert_async().await;
assert_eq!(
h.meta().expect("meta").etag.as_deref(),
Some(etag.as_str()),
"a 304 must never clear the stored validator"
);
assert_eq!(
std::fs::read(store::bundle_path(h.dir.path())).expect("body"),
body_on_disk,
"a 304 must not touch bundle.json"
);
assert_eq!(h.resident_revision(), Some(11));
}
#[tokio::test]
async fn not_modified_advances_the_poll_clock_only() {
let mut server = mockito::Server::new_async().await;
let good = body(5);
let etag = etag_for(&good);
let download = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(200)
.with_header("ETag", &etag)
.with_body(&good)
.expect(1)
.create_async()
.await;
let mut h = Harness::new(server.url());
assert!(matches!(
h.poller.poll_once().await,
PollOutcome::Activated { .. }
));
download.assert_async().await;
let activated_at = h.meta().expect("meta").last_activated_at;
let before = Arc::as_ptr(&h.handle.load_full());
h.last_poll_ok_at.store(1, Ordering::Relaxed);
let revalidate = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(304)
.with_header("ETag", &etag)
.create_async()
.await;
assert_eq!(h.poller.poll_once().await, PollOutcome::NotModified);
revalidate.assert_async().await;
assert!(h.last_poll_ok_at.load(Ordering::Relaxed) > 1);
assert!(h.last_fetch_ok.load(Ordering::Relaxed));
assert_eq!(
before,
Arc::as_ptr(&h.handle.load_full()),
"the rule set must not be reloaded on a 304"
);
let meta = h.meta().expect("meta");
assert_eq!(
meta.last_activated_at, activated_at,
"the activation tier must not move on a 304"
);
assert!(meta.last_poll_ok_at.is_some());
}
#[tokio::test]
async fn not_found_keeps_the_resident_bundle_enforcing() {
let mut server = mockito::Server::new_async().await;
let good = body(3);
let download = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(200)
.with_header("ETag", &etag_for(&good))
.with_body(&good)
.expect(1)
.create_async()
.await;
let mut h = Harness::new(server.url());
assert!(matches!(
h.poller.poll_once().await,
PollOutcome::Activated { .. }
));
download.assert_async().await;
let polled_at = h.last_poll_ok_at.load(Ordering::Relaxed);
let gone = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(404)
.with_body(r#"{"error":{"code":"not_found","message":"no bundle"}}"#)
.create_async()
.await;
assert_eq!(h.poller.poll_once().await, PollOutcome::NoBundle);
gone.assert_async().await;
assert!(h.still_enforcing(), "a 404 must not disarm the host");
assert_eq!(h.resident_revision(), Some(3));
assert_eq!(
h.last_poll_ok_at.load(Ordering::Relaxed),
polled_at,
"a 404 is neither a 2xx nor a 304"
);
assert!(!h.last_fetch_ok.load(Ordering::Relaxed));
}
#[tokio::test]
async fn server_error_keeps_the_resident_bundle_enforcing() {
let mut server = mockito::Server::new_async().await;
let good = body(4);
let download = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(200)
.with_header("ETag", &etag_for(&good))
.with_body(&good)
.expect(1)
.create_async()
.await;
let mut h = Harness::new(server.url());
assert!(matches!(
h.poller.poll_once().await,
PollOutcome::Activated { .. }
));
download.assert_async().await;
let boom = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(503)
.create_async()
.await;
assert_eq!(h.poller.poll_once().await, PollOutcome::Failed);
boom.assert_async().await;
assert!(h.still_enforcing());
assert!(!h.last_fetch_ok.load(Ordering::Relaxed));
assert_eq!(
h.meta().expect("meta").etag.as_deref(),
Some(etag_for(&good).as_str()),
"a transport failure must not disturb the stored validator"
);
}
#[tokio::test]
async fn rate_limit_honours_retry_after() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(429)
.with_header("Retry-After", "45")
.create_async()
.await;
let mut h = Harness::new(server.url());
assert_eq!(
h.poller.poll_once().await,
PollOutcome::RateLimited(Some(Duration::from_secs(45)))
);
mock.assert_async().await;
}
#[tokio::test]
async fn rate_limit_without_a_usable_header_falls_back_to_the_interval() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(429)
.with_header("Retry-After", "when we feel like it")
.create_async()
.await;
let mut h = Harness::new(server.url());
assert_eq!(h.poller.poll_once().await, PollOutcome::RateLimited(None));
mock.assert_async().await;
}
#[tokio::test]
async fn rate_limit_honours_an_http_date_header() {
let future = chrono::Utc::now() + chrono::Duration::seconds(900);
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(429)
.with_header(
"Retry-After",
&future.format("%a, %d %b %Y %H:%M:%S GMT").to_string(),
)
.create_async()
.await;
let mut h = Harness::new(server.url());
match h.poller.poll_once().await {
PollOutcome::RateLimited(Some(d)) => assert!(
d >= Duration::from_secs(880) && d <= Duration::from_secs(900),
"expected ~900s from the http-date, got {d:?}"
),
other => panic!("expected a honoured Retry-After, got {other:?}"),
}
mock.assert_async().await;
}
#[tokio::test]
async fn retry_after_is_capped() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(429)
.with_header("Retry-After", "999999")
.create_async()
.await;
let mut h = Harness::new(server.url());
assert_eq!(
h.poller.poll_once().await,
PollOutcome::RateLimited(Some(Duration::from_secs(MAX_RETRY_AFTER_SECS)))
);
mock.assert_async().await;
}
#[tokio::test]
async fn auth_failure_pauses_until_the_credential_changes() {
let mut server = mockito::Server::new_async().await;
let good = body(6);
let download = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(200)
.with_header("ETag", &etag_for(&good))
.with_body(&good)
.expect(1)
.create_async()
.await;
let mut h = Harness::new(server.url());
assert!(matches!(
h.poller.poll_once().await,
PollOutcome::Activated { .. }
));
download.assert_async().await;
let revoked = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(401)
.expect(1)
.create_async()
.await;
assert_eq!(h.poller.poll_once().await, PollOutcome::AuthFailed);
for _ in 0..3 {
assert_eq!(
h.poller.poll_once().await,
PollOutcome::Skipped("credential_unchanged_after_auth_failure")
);
}
revoked.assert_async().await;
assert!(
h.still_enforcing(),
"a revoked key must not disarm the host"
);
assert!(
!h.cloud_state.is_auth_error(),
"D46: the poller must never write the cloud auth latch"
);
let rotated = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(200)
.with_header("ETag", &etag_for(&good))
.with_body(&good)
.expect(1)
.create_async()
.await;
h.credentials.set_key("ol_org_rotated");
assert!(matches!(
h.poller.poll_once().await,
PollOutcome::Activated { .. }
));
rotated.assert_async().await;
}
#[tokio::test]
async fn latched_cloud_auth_error_skips_the_fetch() {
let mut server = mockito::Server::new_async().await;
let never = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(200)
.expect(0)
.create_async()
.await;
let mut h = Harness::new(server.url());
h.cloud_state.auth_error.store(true, Ordering::Relaxed);
assert_eq!(
h.poller.poll_once().await,
PollOutcome::Skipped("auth_error_latched")
);
assert!(
h.cloud_state.is_auth_error(),
"the poller must not clear a latch it does not own"
);
never.assert_async().await;
}
#[tokio::test]
async fn missing_credential_skips_the_fetch() {
let mut server = mockito::Server::new_async().await;
let never = server
.mock("GET", BUNDLE_ENDPOINT)
.expect(0)
.create_async()
.await;
let mut h = Harness::new(server.url());
*h.credentials.key.lock().expect("lock") = None;
assert_eq!(
h.poller.poll_once().await,
PollOutcome::Skipped("no_credential")
);
never.assert_async().await;
}
#[tokio::test]
async fn staleness_warns_and_keeps_enforcing() {
let mut server = mockito::Server::new_async().await;
let good = body(8);
let etag = etag_for(&good);
let download = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(200)
.with_header("ETag", &etag)
.with_body(&good)
.expect(1)
.create_async()
.await;
let mut h = Harness::new(server.url());
assert!(matches!(
h.poller.poll_once().await,
PollOutcome::Activated { .. }
));
download.assert_async().await;
assert!(!h.poller.check_staleness(), "a fresh poll is not stale");
h.last_poll_ok_at.store(
now_unix_secs() - (h.poller.config.stale_warn_after_secs as i64) - 60,
Ordering::Relaxed,
);
assert!(h.poller.check_staleness(), "OL-1213 must fire");
assert!(
h.still_enforcing(),
"staleness must never stop the host enforcing"
);
let revalidate = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(304)
.create_async()
.await;
assert_eq!(h.poller.poll_once().await, PollOutcome::NotModified);
revalidate.assert_async().await;
assert!(!h.poller.check_staleness());
}
#[test]
fn a_host_that_never_polled_successfully_does_not_warn_as_stale() {
let h = Harness::new("http://127.0.0.1:1".to_string());
assert_eq!(h.last_poll_ok_at.load(Ordering::Relaxed), 0);
assert!(!h.poller.check_staleness());
}
#[tokio::test]
async fn poll_clock_is_seeded_from_disk() {
let mut server = mockito::Server::new_async().await;
let good = body(12);
let download = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(200)
.with_header("ETag", &etag_for(&good))
.with_body(&good)
.expect(1)
.create_async()
.await;
let mut h = Harness::new(server.url());
assert!(matches!(
h.poller.poll_once().await,
PollOutcome::Activated { .. }
));
download.assert_async().await;
let restarted = PolicyPoller::new(
new_handle(None),
Arc::new(AtomicBool::new(false)),
Arc::new(AtomicI64::new(0)),
CloudState::new(),
TestCredentialProvider::with_key("ol_org_test"),
server.url(),
PolicyConfig {
enabled: true,
poll_interval_secs: 300,
stale_warn_after_secs: 86_400,
},
h.dir.path().to_path_buf(),
reqwest::Client::new(),
Some(AGENT_ID.to_string()),
);
restarted.seed_poll_clock();
assert!(restarted.last_poll_ok_at.load(Ordering::Relaxed) > 0);
}
#[tokio::test]
async fn boot_fetch_happens_before_the_first_tick() {
let mut server = mockito::Server::new_async().await;
let good = body(21);
let mock = server
.mock("GET", BUNDLE_ENDPOINT)
.with_status(200)
.with_header("ETag", &etag_for(&good))
.with_body(&good)
.create_async()
.await;
let dir = tempfile::tempdir().expect("tempdir");
let handle = new_handle(None);
let task = tokio::spawn(run_policy_poller(
handle.clone(),
Arc::new(AtomicBool::new(false)),
Arc::new(AtomicI64::new(0)),
CloudState::new(),
TestCredentialProvider::with_key("ol_org_test"),
server.url(),
PolicyConfig {
enabled: true,
poll_interval_secs: 3_600,
stale_warn_after_secs: 86_400,
},
dir.path().to_path_buf(),
reqwest::Client::new(),
Some(AGENT_ID.to_string()),
));
let mut activated = false;
for _ in 0..50 {
if handle.load().is_some() {
activated = true;
break;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
task.abort();
assert!(
activated,
"the poller must fetch on boot, not on the first timer tick"
);
assert_eq!(
handle.load().as_ref().as_ref().map(|b| b.revision),
Some(21)
);
mock.assert_async().await;
}
}