use std::collections::{BTreeMap, HashMap, HashSet};
use std::net::SocketAddr;
use std::sync::{Arc, Mutex};
use ed25519_dalek::SigningKey;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use crate::chain::{self, ChainManager};
use crate::cog_ingest::{IngestError, IngestHooks, IngestLease, InstanceBinding};
use crate::gate::{GateBackend, GateDecision};
use crate::ipc::GlobalPid;
use crate::mesh_artifact::ArtifactExchange;
use crate::mesh_artifact_pkg::PackageExchangeError;
use crate::mesh_artifact_transfer::PeerSet;
use crate::mesh_service_adv::ServiceAdvertisement;
use crate::node_facts_advert::SignedNodeFacts;
use crate::node_registry::node_id_from_pubkey;
use crate::workload_pkg::TrustAnchors;
use crate::workload_pkg::manifest::valid_token;
use crate::workload_pkg::store::StoreSource;
use crate::workload_runtime::{
HostContract, InstanceHandle, RunEvidence, RunMode, RuntimeError, VerifiedWorkload,
WorkloadConfig, WorkloadHost,
};
use super::cog_kind::route_of;
use super::host_instances::InFlight;
use super::msg::{
CTL_VERSION, ControllerPolicy, CtlOutcome, CtlRequest, CtlResponse, NonceGuard, Refusal,
RefusalCode, SignedCtl, WORKLOAD_HOST_SERVICE, method, verify_request,
};
use super::refusal_budget::RefusalBudget;
pub const HOST_CHAIN_SOURCE: &str = "workload.host";
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CtlConfig {
pub mode: RunMode,
#[serde(default)]
pub args: Vec<String>,
#[serde(default = "default_csi_port")]
pub csi_port: u16,
}
fn default_csi_port() -> u16 {
crate::workload_runtime::host_contract::DEFAULT_CSI_PORT
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct PlaceBody {
pub name: String,
pub manifest_hash: String,
pub variant: String,
pub config: CtlConfig,
#[serde(default)]
pub start: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub project_id: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct InstanceBody {
#[serde(default)]
pub instance_id: Option<String>,
#[serde(default)]
pub grace_ms: Option<u64>,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub include_departed: bool,
}
pub(super) struct Placed {
pub(super) handle: InstanceHandle,
pub(super) route: String,
pub(super) name: String,
pub(super) variant: String,
pub(super) decision_id: Option<String>,
pub(super) last: Option<RunEvidence>,
pub(super) ingest: Option<IngestLease>,
pub(super) ingest_state: &'static str,
pub(super) life: super::lifecycle::InstanceLife,
pub(super) continuous: bool,
pub(super) desired_running: bool,
pub(super) lease_stopped: bool,
pub(super) requester: String,
pub(super) licence: Option<super::host_licence::LicensedRun>,
}
pub type FactsSource = Arc<dyn Fn() -> Option<SignedNodeFacts> + Send + Sync>;
pub struct WorkloadHostService {
pub(super) node_id: String,
key: SigningKey,
anchors: TrustAnchors,
controllers: Box<dyn ControllerPolicy>,
nonces: NonceGuard,
pub(super) routes: BTreeMap<String, Arc<WorkloadHost>>,
exchange: Arc<ArtifactExchange>,
gate: Arc<dyn GateBackend>,
chain: Option<Arc<ChainManager>>,
facts: Mutex<Option<SignedNodeFacts>>,
facts_source: Option<FactsSource>,
address: Option<String>,
pub(super) instances: tokio::sync::Mutex<HashMap<String, Placed>>,
pub(super) in_flight: Mutex<HashSet<String>>,
verify_budget: RefusalBudget,
pub(super) ingest: Option<IngestHooks>,
pub(super) revocations: std::sync::OnceLock<Arc<crate::revocation::RevocationList>>,
pub(super) kinds: crate::workload_kind::KindRegistry,
pub(super) restart: super::lifecycle::RestartPolicy,
pub(super) lease: Option<std::time::Duration>,
pub(super) departed: Mutex<std::collections::VecDeque<(String, &'static str)>>,
pub(super) last_contact_ms: std::sync::atomic::AtomicU64,
pub(super) licence_gate: std::sync::OnceLock<Arc<dyn crate::licence::CognitumRunGate>>,
pub(super) node_admin: std::sync::OnceLock<Arc<dyn super::host_node_admin::NodeAdmin>>,
}
fn now_ms() -> u64 {
chrono::Utc::now().timestamp_millis().max(0) as u64
}
pub(super) fn refuse(code: RefusalCode, reason: impl Into<String>) -> Refusal {
Refusal::new(code, reason)
}
pub(super) fn runtime_refusal(e: &RuntimeError) -> Refusal {
let code = match e {
RuntimeError::AdmissionRefused(_) => RefusalCode::Admission,
RuntimeError::Governance(_) => RefusalCode::Governance,
RuntimeError::UnknownInstance(_) => RefusalCode::UnknownInstance,
RuntimeError::InvalidConfig(_) => RefusalCode::InvalidRequest,
_ => RefusalCode::Runtime,
};
Refusal::new(code, e.to_string())
}
impl WorkloadHostService {
pub fn new(
key: SigningKey,
exchange: Arc<ArtifactExchange>,
anchors: TrustAnchors,
gate: Arc<dyn GateBackend>,
) -> Self {
Self {
node_id: node_id_from_pubkey(&key.verifying_key().to_bytes()),
key,
anchors,
controllers: Box::new(Vec::<[u8; 32]>::new()),
nonces: NonceGuard::new(),
routes: BTreeMap::new(),
exchange,
gate,
chain: None,
facts: Mutex::new(None),
facts_source: None,
address: None,
instances: tokio::sync::Mutex::new(HashMap::new()),
in_flight: Mutex::new(HashSet::new()),
verify_budget: RefusalBudget::default(),
ingest: None,
revocations: std::sync::OnceLock::new(),
kinds: crate::workload_kind::KindRegistry::builtin(),
restart: super::lifecycle::RestartPolicy::default(),
lease: None,
departed: Mutex::new(std::collections::VecDeque::new()),
last_contact_ms: std::sync::atomic::AtomicU64::new(now_ms()),
licence_gate: std::sync::OnceLock::new(),
node_admin: std::sync::OnceLock::new(),
}
}
pub fn with_kinds(mut self, kinds: crate::workload_kind::KindRegistry) -> Self {
self.kinds = kinds;
self
}
pub fn set_revocations(&self, list: Arc<crate::revocation::RevocationList>) -> bool {
self.revocations.set(list).is_ok()
}
pub fn revocations(&self) -> Option<&Arc<crate::revocation::RevocationList>> {
self.revocations.get()
}
pub fn with_ingest(mut self, hooks: IngestHooks) -> Self {
self.ingest = Some(hooks);
self
}
pub fn ingest(&self) -> Option<&IngestHooks> {
self.ingest.as_ref()
}
pub fn with_refusal_budget(mut self, b: RefusalBudget) -> Self {
self.verify_budget = b;
self
}
pub fn with_route(mut self, route: &str, host: Arc<WorkloadHost>) -> Self {
self.routes.insert(route.to_string(), host);
self
}
pub fn with_controllers(mut self, c: impl ControllerPolicy + 'static) -> Self {
self.controllers = Box::new(c);
self
}
pub fn with_chain(mut self, cm: Arc<ChainManager>) -> Self {
self.chain = Some(cm);
self
}
pub fn with_address(mut self, addr: impl Into<String>) -> Self {
self.address = Some(addr.into());
self
}
pub fn set_facts(&self, facts: SignedNodeFacts) {
if let Ok(mut f) = self.facts.lock() {
*f = Some(facts);
}
}
pub fn with_facts_source(mut self, source: FactsSource) -> Self {
self.facts_source = Some(source);
self
}
fn current_facts(&self) -> Option<SignedNodeFacts> {
self.facts_source
.as_ref()
.and_then(|f| f())
.or_else(|| self.facts.lock().ok().and_then(|f| f.clone()))
}
pub fn node_id(&self) -> &str {
&self.node_id
}
pub fn advertisement(&self) -> ServiceAdvertisement {
let mut metadata = HashMap::new();
let routes: Vec<&str> = self.routes.keys().map(String::as_str).collect();
metadata.insert("routes".to_string(), routes.join(","));
if let Some(a) = &self.address {
metadata.insert("addr".to_string(), a.clone());
}
metadata.insert(
"ingest".to_string(),
self.ingest.as_ref().map_or("none", |h| h.state()).to_string(),
);
ServiceAdvertisement {
name: WORKLOAD_HOST_SERVICE.to_string(),
methods: method::ALL.iter().map(|m| m.to_string()).collect(),
node_id: self.node_id.clone(),
global_pid: GlobalPid::new(self.node_id.clone(), 0),
version: CTL_VERSION.to_string(),
contract_hash: None,
metadata,
last_updated: chrono::Utc::now().timestamp().max(0) as u64,
}
}
pub(super) fn note_departed(&self, instance_id: &str, why: &'static str) {
if let Ok(mut d) = self.departed.lock() {
if d.len() >= 256 {
d.pop_front();
}
d.push_back((instance_id.to_string(), why));
}
}
pub(super) fn record(&self, kind: &str, payload: Value) {
if let Some(cm) = &self.chain {
cm.append(HOST_CHAIN_SOURCE, kind, Some(payload));
}
}
fn refused(&self, req: Option<&CtlRequest>, phase: &str, r: &Refusal) {
let suppressed = match req {
Some(_) => 0,
None => match self.verify_budget.take() {
Some(n) => n,
None => return,
},
};
self.record(
chain::EVENT_KIND_WORKLOAD_REFUSE,
json!({
"node": self.node_id, "phase": phase, "code": r.code.as_str(),
"reason": r.reason, "method": req.map(|q| q.method.as_str()),
"requester": req.map(|q| q.requester.as_str()),
"decision_id": req.and_then(|q| q.decision_id.as_deref()),
"suppressed": suppressed,
}),
);
}
pub async fn handle(
&self,
header_method: &str,
signed: &SignedCtl,
fetch: Option<&mut PeerSet>,
) -> SignedCtl {
self.handle_checked(header_method, signed, fetch).await.0
}
pub async fn handle_checked(
&self,
header_method: &str,
signed: &SignedCtl,
fetch: Option<&mut PeerSet>,
) -> (SignedCtl, bool) {
let verified = verify_request(
signed,
&self.node_id,
now_ms(),
self.controllers.as_ref(),
&self.nonces,
)
.and_then(|r| {
if r.method == header_method {
Ok(r)
} else {
Err(refuse(
RefusalCode::InvalidRequest,
"envelope method differs from signed method",
))
}
});
let authenticated = verified.is_ok();
let (nonce, method_name, outcome) = match verified {
Err(r) => {
self.refused(None, "verify", &r);
let claimed: Option<CtlRequest> = serde_json::from_str(&signed.payload).ok();
let nonce = claimed.map(|c| c.nonce).unwrap_or_default();
(nonce, header_method.to_string(), Err(r))
}
Ok(req) => {
if req.requester != self.node_id {
self.last_contact_ms
.store(now_ms(), std::sync::atomic::Ordering::Relaxed);
}
let out = self.dispatch(&req, fetch).await;
if let Err(r) = &out {
self.refused(Some(&req), "handle", r);
}
(req.nonce, req.method, out)
}
};
let resp = CtlResponse {
version: CTL_VERSION,
method: method_name,
responder: self.node_id.clone(),
request_nonce: nonce,
outcome: match outcome {
Ok(result) => CtlOutcome::Ok { result },
Err(refusal) => CtlOutcome::Refused { refusal },
},
}
.sign(&self.key);
(resp, authenticated)
}
async fn dispatch(
&self,
req: &CtlRequest,
fetch: Option<&mut PeerSet>,
) -> Result<Value, Refusal> {
let valid_id = req
.decision_id
.as_deref()
.is_some_and(|d| crate::workload_pkg::codec::is_lower_hex(d, 64));
if method::mutates(&req.method) && !valid_id {
return Err(refuse(
RefusalCode::InvalidRequest,
"mutations carry a decision id (64 hex chain hash)",
));
}
match req.method.as_str() {
method::DESCRIBE => {
Ok(json!({ "facts": self.current_facts(), "advertisement": self.advertisement() }))
}
m if method::is_node_admin(m) => self.node_admin_op(req).await,
method::PLACE | method::LOAD => self.place(req, fetch).await,
method::START | method::STOP | method::UNLOAD | method::STATUS | method::LOGS => {
self.instance_op(req).await
}
m if m.starts_with(crate::workload_governance::policy::WORKLOAD_ACTION_PREFIX) => {
let ctx = json!({ "requester": req.requester, "method": m });
let why = match self.gate.check(&req.requester, m, &ctx) {
GateDecision::Deny { reason, .. } => reason,
_ => "not a workload.ctl method".to_string(),
};
Err(refuse(RefusalCode::UnknownMethod, format!("{m}: {why}")))
}
m => Err(refuse(
RefusalCode::InvalidRequest,
format!("{m} is not a workload.ctl method"),
)),
}
}
async fn place(&self, req: &CtlRequest, fetch: Option<&mut PeerSet>) -> Result<Value, Refusal> {
let _in_flight = InFlight::enter(&self.in_flight, req.decision_id.clone());
let list_at_start = self.revocations.get().map(|l| l.generation());
let b: PlaceBody = serde_json::from_value(req.body.clone())
.map_err(|e| refuse(RefusalCode::InvalidRequest, format!("place body: {e}")))?;
if !valid_token(&b.name, 64)
|| !crate::workload_pkg::codec::is_lower_hex(&b.manifest_hash, 64)
{
return Err(refuse(
RefusalCode::InvalidRequest,
"bad name or manifest hash",
));
}
let route = route_of(&b.variant).to_string();
let host = self.routes.get(&route).cloned().ok_or_else(|| {
refuse(
RefusalCode::Admission,
format!("no {route} adapter on this node for {}", b.variant),
)
})?;
let peers = fetch.ok_or_else(|| refuse(RefusalCode::Fetch, "no payload source"))?;
let pkg = self
.exchange
.fetch_package_in(peers, &b.manifest_hash, &self.anchors, &self.kinds)
.await
.map_err(|e| match e {
PackageExchangeError::Verify(v) => refuse(RefusalCode::Verify, v.to_string()),
other => refuse(RefusalCode::Fetch, other.to_string()),
})?;
let w =
VerifiedWorkload::from_package(&pkg.verified, &StoreSource::new(self.exchange.store()))
.map_err(|e| refuse(RefusalCode::Verify, e.to_string()))?;
if w.id != b.name {
return Err(refuse(
RefusalCode::Verify,
"package id differs from the placed name",
));
}
let licence = self.licence_check_place(&pkg.verified, &w, &b.variant, &host, req).await?;
if let Some(p) = &b.project_id
&& !crate::cog_ingest::valid_project_id(p)
{
return Err(refuse(
RefusalCode::InvalidRequest,
"project_id must be a 26-character project id",
));
}
let mut contract = HostContract::new(SocketAddr::from(([0, 0, 0, 0], b.config.csi_port)));
let mut listener = None;
let hooks = self.ingest.as_ref().filter(|_| w.kind == "cog");
let ingest_state: &'static str = hooks.map_or("none", |h| h.state());
if let Some(hk) = hooks {
let probe = InstanceBinding::new("", b.project_id.clone(), &req.requester);
hk.authorize(&probe).map_err(|e| match e {
IngestError::Forbidden => refuse(
RefusalCode::Unauthorized,
"this controller may not place cogs for that project",
),
IngestError::NotRouted(m) => refuse(RefusalCode::Admission, m),
other => refuse(RefusalCode::Runtime, format!("ingest bridge: {other}")),
})?;
}
let hooks = hooks.filter(|h| h.is_enabled());
if ingest_state == "disabled" {
contract = contract.without_ingest();
}
if let Some(hk) = hooks {
(contract, listener) = hk
.prepare(&route, contract)
.await
.map_err(|e| refuse(RefusalCode::Runtime, format!("ingest bridge: {e}")))?;
}
let cfg = WorkloadConfig {
mode: b.config.mode.clone(),
args: b.config.args.clone(),
host: contract.clone(),
node_id: self.node_id.clone(),
};
let h = host.load(&w, &cfg).await.map_err(|e| runtime_refusal(&e))?;
let iid = h.instance_id.clone();
let lease = match hooks {
Some(hk) => {
let binding = InstanceBinding::new(&iid, b.project_id.clone(), &req.requester);
match hk.lease(binding, contract, listener) {
Ok(l) => Some(l),
Err(e) => {
let mut r = refuse(RefusalCode::Runtime, format!("ingest bridge: {e}"));
if let Err(u) = host.unload(h.clone()).await {
r.reason = format!("{}; unload of {iid} failed too ({u})", r.reason);
}
return Err(r);
}
}
}
None => None,
};
if let Ok(mut d) = self.departed.lock() {
d.retain(|(i, _)| i != &iid);
}
let continuous = !matches!(b.config.mode, crate::workload_runtime::RunMode::Once);
let will_start = req.method == method::PLACE && b.start;
let mut life = super::lifecycle::InstanceLife::new(
super::lifecycle::LifecycleState::Loaded,
now_ms(),
);
if will_start {
let _ = life.transition(super::lifecycle::LifecycleState::Running, now_ms());
}
self.instances.lock().await.insert(
iid.clone(),
Placed {
handle: h.clone(),
route: route.clone(),
name: b.name.clone(),
variant: b.variant.clone(),
decision_id: req.decision_id.clone(),
last: None,
ingest: lease,
ingest_state,
life,
continuous,
desired_running: will_start,
lease_stopped: false,
requester: req.requester.clone(),
licence,
},
);
if let (Some(list), Some(g0)) = (self.revocations.get(), list_at_start)
&& list.generation() != g0
{
self.enforce_revocations(list).await;
if !self.instances.lock().await.contains_key(&iid) {
return Err(refuse(
RefusalCode::Governance,
format!("revoked while it was being placed: instance {iid} torn down"),
));
}
}
let started = req.method == method::PLACE && b.start;
if started && let Err(e) = host.start(&h).await {
let revoked = match (self.revocations.get(), host.workload_for(&h).await) {
(Some(l), Some(w)) => {
let (p, k, a) = w.revocation_refs();
l.first_revoked(Some(&p), &k, &a)
}
_ => None,
};
return Err(self.roll_back(&host, &h, &e, revoked).await);
}
let status = host.status(&h).await;
self.record(
chain::EVENT_KIND_WORKLOAD_PLACE,
json!({
"phase": "placed", "node": self.node_id, "requester": req.requester,
"decision_id": req.decision_id, "workload": b.name, "variant": b.variant,
"runtime": h.runtime, "instance_id": iid, "package_id": pkg.package_id,
"started": started,
}),
);
Ok(json!({
"instance_id": iid, "runtime": h.runtime, "variant": b.variant,
"package_id": pkg.package_id, "status": status, "node": self.node_id,
"ingest": ingest_state,
}))
}
async fn roll_back(
&self,
host: &WorkloadHost,
h: &InstanceHandle,
e: &RuntimeError,
revoked: Option<crate::revocation::RevokedSubject>,
) -> Refusal {
let mut r = runtime_refusal(e);
let iid = h.instance_id.clone();
if let (Some(hk), Some(p)) = (&self.ingest, self.instances.lock().await.get(&iid))
&& let Some(l) = &p.ingest
{
hk.deactivate(l);
}
let unloaded = match &revoked {
Some(subject) => {
host.revoke_teardown(h, std::time::Duration::from_secs(2), json!(subject))
.await
}
None => host.unload(h.clone()).await,
};
match unloaded {
Ok(()) => {
self.instances.lock().await.remove(&iid);
r.reason = format!("start failed: {}; loaded instance {iid} unloaded", r.reason);
}
Err(u) => {
r.reason = format!(
"start failed: {}; unload of {iid} failed too ({u}), it stays listed",
r.reason
);
}
}
r
}
}