1use std::{collections::HashMap, env, future::Future, net::SocketAddr, sync::Arc, time::Duration};
2
3use anyhow::{Context, Result, ensure};
4use tracing::info;
5
6use crate::{
7 host_leases::PostgresHostLeaseStore,
8 placement::PostgresObjectPlacementStore,
9 postgres::PostgresDatabase,
10 sandbox::{CommandSandboxProvider, HostSandboxRuntimeConfig},
11 storage_urls::{GcsStorageUrlSigner, validate_buckets},
12};
13
14use super::{ActorJwtVerifier, ControlPlaneService};
15
16const DEFAULT_JWT_ISSUER: &str = "durable-object-control-plane";
17const DEFAULT_AUTHORITY_AUDIENCE: &str = "durable-object-authority";
18const DEFAULT_INVOCATION_AUDIENCE: &str = "durable-object-invoke";
19const DEFAULT_JWT_TTL_SECONDS: u64 = 1_800;
20const DEFAULT_ACTOR_IDLE_TIMEOUT_MS: u64 = 60_000;
21const DEFAULT_HOST_IDLE_TIMEOUT_MS: u64 = 300_000;
22const MAX_IDLE_TIMEOUT_MS: u64 = 86_400_000;
23
24pub struct ControlPlaneProcessConfig {
25 pub bind: SocketAddr,
26 pub jwt_signing_key: String,
27 pub jwt_key_id: String,
28 pub jwt_issuer: String,
29 pub authority_audience: String,
30 pub invocation_audience: String,
31 pub jwt_max_lifetime: Duration,
32 pub admin_token: String,
33 pub storage: ControlPlaneStorageConfig,
34 pub sandbox_provider: SandboxProviderConfig,
35 pub socket_event_sink: Option<SocketEventSinkConfig>,
36 pub socket_authenticator: Option<SocketAuthenticatorConfig>,
37}
38
39pub struct ControlPlaneStorageConfig {
40 pub postgres_url: String,
41 pub standard_buckets: HashMap<String, String>,
42}
43
44pub struct SandboxProviderConfig {
45 pub provider_name: String,
46 pub command: String,
47 pub environment: HashMap<String, String>,
48 pub runtime: HostSandboxRuntimeConfig,
49}
50
51pub struct SocketEventSinkConfig {
52 pub url: String,
53 pub token: String,
54}
55
56pub struct SocketAuthenticatorConfig {
57 pub url: String,
58 pub token: String,
59}
60
61impl ControlPlaneProcessConfig {
62 pub fn from_env() -> Result<Self> {
63 Self::from_lookup(|name| env::var(name).ok())
64 }
65}
66
67pub async fn serve_control_plane(
68 config: ControlPlaneProcessConfig,
69 shutdown: impl Future<Output = ()> + Send + 'static,
70) -> Result<()> {
71 let bind = config.bind;
72 let routes = control_plane_routes(config).await?;
73 info!(bind = %bind, "durable-object control plane is ready");
74 let listener = tokio::net::TcpListener::bind(bind)
75 .await
76 .context("bind durable-object control plane")?;
77 serve_routes(listener, routes, shutdown).await
78}
79
80async fn serve_routes(
81 listener: tokio::net::TcpListener,
82 routes: tonic::service::Routes,
83 shutdown: impl Future<Output = ()> + Send + 'static,
84) -> Result<()> {
85 axum::serve(listener, routes.into_axum_router())
86 .with_graceful_shutdown(shutdown)
87 .await
88 .context("serve durable-object control plane")
89}
90
91async fn control_plane_routes(config: ControlPlaneProcessConfig) -> Result<tonic::service::Routes> {
92 let issuer = super::ActorJwtIssuer::from_base64_pkcs8(
93 &config.jwt_signing_key,
94 config.jwt_key_id,
95 config.jwt_issuer.clone(),
96 config.authority_audience.clone(),
97 config.invocation_audience,
98 config.jwt_max_lifetime,
99 )?;
100 let auth = ActorJwtVerifier::for_scope(
101 issuer.verifier_keys_json()?,
102 config.jwt_issuer,
103 config.authority_audience,
104 super::ActorTokenPurpose::ControlPlane,
105 config.jwt_max_lifetime,
106 )?;
107 let database = PostgresDatabase::connect(&config.storage.postgres_url).await?;
108 let leases = Arc::new(PostgresHostLeaseStore::from_database(database.clone()));
109 let placements = Arc::new(PostgresObjectPlacementStore::from_database(
110 database.clone(),
111 ));
112 let registry = Arc::new(super::PostgresAdminRegistry::from_database(database));
113 let storage_urls =
114 Arc::new(GcsStorageUrlSigner::from_adc(config.storage.standard_buckets).await?);
115 let provisioner = sandbox_provisioner(config.sandbox_provider, &issuer, &leases)?;
116 let socket_events = config
117 .socket_event_sink
118 .map(|sink| super::event_sink::HttpSocketMessageEventSink::new(sink.url, sink.token))
119 .transpose()?
120 .map(|sink| Arc::new(sink) as Arc<dyn super::event_sink::SocketMessageEventSink>);
121 let socket_authenticator = config
122 .socket_authenticator
123 .map(|auth| super::socket_auth::HttpSocketAuthenticator::new(auth.url, auth.token))
124 .transpose()?
125 .map(|auth| Arc::new(auth) as Arc<dyn super::socket_auth::SocketAuthenticator>);
126 let service = ControlPlaneService::new(
127 leases,
128 placements,
129 storage_urls,
130 auth,
131 registry.clone(),
132 issuer.clone(),
133 provisioner,
134 )
135 .with_socket_event_sink(socket_events)
136 .with_socket_authenticator(socket_authenticator);
137 let admin = super::admin::AdminService::new(config.admin_token, registry, issuer)?;
138 let public_api = super::public_api::router(service.clone(), admin);
139 let internal_api = service.into_internal_service();
140 Ok(tonic::service::Routes::from(public_api).add_service(internal_api))
141}
142
143fn sandbox_provisioner(
144 config: SandboxProviderConfig,
145 issuer: &super::ActorJwtIssuer,
146 leases: &Arc<PostgresHostLeaseStore>,
147) -> Result<Arc<dyn super::service::HostProvisioner>> {
148 let provider = Arc::new(CommandSandboxProvider::new(
149 config.provider_name,
150 config.command,
151 config.environment,
152 )?);
153 Ok(Arc::new(super::service::SandboxHostProvisioner::new(
154 provider,
155 config.runtime,
156 issuer.clone(),
157 leases.clone(),
158 )))
159}
160
161impl ControlPlaneProcessConfig {
162 fn from_lookup(mut get: impl FnMut(&str) -> Option<String>) -> Result<Self> {
163 let bind = get("DURABLE_OBJECT_CONTROL_PLANE_BIND")
164 .unwrap_or_else(|| "127.0.0.1:7100".into())
165 .parse()
166 .context("DURABLE_OBJECT_CONTROL_PLANE_BIND must be a socket address")?;
167 let jwt_signing_key = required(&mut get, "DURABLE_OBJECT_JWT_SIGNING_KEY")?;
168 let jwt_key_id = get("DURABLE_OBJECT_JWT_KEY_ID").unwrap_or_else(|| "primary".into());
169 let jwt_issuer =
170 get("DURABLE_OBJECT_JWT_ISSUER").unwrap_or_else(|| DEFAULT_JWT_ISSUER.into());
171 let authority_audience = get("DURABLE_OBJECT_AUTHORITY_JWT_AUDIENCE")
172 .unwrap_or_else(|| DEFAULT_AUTHORITY_AUDIENCE.into());
173 let invocation_audience = get("DURABLE_OBJECT_INVOKE_JWT_AUDIENCE")
174 .unwrap_or_else(|| DEFAULT_INVOCATION_AUDIENCE.into());
175 let jwt_max_lifetime = Duration::from_secs(
176 get("DURABLE_OBJECT_JWT_MAX_TTL_SECONDS")
177 .map(|value| value.parse())
178 .transpose()
179 .context("DURABLE_OBJECT_JWT_MAX_TTL_SECONDS must be an integer")?
180 .unwrap_or(DEFAULT_JWT_TTL_SECONDS),
181 );
182 ensure!(
183 !jwt_max_lifetime.is_zero(),
184 "DURABLE_OBJECT_JWT_MAX_TTL_SECONDS must be positive"
185 );
186 let admin_token = required(&mut get, "DURABLE_OBJECT_ADMIN_TOKEN")?;
187 ensure!(
188 admin_token.trim() == admin_token,
189 "DURABLE_OBJECT_ADMIN_TOKEN has surrounding whitespace"
190 );
191 let standard_buckets: HashMap<String, String> =
192 serde_json::from_str(&required(&mut get, "DURABLE_OBJECT_STANDARD_BUCKETS")?)
193 .context("DURABLE_OBJECT_STANDARD_BUCKETS must be a JSON region-to-bucket map")?;
194 validate_buckets(&standard_buckets)?;
195 let storage = ControlPlaneStorageConfig {
196 postgres_url: required(&mut get, "DURABLE_OBJECT_POSTGRES_URL")?,
197 standard_buckets,
198 };
199 let sandbox_provider =
200 sandbox_provider_config(&mut get, &jwt_issuer, &invocation_audience)?;
201 let socket_event_sink = socket_event_sink_config(&mut get)?;
202 let socket_authenticator = socket_authenticator_config(&mut get)?;
203 Ok(Self {
204 bind,
205 jwt_signing_key,
206 jwt_key_id,
207 jwt_issuer,
208 authority_audience,
209 invocation_audience,
210 jwt_max_lifetime,
211 admin_token,
212 storage,
213 sandbox_provider,
214 socket_event_sink,
215 socket_authenticator,
216 })
217 }
218}
219
220fn socket_authenticator_config(
221 get: &mut impl FnMut(&str) -> Option<String>,
222) -> Result<Option<SocketAuthenticatorConfig>> {
223 let url = get("DURABLE_OBJECT_SOCKET_AUTH_URL");
224 let token = get("DURABLE_OBJECT_SOCKET_AUTH_TOKEN");
225 match (url, token) {
226 (None, None) => Ok(None),
227 (Some(url), Some(token)) => Ok(Some(SocketAuthenticatorConfig {
228 url: validated_http_url(&url, "DURABLE_OBJECT_SOCKET_AUTH_URL")?,
229 token,
230 })),
231 _ => anyhow::bail!(
232 "DURABLE_OBJECT_SOCKET_AUTH_URL and DURABLE_OBJECT_SOCKET_AUTH_TOKEN must be configured together"
233 ),
234 }
235}
236
237fn socket_event_sink_config(
238 get: &mut impl FnMut(&str) -> Option<String>,
239) -> Result<Option<SocketEventSinkConfig>> {
240 let url = get("DURABLE_OBJECT_SOCKET_EVENT_URL");
241 let token = get("DURABLE_OBJECT_SOCKET_EVENT_TOKEN");
242 match (url, token) {
243 (None, None) => Ok(None),
244 (Some(url), Some(token)) => Ok(Some(SocketEventSinkConfig {
245 url: validated_http_url(&url, "DURABLE_OBJECT_SOCKET_EVENT_URL")?,
246 token,
247 })),
248 _ => anyhow::bail!(
249 "DURABLE_OBJECT_SOCKET_EVENT_URL and DURABLE_OBJECT_SOCKET_EVENT_TOKEN must be configured together"
250 ),
251 }
252}
253
254fn sandbox_provider_config(
255 get: &mut impl FnMut(&str) -> Option<String>,
256 jwt_issuer: &str,
257 invocation_audience: &str,
258) -> Result<SandboxProviderConfig> {
259 let provider_name = required(get, "DURABLE_OBJECT_SANDBOX_PROVIDER")?;
260 ensure!(
261 provider_name == "modal",
262 "unsupported sandbox provider {provider_name:?}"
263 );
264 let environment = HashMap::from([
265 (
266 "MODAL_TOKEN_ID".into(),
267 provider_credential(get, "MODAL_TOKEN_ID")?,
268 ),
269 (
270 "MODAL_TOKEN_SECRET".into(),
271 provider_credential(get, "MODAL_TOKEN_SECRET")?,
272 ),
273 ]);
274 let control_plane_url = validated_http_url(
275 &required(get, "DURABLE_OBJECT_CONTROL_PLANE_URL")?,
276 "DURABLE_OBJECT_CONTROL_PLANE_URL",
277 )?;
278 Ok(SandboxProviderConfig {
279 provider_name,
280 command: get("DURABLE_OBJECT_SANDBOX_COMMAND")
281 .unwrap_or_else(|| "little-durable-objects-modal-go".into()),
282 environment,
283 runtime: HostSandboxRuntimeConfig {
284 control_plane_url,
285 jwt_issuer: jwt_issuer.into(),
286 invocation_jwt_audience: invocation_audience.into(),
287 actor_idle_timeout_ms: idle_timeout(
288 get,
289 "DURABLE_OBJECT_ACTOR_IDLE_TIMEOUT_MS",
290 DEFAULT_ACTOR_IDLE_TIMEOUT_MS,
291 )?,
292 host_idle_timeout_ms: idle_timeout(
293 get,
294 "DURABLE_OBJECT_HOST_IDLE_TIMEOUT_MS",
295 DEFAULT_HOST_IDLE_TIMEOUT_MS,
296 )?,
297 },
298 })
299}
300
301fn provider_credential(get: &mut impl FnMut(&str) -> Option<String>, name: &str) -> Result<String> {
302 let value = required(get, name)?;
303 ensure!(value.trim() == value, "{name} has surrounding whitespace");
304 Ok(value)
305}
306
307fn required(get: &mut impl FnMut(&str) -> Option<String>, name: &str) -> Result<String> {
308 let value = get(name).with_context(|| format!("{name} is required"))?;
309 ensure!(!value.is_empty(), "{name} must not be empty");
310 Ok(value)
311}
312
313fn validated_http_url(value: &str, name: &str) -> Result<String> {
314 let url = reqwest::Url::parse(value).with_context(|| format!("{name} must be a URL"))?;
315 ensure!(
316 matches!(url.scheme(), "http" | "https") && url.host_str().is_some(),
317 "{name} must be HTTP or HTTPS"
318 );
319 Ok(url.to_string())
320}
321
322fn idle_timeout(
323 get: &mut impl FnMut(&str) -> Option<String>,
324 name: &str,
325 default: u64,
326) -> Result<u64> {
327 let value = get(name)
328 .map(|value| value.parse())
329 .transpose()
330 .with_context(|| format!("{name} must be an integer"))?
331 .unwrap_or(default);
332 ensure!(
333 (1..=MAX_IDLE_TIMEOUT_MS).contains(&value),
334 "{name} is outside the supported range"
335 );
336 Ok(value)
337}
338
339#[cfg(test)]
340mod tests {
341 use axum::{Router, extract::WebSocketUpgrade, response::Response, routing::get};
342 use futures_util::{SinkExt, StreamExt};
343 use tokio::sync::oneshot;
344 use tokio_tungstenite::{connect_async, tungstenite::Message};
345
346 use super::*;
347
348 #[tokio::test]
349 async fn server_carries_websocket_upgrades() -> Result<()> {
350 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
351 let address = listener.local_addr()?;
352 let routes =
353 tonic::service::Routes::from(Router::new().route("/socket", get(echo_websocket)));
354 let (shutdown_tx, shutdown_rx) = oneshot::channel();
355 let server = tokio::spawn(serve_routes(listener, routes, async {
356 let _ = shutdown_rx.await;
357 }));
358
359 let (mut socket, _) = connect_async(format!("ws://{address}/socket")).await?;
360 socket.send(Message::Text("hello".into())).await?;
361 assert_eq!(
362 socket.next().await.transpose()?,
363 Some(Message::Text("hello".into()))
364 );
365 socket.close(None).await?;
366 let _ = shutdown_tx.send(());
367 server.await??;
368 Ok(())
369 }
370
371 #[test]
372 fn parses_the_minimal_storage_configuration() -> Result<()> {
373 let values = HashMap::from([
374 ("DURABLE_OBJECT_JWT_SIGNING_KEY", "c2lnbmluZw=="),
375 ("DURABLE_OBJECT_ADMIN_TOKEN", "admin-token"),
376 ("DURABLE_OBJECT_SANDBOX_PROVIDER", "modal"),
377 (
378 "DURABLE_OBJECT_CONTROL_PLANE_URL",
379 "https://objects.example.com",
380 ),
381 ("MODAL_TOKEN_ID", "modal-token-id"),
382 ("MODAL_TOKEN_SECRET", "modal-token-secret"),
383 (
384 "DURABLE_OBJECT_POSTGRES_URL",
385 "postgresql://localhost/actors",
386 ),
387 (
388 "DURABLE_OBJECT_STANDARD_BUCKETS",
389 "{\"us-east\":\"actor-state-test\"}",
390 ),
391 ]);
392 let config = ControlPlaneProcessConfig::from_lookup(|name| {
393 values.get(name).map(|value| (*value).into())
394 })?;
395 assert_eq!(
396 config.storage.standard_buckets["us-east"],
397 "actor-state-test"
398 );
399 Ok(())
400 }
401
402 #[test]
403 fn parses_socket_event_sink_only_when_url_and_token_are_present() -> Result<()> {
404 let mut complete = HashMap::from([
405 (
406 "DURABLE_OBJECT_SOCKET_EVENT_URL",
407 "https://api.example.com/events",
408 ),
409 ("DURABLE_OBJECT_SOCKET_EVENT_TOKEN", "event-token"),
410 ]);
411 let sink =
412 socket_event_sink_config(&mut |name| complete.get(name).map(|value| (*value).into()))?
413 .context("socket event sink was not configured")?;
414 assert_eq!(sink.url, "https://api.example.com/events");
415 assert_eq!(sink.token, "event-token");
416
417 complete.remove("DURABLE_OBJECT_SOCKET_EVENT_TOKEN");
418 assert!(
419 socket_event_sink_config(&mut |name| complete.get(name).map(|value| (*value).into()))
420 .is_err()
421 );
422 Ok(())
423 }
424
425 #[test]
426 fn parses_socket_authenticator_only_when_url_and_token_are_present() -> Result<()> {
427 let mut complete = HashMap::from([
428 (
429 "DURABLE_OBJECT_SOCKET_AUTH_URL",
430 "https://api.example.com/authorize",
431 ),
432 ("DURABLE_OBJECT_SOCKET_AUTH_TOKEN", "auth-token"),
433 ]);
434 let auth = socket_authenticator_config(&mut |name| {
435 complete.get(name).map(|value| (*value).into())
436 })?
437 .context("socket authenticator was not configured")?;
438 assert_eq!(auth.url, "https://api.example.com/authorize");
439 assert_eq!(auth.token, "auth-token");
440
441 complete.remove("DURABLE_OBJECT_SOCKET_AUTH_TOKEN");
442 assert!(
443 socket_authenticator_config(&mut |name| complete
444 .get(name)
445 .map(|value| (*value).into()))
446 .is_err()
447 );
448 Ok(())
449 }
450
451 async fn echo_websocket(upgrade: WebSocketUpgrade) -> Response {
452 upgrade.on_upgrade(async |mut socket| {
453 if let Some(Ok(message)) = socket.recv().await {
454 let _ = socket.send(message).await;
455 }
456 })
457 }
458}