1#[cfg(test)]
2use std::sync::{LazyLock, Mutex};
3use std::{
4 fmt,
5 io::Error as IoError,
6 net::{AddrParseError, SocketAddr},
7 num::{NonZeroU64, NonZeroUsize, ParseIntError},
8 path::{Path, PathBuf},
9 thread::available_parallelism,
10};
11
12mod env;
13mod secrets;
14
15use shardline_protocol::{SecretBytes, SecretString};
16use shardline_storage::S3ObjectStoreConfig;
17use thiserror::Error;
18
19use crate::{
20 reconstruction_cache::{
21 DEFAULT_RECONSTRUCTION_CACHE_MEMORY_MAX_ENTRIES, DEFAULT_RECONSTRUCTION_CACHE_TTL_SECONDS,
22 ReconstructionCacheAdapter,
23 },
24 server_frontend::ServerFrontend,
25 server_role::ServerRole,
26};
27use secrets::ensure_secret_size_within_limit;
28#[cfg(test)]
29use secrets::{
30 PendingS3ObjectStoreConfig, configure_s3_object_store_config, optional_s3_secret_from_sources,
31};
32#[cfg(test)]
33use secrets::{configure_provider_runtime_from_paths, read_secret_file_bytes};
34
35#[derive(Clone, PartialEq, Eq)]
37pub(crate) struct AuthConfig {
38 pub(crate) token_signing_key: Option<SecretBytes>,
39 pub(crate) auth_provider: AuthProviderKind,
40 pub(crate) auth_oidc_issuer: Option<String>,
41 pub(crate) auth_jwks_url: Option<String>,
42 pub(crate) auth_jwks_issuer: Option<String>,
43}
44
45#[derive(Clone, PartialEq, Eq)]
47pub(crate) struct OciConfig {
48 pub(crate) upload_session_ttl_seconds: NonZeroU64,
49 pub(crate) upload_max_active_sessions: NonZeroUsize,
50 pub(crate) registry_token_ttl_seconds: NonZeroU64,
51 pub(crate) registry_token_max_in_flight_requests: NonZeroUsize,
52}
53
54#[derive(Clone, PartialEq, Eq)]
56pub(crate) struct CacheConfig {
57 pub(crate) adapter: ReconstructionCacheAdapter,
58 pub(crate) ttl_seconds: NonZeroU64,
59 pub(crate) memory_max_entries: NonZeroUsize,
60 pub(crate) redis_url: Option<SecretString>,
61}
62
63#[derive(Clone, PartialEq, Eq)]
65pub(crate) struct ProviderConfig {
66 pub(crate) config_path: Option<PathBuf>,
67 pub(crate) api_key: Option<SecretBytes>,
68 pub(crate) token_issuer: Option<String>,
69 pub(crate) token_ttl_seconds: Option<NonZeroU64>,
70}
71
72#[derive(Clone, PartialEq, Eq)]
74pub struct ServerConfig {
75 bind_addr: SocketAddr,
76 server_role: ServerRole,
77 server_frontends: Vec<ServerFrontend>,
78 public_base_url: String,
79 root_dir: PathBuf,
80 object_storage_adapter: ObjectStorageAdapter,
81 s3_object_store_config: Option<S3ObjectStoreConfig>,
82 max_request_body_bytes: NonZeroUsize,
83 shard_metadata_limits: ShardMetadataLimits,
84 chunk_size: NonZeroUsize,
85 upload_max_in_flight_chunks: NonZeroUsize,
86 transfer_max_in_flight_chunks: NonZeroUsize,
87 index_postgres_url: Option<SecretString>,
88 metrics_token: Option<SecretBytes>,
89 auth: AuthConfig,
90 oci: OciConfig,
91 cache: CacheConfig,
92 provider: ProviderConfig,
93}
94
95impl fmt::Debug for ServerConfig {
96 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
97 formatter
98 .debug_struct("ServerConfig")
99 .field("bind_addr", &self.bind_addr)
100 .field("server_role", &self.server_role)
101 .field("server_frontends", &self.server_frontends)
102 .field("public_base_url", &self.public_base_url)
103 .field("root_dir", &self.root_dir)
104 .field("object_storage_adapter", &self.object_storage_adapter)
105 .field("s3_object_store_config", &self.s3_object_store_config)
106 .field("max_request_body_bytes", &self.max_request_body_bytes)
107 .field("shard_metadata_limits", &self.shard_metadata_limits)
108 .field("chunk_size", &self.chunk_size)
109 .field(
110 "upload_max_in_flight_chunks",
111 &self.upload_max_in_flight_chunks,
112 )
113 .field(
114 "transfer_max_in_flight_chunks",
115 &self.transfer_max_in_flight_chunks,
116 )
117 .field(
118 "index_postgres_url",
119 &self.index_postgres_url.as_ref().map(|_url| "***"),
120 )
121 .field(
122 "metrics_token",
123 &self.metrics_token.as_ref().map(|_token| "***"),
124 )
125 .field("auth", &self.auth)
126 .field("oci", &self.oci)
127 .field("cache", &self.cache)
128 .field("provider", &self.provider)
129 .finish()
130 }
131}
132
133impl fmt::Debug for AuthConfig {
134 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
135 formatter
136 .debug_struct("AuthConfig")
137 .field(
138 "token_signing_key",
139 &self.token_signing_key.as_ref().map(|_key| "***"),
140 )
141 .field("auth_provider", &self.auth_provider)
142 .field("auth_oidc_issuer", &self.auth_oidc_issuer)
143 .field("auth_jwks_url", &self.auth_jwks_url)
144 .field("auth_jwks_issuer", &self.auth_jwks_issuer)
145 .finish()
146 }
147}
148
149impl fmt::Debug for OciConfig {
150 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
151 formatter
152 .debug_struct("OciConfig")
153 .field(
154 "upload_session_ttl_seconds",
155 &self.upload_session_ttl_seconds,
156 )
157 .field(
158 "upload_max_active_sessions",
159 &self.upload_max_active_sessions,
160 )
161 .field(
162 "registry_token_ttl_seconds",
163 &self.registry_token_ttl_seconds,
164 )
165 .field(
166 "registry_token_max_in_flight_requests",
167 &self.registry_token_max_in_flight_requests,
168 )
169 .finish()
170 }
171}
172
173impl fmt::Debug for CacheConfig {
174 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
175 formatter
176 .debug_struct("CacheConfig")
177 .field("adapter", &self.adapter)
178 .field("ttl_seconds", &self.ttl_seconds)
179 .field("memory_max_entries", &self.memory_max_entries)
180 .field("redis_url", &self.redis_url.as_ref().map(|_url| "***"))
181 .finish()
182 }
183}
184
185impl fmt::Debug for ProviderConfig {
186 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
187 formatter
188 .debug_struct("ProviderConfig")
189 .field("config_path", &self.config_path)
190 .field("api_key", &self.api_key.as_ref().map(|_key| "***"))
191 .field("token_issuer", &self.token_issuer)
192 .field("token_ttl_seconds", &self.token_ttl_seconds)
193 .finish()
194 }
195}
196
197const MIN_DEFAULT_TRANSFER_MAX_IN_FLIGHT_CHUNKS: NonZeroUsize = match NonZeroUsize::new(64) {
198 Some(value) => value,
199 None => NonZeroUsize::MIN,
200};
201
202const MAX_DEFAULT_TRANSFER_MAX_IN_FLIGHT_CHUNKS: NonZeroUsize = match NonZeroUsize::new(1024) {
203 Some(value) => value,
204 None => NonZeroUsize::MIN,
205};
206
207const MIN_DEFAULT_UPLOAD_MAX_IN_FLIGHT_CHUNKS: NonZeroUsize = match NonZeroUsize::new(64) {
208 Some(value) => value,
209 None => NonZeroUsize::MIN,
210};
211
212const MAX_DEFAULT_UPLOAD_MAX_IN_FLIGHT_CHUNKS: NonZeroUsize = match NonZeroUsize::new(256) {
213 Some(value) => value,
214 None => NonZeroUsize::MIN,
215};
216
217const DEFAULT_MAX_REQUEST_BODY_BYTES: NonZeroUsize = match NonZeroUsize::new(67_108_864) {
218 Some(value) => value,
219 None => NonZeroUsize::MIN,
220};
221
222const DEFAULT_MAX_SHARD_FILES: NonZeroUsize = match NonZeroUsize::new(16_384) {
223 Some(value) => value,
224 None => NonZeroUsize::MIN,
225};
226
227const DEFAULT_MAX_SHARD_XORBS: NonZeroUsize = match NonZeroUsize::new(16_384) {
228 Some(value) => value,
229 None => NonZeroUsize::MIN,
230};
231
232const DEFAULT_MAX_SHARD_RECONSTRUCTION_TERMS: NonZeroUsize = match NonZeroUsize::new(65_536) {
233 Some(value) => value,
234 None => NonZeroUsize::MIN,
235};
236
237const DEFAULT_MAX_SHARD_XORB_CHUNKS: NonZeroUsize = match NonZeroUsize::new(65_536) {
238 Some(value) => value,
239 None => NonZeroUsize::MIN,
240};
241const MAX_TOKEN_SIGNING_KEY_BYTES: u64 = 1_048_576;
242const MAX_PROVIDER_API_KEY_BYTES: u64 = 4096;
243const MAX_METRICS_TOKEN_BYTES: u64 = 4096;
244const MAX_S3_CREDENTIAL_BYTES: u64 = 4096;
245const DEFAULT_PARALLELISM_FALLBACK: NonZeroUsize = match NonZeroUsize::new(8) {
246 Some(value) => value,
247 None => NonZeroUsize::MIN,
248};
249const DEFAULT_OCI_UPLOAD_SESSION_TTL_SECONDS: NonZeroU64 = match NonZeroU64::new(3_600) {
250 Some(value) => value,
251 None => NonZeroU64::MIN,
252};
253const DEFAULT_OCI_UPLOAD_MAX_ACTIVE_SESSIONS: NonZeroUsize = match NonZeroUsize::new(1_024) {
254 Some(value) => value,
255 None => NonZeroUsize::MIN,
256};
257const DEFAULT_OCI_REGISTRY_TOKEN_TTL_SECONDS: NonZeroU64 = match NonZeroU64::new(300) {
258 Some(value) => value,
259 None => NonZeroU64::MIN,
260};
261const DEFAULT_OCI_REGISTRY_TOKEN_MAX_IN_FLIGHT_REQUESTS: NonZeroUsize = match NonZeroUsize::new(64)
262{
263 Some(value) => value,
264 None => NonZeroUsize::MIN,
265};
266
267#[cfg(test)]
268type SecretFileReadHook = Box<dyn FnOnce() + Send>;
269
270#[cfg(test)]
271struct SecretFileReadHookRegistration {
272 path: PathBuf,
273 hook: SecretFileReadHook,
274}
275
276#[cfg(test)]
277type SecretFileReadHookSlot = Vec<SecretFileReadHookRegistration>;
278
279#[cfg(test)]
280static BEFORE_SECRET_FILE_READ_HOOK: LazyLock<Mutex<SecretFileReadHookSlot>> =
281 LazyLock::new(|| Mutex::new(Vec::new()));
282
283pub use shardline_server_core::DEFAULT_SHARD_METADATA_LIMITS;
285
286impl ServerConfig {
287 #[must_use]
289 pub fn new(
290 bind_addr: SocketAddr,
291 public_base_url: String,
292 root_dir: PathBuf,
293 chunk_size: NonZeroUsize,
294 ) -> Self {
295 Self {
296 bind_addr,
297 server_role: ServerRole::All,
298 server_frontends: vec![ServerFrontend::Xet],
299 public_base_url,
300 root_dir,
301 object_storage_adapter: ObjectStorageAdapter::Local,
302 s3_object_store_config: None,
303 max_request_body_bytes: DEFAULT_MAX_REQUEST_BODY_BYTES,
304 shard_metadata_limits: DEFAULT_SHARD_METADATA_LIMITS,
305 chunk_size,
306 upload_max_in_flight_chunks: default_upload_max_in_flight_chunks(),
307 transfer_max_in_flight_chunks: default_transfer_max_in_flight_chunks(),
308 index_postgres_url: None,
309 metrics_token: None,
310 auth: AuthConfig {
311 token_signing_key: None,
312 auth_provider: AuthProviderKind::Local,
313 auth_oidc_issuer: None,
314 auth_jwks_url: None,
315 auth_jwks_issuer: None,
316 },
317 oci: OciConfig {
318 upload_session_ttl_seconds: DEFAULT_OCI_UPLOAD_SESSION_TTL_SECONDS,
319 upload_max_active_sessions: DEFAULT_OCI_UPLOAD_MAX_ACTIVE_SESSIONS,
320 registry_token_ttl_seconds: DEFAULT_OCI_REGISTRY_TOKEN_TTL_SECONDS,
321 registry_token_max_in_flight_requests:
322 DEFAULT_OCI_REGISTRY_TOKEN_MAX_IN_FLIGHT_REQUESTS,
323 },
324 cache: CacheConfig {
325 adapter: ReconstructionCacheAdapter::Memory,
326 ttl_seconds: DEFAULT_RECONSTRUCTION_CACHE_TTL_SECONDS,
327 memory_max_entries: DEFAULT_RECONSTRUCTION_CACHE_MEMORY_MAX_ENTRIES,
328 redis_url: None,
329 },
330 provider: ProviderConfig {
331 config_path: None,
332 api_key: None,
333 token_issuer: None,
334 token_ttl_seconds: None,
335 },
336 }
337 }
338
339 pub fn from_env() -> Result<Self, ServerConfigError> {
346 env::load_server_config_from_env()
347 }
348
349 #[must_use]
351 pub const fn bind_addr(&self) -> SocketAddr {
352 self.bind_addr
353 }
354
355 #[must_use]
357 pub const fn server_role(&self) -> ServerRole {
358 self.server_role
359 }
360
361 #[must_use]
363 pub fn server_frontends(&self) -> &[ServerFrontend] {
364 &self.server_frontends
365 }
366
367 #[must_use]
369 pub fn public_base_url(&self) -> &str {
370 &self.public_base_url
371 }
372
373 #[must_use]
375 pub fn root_dir(&self) -> &Path {
376 &self.root_dir
377 }
378
379 #[must_use]
381 pub(crate) const fn object_storage_adapter(&self) -> ObjectStorageAdapter {
382 self.object_storage_adapter
383 }
384
385 #[must_use]
387 pub(crate) const fn s3_object_store_config(&self) -> Option<&S3ObjectStoreConfig> {
388 self.s3_object_store_config.as_ref()
389 }
390
391 #[must_use]
393 pub const fn max_request_body_bytes(&self) -> NonZeroUsize {
394 self.max_request_body_bytes
395 }
396
397 #[must_use]
399 pub const fn shard_metadata_limits(&self) -> ShardMetadataLimits {
400 self.shard_metadata_limits
401 }
402
403 #[must_use]
405 pub const fn chunk_size(&self) -> NonZeroUsize {
406 self.chunk_size
407 }
408
409 #[must_use]
411 pub const fn with_chunk_size(mut self, chunk_size: NonZeroUsize) -> Self {
412 self.chunk_size = chunk_size;
413 self
414 }
415
416 #[must_use]
418 pub const fn upload_max_in_flight_chunks(&self) -> NonZeroUsize {
419 self.upload_max_in_flight_chunks
420 }
421
422 #[must_use]
425 pub const fn transfer_max_in_flight_chunks(&self) -> NonZeroUsize {
426 self.transfer_max_in_flight_chunks
427 }
428
429 #[must_use]
431 pub(crate) const fn reconstruction_cache_adapter(&self) -> ReconstructionCacheAdapter {
432 self.cache.adapter
433 }
434
435 #[must_use]
437 pub(crate) const fn reconstruction_cache_ttl_seconds(&self) -> NonZeroU64 {
438 self.cache.ttl_seconds
439 }
440
441 #[must_use]
443 pub(crate) const fn reconstruction_cache_memory_max_entries(&self) -> NonZeroUsize {
444 self.cache.memory_max_entries
445 }
446
447 #[must_use]
449 pub(crate) const fn oci_upload_session_ttl_seconds(&self) -> NonZeroU64 {
450 self.oci.upload_session_ttl_seconds
451 }
452
453 #[must_use]
455 pub(crate) const fn oci_upload_max_active_sessions(&self) -> NonZeroUsize {
456 self.oci.upload_max_active_sessions
457 }
458
459 #[must_use]
460 pub(crate) const fn oci_registry_token_ttl_seconds(&self) -> NonZeroU64 {
461 self.oci.registry_token_ttl_seconds
462 }
463
464 #[must_use]
465 pub(crate) const fn oci_registry_token_max_in_flight_requests(&self) -> NonZeroUsize {
466 self.oci.registry_token_max_in_flight_requests
467 }
468
469 #[must_use]
471 pub(crate) fn reconstruction_cache_redis_url(&self) -> Option<&str> {
472 self.cache
473 .redis_url
474 .as_ref()
475 .map(SecretString::expose_secret)
476 }
477
478 #[must_use]
480 pub fn with_root_dir(mut self, root_dir: PathBuf) -> Self {
481 self.root_dir = root_dir;
482 self
483 }
484
485 #[must_use]
487 pub fn with_object_storage(
488 mut self,
489 adapter: ObjectStorageAdapter,
490 s3_config: Option<S3ObjectStoreConfig>,
491 ) -> Self {
492 self.object_storage_adapter = adapter;
493 self.s3_object_store_config = s3_config;
494 self
495 }
496
497 #[must_use]
499 pub const fn with_max_request_body_bytes(
500 mut self,
501 max_request_body_bytes: NonZeroUsize,
502 ) -> Self {
503 self.max_request_body_bytes = max_request_body_bytes;
504 self
505 }
506
507 #[must_use]
509 pub const fn with_shard_metadata_limits(
510 mut self,
511 shard_metadata_limits: ShardMetadataLimits,
512 ) -> Self {
513 self.shard_metadata_limits = shard_metadata_limits;
514 self
515 }
516
517 #[must_use]
519 pub const fn with_server_role(mut self, server_role: ServerRole) -> Self {
520 self.server_role = server_role;
521 self
522 }
523
524 pub fn with_server_frontends(
531 mut self,
532 server_frontends: impl IntoIterator<Item = ServerFrontend>,
533 ) -> Result<Self, ServerConfigError> {
534 let server_frontends = deduplicated_server_frontends(server_frontends);
535 if server_frontends.is_empty() {
536 return Err(ServerConfigError::MissingServerFrontends);
537 }
538
539 self.server_frontends = server_frontends;
540 Ok(self)
541 }
542
543 #[must_use]
545 pub const fn with_upload_max_in_flight_chunks(
546 mut self,
547 upload_max_in_flight_chunks: NonZeroUsize,
548 ) -> Self {
549 self.upload_max_in_flight_chunks = upload_max_in_flight_chunks;
550 self
551 }
552
553 #[must_use]
555 pub const fn with_transfer_max_in_flight_chunks(
556 mut self,
557 transfer_max_in_flight_chunks: NonZeroUsize,
558 ) -> Self {
559 self.transfer_max_in_flight_chunks = transfer_max_in_flight_chunks;
560 self
561 }
562
563 #[must_use]
565 pub fn with_reconstruction_cache_disabled(mut self) -> Self {
566 self.cache.adapter = ReconstructionCacheAdapter::Disabled;
567 self.cache.redis_url = None;
568 self
569 }
570
571 #[must_use]
573 pub fn with_reconstruction_cache_memory(
574 mut self,
575 reconstruction_cache_ttl_seconds: NonZeroU64,
576 reconstruction_cache_memory_max_entries: NonZeroUsize,
577 ) -> Self {
578 self.cache.adapter = ReconstructionCacheAdapter::Memory;
579 self.cache.ttl_seconds = reconstruction_cache_ttl_seconds;
580 self.cache.memory_max_entries = reconstruction_cache_memory_max_entries;
581 self.cache.redis_url = None;
582 self
583 }
584
585 #[must_use]
587 pub const fn with_oci_upload_session_ttl_seconds(
588 mut self,
589 oci_upload_session_ttl_seconds: NonZeroU64,
590 ) -> Self {
591 self.oci.upload_session_ttl_seconds = oci_upload_session_ttl_seconds;
592 self
593 }
594
595 #[must_use]
597 pub const fn with_oci_upload_max_active_sessions(
598 mut self,
599 oci_upload_max_active_sessions: NonZeroUsize,
600 ) -> Self {
601 self.oci.upload_max_active_sessions = oci_upload_max_active_sessions;
602 self
603 }
604
605 #[must_use]
606 pub const fn with_oci_registry_token_ttl_seconds(
607 mut self,
608 oci_registry_token_ttl_seconds: NonZeroU64,
609 ) -> Self {
610 self.oci.registry_token_ttl_seconds = oci_registry_token_ttl_seconds;
611 self
612 }
613
614 #[must_use]
615 pub const fn with_oci_registry_token_max_in_flight_requests(
616 mut self,
617 oci_registry_token_max_in_flight_requests: NonZeroUsize,
618 ) -> Self {
619 self.oci.registry_token_max_in_flight_requests = oci_registry_token_max_in_flight_requests;
620 self
621 }
622
623 pub fn with_reconstruction_cache_redis(
630 mut self,
631 reconstruction_cache_redis_url: String,
632 reconstruction_cache_ttl_seconds: NonZeroU64,
633 ) -> Result<Self, ServerConfigError> {
634 if reconstruction_cache_redis_url.trim().is_empty() {
635 return Err(ServerConfigError::EmptyReconstructionCacheRedisUrl);
636 }
637
638 self.cache.adapter = ReconstructionCacheAdapter::Redis;
639 self.cache.ttl_seconds = reconstruction_cache_ttl_seconds;
640 self.cache.redis_url = Some(SecretString::new(reconstruction_cache_redis_url));
641 Ok(self)
642 }
643
644 #[must_use]
646 pub fn index_postgres_url(&self) -> Option<&str> {
647 self.index_postgres_url
648 .as_ref()
649 .map(SecretString::expose_secret)
650 }
651
652 #[must_use]
654 pub const fn auth_provider(&self) -> AuthProviderKind {
655 self.auth.auth_provider
656 }
657
658 #[must_use]
660 pub fn auth_oidc_issuer(&self) -> Option<&str> {
661 self.auth.auth_oidc_issuer.as_deref()
662 }
663
664 #[must_use]
666 pub fn auth_jwks_url(&self) -> Option<&str> {
667 self.auth.auth_jwks_url.as_deref()
668 }
669
670 #[must_use]
672 pub fn auth_jwks_issuer(&self) -> Option<&str> {
673 self.auth.auth_jwks_issuer.as_deref()
674 }
675
676 #[must_use]
678 pub fn token_signing_key(&self) -> Option<&[u8]> {
679 self.auth
680 .token_signing_key
681 .as_ref()
682 .map(SecretBytes::expose_secret)
683 }
684
685 #[must_use]
687 pub fn metrics_token(&self) -> Option<&[u8]> {
688 self.metrics_token.as_ref().map(SecretBytes::expose_secret)
689 }
690
691 #[must_use]
693 pub fn provider_config_path(&self) -> Option<&Path> {
694 self.provider.config_path.as_deref()
695 }
696
697 #[must_use]
699 pub fn provider_api_key(&self) -> Option<&[u8]> {
700 self.provider
701 .api_key
702 .as_ref()
703 .map(SecretBytes::expose_secret)
704 }
705
706 #[must_use]
708 pub fn provider_token_issuer(&self) -> Option<&str> {
709 self.provider.token_issuer.as_deref()
710 }
711
712 #[must_use]
714 pub const fn provider_token_ttl_seconds(&self) -> Option<NonZeroU64> {
715 self.provider.token_ttl_seconds
716 }
717
718 pub fn with_index_postgres_url(
725 mut self,
726 index_postgres_url: String,
727 ) -> Result<Self, ServerConfigError> {
728 if index_postgres_url.trim().is_empty() {
729 return Err(ServerConfigError::EmptyIndexPostgresUrl);
730 }
731
732 self.index_postgres_url = Some(SecretString::new(index_postgres_url));
733 Ok(self)
734 }
735
736 pub fn with_token_signing_key(
743 mut self,
744 token_signing_key: Vec<u8>,
745 ) -> Result<Self, ServerConfigError> {
746 if token_signing_key.is_empty() {
747 return Err(ServerConfigError::EmptyTokenSigningKey);
748 }
749 ensure_secret_size_within_limit(
750 u64::try_from(token_signing_key.len()).unwrap_or(u64::MAX),
751 MAX_TOKEN_SIGNING_KEY_BYTES,
752 |observed_bytes, maximum_bytes| ServerConfigError::TokenSigningKeyTooLarge {
753 observed_bytes,
754 maximum_bytes,
755 },
756 )?;
757
758 self.auth.token_signing_key = Some(SecretBytes::new(token_signing_key));
759 Ok(self)
760 }
761
762 #[must_use]
764 pub const fn with_auth_provider(mut self, auth_provider: AuthProviderKind) -> Self {
765 self.auth.auth_provider = auth_provider;
766 self
767 }
768
769 #[must_use]
771 pub fn with_auth_oidc_issuer(mut self, issuer: String) -> Self {
772 self.auth.auth_oidc_issuer = Some(issuer);
773 self
774 }
775
776 #[must_use]
778 pub fn with_auth_jwks_url(mut self, url: String) -> Self {
779 self.auth.auth_jwks_url = Some(url);
780 self
781 }
782
783 #[must_use]
785 pub fn with_auth_jwks_issuer(mut self, issuer: String) -> Self {
786 self.auth.auth_jwks_issuer = Some(issuer);
787 self
788 }
789
790 pub fn with_metrics_token(mut self, metrics_token: Vec<u8>) -> Result<Self, ServerConfigError> {
797 if metrics_token.is_empty() {
798 return Err(ServerConfigError::EmptyMetricsToken);
799 }
800 ensure_secret_size_within_limit(
801 u64::try_from(metrics_token.len()).unwrap_or(u64::MAX),
802 MAX_METRICS_TOKEN_BYTES,
803 |observed_bytes, maximum_bytes| ServerConfigError::MetricsTokenTooLarge {
804 observed_bytes,
805 maximum_bytes,
806 },
807 )?;
808
809 self.metrics_token = Some(SecretBytes::new(metrics_token));
810 Ok(self)
811 }
812
813 pub fn with_provider_runtime(
820 mut self,
821 provider_config_path: PathBuf,
822 provider_api_key: Vec<u8>,
823 issuer_identity: String,
824 ttl_seconds: NonZeroU64,
825 ) -> Result<Self, ServerConfigError> {
826 if provider_api_key.is_empty() {
827 return Err(ServerConfigError::EmptyProviderApiKey);
828 }
829 ensure_secret_size_within_limit(
830 u64::try_from(provider_api_key.len()).unwrap_or(u64::MAX),
831 MAX_PROVIDER_API_KEY_BYTES,
832 |observed_bytes, maximum_bytes| ServerConfigError::ProviderApiKeyTooLarge {
833 observed_bytes,
834 maximum_bytes,
835 },
836 )?;
837 if issuer_identity.trim().is_empty() {
838 return Err(ServerConfigError::EmptyProviderTokenIssuer);
839 }
840 if self.auth.token_signing_key.is_none() {
841 return Err(ServerConfigError::ProviderTokensRequireSigningKey);
842 }
843
844 self.provider.config_path = Some(provider_config_path);
845 self.provider.api_key = Some(SecretBytes::new(provider_api_key));
846 self.provider.token_issuer = Some(issuer_identity);
847 self.provider.token_ttl_seconds = Some(ttl_seconds);
848 Ok(self)
849 }
850
851 pub const fn validate_runtime_requirements(&self) -> Result<(), ServerConfigError> {
858 if self.auth.token_signing_key.is_none()
859 && (self.server_role.serves_api() || self.server_role.serves_transfer())
860 {
861 return Err(ServerConfigError::MissingTokenSigningKeyForServedRoutes);
862 }
863
864 Ok(())
865 }
866}
867
868#[must_use]
870pub(crate) fn default_upload_max_in_flight_chunks() -> NonZeroUsize {
871 adaptive_default_in_flight_chunks(
872 2,
873 MIN_DEFAULT_UPLOAD_MAX_IN_FLIGHT_CHUNKS,
874 MAX_DEFAULT_UPLOAD_MAX_IN_FLIGHT_CHUNKS,
875 )
876}
877
878#[must_use]
880pub(crate) fn default_transfer_max_in_flight_chunks() -> NonZeroUsize {
881 adaptive_default_in_flight_chunks(
882 8,
883 MIN_DEFAULT_TRANSFER_MAX_IN_FLIGHT_CHUNKS,
884 MAX_DEFAULT_TRANSFER_MAX_IN_FLIGHT_CHUNKS,
885 )
886}
887
888fn deduplicated_server_frontends(
889 server_frontends: impl IntoIterator<Item = ServerFrontend>,
890) -> Vec<ServerFrontend> {
891 let mut deduplicated = Vec::new();
892 for frontend in server_frontends {
893 if !deduplicated.contains(&frontend) {
894 deduplicated.push(frontend);
895 }
896 }
897 deduplicated
898}
899
900fn adaptive_default_in_flight_chunks(
901 multiplier: usize,
902 minimum: NonZeroUsize,
903 maximum: NonZeroUsize,
904) -> NonZeroUsize {
905 let parallelism = available_parallelism().unwrap_or(DEFAULT_PARALLELISM_FALLBACK);
906 adaptive_default_in_flight_chunks_for_parallelism(
907 parallelism.get(),
908 multiplier,
909 minimum,
910 maximum,
911 )
912}
913
914fn adaptive_default_in_flight_chunks_for_parallelism(
915 parallelism: usize,
916 multiplier: usize,
917 minimum: NonZeroUsize,
918 maximum: NonZeroUsize,
919) -> NonZeroUsize {
920 let scaled = parallelism.saturating_mul(multiplier);
921 let bounded = scaled.clamp(minimum.get(), maximum.get());
922 NonZeroUsize::new(bounded).unwrap_or(minimum)
923}
924
925pub use shardline_server_core::ShardMetadataLimits;
927
928#[derive(Debug, Clone, Copy, PartialEq, Eq)]
930pub enum AuthProviderKind {
931 Local,
933 Oidc,
935 Jwks,
937 Passthrough,
939}
940
941impl AuthProviderKind {
942 pub fn parse(value: &str) -> Result<Self, ServerConfigError> {
949 match value {
950 "local" => Ok(Self::Local),
951 "oidc" => Ok(Self::Oidc),
952 "jwks" => Ok(Self::Jwks),
953 "passthrough" => Ok(Self::Passthrough),
954 _other => Err(ServerConfigError::InvalidAuthProvider),
955 }
956 }
957}
958
959#[derive(Debug, Clone, Copy, PartialEq, Eq)]
961pub enum ObjectStorageAdapter {
962 Local,
964 S3,
966}
967
968impl ObjectStorageAdapter {
969 pub fn parse(value: &str) -> Result<Self, ServerConfigError> {
976 match value {
977 "local" => Ok(Self::Local),
978 "s3" => Ok(Self::S3),
979 _other => Err(ServerConfigError::InvalidObjectStorageAdapter),
980 }
981 }
982}
983
984#[derive(Debug, Error)]
986pub enum ServerConfigError {
987 #[error("invalid bind address")]
989 BindAddress(#[from] AddrParseError),
990 #[error("invalid local deployment root")]
992 RootDir(#[source] IoError),
993 #[error("invalid server role")]
995 InvalidServerRole,
996 #[error("invalid server frontend")]
998 InvalidServerFrontend,
999 #[error("at least one server frontend must be enabled")]
1001 MissingServerFrontends,
1002 #[error("invalid object storage adapter")]
1004 InvalidObjectStorageAdapter,
1005 #[error("invalid auth provider")]
1007 InvalidAuthProvider,
1008 #[error("s3 object storage requires SHARDLINE_S3_BUCKET")]
1010 MissingS3Bucket,
1011 #[error("invalid s3 allow-http flag")]
1013 InvalidS3AllowHttp,
1014 #[error("invalid s3 virtual-hosted-style request flag")]
1016 InvalidS3VirtualHostedStyleRequest,
1017 #[error("s3 credential source conflict: both {env} and {file_env} are set")]
1019 S3CredentialSourceConflict {
1020 env: &'static str,
1022 file_env: &'static str,
1024 },
1025 #[error("s3 credential file {name} could not be read")]
1027 S3CredentialFile {
1028 name: &'static str,
1030 #[source]
1032 source: IoError,
1033 },
1034 #[error("s3 credential file {name} exceeded the bounded parser ceiling")]
1036 S3CredentialTooLarge {
1037 name: &'static str,
1039 observed_bytes: u64,
1041 maximum_bytes: u64,
1043 },
1044 #[error("s3 credential file {name} changed during bounded read")]
1046 S3CredentialLengthMismatch {
1047 name: &'static str,
1049 expected_bytes: u64,
1051 observed_bytes: u64,
1053 },
1054 #[error("s3 credential file {name} was not valid utf-8")]
1056 S3CredentialUtf8 {
1057 name: &'static str,
1059 },
1060 #[error("invalid chunk size")]
1062 ChunkSize(#[from] ParseIntError),
1063 #[error("invalid max request body size")]
1065 MaxRequestBodyBytes(ParseIntError),
1066 #[error("max request body size must be greater than zero")]
1068 ZeroMaxRequestBodyBytes,
1069 #[error("invalid max shard file section count")]
1071 MaxShardFiles(ParseIntError),
1072 #[error("max shard file section count must be greater than zero")]
1074 ZeroMaxShardFiles,
1075 #[error("invalid max shard xorb section count")]
1077 MaxShardXorbs(ParseIntError),
1078 #[error("max shard xorb section count must be greater than zero")]
1080 ZeroMaxShardXorbs,
1081 #[error("invalid max shard reconstruction term count")]
1083 MaxShardReconstructionTerms(ParseIntError),
1084 #[error("max shard reconstruction term count must be greater than zero")]
1086 ZeroMaxShardReconstructionTerms,
1087 #[error("invalid max shard xorb chunk record count")]
1089 MaxShardXorbChunks(ParseIntError),
1090 #[error("max shard xorb chunk record count must be greater than zero")]
1092 ZeroMaxShardXorbChunks,
1093 #[error("chunk size must be greater than zero")]
1095 ZeroChunkSize,
1096 #[error("upload max in-flight chunks must be greater than zero")]
1098 ZeroUploadMaxInFlightChunks,
1099 #[error("transfer max in-flight chunks must be greater than zero")]
1101 ZeroTransferMaxInFlightChunks,
1102 #[error("invalid reconstruction cache adapter")]
1104 InvalidReconstructionCacheAdapter,
1105 #[error("reconstruction cache ttl must be greater than zero")]
1107 ZeroReconstructionCacheTtlSeconds,
1108 #[error("reconstruction cache memory max entries must be greater than zero")]
1110 ZeroReconstructionCacheMemoryMaxEntries,
1111 #[error("invalid oci upload session ttl")]
1113 OciUploadSessionTtl(ParseIntError),
1114 #[error("oci upload session ttl must be greater than zero")]
1116 ZeroOciUploadSessionTtlSeconds,
1117 #[error("invalid oci upload max active sessions")]
1119 OciUploadMaxActiveSessions(ParseIntError),
1120 #[error("oci upload max active sessions must be greater than zero")]
1122 ZeroOciUploadMaxActiveSessions,
1123 #[error("invalid oci registry token ttl")]
1125 OciRegistryTokenTtl(ParseIntError),
1126 #[error("oci registry token ttl must be greater than zero")]
1128 ZeroOciRegistryTokenTtlSeconds,
1129 #[error("invalid oci registry token max in-flight requests")]
1131 OciRegistryTokenMaxInFlightRequests(ParseIntError),
1132 #[error("oci registry token max in-flight requests must be greater than zero")]
1134 ZeroOciRegistryTokenMaxInFlightRequests,
1135 #[error("reconstruction cache redis url must not be empty")]
1137 EmptyReconstructionCacheRedisUrl,
1138 #[error("redis reconstruction cache requires SHARDLINE_RECONSTRUCTION_CACHE_REDIS_URL")]
1140 MissingReconstructionCacheRedisUrl,
1141 #[error("postgres metadata url must not be empty")]
1143 EmptyIndexPostgresUrl,
1144 #[error("token signing key could not be read")]
1146 TokenSigningKey(#[source] IoError),
1147 #[error("token signing key source conflict: both {env} and {file_env} are set")]
1149 TokenSigningKeySourceConflict {
1150 env: &'static str,
1152 file_env: &'static str,
1154 },
1155 #[error("token signing key exceeded the bounded parser ceiling")]
1157 TokenSigningKeyTooLarge {
1158 observed_bytes: u64,
1160 maximum_bytes: u64,
1162 },
1163 #[error("token signing key changed during bounded read")]
1165 TokenSigningKeyLengthMismatch {
1166 expected_bytes: u64,
1168 observed_bytes: u64,
1170 },
1171 #[error("invalid provider token ttl")]
1173 ProviderTokenTtl,
1174 #[error("token signing key must not be empty")]
1176 EmptyTokenSigningKey,
1177 #[error("metrics token could not be read")]
1179 MetricsToken(#[source] IoError),
1180 #[error("metrics token must not be empty")]
1182 EmptyMetricsToken,
1183 #[error("metrics token exceeded the bounded parser ceiling")]
1185 MetricsTokenTooLarge {
1186 observed_bytes: u64,
1188 maximum_bytes: u64,
1190 },
1191 #[error("metrics token changed during bounded read")]
1193 MetricsTokenLengthMismatch {
1194 expected_bytes: u64,
1196 observed_bytes: u64,
1198 },
1199 #[error("served shardline routes require shardline token signing key configuration")]
1201 MissingTokenSigningKeyForServedRoutes,
1202 #[error("provider bootstrap key must not be empty")]
1204 EmptyProviderApiKey,
1205 #[error("provider bootstrap key could not be read")]
1207 ProviderApiKey(#[source] IoError),
1208 #[error("provider bootstrap key exceeded the bounded parser ceiling")]
1210 ProviderApiKeyTooLarge {
1211 observed_bytes: u64,
1213 maximum_bytes: u64,
1215 },
1216 #[error("provider bootstrap key changed during bounded read")]
1218 ProviderApiKeyLengthMismatch {
1219 expected_bytes: u64,
1221 observed_bytes: u64,
1223 },
1224 #[error("provider token issuer must not be empty")]
1226 EmptyProviderTokenIssuer,
1227 #[error("provider token ttl must be greater than zero")]
1229 ZeroProviderTokenTtl,
1230 #[error("provider token issuance requires both provider config and provider api key files")]
1232 IncompleteProviderTokenConfig,
1233 #[error("provider token issuance requires shardline token signing key configuration")]
1235 ProviderTokensRequireSigningKey,
1236 #[error("chunk size must not exceed 1 GB")]
1238 ChunkSizeTooLarge,
1239 #[error("SHARDLINE_PUBLIC_BASE_URL is not a valid URL: {0}")]
1241 InvalidPublicBaseUrl(String),
1242 #[error("oidc auth provider requires SHARDLINE_AUTH_OIDC_ISSUER")]
1244 MissingOidcIssuer,
1245 #[error("jwks auth provider requires SHARDLINE_AUTH_JWKS_URL")]
1247 MissingJwksUrl,
1248 #[error(
1250 "hub frontend requires auth configuration (SHARDLINE_AUTH_PROVIDER with token signing key or oidc/jwks)"
1251 )]
1252 HubRequiresAuth,
1253}
1254
1255#[cfg(test)]
1256fn run_before_secret_file_read_hook_for_tests(path: &Path) {
1257 let hook = match BEFORE_SECRET_FILE_READ_HOOK.lock() {
1258 Ok(mut guard) => take_secret_file_read_hook_for_path(&mut guard, path),
1259 Err(poisoned) => take_secret_file_read_hook_for_path(&mut poisoned.into_inner(), path),
1260 };
1261
1262 if let Some(hook) = hook {
1263 hook();
1264 }
1265}
1266
1267#[cfg(test)]
1268fn take_secret_file_read_hook_for_path(
1269 slot: &mut SecretFileReadHookSlot,
1270 path: &Path,
1271) -> Option<SecretFileReadHook> {
1272 let index = slot
1273 .iter()
1274 .position(|registration| registration.path == path)?;
1275 Some(slot.remove(index).hook)
1276}
1277
1278#[cfg(not(test))]
1279const fn run_before_secret_file_read_hook_for_tests(_path: &Path) {}
1280
1281#[cfg(test)]
1282mod tests;