ordinary_api/server/

mod.rs

// Copyright (C) 2026 Ordinary Labs, LLC.
//
// SPDX-License-Identifier: AGPL-3.0-only

mod ops;

use axum::extract::ConnectInfo;
use axum::http::{HeaderValue, Uri};
use axum::http::{Request, header};
use axum::response::Redirect;
use hashbrown::{HashMap, HashSet};
use hyper::HeaderMap;
use hyper_util::rt::TokioIo;
use ordinary_app::server::OrdinaryAppServer;
#[cfg(feature = "server")]
use ordinary_config::OrdinaryApiLimits;
use ordinary_monitor::service::OrdinaryMonitorService;
use ordinary_storage::Storage;
use parking_lot::{Mutex, RwLock};
use rand_chacha::rand_core::Rng;
use rand_chacha::rand_core::SeedableRng;
use saferlmdb::{
    self as lmdb, Database, DatabaseOptions, EnvBuilder, Environment, ReadTransaction,
    WriteTransaction, put,
};
use sha2::{Digest, Sha256};
use std::io::Write;
use std::process;
use tokio::net::TcpListener;
use tokio::net::TcpStream;
use tokio_rustls::LazyConfigAcceptor;
use tokio_rustls::StartHandshake;

use axum::Router;
use axum::http::HeaderName;
use axum::response::Response;
use axum::routing::{delete, get, post, put};

use axum::http::StatusCode;
use tower_http::classify::ServerErrorsFailureClass;
use tower_http::request_id::{MakeRequestUuid, PropagateRequestIdLayer, SetRequestIdLayer};
use uuid::Uuid;

use std::net::SocketAddr;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;

use rustls_acme::{AcmeConfig, caches::DirCache};
use tokio_rustls::{TlsAcceptor, rustls::ServerConfig};
use tower::ServiceBuilder;
use tower_http::compression::CompressionLayer;
use tower_http::decompression::RequestDecompressionLayer;
use tower_http::trace::TraceLayer;
use tracing::{Instrument, Span};

use ordinary_auth::{Auth, AuthClient, OsRng};

use std::fs;

use crate::{api_account_claims, api_invite_claims};
use anyhow::bail;
use axum::middleware::Next;
use getrandom::SysRng;
use ordinary_config::{
    AccessTokenConfig, AuthConfig, InviteConfig, InviteMode, MfaConfig, OrdinaryApiConfig,
    OrdinaryConfig, PasswordConfig, RedactedHashAlg, RefreshTokenConfig, TotpConfig,
};
use ordinary_monitor::tracing::logger::OrdinaryLogger;
use ordinary_utils::middleware::modify_etag_for_encoding;
use ordinary_utils::{
    GMT_FORMAT, HeadersDebug, LatencyDisplay, REQUEST_ID_HEADER, WrappedRedactedHashingAlg,
    get_bearer_token_as_bytes, get_host, redirect_service, response_for_panic,
    rustls_server_config, shutdown_signal,
};
use sysinfo::{Pid, System};
use time::UtcDateTime;
use tokio::sync::watch;
use tokio::sync::watch::{Receiver, Sender};
use tokio::time::timeout;
use tokio_rustls::rustls::crypto::ring;
use tokio_rustls::rustls::server::Acceptor;
use tower_http::catch_panic::CatchPanicLayer;
use tower_http::set_header::SetResponseHeaderLayer;
use tower_http::timeout::TimeoutLayer;
use utoipa::openapi::security::{Http, HttpAuthScheme, SecurityScheme};
use utoipa::{Modify, OpenApi};
use utoipa_swagger_ui::{Config, SwaggerUi};
use x25519_dalek::{PublicKey, StaticSecret};

pub const ROOT: &str = "root";
pub const ADMIN: &str = "admin";
pub const APPLICATION: &str = "application";
pub const AUTHENTICATION: &str = "authentication";

#[derive(OpenApi)]
#[openapi(
    info(
        title = "Ordinary API Server",
        description = "
## ordinary-api

[![docs.rs](https://img.shields.io/docsrs/ordinary-api/0.6.0-pre.13)](https://docs.rs/ordinary-api/0.6.0-pre.13)
[![dependency status](https://img.shields.io/deps-rs/ordinary-api/0.6.0-pre.13)](https://deps.rs/crate/ordinary-api/0.6.0-pre.13)

### Access Token

Use the [Ordinary CLI](https://crates.io/crates/ordinary) to get an access token.

```sh
ordinary accounts access get --min 15
```
",
        contact(),
    ),
    modifiers(&Security),
    nest(
        (path = "/v1", api = OrdinaryApiOpenApi),
    ),
    tags(
        (name = APPLICATION, description = "Application API"),
        (name = ADMIN, description = "Admin API"),
        (name = ROOT, description = "Root API"),
        (name = AUTHENTICATION, description = "Authentication API"),
    )
)]
struct ApiDoc;

struct Security;

impl Modify for Security {
    fn modify(&self, openapi: &mut utoipa::openapi::OpenApi) {
        if let Some(components) = openapi.components.as_mut() {
            components.add_security_scheme(
                "invite",
                SecurityScheme::Http(Http::new(HttpAuthScheme::Bearer)),
            );

            components.add_security_scheme(
                "refresh",
                SecurityScheme::Http(Http::new(HttpAuthScheme::Bearer)),
            );

            components.add_security_scheme(
                "access",
                SecurityScheme::Http(Http::new(HttpAuthScheme::Bearer)),
            );
        }
    }
}

#[derive(OpenApi)]
#[openapi(paths(
    ops::accounts::invite,
    ops::accounts::list,
    ops::accounts::delete,
    ops::accounts::registration_start,
    ops::accounts::registration_finish,
    ops::accounts::login_start,
    ops::accounts::login_finish,
    ops::accounts::access_get,
    ops::accounts::password_reset_login_start,
    ops::accounts::password_reset_login_finish,
    ops::accounts::password_reset_registration_start,
    ops::accounts::password_reset_registration_finish,
    ops::accounts::password_forgot_start,
    ops::accounts::password_forgot_finish,
    ops::accounts::mfa_reset_totp_start,
    ops::accounts::mfa_reset_totp_finish,
    ops::accounts::mfa_lost_totp_start,
    ops::accounts::mfa_lost_totp_finish,
    ops::accounts::recovery_reset_codes_start,
    ops::accounts::recovery_reset_codes_finish,
    ops::accounts::account_delete_start,
    ops::accounts::account_delete_finish,
    ops::root::log_files_metadata,
    ops::root::log_files,
    ops::root::info,
    ops::root::lock,
    ops::root::unlock,
    ops::app::log_files_metadata,
    ops::app::log_files,
    ops::app::deploy,
    ops::app::kill,
    ops::app::restart,
    ops::app::erase,
    ops::templates::upload,
    ops::secrets::store,
    ops::keys::dh_public_key,
    ops::content::update,
    ops::assets::write,
    ops::models::items_list,
    ops::actions::install,
))]
pub struct OrdinaryApiOpenApi;

pub struct AccountLockManager {
    env: Arc<Environment>,
    lock_db: Arc<Database<'static>>,
}

impl AccountLockManager {
    pub fn new(env: &Arc<Environment>) -> anyhow::Result<Self> {
        let lock_db = Arc::new(Database::open(
            env.clone(),
            Some("locked"),
            &DatabaseOptions::new(lmdb::db::Flags::CREATE),
        )?);

        Ok(Self {
            env: env.clone(),
            lock_db,
        })
    }

    pub fn lock_account(&self, account: &str) -> anyhow::Result<()> {
        if account == "root" {
            bail!("cannot lock the root account");
        }

        let txn = WriteTransaction::new(self.env.clone())?;

        {
            let mut access = txn.access();
            access.put(
                &self.lock_db,
                account.as_bytes(),
                &[0u8],
                &put::Flags::empty(),
            )?;
        }

        txn.commit()?;
        Ok(())
    }

    pub fn unlock_account(&self, account: &str) -> anyhow::Result<()> {
        let txn = WriteTransaction::new(self.env.clone())?;

        {
            let mut access = txn.access();
            access.del_key(&self.lock_db, account.as_bytes())?;
        }

        txn.commit()?;

        Ok(())
    }

    pub fn is_locked(&self, account: &str) -> anyhow::Result<bool> {
        let txn = ReadTransaction::new(self.env.clone())?;
        let access = txn.access();

        let is_locked = access
            .get::<[u8], [u8]>(&self.lock_db, account.as_bytes())
            .is_ok();

        Ok(is_locked)
    }
}

/// "admin" 0 | "read" 1 | "write" 2 | "update" 3 | "upload" 4 | "install" 5 | "deploy" 6 | "bridge" 7 | "kill" 8 | "erase" 9
pub(crate) fn check_ordinary_auth(
    state: &OrdinaryApiServerState,
    headers: &HeaderMap,
    permission: u8,
    domain: &str,
) -> Result<String, StatusCode> {
    if let Ok(token) = get_bearer_token_as_bytes(headers) {
        match state.auth.verify_access_token(token.as_ref()) {
            Ok((account, claims)) => {
                let span = tracing::info_span!("permission", account);

                span.in_scope(|| {
                    if account == "root" {
                        return Ok(account.to_string());
                    }

                    if domain == "root" {
                        tracing::error!("root-only operation disallowed");
                        return Err(StatusCode::UNAUTHORIZED);
                    }

                    let check_domain = claims.idx(1).as_str();

                    if check_domain != domain {
                        tracing::error!(domain, "token domain '{check_domain}' does not match");
                        return Err(StatusCode::UNAUTHORIZED);
                    }

                    let permissions = claims.idx(2).as_vector();

                    for claim_permission in &permissions {
                        if claim_permission.as_u8() == 0 || claim_permission.as_u8() == permission {
                            tracing::info!(
                                role = match claim_permission.as_u8() {
                                    0 => "admin",
                                    1 => "read",
                                    2 => "write",
                                    3 => "update",
                                    4 => "upload",
                                    5 => "install",
                                    6 => "deploy",
                                    7 => "bridge",
                                    8 => "kill",
                                    9 => "erase",
                                    _ => "invalid",
                                },
                                "granted"
                            );

                            match state.account_lock_manager.is_locked(account) {
                                Ok(is_locked) => {
                                    if is_locked {
                                        tracing::error!(%account, "locked");
                                        return Err(StatusCode::UNAUTHORIZED);
                                    }
                                }
                                Err(err) => {
                                    tracing::error!(%err);
                                    return Err(StatusCode::UNAUTHORIZED);
                                }
                            }

                            return Ok(account.to_string());
                        }
                    }

                    tracing::error!(
                        permission = match permission {
                            0 => "admin",
                            1 => "read",
                            2 => "write",
                            3 => "update",
                            4 => "upload",
                            5 => "install",
                            6 => "deploy",
                            7 => "bridge",
                            8 => "kill",
                            9 => "erase",
                            _ => "invalid",
                        },
                        "not in token permissions"
                    );
                    Err(StatusCode::UNAUTHORIZED)
                })
            }
            Err(err) => {
                tracing::error!(%err, "access denied");
                Err(StatusCode::UNAUTHORIZED)
            }
        }
    } else {
        tracing::error!("invalid token");
        Err(StatusCode::UNAUTHORIZED)
    }
}

pub struct WrappedOrdinaryAppServer {
    port: u16,
    app: Arc<OrdinaryAppServer>,
    terminate_tx: Sender<bool>,
    stream_tx: tokio::sync::mpsc::UnboundedSender<(StartHandshake<TcpStream>, SocketAddr, Span)>,
    dh_keypair: (StaticSecret, PublicKey),
}

type Dbs = Arc<tokio::sync::Mutex<HashMap<String, (Arc<Environment>, Arc<Auth>, Arc<Storage>)>>>;
type AppServers = Arc<tokio::sync::RwLock<HashMap<String, Arc<WrappedOrdinaryAppServer>>>>;

#[allow(clippy::struct_excessive_bools)]
pub struct OrdinaryApiServerState {
    pub domain: String,
    pub app_domains: Arc<Vec<String>>,
    pub secure: bool,
    pub secure_cookies: bool,
    pub log_headers: bool,
    pub log_ips: bool,
    pub log_size: bool,
    pub auth: Arc<Auth>,
    pub apps: AppServers,
    pub apps_dir: PathBuf,
    pub dbs: Dbs,
    pub env_name: String,
    pub provision_mode: Option<ProvisionMode>,
    pub dedicated_ports: bool,
    pub server_span: Span,
    pub monitor: Arc<Option<OrdinaryMonitorService>>,
    pub stored_logs: bool,
    pub limits: String,
    pub config: OrdinaryApiConfig,

    pub signal_tx: Arc<RwLock<Option<Sender<()>>>>,
    pub close_rx: Arc<RwLock<Option<Receiver<()>>>>,

    pub privileged_domains: HashSet<String>,

    pub pid: Pid,
    pub system: Arc<Mutex<System>>,

    pub account_lock_manager: Arc<AccountLockManager>,
}

#[derive(PartialEq, Clone)]
pub enum ProvisionMode {
    Localhost,
    Staging,
    Production,
}

pub enum SecurityMode<T: AsRef<Path>> {
    Insecure,
    Secure(T, ProvisionMode),
}

pub struct OrdinaryApiServer {
    auth: Arc<Auth>,
    env: Arc<Environment>,
    config: OrdinaryApiConfig,
    apps: AppServers,
    apps_dir: PathBuf,
    monitor: Arc<Option<OrdinaryMonitorService>>,
}

#[cfg(feature = "server")]
impl OrdinaryApiServer {
    #[allow(clippy::similar_names, clippy::too_many_arguments)]
    pub async fn init(
        env_name: &str,
        domain: &str,
        password: &str,
        env_path: impl AsRef<Path>,
        storage_size: usize,
        api_contacts: &[String],
        app_domains: &[String],
        privileged_domains: &Option<Vec<String>>,
        logger: Option<OrdinaryLogger>,
    ) -> anyhow::Result<(String, Vec<u8>)> {
        let span = tracing::info_span!("init", env = %env_name, pid = process::id());

        let mut limits = OrdinaryApiLimits::default();
        if let Some(pd) = privileged_domains {
            pd.clone_into(&mut limits.privileged_domains);
        }
        limits.app_domains = app_domains.to_vec();

        async {
            let config = OrdinaryApiConfig {
                domain: domain.into(),
                contacts: api_contacts.to_vec(),
                // todo: pass in as a flag
                public_dns_ip: None,
                env_name: env_name.to_string(),
                limits,
            };
            let config_file = serde_json::to_string_pretty(&config)?;

            let data_path = env_path.as_ref().join("data");

            if data_path.exists() {
                bail!("environment already initialized");
            }

            fs::write(env_path.as_ref().join("ordinaryd.json"), config_file)?;

            let api_server =
                OrdinaryApiServer::new(env_name, env_path, storage_size, logger).await?;

            let mut input = domain.as_bytes().to_vec();
            input.extend_from_slice(b"root");

            let mut password_input = input.clone();
            password_input.extend_from_slice(password.as_bytes());

            let mut hasher = Sha256::new();
            hasher.update(&password_input);
            let password = hasher.finalize().to_vec();

            let invite_token = api_server.auth.api_invite_get(domain, "root", None)?;

            let (state, reg_start_req) = AuthClient::registration_start_req(b"root", &password)?;

            let checked_claims = api_server.auth.invite_check(&invite_token)?;
            let reg_start_res = api_server.auth.registration_start(
                reg_start_req,
                None,
                None,
                Some(checked_claims),
            )?;

            let (private_key, reg_finish_req) =
                AuthClient::registration_finish_req(b"root", &password, &state, &reg_start_res)?;
            let (reg_finish_res, ..) = api_server.auth.registration_finish(reg_finish_req, None)?;

            let (totp, _recovery_codes) = AuthClient::decrypt_totp_mfa(
                &reg_finish_res,
                private_key,
                env_name.to_string(),
                "root".into(),
            )?;

            Ok((totp.get_url(), totp.secret))
        }
        .instrument(span.clone())
        .await
    }

    #[allow(clippy::too_many_lines, clippy::missing_panics_doc)]
    pub async fn new(
        env_name: &str,
        env_path: impl AsRef<Path>,
        storage_size: usize,
        logger: Option<OrdinaryLogger>,
    ) -> anyhow::Result<OrdinaryApiServer> {
        let span = tracing::info_span!("setup", env = %env_name, pid = process::id());

        async {
            let config: OrdinaryApiConfig = serde_json::from_str(&fs_err::read_to_string(
                env_path.as_ref().join("ordinaryd.json"),
            )?)?;

            let data_path = env_path.as_ref().join("data");

            fs_err::create_dir_all(&data_path)?;

            let ps = page_size::get();

            // round up to full OS page
            let remainder = storage_size % ps;
            let mapsize = (storage_size - remainder) + ps;

            tracing::info!(mapsize = %bytesize::ByteSize(mapsize as u64).display().si_short());

            let env = Arc::new(unsafe {
                let mut env_builder = EnvBuilder::new()?;
                env_builder.set_maxreaders(126)?;
                env_builder.set_mapsize(mapsize)?;
                env_builder.set_maxdbs(13)?;
                env_builder.open(
                    match data_path.to_str() {
                        Some(v) => v,
                        None => bail!("data_path not a str"),
                    },
                    &saferlmdb::open::Flags::empty(),
                    0o600,
                )?
            });

            let keys_dir = env_path.as_ref().join("keys");
            fs_err::create_dir_all(&keys_dir)?;

            let auth_key_path = keys_dir.join("auth");

            let auth_key: [u8; 32] = if auth_key_path.exists() && auth_key_path.is_file() {
                let auth_key = fs_err::read(&auth_key_path)?;
                let auth_key: [u8; 32] = auth_key[..].try_into()?;
                auth_key
            } else {
                let mut auth_key = [0u8; 32];
                let mut rng = rand_chacha::ChaCha20Rng::try_from_rng(&mut SysRng)?;

                rng.fill_bytes(&mut auth_key[..]);

                let mut auth_key_file = fs_err::File::create(auth_key_path)?;
                auth_key_file.write_all(&auth_key)?;
                auth_key_file.flush()?;

                auth_key
            };

            let auth = Arc::new(Auth::new(
                config.domain.clone(),
                Some(AuthConfig {
                    password: PasswordConfig {
                        protocol: ordinary_config::PasswordProtocol::Opaque,
                    },
                    mfa: MfaConfig {
                        totp: TotpConfig {
                            template: None,
                            algorithm: ordinary_config::TotpAlgorithm::Sha1,
                        },
                    },
                    refresh_token: RefreshTokenConfig::default(),
                    access_token: AccessTokenConfig {
                        claims: api_account_claims(),
                        ..AccessTokenConfig::default()
                    },
                    client_hash: ordinary_config::ClientPasswordHash::Sha256,
                    cookies_enabled: true,
                    invite: Some(InviteConfig {
                        // todo: make `InviteMode` configurable
                        mode: InviteMode::Root,
                        lifetime: 60 * 60 * 24,
                        clean_interval: (30, 90),
                        claims: Some(api_invite_claims()),
                    }),
                }),
                auth_key,
                env.clone(),
            )?);

            let apps_dir = env_path.as_ref().join("apps");
            fs_err::create_dir_all(&apps_dir)?;

            let monitor = Arc::new(logger.map(|logger| {
                OrdinaryMonitorService::new(logger).expect("failed to set up monitor service")
            }));

            Ok(OrdinaryApiServer {
                config,
                auth,
                env,
                apps: Arc::new(tokio::sync::RwLock::new(HashMap::new())),
                apps_dir,
                monitor,
            })
        }
        .instrument(span)
        .await
    }

    #[allow(
        clippy::too_many_arguments,
        clippy::fn_params_excessive_bools,
        clippy::too_many_lines
    )]
    pub async fn start<P, F>(
        &self,
        server_span: Span,
        mode: SecurityMode<P>,
        listener: TcpListener,
        secure_cookies: bool,
        log_headers: bool,
        log_ips: bool,
        log_size: bool,
        redirect_listener: Option<TcpListener>,
        dedicated_ports: bool,
        stored_logs: bool,
        redacted_hash: Option<RedactedHashAlg>,
        swagger: bool,
        signal: fn() -> F,
    ) -> anyhow::Result<()>
    where
        P: AsRef<Path>,
        F: Future<Output = ()> + Send + 'static,
    {
        use axum::extract::State;

        let pid = process::id();

        let start_span = tracing::info_span!(
            "start",
            env = &self.config.env_name,
            pid,
            domain = %self.config.domain
        );

        let server_span_clone = server_span.clone();
        let server_span_clone2 = server_span.clone();
        let server_span_clone3 = server_span.clone();

        let api_domain_clone = self.config.domain.clone();

        let limits = serde_json::to_string(&self.config.limits)?;

        let system = System::new_all();

        let account_lock_manager = Arc::new(AccountLockManager::new(&self.env)?);

        let state = start_span.in_scope(|| {
            Arc::new(OrdinaryApiServerState {
                domain: self.config.domain.clone(),
                app_domains: Arc::new(self.config.limits.app_domains.clone()),
                secure: !matches!(mode, SecurityMode::Insecure),
                secure_cookies,
                log_headers,
                log_ips,
                log_size,
                auth: self.auth.clone(),
                apps: self.apps.clone(),
                apps_dir: self.apps_dir.clone(),
                dbs: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
                env_name: self.config.env_name.clone(),
                provision_mode: match &mode {
                    SecurityMode::Insecure => None,
                    SecurityMode::Secure(_, provision) => Some(provision.clone()),
                },
                dedicated_ports,
                server_span: server_span.clone(),
                monitor: self.monitor.clone(),
                stored_logs,
                limits,
                config: self.config.clone(),

                signal_tx: Arc::new(RwLock::new(None)),
                close_rx: Arc::new(RwLock::new(None)),

                privileged_domains: self
                    .config
                    .limits
                    .privileged_domains
                    .iter()
                    .cloned()
                    .collect::<HashSet<_>>(),

                pid: Pid::from(pid as usize),
                system: Arc::new(Mutex::new(system)),
                account_lock_manager,
            })
        });

        let request_id = HeaderName::from_static(REQUEST_ID_HEADER);

        let redacted_hash = if let Some(redacted_hash) = redacted_hash {
            Arc::new(Some(WrappedRedactedHashingAlg(redacted_hash)))
        } else {
            Arc::new(None)
        };

        let redacted_hash_clone = redacted_hash.clone();
        let redacted_hash_clone1 = redacted_hash.clone();

        let last_modified = UtcDateTime::now();

        let last_modified_string = last_modified.format(&GMT_FORMAT)?;
        let last_modified_header = HeaderValue::from_str(last_modified_string.as_str())?;

        let ordinaryd_resource_layers = ServiceBuilder::new()
            .layer(axum::middleware::from_fn(
                move |req_headers: HeaderMap, request: axum::extract::Request, next: Next| {
                    ordinary_utils::middleware::http_cache_middleware(
                        last_modified,
                        req_headers,
                        request,
                        next,
                    )
                },
            ))
            .layer(SetResponseHeaderLayer::if_not_present(
                header::LAST_MODIFIED,
                last_modified_header.clone(),
            ))
            .layer(SetResponseHeaderLayer::if_not_present(
                header::EXPIRES,
                |_: &Response<_>| {
                    let future = UtcDateTime::now() + time::Duration::minutes(10);

                    if let Ok(formatted) = future.format(&GMT_FORMAT)
                        && let Ok(expires) = HeaderValue::from_str(formatted.as_str())
                    {
                        return Some(expires);
                    }

                    None
                },
            ))
            .layer(SetResponseHeaderLayer::if_not_present(
                header::VARY,
                HeaderValue::from_static(header::ACCEPT_ENCODING.as_str()),
            ));

        let mut service = Router::new();

        if swagger {
            // todo: rapidoc
            // todo: scalar
            // todo: redoc

            service = service
                .merge(
                    SwaggerUi::new("/.ordinary/swagger").config(Config::from("/.ordinary/openapi")),
                )
                .route_layer(ordinaryd_resource_layers.clone().layer(
                    SetResponseHeaderLayer::if_not_present(
                        header::CONTENT_SECURITY_POLICY,
                        HeaderValue::from_static(
                            "default-src 'self'; style-src 'self' 'unsafe-inline'; img-src https://img.shields.io",
                        ),
                    ),
                ));
        }

        // todo: || openapi
        if swagger {
            let api = ApiDoc::openapi().to_json()?;

            service = service.route(
                "/.ordinary/openapi",
                get(|| async move {
                    (
                        StatusCode::OK,
                        [(header::CONTENT_TYPE, "application/json")],
                        api,
                    )
                })
                .route_layer(ordinaryd_resource_layers.clone()),
            );
        }

        service = service.route(
            "/.ordinary/limits",
            get(
                |State(state): State<Arc<OrdinaryApiServerState>>| async move {
                    (
                        StatusCode::OK,
                        [(header::CONTENT_TYPE, "application/json")],
                        state.limits.clone(),
                    )
                },
            )
            .route_layer(ordinaryd_resource_layers.clone()),
        );

        let mut service = service
            // healthcheck
            .route("/healthz", get(|| async { StatusCode::OK }))
            // keys
            .route("/v1/keys/dh/public", get(ops::keys::dh_public_key))
            // auth ops
            .route(
                "/v1/accounts/registration/start",
                post(ops::accounts::registration_start),
            )
            .route(
                "/v1/accounts/registration/finish",
                post(ops::accounts::registration_finish),
            )
            .route("/v1/accounts/login/start", post(ops::accounts::login_start))
            .route(
                "/v1/accounts/login/finish",
                post(ops::accounts::login_finish),
            )
            .route("/v1/accounts/access", get(ops::accounts::access_get))
            .route("/v1/accounts/logout", put(ops::accounts::logout))
            // reset password
            .route(
                "/v1/accounts/password/reset/login/start",
                post(ops::accounts::password_reset_login_start),
            )
            .route(
                "/v1/accounts/password/reset/login/finish",
                post(ops::accounts::password_reset_login_finish),
            )
            .route(
                "/v1/accounts/password/reset/registration/start",
                post(ops::accounts::password_reset_registration_start),
            )
            .route(
                "/v1/accounts/password/reset/registration/finish",
                post(ops::accounts::password_reset_registration_finish),
            )
            // forgot password
            .route(
                "/v1/accounts/password/forgot/start",
                post(ops::accounts::password_forgot_start),
            )
            .route(
                "/v1/accounts/password/forgot/finish",
                post(ops::accounts::password_forgot_finish),
            )
            // reset totp mfa
            .route(
                "/v1/accounts/mfa/totp/reset/start",
                post(ops::accounts::mfa_reset_totp_start),
            )
            .route(
                "/v1/accounts/mfa/totp/reset/finish",
                post(ops::accounts::mfa_reset_totp_finish),
            )
            // lost totp mfa
            .route(
                "/v1/accounts/mfa/totp/lost/start",
                post(ops::accounts::mfa_lost_totp_start),
            )
            .route(
                "/v1/accounts/mfa/totp/lost/finish",
                post(ops::accounts::mfa_lost_totp_finish),
            )
            // delete account
            .route(
                "/v1/accounts/delete/start",
                post(ops::accounts::account_delete_start),
            )
            .route(
                "/v1/accounts/delete/finish",
                post(ops::accounts::account_delete_finish),
            )
            // reset recovery codes
            .route(
                "/v1/accounts/recovery-codes/reset/start",
                post(ops::accounts::recovery_reset_codes_start),
            )
            .route(
                "/v1/accounts/recovery-codes/reset/finish",
                post(ops::accounts::recovery_reset_codes_finish),
            )
            // root ops
            .route("/v1/apps", get(ops::app::list))
            .route("/v1/logs/metadata", get(ops::root::log_files_metadata))
            .route("/v1/logs/files/{file}", get(ops::root::log_files))
            .route("/v1/info", get(ops::root::info))
            .route("/v1/lock", post(ops::root::lock))
            .route("/v1/unlock", post(ops::root::unlock))
            // admin ops
            .route("/v1/accounts/invite", get(ops::accounts::invite))
            .route("/v1/accounts", get(ops::accounts::list))
            .route("/v1/accounts", delete(ops::accounts::delete))
            // app account ops
            .route("/v1/app/accounts/invite", get(ops::app::invite))
            .route("/v1/app/accounts", get(ops::app::accounts_list))
            // app ops
            .route("/v1/app/deploy", put(ops::app::deploy))
            .route("/v1/app/migrate", put(ops::app::migrate))
            .route("/v1/app/bridge", put(ops::app::bridge))
            .route("/v1/app/kill", put(ops::app::kill))
            .route("/v1/app/restart", post(ops::app::restart))
            .route("/v1/app/erase", delete(ops::app::erase))
            .route("/v1/app/logs/metadata", get(ops::app::log_files_metadata))
            .route("/v1/app/logs/files/{file}", get(ops::app::log_files))
            // template ops
            .route("/v1/templates", put(ops::templates::upload))
            // content ops
            .route("/v1/content", put(ops::content::update))
            // secret ops
            .route("/v1/secrets", put(ops::secrets::store))
            // assets ops
            .route("/v1/assets", put(ops::assets::write))
            // action ops
            .route("/v1/actions", put(ops::actions::install))
            // models ops
            .route("/v1/models/items", get(ops::models::items_list))
            .fallback(|| async { StatusCode::NOT_FOUND })
            .with_state(state.clone())
            .layer(
                ServiceBuilder::new()
                    .layer(CatchPanicLayer::custom(response_for_panic))
                    .layer(RequestDecompressionLayer::new())
                    .layer(CompressionLayer::new()),
            )
            .layer(
                ServiceBuilder::new()
                    .layer(SetRequestIdLayer::new(request_id.clone(), MakeRequestUuid))
                    .layer(
                        TraceLayer::new_for_http()
                            .make_span_with(move |req: &Request<_>| {
                                let request_id = req.headers().get(REQUEST_ID_HEADER);

                                let ip = log_ips.then(|| {
                                    req.extensions()
                                        .get::<ConnectInfo<SocketAddr>>()
                                        .map(|addr| tracing::field::display(addr.ip()))
                                });

                                let query = req.uri().query().map(tracing::field::display);

                                server_span_clone.in_scope(|| match request_id {
                                    Some(rid) => {
                                        tracing::info_span!(
                                            "admin",
                                            domain = %api_domain_clone,
                                            rid = %rid
                                                .to_str()
                                                .unwrap_or(Uuid::new_v4().to_string().as_str()),
                                            ip,
                                            path = %req.uri().path(),
                                            query,
                                        )
                                    }
                                    None => {
                                        tracing::info_span!(
                                            "admin",
                                            domain = %api_domain_clone,
                                            rid = %Uuid::new_v4(),
                                            ip,
                                            path = %req.uri().path(),
                                            query,
                                        )
                                    }
                                })
                            })
                            .on_request(move |req: &Request<_>, _: &Span| {
                                let hd = log_headers
                                    .then_some(HeadersDebug(req.headers(), redacted_hash.clone()));

                                #[cfg(tracing_unstable)]
                                let headers = log_headers.then_some(tracing::field::valuable(&hd));

                                #[cfg(not(tracing_unstable))]
                                let headers = log_headers.then_some(tracing::field::debug(&hd));

                                tracing::info!(
                                    version = ?req.version(),
                                    method = %req.method(),
                                    headers,
                                    "req"
                                );
                            })
                            .on_response(move |res: &Response<_>, latency: Duration, _: &Span| {
                                let hd = log_headers.then_some(HeadersDebug(
                                    res.headers(),
                                    redacted_hash_clone.clone(),
                                ));

                                #[cfg(tracing_unstable)]
                                let headers = log_headers.then_some(tracing::field::valuable(&hd));

                                #[cfg(not(tracing_unstable))]
                                let headers = log_headers.then_some(tracing::field::debug(&hd));

                                let status = res.status().as_u16();
                                let latency = LatencyDisplay(latency.as_nanos() as f64);

                                if status >= 500 {
                                    tracing::error!(status, headers, %latency, "res");
                                } else if status >= 400 {
                                    tracing::warn!(status, headers, %latency, "res");
                                } else {
                                    tracing::info!(status, headers, %latency, "res");
                                }
                            })
                            .on_failure(
                                |error: ServerErrorsFailureClass, _: Duration, _: &Span| {
                                    tracing::error!(
                                        err = %error,
                                        "fail"
                                    );
                                },
                            ),
                    )
                    .layer(TimeoutLayer::with_status_code(
                        StatusCode::REQUEST_TIMEOUT,
                        Duration::from_secs(10),
                    ))
                    .layer(PropagateRequestIdLayer::new(request_id))
                    .layer(SetResponseHeaderLayer::if_not_present(
                        header::SERVER,
                        HeaderValue::from_static("Ordinary"),
                    ))
                    .layer(SetResponseHeaderLayer::overriding(
                        header::ETAG,
                        modify_etag_for_encoding,
                    )),
            );

        if state.secure {
            service = service.layer(SetResponseHeaderLayer::if_not_present(
                header::STRICT_TRANSPORT_SECURITY,
                HeaderValue::from_static("max-age=31536000"),
            ));
        }

        if let Some(listener) = redirect_listener {
            let api_domain = self.config.domain.clone();

            let request_id_header = HeaderName::from_static(REQUEST_ID_HEADER);

            let local_addr = listener.local_addr()?;
            let state_clone = state.clone();

            tokio::spawn(async move {
                use axum::extract::State;

                let span = server_span_clone3;

                span.in_scope(
                    || tracing::info!(server = %"redirect", addr = %local_addr, "listening"),
                );

                let handler = async move |uri: Uri,
                                          headers: HeaderMap,
                                          State(state): State<Arc<OrdinaryApiServerState>>|
                            -> Result<Redirect, StatusCode> {
                    // todo: handle localhost redirecting

                    if let Some(host) = get_host(&headers, &uri) {
                        if host == api_domain
                            && let Some(path_and_query) = uri.path_and_query()
                        {
                            return Ok(Redirect::permanent(&format!(
                                "https://{api_domain}{}",
                                path_and_query.as_str()
                            )));
                        }

                        let apps = state.apps.read().await;

                        if apps.contains_key(&host)
                            && let Some(path_and_query) = uri.path_and_query()
                        {
                            return Ok(Redirect::permanent(&format!(
                                "https://{host}{}",
                                path_and_query.as_str()
                            )));
                        }
                    }

                    Err(StatusCode::NOT_FOUND)
                };

                let service = redirect_service(
                    span,
                    redacted_hash_clone1,
                    log_ips,
                    log_headers,
                    request_id_header,
                    handler,
                    state_clone,
                );

                if let Err(err) = axum::serve(
                    listener,
                    service.into_make_service_with_connect_info::<SocketAddr>(),
                )
                .with_graceful_shutdown(shutdown_signal())
                .await
                {
                    tracing::error!(%err, "server closed");
                }
            });
        }

        let local_addr = listener.local_addr()?;

        let span = server_span_clone2;

        match mode {
            SecurityMode::Insecure => {
                span.in_scope(|| {
                    tracing::info!(
                        server = %"primary",
                        addr = %local_addr,
                        provision = %"none",
                        "listening"
                    );
                });

                self.start_apps(&span, &state).await?;

                axum::serve(
                    listener,
                    service.into_make_service_with_connect_info::<SocketAddr>(),
                )
                .with_graceful_shutdown(signal())
                .await?;
            }
            SecurityMode::Secure(cert_dir_path, provision) => {
                let span_clone = span.clone();

                let (signal_tx, signal_rx) = watch::channel(());
                let (close_tx, close_rx) = watch::channel(());

                {
                    let mut signal_tx_lock = state.signal_tx.write();
                    *signal_tx_lock = Some(signal_tx.clone());

                    let mut close_rx_lock = state.close_rx.write();
                    *close_rx_lock = Some(close_rx.clone());
                }

                let signal_tx_clone = state.signal_tx.clone();

                tokio::spawn(async move {
                    signal().await;

                    span_clone.in_scope(|| {
                        tracing::warn!("graceful shutdown");
                    });

                    {
                        let mut signal_tx_lock = signal_tx_clone.write();
                        *signal_tx_lock = None;
                    }

                    drop(signal_rx);
                });

                match provision {
                    ProvisionMode::Localhost => {
                        let cert_dir_path = cert_dir_path.as_ref();

                        ordinary_utils::generate_self_signed_localhost_certs(cert_dir_path)?;

                        let rustls_config = rustls_server_config(
                            cert_dir_path.join("key.pem"),
                            cert_dir_path.join("crt.pem"),
                        )?;

                        let tls_acceptor = TlsAcceptor::from(rustls_config);
                        span.in_scope(|| {
                            tracing::info!(
                                server = %"primary",
                                addr = %local_addr,
                                provision = %"localhost",
                                "listening"
                            );
                        });

                        self.start_apps(&span, &state).await?;

                        loop {
                            let span = span.clone();
                            let (cnx, addr) = tokio::select! {
                                conn = listener.accept() => conn?,
                                () = signal_tx.closed() => {
                                    span.in_scope(|| {
                                       tracing::warn!("not accepting new connections");
                                    });
                                    break;
                                }
                            };

                            let tower_service = service.clone();
                            let tls_acceptor = tls_acceptor.clone();

                            let signal_tx = signal_tx.clone();
                            let close_rx = close_rx.clone();

                            tokio::spawn(async move {
                                // todo: determine a better timeout
                                match timeout(Duration::from_secs(3), tls_acceptor.accept(cnx))
                                    .await
                                {
                                    Ok(stream) => {
                                        let stream = match stream {
                                            Ok(stream) => TokioIo::new(stream),
                                            Err(err) => {
                                                span.in_scope(|| {
                                                    if log_ips {
                                                        tracing::warn!(ip = %addr.ip(), port = addr.port(), %err, "accept");
                                                    } else {
                                                        tracing::warn!(%err, "accept");
                                                    }
                                                });

                                                return;
                                            }
                                        };

                                        OrdinaryAppServer::handle_stream(
                                            &span,
                                            tower_service,
                                            addr,
                                            stream,
                                            &signal_tx,
                                            &close_rx,
                                            None,
                                        )
                                        .await;
                                    }
                                    Err(_) => {
                                        span.in_scope(|| {
                                            if log_ips {
                                                tracing::warn!(ip = %addr.ip(), port = addr.port(), "timeout");
                                            } else {
                                                tracing::warn!("timeout");
                                            }
                                        });
                                    }
                                }
                            });
                        }
                    }
                    ProvisionMode::Staging | ProvisionMode::Production => {
                        let contacts = self
                            .config
                            .contacts
                            .iter()
                            .map(|c| format!("mailto:{c}"))
                            .collect::<Vec<String>>();

                        let cache_path = Path::new(cert_dir_path.as_ref()).join("cache");

                        let acme_state = AcmeConfig::new([&self.config.domain])
                            .contact(contacts)
                            .cache_option(Some(DirCache::new(cache_path)))
                            .directory_lets_encrypt(provision == ProvisionMode::Production)
                            .state();

                        let challenge_rustls_config = acme_state.challenge_rustls_config();
                        let mut default_rustls_config =
                            ServerConfig::builder_with_provider(Arc::new(ring::default_provider()))
                                .with_safe_default_protocol_versions()?
                                .with_no_client_auth()
                                .with_cert_resolver(acme_state.resolver());

                        default_rustls_config.alpn_protocols =
                            vec![b"h2".to_vec(), b"http/1.1".to_vec()];

                        let default_rustls_config = Arc::new(default_rustls_config);

                        let api_domain_clone = self.config.domain.clone();
                        let span_clone = span.clone();

                        let provision_str = match provision {
                            ProvisionMode::Production => "production",
                            ProvisionMode::Staging => "staging",
                            _ => unreachable!(),
                        };

                        let acme_span = span_clone.in_scope(|| {
                            tracing::info_span!(
                                "acme",
                                domain = %api_domain_clone,
                                provision = %provision_str
                            )
                        });

                        self.start_apps(&span, &state).await?;
                        ordinary_utils::acme_task(acme_span.clone(), acme_state, signal_tx.clone());

                        span.in_scope(|| {
                            tracing::info!(
                                server = %"primary",
                                addr = %local_addr,
                                provision = %provision_str,
                                "listening"
                            );
                        });

                        let api_domain = Arc::new(self.config.domain.clone());

                        loop {
                            let span = span.clone();
                            let (cnx, addr) = tokio::select! {
                                conn = listener.accept() => conn?,
                                () = signal_tx.closed() => {
                                    span.in_scope(|| {
                                       tracing::warn!("not accepting new connections");
                                    });
                                    break;
                                }
                            };

                            let tower_service = service.clone();
                            let acme_span = acme_span.clone();

                            let challenge_rustls_config = challenge_rustls_config.clone();
                            let default_rustls_config = default_rustls_config.clone();

                            let api_domain = api_domain.clone();
                            let apps = self.apps.clone();

                            let signal_tx = signal_tx.clone();
                            let close_rx = close_rx.clone();

                            tokio::spawn(async move {
                                // todo: determine a better timeout
                                match timeout(
                                    Duration::from_secs(3),
                                    LazyConfigAcceptor::new(Acceptor::default(), cnx),
                                )
                                .await
                                {
                                    Ok(start_handshake) => {
                                        let start_handshake = match start_handshake {
                                            Ok(start_handshake) => start_handshake,
                                            Err(err) => {
                                                if log_ips {
                                                    let tls_span = span.in_scope(|| { tracing::error_span!("tls", ip = %addr.ip(), port = addr.port()) });

                                                    tls_span.in_scope(
                                                        || tracing::error!(%err, "accept"),
                                                    );
                                                } else {
                                                    let tls_span = span
                                                        .in_scope(|| tracing::error_span!("tls"));

                                                    tls_span.in_scope(
                                                        || tracing::error!(%err, "accept"),
                                                    );
                                                }

                                                return;
                                            }
                                        };

                                        if let Some(sni) =
                                            start_handshake.client_hello().server_name()
                                        {
                                            let tls_span = span.in_scope(|| {
                                                if log_ips {
                                                    tracing::info_span!("tls", %sni, ip = %addr.ip(), port = addr.port())
                                                } else {
                                                    tracing::info_span!("tls", %sni)
                                                }
                                            });

                                            if sni == *api_domain {
                                                if let Err(err) =
                                                    OrdinaryAppServer::handle_handshake(
                                                        &tls_span,
                                                        &acme_span,
                                                        start_handshake,
                                                        tower_service,
                                                        addr,
                                                        challenge_rustls_config,
                                                        default_rustls_config,
                                                        &signal_tx,
                                                        &close_rx,
                                                        None,
                                                    )
                                                    .await
                                                {
                                                    tls_span.in_scope(|| {
                                                        tracing::error!(%err, "handshake");
                                                    });
                                                }
                                            } else {
                                                let apps = apps.read().await;

                                                if let Some(app) = apps.get(sni) {
                                                    // todo: WrappedApp.live: Arc<RwLock<bool>>
                                                    // todo: just handle the stream here
                                                    if let Err(err) = app.stream_tx.send((
                                                        start_handshake,
                                                        addr,
                                                        tls_span.clone(),
                                                    )) {
                                                        tls_span.in_scope(
                                                            || tracing::error!(%err, "handoff"),
                                                        );
                                                    }
                                                } else {
                                                    tls_span
                                                        .in_scope(|| tracing::warn!("no match"));
                                                }
                                            }
                                        } else {
                                            let tls_span = span.in_scope(|| {
                                                if log_ips {
                                                    tracing::info_span!("tls", ip = %addr.ip(), port = addr.port())
                                                } else {
                                                    tracing::info_span!("tls")
                                                }
                                            });

                                            tls_span.in_scope(|| tracing::warn!("no sni"));
                                        }
                                    }
                                    Err(_) => {
                                        if log_ips {
                                            let tls_span = span.in_scope(|| { tracing::error_span!("tls", ip = %addr.ip(), port = addr.port()) });

                                            tls_span.in_scope(|| tracing::error!("timeout"));
                                        } else {
                                            let tls_span =
                                                span.in_scope(|| tracing::error_span!("tls"));

                                            tls_span.in_scope(|| tracing::error!("timeout"));
                                        }
                                    }
                                }
                            });
                        }
                    }
                }

                {
                    let mut close_rx_lock = state.close_rx.write();
                    *close_rx_lock = None;
                }

                drop(close_rx);
                drop(listener);

                span.in_scope(|| {
                    tracing::warn!(
                        "waiting for {} task(s) to finish",
                        close_tx.receiver_count()
                    );
                });
                close_tx.closed().await;
            }
        }

        Ok(())
    }

    #[allow(clippy::too_many_lines)]
    async fn start_apps(
        &self,
        server_span: &Span,
        state: &Arc<OrdinaryApiServerState>,
    ) -> anyhow::Result<()> {
        let state_clone = state.clone();

        let Ok(shared_rt) = tokio::runtime::Handle::try_current() else {
            bail!("failed to get shared runtime");
        };

        Box::pin(async {
            // todo: sleep this for 2 seconds or something to wait for the server to get started up

            let mut apps = state_clone.apps.write().await;

            for entry in fs::read_dir(&self.apps_dir)?.flatten() {
                let config_path = entry.path().join("ordinary.json");
                let killed_path = entry.path().join("killed");

                match fs::read(&config_path) {
                    Ok(mut config_bytes) => match simd_json::from_slice::<OrdinaryConfig>(&mut config_bytes[..]) {
                        Ok(config) => {
                            let domain = config.domain.clone();

                            if state.stored_logs
                                && let Some(monitor) = &*state.monitor
                                && let Err(err) = monitor
                                .add(
                                    config.domain.as_str(),
                                    &config.cnames.clone().unwrap_or_default(),
                                )
                            {
                                tracing::error!(%err, "failed to add domain to log manager");
                            }

                            let app_span = Some(state.server_span.clone().in_scope(|| {
                                tracing::info_span!("app", domain = %domain)
                            }));

                            match ops::app::dbs(&state_clone, &config, &entry.path(), app_span.clone()).await {
                                Ok((auth, storage)) => {
                                    match ops::app::start(
                                        &state_clone,
                                        &config,
                                        auth,
                                        storage,
                                        match state_clone
                                            .provision_mode
                                            .as_ref()
                                            .unwrap_or(&ProvisionMode::Localhost)
                                        {
                                            ProvisionMode::Localhost => {
                                                ordinary_app::server::ProvisionMode::Localhost
                                            }
                                            ProvisionMode::Staging => {
                                                ordinary_app::server::ProvisionMode::Staging
                                            }
                                            ProvisionMode::Production => {
                                                ordinary_app::server::ProvisionMode::Production
                                            }
                                        },
                                        entry.path().join("certs"),
                                        shared_rt.clone(),
                                        app_span.clone(),
                                    ).await {
                                        Ok((port, app, terminate_tx, stream_tx)) => {
                                            let cnames = app.config.cnames.clone();

                                            let static_secret = StaticSecret::random_from_rng(OsRng);
                                            let public_key = PublicKey::from(&static_secret);

                                            let app = Arc::new(WrappedOrdinaryAppServer {
                                                port,
                                                app,
                                                terminate_tx: terminate_tx.clone(),
                                                stream_tx,
                                                dh_keypair: (static_secret, public_key),
                                            });

                                            if let Some(cnames) = cnames {
                                                for cname in cnames {
                                                    apps.insert(cname, app.clone());
                                                }
                                            }

                                            apps.insert(
                                                domain,
                                                app,
                                            );

                                            if killed_path.exists() && let Some(app_span) = app_span {
                                                async {
                                                    if let Ok(since) = tokio::fs::read_to_string(killed_path).await {
                                                        tracing::info!(%since, "killed");
                                                    }
                                                }.instrument(app_span).await;
                                            }
                                        }
                                        Err(err) => {
                                            if let Some(app_span) = app_span {
                                                app_span.in_scope(|| {
                                                    tracing::error!(%err, "failed to start app");
                                                });
                                            }
                                        }
                                    }
                                }
                                Err(err) => {
                                    if let Some(app_span) = app_span {
                                        app_span.in_scope(|| {
                                            tracing::error!(%err, "failed to init dbs");
                                        });
                                    }
                                }
                            }
                        }
                        Err(err) => {
                            if let Some(path_str) = config_path.to_str() {
                                tracing::error!(%err, path = path_str, "failed to parse config file");
                            } else {
                                tracing::error!(%err, "failed to parse config file");
                            }
                        }
                    },
                    Err(err) => {
                        if !entry.path().join("erased").exists() {
                            if let Some(path_str) = config_path.to_str() {
                                tracing::warn!(%err, path = path_str, "failed to read config file");
                            } else {
                                tracing::warn!(%err, "failed to read config file");
                            }
                        }
                    }
                }
            }

            drop(apps);

            anyhow::Ok(())

        }.instrument(server_span.clone())).await
    }
}