malwaredb_server/

lib.rs

// SPDX-License-Identifier: Apache-2.0

#![doc = include_str!("../README.md")]
#![cfg_attr(docsrs, feature(doc_cfg))]
#![deny(missing_docs)]
#![deny(clippy::all)]
#![deny(clippy::pedantic)]

/// Cryptographic functionality for file storage
pub mod crypto;

/// Database I/O
pub mod db;

/// HTTP Server
pub mod http;

/// Entropy functions
pub mod utils;

/// Virus Total integration
#[cfg_attr(docsrs, doc(cfg(feature = "vt")))]
#[cfg(feature = "vt")]
pub mod vt;

use crate::crypto::FileEncryption;
use crate::db::MDBConfig;
use malwaredb_api::ServerInfo;
//use utils::HashPath;

use std::collections::HashMap;
use std::fmt::{Debug, Formatter};
use std::io::{Cursor, Read};
use std::net::{IpAddr, SocketAddr};
use std::path::PathBuf;
use std::sync::{Arc, LazyLock};
use std::time::{Duration, SystemTime};

use anyhow::{bail, ensure, Context, Result};
use axum_server::tls_rustls::RustlsConfig;
use chrono::Local;
use chrono_humanize::{Accuracy, HumanTime, Tense};
use flate2::read::GzDecoder;
use mdns_sd::{ServiceDaemon, ServiceInfo};
use sha2::{Digest, Sha256};
use tokio::net::TcpListener;
use tracing::{debug, error, trace, warn};

/// MDB version
pub const MDB_VERSION: &str = env!("CARGO_PKG_VERSION");

/// MDB version as a semantic version object
pub static MDB_VERSION_SEMVER: LazyLock<semver::Version> =
    LazyLock::new(|| semver::Version::parse(MDB_VERSION).unwrap());

/// How often stale pagination searches should be cleaned up: 1 day
/// Could be used if future cleanup operations are needed.
pub(crate) const DB_CLEANUP_INTERVAL: Duration = Duration::from_secs(60 * 60 * 24);

/// Gzip's magic number to see if a file is compressed
pub const GZIP_MAGIC: [u8; 2] = [0x1fu8, 0x8bu8];

/// Zstd magic number to see if a file is compressed
pub const ZSTD_MAGIC: [u8; 4] = [0x28u8, 0xb5u8, 0x2fu8, 0xfdu8];

/// State & configuration of the running server instance
pub struct State {
    /// The port which will be used to listen for connections.
    pub port: u16,

    /// The directory to store malware samples if we're keeping them.
    pub directory: Option<PathBuf>,

    /// Maximum upload size
    pub max_upload: usize,

    /// The IP to use for listening for connections
    pub ip: IpAddr,

    /// Handle to the database connection
    pub db_type: Arc<db::DatabaseType>,

    /// Start time of the server
    pub started: SystemTime,

    /// Configuration which is stored in the database
    pub db_config: MDBConfig,

    /// File encryption keys, may be empty
    pub(crate) keys: HashMap<u32, FileEncryption>,

    /// Virus Total API key
    #[cfg(feature = "vt")]
    pub(crate) vt_client: Option<malwaredb_virustotal::VirusTotalClient>,

    /// Https certificate file, optionally containing the CA cert as well (server and CA certs
    /// concatenated)
    cert: Option<PathBuf>,

    /// Https private key file
    key: Option<PathBuf>,

    /// If Malware DB should be advertised via Multicast DNS (also known as Bonjour or Zeroconf)
    mdns: bool,
}

impl State {
    /// New server state object given a few configuration parameters
    ///
    /// # Errors
    /// * If there's a certificate and not a key, or a key and not a certificate; if either don't exist.
    /// * If the sample storage directory doesn't exist.
    /// * if the database configuration isn't valid or if the Postgres server can't be reached.
    #[allow(clippy::too_many_arguments)]
    pub async fn new(
        port: u16,
        directory: Option<PathBuf>,
        max_upload: usize,
        ip: IpAddr,
        db_string: &str,
        cert: Option<PathBuf>,
        key: Option<PathBuf>,
        pg_cert: Option<PathBuf>,
        mdns: bool,
        #[cfg(feature = "vt")] vt_client: Option<malwaredb_virustotal::VirusTotalClient>,
    ) -> Result<Self> {
        if let Some(dir) = &directory {
            if !dir.exists() {
                bail!("data directory {} does not exist!", dir.display());
            }
        }

        if cert.is_some() != key.is_some() {
            bail!("Either both the https cert & key are provided, or none are provided.");
        }

        if let Some(cert_file) = &cert {
            ensure!(
                cert_file.exists(),
                "Certificate file {} does not exist!",
                cert_file.display()
            );
        }

        if let Some(key_file) = &key {
            ensure!(
                key_file.exists(),
                "Key file {} does not exist!",
                key_file.display()
            );
        }

        let db_type = db::DatabaseType::from_string(db_string, pg_cert).await?;
        // TODO: allow config and keys to be refreshed so changes don't require a restart
        let db_config = db_type.get_config().await?;
        let keys = db_type.get_encryption_keys().await?;

        Ok(Self {
            port,
            directory,
            max_upload,
            ip,
            db_type: Arc::new(db_type),
            db_config,
            keys,
            #[cfg(feature = "vt")]
            vt_client,
            started: SystemTime::now(),
            cert,
            key,
            mdns,
        })
    }

    /// Store the sample with a depth of three based on the sample's SHA-256 hash, even if compressed
    ///
    /// # Errors
    ///
    /// * If the file can't be written.
    /// * If a necessary sub-directory can't be created.
    pub async fn store_bytes(&self, data: &[u8]) -> Result<bool> {
        if let Some(dest_path) = &self.directory {
            let mut hasher = Sha256::new();
            hasher.update(data);
            let sha256 = hex::encode(hasher.finalize());

            // Trait `HashPath` needs to be re-worked so it can work with Strings.
            // This code below ends up making the String into ASCII representations of the hash
            // See: https://github.com/malwaredb/malwaredb-rs/issues/60
            let hashed_path = format!(
                "{}/{}/{}/{}",
                &sha256[0..2],
                &sha256[2..4],
                &sha256[4..6],
                sha256
            );

            // The path which has the file name included, with the storage directory prepended.
            //let hashed_path = result.hashed_path(3);
            let mut dest_path = dest_path.clone();
            dest_path.push(hashed_path);

            // Remove the file name so we can just have the directory path.
            let mut just_the_dir = dest_path.clone();
            just_the_dir.pop();
            std::fs::create_dir_all(just_the_dir)?;

            let data = if self.db_config.compression {
                let buff = Cursor::new(data);
                let mut compressed = Vec::with_capacity(data.len() / 2);
                zstd::stream::copy_encode(buff, &mut compressed, 4)?;
                compressed
            } else {
                data.to_vec()
            };

            let data = if let Some(key_id) = self.db_config.default_key {
                if let Some(key) = self.keys.get(&key_id) {
                    let nonce = key.nonce();
                    self.db_type
                        .set_file_nonce(&sha256, nonce.as_deref())
                        .await?;
                    key.encrypt(&data, nonce)?
                } else {
                    bail!("Key not available!")
                }
            } else {
                data
            };

            std::fs::write(dest_path, data)?;

            Ok(true)
        } else {
            Ok(false)
        }
    }

    /// Retrieve a sample given the SHA-256 hash
    /// Assumes that `MalwareDB` permissions have already been checked to ensure this is permitted.
    ///
    /// # Errors
    ///
    /// * The file could not be read, maybe because it doesn't exist.
    /// * Failure to decrypt or decompress (corruption).
    pub async fn retrieve_bytes(&self, sha256: &String) -> Result<Vec<u8>> {
        if let Some(dest_path) = &self.directory {
            let path = format!(
                "{}/{}/{}/{}",
                &sha256[0..2],
                &sha256[2..4],
                &sha256[4..6],
                sha256
            );
            // Trait `HashPath` needs to be re-worked so it can work with Strings.
            // This code below ends up making the String into ASCII representations of the hash
            // See: https://github.com/malwaredb/malwaredb-rs/issues/60
            //let path = sha256.as_bytes().iter().hashed_path(3);
            let contents = std::fs::read(dest_path.join(path))?;

            let contents = if self.keys.is_empty() {
                // We don't have file encryption enabled
                contents
            } else {
                let (key_id, nonce) = self.db_type.get_file_encryption_key_id(sha256).await?;
                if let Some(key_id) = key_id {
                    if let Some(key) = self.keys.get(&key_id) {
                        key.decrypt(&contents, nonce)?
                    } else {
                        bail!("File was encrypted but we don't have tke key!")
                    }
                } else {
                    // File was not encrypted
                    contents
                }
            };

            if contents.starts_with(&GZIP_MAGIC) {
                let buff = Cursor::new(contents);
                let mut decompressor = GzDecoder::new(buff);
                let mut decompressed: Vec<u8> = vec![];
                decompressor.read_to_end(&mut decompressed)?;
                Ok(decompressed)
            } else if contents.starts_with(&ZSTD_MAGIC) {
                let buff = Cursor::new(contents);
                let mut decompressed: Vec<u8> = vec![];
                zstd::stream::copy_decode(buff, &mut decompressed)?;
                Ok(decompressed)
            } else {
                Ok(contents)
            }
        } else {
            bail!("files are not saved")
        }
    }

    /// Get the duration for which the server has been running
    ///
    /// # Panics
    ///
    /// Despite the `unwrap()` this function will not panic as the data used is guaranteed to be valid.
    #[must_use]
    pub fn since(&self) -> Duration {
        let now = SystemTime::now();
        now.duration_since(self.started).unwrap()
    }

    /// Get server information
    ///
    /// # Errors
    ///
    /// An error would occur if the Postgres server could not be reached.
    pub async fn get_info(&self) -> Result<ServerInfo> {
        let db_info = self.db_type.db_info().await?;
        let uptime = Local::now() - self.since();
        let mem_size = if let Some(mem_size) = app_memory_usage_fetcher::get_memory_usage_bytes() {
            humansize::SizeFormatter::new(mem_size.get(), humansize::BINARY).to_string()
        } else {
            String::new()
        };

        Ok(ServerInfo {
            os_name: std::env::consts::OS.into(),
            memory_used: mem_size,
            num_samples: db_info.num_files,
            num_users: db_info.num_users,
            uptime: HumanTime::from(uptime).to_text_en(Accuracy::Rough, Tense::Present),
            mdb_version: MDB_VERSION_SEMVER.clone(),
            db_version: db_info.version,
            db_size: db_info.size,
            instance_name: self.db_config.name.clone(),
        })
    }

    /// The server listens and responds to requests. Does not return unless there's an error.
    ///
    /// # Errors
    ///
    /// * If the certificate and private key could not be parsed or are not valid.
    /// * If the IP address and port are already in use.
    /// * If the service doesn't have permission to open the port.
    ///
    /// # Panics
    ///
    /// * The `.unwrap()` calls won't panic because the data would have already been validated.
    pub async fn serve(
        self,
        #[cfg(target_family = "windows")] rx: Option<tokio::sync::mpsc::Receiver<()>>,
    ) -> Result<()> {
        let socket = SocketAddr::new(self.ip, self.port);
        let arc_self = Arc::new(self);
        let db_ifo = arc_self.db_type.clone();

        tokio::spawn(async move {
            loop {
                match db_ifo.cleanup().await {
                    Ok(removed) => {
                        trace!("Pagination cleanup succeeded, {removed} searches removed");
                    }
                    Err(e) => warn!("Pagination cleanup failed: {e}"),
                }

                tokio::time::sleep(DB_CLEANUP_INTERVAL).await;
            }
        });

        if arc_self.mdns {
            if arc_self.ip.is_loopback() {
                debug!("Refusing to start mdns responder for localhost");
            } else {
                trace!("Enabling MDNS advertising");
                match ServiceDaemon::new() {
                    Ok(mdns) => {
                        let host_name = format!("{}.local.", arc_self.ip);
                        let ssl = arc_self.cert.is_some() && arc_self.key.is_some();
                        let properties = [("ssl", ssl.to_string())];
                        match ServiceInfo::new(
                            malwaredb_api::MDNS_NAME,
                            &arc_self.db_config.name,
                            &host_name,
                            arc_self.ip.to_string(),
                            arc_self.port,
                            &properties[..],
                        ) {
                            Ok(service) => match mdns.register(service) {
                                Ok(()) => trace!("MalwareDB mdns registered"),
                                Err(e) => error!("Failed to register service: {e}"),
                            },
                            Err(e) => error!("Failed to publish MDNS service: {e}"),
                        }
                    }
                    Err(e) => error!("Failed to open port for MDNS responder: {e}"),
                }
            }
        }

        if arc_self.cert.is_some() && arc_self.key.is_some() {
            let cert_path = arc_self.cert.as_ref().unwrap();
            let key_path = arc_self.key.as_ref().unwrap();
            let cert_ext_str = cert_path
                .extension()
                .context("failed to get certificate extension")?;
            let key_ext_str = key_path
                .extension()
                .context("failed to get key extension")?;

            // Unnecessary for running MalwareDB, but some unit tests fail without this check.
            if rustls::crypto::CryptoProvider::get_default().is_none() {
                rustls::crypto::aws_lc_rs::default_provider()
                    .install_default()
                    .expect("Failed to load crypto provider");
            }

            let config = if (cert_ext_str == "pem" || cert_ext_str == "crt") && key_ext_str == "pem"
            {
                RustlsConfig::from_pem_file(cert_path, key_path)
                    .await
                    .context("failed to load or parse certificate and key files")?
            } else if cert_ext_str == "der" && key_ext_str == "der" {
                let cert_contents =
                    std::fs::read(cert_path).context("failed to read certificate file")?;
                let key_contents =
                    std::fs::read(key_path).context("failed to read certificate file")?;
                RustlsConfig::from_der(vec![cert_contents], key_contents)
                    .await
                    .context("failed to parse certificate and key files as DER")?
            } else {
                bail!("Unknown or unmatched certificate and key file extensions {cert_ext_str:?} and {key_ext_str:?}");
            };

            println!("Listening on https://{socket:?}");
            let handle = axum_server::Handle::new();
            let server_future = axum_server::bind_rustls(socket, config)
                .serve(http::app(arc_self).into_make_service());
            tokio::select! {
                () = shutdown_signal(#[cfg(target_family = "windows")]rx) =>
                    handle.graceful_shutdown(Some(Duration::from_secs(30))),
                res = server_future => res?,
            }
            warn!("Terminate signal received");
        } else {
            println!("Listening on http://{socket:?}");
            let listener = TcpListener::bind(socket)
                .await
                .context(format!("failed to bind socket {socket}"))?;
            axum::serve(listener, http::app(arc_self).into_make_service())
                .with_graceful_shutdown(shutdown_signal(
                    #[cfg(target_family = "windows")]
                    rx,
                ))
                .await?;
            warn!("Terminate signal received");
        }
        Ok(())
    }
}

impl Debug for State {
    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
        write!(
            f,
            "MDB state, port {}, database {:?}",
            self.port, self.db_type
        )
    }
}

/// Enable graceful shutdown
/// <https://github.com/tokio-rs/axum/discussions/1500>
async fn shutdown_signal(
    #[cfg(target_family = "windows")] mut rx: Option<tokio::sync::mpsc::Receiver<()>>,
) {
    let ctrl_c = async {
        tokio::signal::ctrl_c()
            .await
            .expect("failed to install Ctrl+C handler");
    };

    #[cfg(unix)]
    let terminate = async {
        tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
            .expect("failed to install signal handler")
            .recv()
            .await;
    };

    #[cfg(not(unix))]
    let terminate = std::future::pending::<()>();

    #[cfg(target_family = "windows")]
    if let Some(rx_inner) = &mut rx {
        let terminate_rx = rx_inner.recv();

        tokio::select! {
            () = ctrl_c => {},
            () = terminate => {},
            Some(()) = terminate_rx => {},
        }
    } else {
        tokio::select! {
            () = ctrl_c => {},
            () = terminate => {},
        }
    }

    #[cfg(not(target_family = "windows"))]
    tokio::select! {
        () = ctrl_c => {},
        () = terminate => {},
    }
}