galaxy_rs/

lib.rs

//! # Galaxy
//!
//! `galaxy-rs` is a library providing bindings to the Python `galaxy` project, found [here](https://github.com/quinntyx/galaxy). 
//! Galaxy is a node-based, extensible, multimedia database system. 
//!
//! This crate provides a way to interact with a Galaxy database through Rust using PyO3,
//! preserving compatibility with Galaxy's extensibility through its Python API while allowing use
//! with Rust programs. 

use pyo3::prelude::*;
use pyo3::types::{PyString, PyBool, PyIterator};
use std::collections::HashMap;
pub use pyo3::exceptions::*;

/// A struct representing a Galaxy database. 
/// Because this interfaces with files on disk, ever having >1 instance of a Galaxy database
/// pointing to the same data is an error and will cause UB. 
///
/// A Galaxy DB can be created with `galaxy_rs::Galaxy::new(module_path: &str, api_path: &str, silent: bool)`. 
///
/// Because every method in Galaxy requires acquiring the Python Global Interpreter Lock (GIL),
/// Galaxy is technically threadsafe for shared memory (although running multiple instances of
/// Galaxy is not recommended as the database is lazy and may become out of sync with disk until
/// flushed, causing likely data races). 
pub struct Galaxy {
    db: Py<PyAny>
}

impl Galaxy {
    /// Creates a new Galaxy database pointing to the given data. 
    ///
    /// `module_path: &str` is the path to the galaxy database's extension module folder, relative to the 
    /// location the Binary is being executed from. 
    ///
    /// `api_path: &str` is the path to the Galaxy API, for use by modules as an imported module. This can
    /// be left as a blank string if Galaxy is installed to the system interpreter or as a venv in
    /// the current environment (as is the default setup when using Cargo).
    ///
    /// `silent: bool` is a boolean that will suppress prints from Python while initializing the
    /// database through a flag on the Python-internal Galaxy.post() method. Note that this is not
    /// guaranteed to suppress all output, depending on the implementation of added modules.
    ///
    /// All data is stored in the `data` folder relative to the execution location of the final
    /// binary, and when instancing a Galaxy database it automatically loads all data from that
    /// directory. 
    pub fn new (module_path: &str, api_path: &str, silent: bool) -> Self {
        match Python::with_gil(|py| -> PyResult<Self> {
            let db: Py<PyAny> = PyModule::import(py, "galaxy")?
                .getattr("galaxy")?
                .getattr("Galaxy")?
                .call1((silent, module_path, api_path))?
                .into();
            Ok(Galaxy { db })
        }) {
            Ok(x) => x,
            Err(_) => todo!(),
        }
    }

    /// Creates a new Galaxy database pointing to the given data inside of the given directory. 
    ///
    /// `module_path: &str` is the path to the galaxy database's extension module folder, relative to the 
    /// location the Binary is being executed from. 
    ///
    /// `api_path: &str` is the path to the Galaxy API, for use by modules as an imported module. This can
    /// be left as a blank string if Galaxy is installed to the system interpreter or as a venv in
    /// the current environment (as is the default setup when using Cargo).
    ///
    /// `silent: bool` is a boolean that will suppress prints from Python while initializing the
    /// database through a flag on the Python-internal Galaxy.post() method. Note that this is not
    /// guaranteed to suppress all output, depending on the implementation of added modules.
    ///
    /// All data is stored in the `data` folder relative to the passed directory. 
    pub fn with_dir (dir: &str, module_path: &str, api_path: &str, silent: bool) -> Self {
        Python::with_gil(|py| {
            PyModule::import(py, "os").unwrap()
                .getattr("chdir").unwrap()
                .call1((dir,)).unwrap();
        });
        Self::new(module_path, api_path, silent)
    }

    /// Gets the Registry Handler. 
    pub fn registry_handler (&self) -> RegistryHandler {
        let res: PyResult<RegistryHandler> = Python::with_gil(|py| {
            let handler: Py<PyAny> = PyModule::import(py, "galaxy")?
                .getattr("registry")?
                .into();
            Ok(RegistryHandler::new(handler))
        });
        match res {
            Ok(x) => x,
            Err(_) => todo!()
        }
    }


    /// Loads a node to the Galaxy database and returns a `Result<(), PyErr>` holding any resulting
    /// Python errors that may have arisen from attempting to load that node. 
    ///
    /// `loc: &str` is the location of the node, in the format `source:name`. This is expanded to
    /// `data/source:name.json` and `data/source:name.match` to instance the (lazy) Galaxy node and
    /// add it to the nodes in the Galaxy db. 
    pub fn load_node (&mut self, loc: &str) -> Result<(), PyErr> {
       Python::with_gil(|py| {
            self.db.as_ref(py)
                .getattr("load_node")
                .expect("Galaxy object should have attribute load_node")
                .call1((loc,))?;
            
            Ok(())
        })
    }

    /// Calling this makes the Galaxy database aware that a new node has been fully initialized and
    /// added to the database and that it should now begin making connections using the match
    /// files stored in that node. 
    ///
    /// `srcnode: &str` is the ID of the node that was just added, which is the name of the node
    /// (without the source). This is subject to change, and may become `source:name` in the future
    /// to disambiguate nodes with the same name (e.g. Nuclear Energy from Britannica and Nuclear
    /// Energy from Wikipedia). For now, it is impossible to have two nodes with identical names. 
    pub fn process_new_match (&mut self, srcnode: &str) -> Result<(), PyErr> {
        Python::with_gil(|py| {
            self.db.as_ref(py)
                .getattr("process_new_match")
                .expect("Galaxy object should have attribute process_new_match")
                .call1((srcnode,))?;

            Ok(())
        })
    }

    /// Flushes the data of the nodes to disk. Because the implementation of Galaxy is as lazy as
    /// possible in the API, minimal disk writes are made to make it faster.
    ///
    /// Therefore, the `flush` method is provided to tell each node to dump its contents to disk.
    /// There is however the caveat that while nodes provided in modules are required to implement
    /// this method as part of their API, it may be a no-op in certain cases where nodes need to
    /// flush immediately after writes for one reason or another and therefore are already in sync
    /// with the version on disk. 
    ///
    /// This may be optimized later for larger numbers of nodes using timestamps, though at the
    /// moment this is not the case. 
    pub fn flush (&mut self) -> Result<(), PyErr> {
        Python::with_gil(|py| {
            self.db.as_ref(py)
                .getattr("flush")
                .expect("Galaxy object should have attribute flush")
                .call0()?;

            Ok(())
        })
    }

    /// Returns a `Result<Node, PyErr>` enum of the node name given. 
    ///
    /// This name is unqualified with the source, meaning it cannot disambiguate between two nodes
    /// with identical names. This is subject to change in future updates.
    ///
    /// `node: &str` is a string name of the node to be retrieved.
    pub fn get (&self, node: &str) -> Result<Node, PyErr> {
        Python::with_gil(|py| {
            let output = self.db.as_ref(py)
                .getattr("get")
                .expect("Galaxy object should have attribute get")
                .call1((node,))?;

            Ok(Node::new(output.into()))
        })
    }

    /// Returns a `Result<HashMap<String, Node>, PyErr>` representing all of the nodes registered
    /// in the Galaxy db. 
    pub fn nodes (&self) -> Result<HashMap<String, Node>, PyErr> {
        Python::with_gil(|py| {
            let db = self.db.as_ref(py);
            let nodes = db.getattr("nodes").expect("Galaxy object should have nodes");
            

            let mut output: HashMap<String, Node> = HashMap::new();

            for i in nodes.iter()? {
                let key = i?.downcast::<PyString>()?.to_str()?;
                let node: Node = Node::new(nodes.get_item(key)?.into());

                output.insert(String::from(key), node);
            }

            Ok(output)
        })
    }

    /// Returns a `Result<Vec<String>, PyErr>` containing all of the names of the registered
    /// Galaxy modules. 
    ///
    /// In the future I plan to add a way to actually access functions inside modules for more
    /// advanced behavior, but it's hard to manage because modules are not required to present any
    /// API, really. 
    pub fn modules (&self) -> Result<Vec<String>, PyErr> {
        Python::with_gil(|py| {
            Ok(self.db.as_ref(py)
                .getattr("modules").expect("Galaxy object should have modules")
                .getattr("keys").expect("Python dict object should have keys method")
                .call0().expect("Python dict.keys() should not error")
                .iter().expect("Python dict.keys() should be iterable")
                .map(|x| String::from(x.unwrap().downcast::<PyString>().unwrap().to_str().unwrap()))
                .collect())
        })
    }
    
    /// Returns `Result<bool, PyErr>` containing whether or not the Galaxy db is currently
    /// `silent`. This refers to the boolean the Galaxy db was initialized with; for more
    /// information, see the constructor `galaxy_rs::Galaxy::new`. 
    pub fn silent (&self) -> Result<bool, PyErr> {
        Python::with_gil(|py| {
            Ok(self.db.as_ref(py)
               .getattr("silent").expect("Galaxy object should have bool silent")
               .downcast::<PyBool>().expect("silent flag should be bool")
               .extract().unwrap())
        })
    }
}


/// A struct representing a Galaxy db data node. 
pub struct Node {
    py_obj: Py<PyAny>,
}

impl Node {
    fn new (py_obj: Py<PyAny>) -> Self {
        Node {
            py_obj
        }
    }

    /// Gets the content of this `Node` as a `Result<String, PyErr>`.
    ///
    /// Galaxy encourages module implementations of Nodes to be lazy, so this may cause an IO
    /// operation. 
    pub fn content (&self) -> Result<String, PyErr> {
        Python::with_gil(|py| {
            let result = self.py_obj.as_ref(py)
                .getattr("content")?
                .downcast::<PyString>()?
                .to_str()?;

            Ok(String::from(result))
        })
    }

    /// Gets the match data of this `Node` as a `Result<String, PyErr>`. 
    ///
    /// Galaxy encourages module implementations of Nodes to be lazy, so this may cause an IO
    /// operation.
    pub fn match_data (&self) -> Result<String, PyErr> {
        Python::with_gil(|py| {
            let result = self.py_obj.as_ref(py)
                .getattr("match_data")?
                .downcast::<PyString>()?
                .to_str()?;

            Ok(String::from(result))
        })
    }

    /// Gets the parsed data of this `Node` as a `Result<NodeData, PyErr>`. 
    ///
    /// Galaxy encourages module implementations of Nodes to be lazy, so this may cause an IO
    /// operation. 
    pub fn parsed_data (&self) -> Result<NodeData, PyErr> {
        Python::with_gil(|py| {
            let result = self.py_obj.as_ref(py)
                .getattr("parsed_data")?
                .into();

            Ok(NodeData::new(result))
        })
    }
}

/// A representation of the internal data of the Node. 
/// Cannot be constructed, but is returned by `galaxy_rs::Node::parsed_data`. 
pub struct NodeData {
    py_obj: Py<PyAny>,
}

impl NodeData {
    fn new (py_obj: Py<PyAny>) -> Self {
        Self {
            py_obj
        }
    }

    /// Gets the title stored in the `NodeData` as a `Result<String, PyErr>`.
    pub fn title (&self) -> Result<String, PyErr> {
        Python::with_gil(|py| {
            let result = self.py_obj.as_ref(py)
                .get_item("title")?
                .downcast::<PyString>()?
                .to_str()?;

            Ok(String::from(result))
        })
    }

    /// Gets the data type stored in the `NodeData` as a `Result<String, PyErr>`.
    ///
    /// By default, Galaxy databases support a "txt" datatype, which is used to ingest all 
    /// input data. Modules may implement more using the extensible Galaxy Python API. 
    pub fn data_type (&self) -> Result<String, PyErr> {
        Python::with_gil(|py| {
            let result = self.py_obj.as_ref(py)
                .get_item("type")?
                .downcast::<PyString>()?
                .to_str()?;

            Ok(String::from(result))
        })
    }

    /// Returns the source the data was retrieved from as a `Result<String, PyErr>`.
    pub fn source (&self) -> Result<String, PyErr> {
        Python::with_gil(|py| {
            let result = self.py_obj.as_ref(py)
                .get_item("source")?
                .downcast::<PyString>()?
                .to_str()?;

            Ok(String::from(result))
        })
    }

    /// Returns the links stored in this `NodeData` as a `Result<Vec<Link>, PyErr>`. 
    pub fn links (&self) -> Result<Vec<Link>, PyErr> {
        Python::with_gil(|py| {
            let result = self.py_obj.as_ref(py)
                .get_item("links")?
                .iter()?
                .map(|i| Link::new((i.unwrap()).into()))
                .collect();

            Ok(result)
        })
    }

    /// Flushes an individual node to disk. Returns a `Result<(), PyErr>` for error handling.
    ///
    /// Galaxy demands that all module-added node types implement this method, but dependent on the
    /// implementation this may or may not actually flush the node's cache to disk. See
    /// `galaxy_rs::Galaxy::flush` for more details. 
    pub fn flush (&self) -> Result<(), PyErr> {
        Python::with_gil(|py| {
            self.py_obj.as_ref(py)
                .getattr("flush")?
                .call0()?;
            Ok(())
        })
    }
                
}

/// Represents a single link between two nodes in the Galaxy db. 
///
/// Individual links don't know what node they point from, they merely have a weight and point to
/// another node. Links should never be stored separate from the `Node` (or `NodeData`) that they
/// are tied to for this reason. 
pub struct Link {
    py_obj: Py<PyAny>
}

impl Link {
    fn new (py_obj: Py<PyAny>) -> Self {
        Self {
            py_obj
        }
    }

    /// Returns the node that is being targeted by this link, as a `Result<String, PyErr>`. 
    pub fn target (&self) -> Result<String, PyErr> {
        Python::with_gil(|py| {
            let result = self.py_obj.as_ref(py)
                .get_item("target")?
                .downcast::<PyString>()?
                .to_str()?;

            Ok(String::from(result))
        })
    }

    /// Returns the strength (or "weight") of this `Link`, as a `Result<i64, PyErr>`. 
    pub fn strength (&self) -> Result<i64, PyErr> {
        Python::with_gil(|py| {
            let result = self.py_obj.as_ref(py)
                .get_item("strength")?
                .extract()?;
            
            Ok(result)
        })
    }

}

fn from_pystring_unchecked (x: PyResult<&PyAny>) -> String {
    String::from(x.unwrap().downcast::<PyString>().unwrap().to_str().unwrap())
}

/// Wraps the registries in the Galaxy db. 
pub struct RegistryHandler {
    py_obj: Py<PyAny>
}

impl RegistryHandler {
    fn new (py_obj: Py<PyAny>) -> Self {
        Self {
            py_obj
        }
    }

    fn node_registry (&self) -> Py<PyAny> {
        Python::with_gil(|py| {
            self.py_obj.as_ref(py)
                .getattr("NODE_REGISTRY").expect("Node Registry should exist")
                .into()
        })
    }

    fn ingest_manager_registry (&self) -> Py<PyAny> {
        Python::with_gil(|py| {
            self.py_obj.as_ref(py)
                .getattr("INGEST_MANAGER_REGISTRY").expect("Ingest Manager Registry should exist")
                .into()
        })
    }

    fn keys (x: &PyAny) -> &PyIterator {
        x.getattr("keys").expect("Registry object should have keys")
            .iter().expect("Python list should be iterable")
    }
        

    /// Gets a list of the IDs of all registered node types as a `Vec<String>`. Can sometimes
    /// be more reliable than getting the loaded modules, as there is no guarantee that each module
    /// registers a node, or that each module registers only one node. Each node is targeted by a
    /// type field in the JSON data on disk, so there is a 1:1 correlation of this list to all of
    /// the currently supported data types in the Galaxy db. 
    ///
    /// Identifiers are namespaced as `module::nodetype`, as enforced by the Registry in Python;
    /// however, Identifiers default to the `core` namespace when no namespace is provided, meaning
    /// that some nodes in badly written, non-idiomatic ways may be registered under `core`.
    /// Looking at the registry namespaces should illustrate that modules with the same namespace
    /// may register multiple, or no, nodes. 
    pub fn registered_nodes (&self) -> Vec<String> {
        Python::with_gil(|py| {
            Self::keys(self.node_registry().as_ref(py))
                .map(from_pystring_unchecked)
                .collect()
        })
    }

    /// Gets a list of the IDs of all the registered ingest helpers as a `Vec<String>`. Can
    /// sometimes be more reliable than getting all the loaded modules, as there is no guarantee
    /// that each module registers an ingest manager, or that each module registers only one ingest
    /// manager. Each ingest manager represents a single source of data. This may not necessary map
    /// onto all possible nominal sources in node metadata (ex. a WebIngestManager may generate the 
    /// source from the scraped web data) but it represents all possible ways to obtain data. 
    ///
    /// Identifiers are namespaced as `module::nodetype`, as enforced by the Registry in Python;
    /// however, Identifiers default to the `core` namespace when no namespace is provided, meaning
    /// that some nodes in badly written, non-idiomatic ways may be registered under `core`. 
    /// Looking at the registry namespaces should illustrate that modules with the same namespace
    /// may register multiple, or no, ingest managers. 
    pub fn registered_ingest_managers (&self) -> Vec<String> {
        Python::with_gil(|py| {
            Self::keys(self.ingest_manager_registry().as_ref(py))
                .map(from_pystring_unchecked)
                .collect()
        })
    }

}