operon 0.7.0

A workflow engine for parallel, incremental scheduling of DAG-defined multiplex tasks.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111

use async_trait::async_trait;
use secrecy::ExposeSecret;

use crate::schema::RunFootprint;
use crate::storage::OperonStorage;
use crate::storage::psql::client::StorageClient;
use crate::storage::psql::{
    EntityQueries, PsqlStorageError, PsqlStorageOptions, PsqlStorageResult,
};
use crate::utils::SchemaPrefix;

/// The SQL storage that can be used with the service.
#[derive(Debug, Clone)]
pub struct PsqlStorage<T> {
    /// Underlying connection pool to the PostgreSQL database.
    pub pool: deadpool_postgres::Pool,
    /// Optional schema name to use for this storage.
    pub schema: Option<String>,
    /// Metadata about the entities in the storage.
    pub entities_meta: T,
}

impl<T> PsqlStorage<T> {
    /// Returns a connection to the PostgreSQL database.
    pub async fn conn(&self) -> PsqlStorageResult<StorageClient<'_>> {
        let client = self.pool.get().await?;
        let schema = self.schema.as_deref();

        Ok(StorageClient::new(client, schema))
    }

    /// Returns the SQL-printable schema prefix for this storage.
    pub fn schema_prefix(&self) -> SchemaPrefix<'_> {
        SchemaPrefix(self.schema.as_deref())
    }
}

/// Constructs a concrete [`PsqlStorage`] alias from [`PsqlStorageOptions`].
///
/// This lets [`PsqlStorageOptions::build`](crate::options::PsqlStorageOptions::build) be
/// turbofished on the generated storage alias, e.g. `build::<PsqlCookingStorage>()`.
pub trait FromPsqlStorageOptions: Sized {
    /// Opens the connection pool `options` describes and builds the storage over it.
    fn from_options(options: PsqlStorageOptions) -> PsqlStorageResult<Self>;
}

impl<T: Default> FromPsqlStorageOptions for PsqlStorage<T> {
    fn from_options(options: PsqlStorageOptions) -> PsqlStorageResult<Self> {
        PsqlStorage::<T>::new(options)
    }
}

impl<T: Default> PsqlStorage<T> {
    pub(crate) fn new(options: PsqlStorageOptions) -> PsqlStorageResult<Self> {
        let pg_config: tokio_postgres::Config = {
            let mut config = options
                .database_uri
                .expose_secret()
                .parse::<tokio_postgres::Config>()?;

            let _ = config
                .keepalives(true)
                .keepalives_idle(options.keepalives_idle)
                .keepalives_interval(options.keepalives_interval);
            config
        };
        let manager_config = deadpool_postgres::ManagerConfig {
            recycling_method: deadpool_postgres::RecyclingMethod::Clean,
        };
        let manager = deadpool_postgres::Manager::from_config(
            pg_config,
            tokio_postgres::NoTls,
            manager_config,
        );
        let pool = deadpool_postgres::Pool::builder(manager)
            .max_size(options.pool_size)
            .build()?;

        Ok(Self {
            pool,
            schema: options.schema,
            entities_meta: T::default(),
        })
    }
}

#[async_trait]
impl<T: EntityQueries> OperonStorage for PsqlStorage<T> {
    type Error = PsqlStorageError;

    /// If the storage schema is specified, initialize the schema in the database.
    async fn init(&self) -> PsqlStorageResult<()> {
        let client = self.conn().await?;

        client.init_schema().await?;
        client.init_entity_hash().await?;
        client.init_footprint().await?;
        self.entities_meta.init(&client).await?;
        Ok(())
    }

    async fn get_footprint(&self) -> PsqlStorageResult<Option<RunFootprint>> {
        let client = self.conn().await?;
        client.get_footprint().await
    }

    async fn put_footprint(&self, footprint: &RunFootprint) -> PsqlStorageResult<()> {
        let client = self.conn().await?;
        client.put_footprint(footprint).await
    }
}