qrlew/io/
mod.rs

1//! # Utilities to run tests on real data
2//!
3//! For now supports:
4//! - PstgresSql
5//! - SQLite using the ["sqlite"] feature.
6//! - MsSql using the ["mssql"] feature.
7//! - BigQuery using the ["bigquery"] feature.
8//!
9
10#[cfg(feature = "bigquery")]
11pub mod bigquery;
12#[cfg(feature = "mssql")]
13pub mod mssql;
14#[cfg(feature = "mysql")]
15pub mod mysql;
16pub mod postgresql;
17#[cfg(feature = "sqlite")]
18pub mod sqlite;
19
20use crate::{
21    builder::{Ready, With},
22    data_type::{
23        self,
24        value::{self},
25        DataType,
26    },
27    expr::identifier::Identifier,
28    hierarchy::Hierarchy,
29    relation::{
30        builder::TableBuilder, field::Constraint, schema::Schema, Relation, Table, Variant as _,
31    },
32};
33use std::{convert::Infallible, error, fmt, io, num, result, sync::Arc, thread, time};
34
35const DATA_GENERATION_SEED: u64 = 1234;
36
37// Error management
38#[derive(Debug)]
39pub enum Error {
40    Database(String),
41    Dataset(String),
42    Other(String),
43}
44
45impl Error {
46    pub fn database(database: impl fmt::Display) -> Error {
47        Error::Database(format!("Database error {}", database))
48    }
49    pub fn dataset(dataset: impl fmt::Display) -> Error {
50        Error::Dataset(format!("Dataset error {}", dataset))
51    }
52    pub fn other(desc: impl fmt::Display) -> Error {
53        Error::Other(format!("{}", desc))
54    }
55}
56
57impl fmt::Display for Error {
58    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
59        match self {
60            Error::Database(database) => writeln!(f, "Database: {}", database),
61            Error::Dataset(dataset) => writeln!(f, "Dataset: {}", dataset),
62            Error::Other(desc) => writeln!(f, "{}", desc),
63        }
64    }
65}
66
67impl error::Error for Error {}
68
69impl From<Infallible> for Error {
70    fn from(err: Infallible) -> Self {
71        Error::Other(err.to_string())
72    }
73}
74impl From<io::Error> for Error {
75    fn from(err: io::Error) -> Self {
76        Error::Other(err.to_string())
77    }
78}
79impl From<num::ParseIntError> for Error {
80    fn from(err: num::ParseIntError) -> Self {
81        Error::Other(err.to_string())
82    }
83}
84impl From<data_type::Error> for Error {
85    fn from(err: data_type::Error) -> Self {
86        Error::Other(err.to_string())
87    }
88}
89impl From<data_type::value::Error> for Error {
90    fn from(err: data_type::value::Error) -> Self {
91        Error::Other(err.to_string())
92    }
93}
94impl From<r2d2::Error> for Error {
95    fn from(err: r2d2::Error) -> Self {
96        Error::Other(err.to_string())
97    }
98}
99
100pub type Result<T> = result::Result<T, Error>;
101
102/// A utility function to try many times
103pub fn try_some_times<T, F: Fn() -> Result<T>>(max_retry: usize, f: F) -> Result<T> {
104    let mut num_retry: usize = 0;
105    loop {
106        match f() {
107            Ok(value) => {
108                return Ok(value);
109            }
110            Err(err) => {
111                thread::sleep(time::Duration::from_secs(1));
112                num_retry += 1;
113                log::info!("Retrying {num_retry} times.");
114                if num_retry > max_retry {
115                    return Err(err);
116                }
117            }
118        }
119    }
120}
121
122pub trait Database: Sized {
123    const MAX_SIZE: usize = 1 << 14;
124    /// Create a database
125    fn new(name: String, tables: Vec<Table>) -> Result<Self>;
126    /// Get the name
127    fn name(&self) -> &str;
128    /// Get the tables
129    fn tables(&self) -> &[Table];
130    /// Get a mutable reference to the tables
131    fn tables_mut(&mut self) -> &mut Vec<Table>;
132    /// Get a dictionary of relations
133    /// A relation can be adressed by its SQL path or its Qrlew name
134    fn relations(&self) -> Hierarchy<Arc<Relation>> {
135        self.tables()
136            .iter()
137            .map(|t| (Identifier::from(t.name()), Arc::new(t.clone().into()))) // Tables can be accessed from their name or path
138            .chain(
139                self.tables()
140                    .iter()
141                    .map(|t| (t.path().clone(), Arc::new(t.clone().into()))),
142            )
143            .collect()
144    }
145    /// Create an empty db
146    fn empty(name: String) -> Result<Self> {
147        Self::new(name, vec![])
148    }
149    /// Create a table from a table object
150    fn create_table(&mut self, table: &Table) -> Result<usize>;
151    /// Insert data in the tables
152    fn insert_data(&mut self, table: &Table) -> Result<()>;
153    /// Execute a query
154    fn query(&mut self, query: &str) -> Result<Vec<value::List>>;
155    /// Test the equivalence of queries regarding the output with this DB
156    fn eq(&mut self, left: &str, right: &str) -> bool {
157        if let (Ok(left), Ok(right)) = (self.query(left), self.query(right)) {
158            left == right
159        } else {
160            false
161        }
162    }
163    /// A basic test DB
164    fn test_tables() -> Vec<Table> {
165        vec![
166            TableBuilder::new()
167                .path(["table_1"])
168                .name("table_1")
169                .size(10)
170                .schema(
171                    Schema::empty()
172                        .with(("a", DataType::float_interval(0., 10.)))
173                        .with(("b", DataType::optional(DataType::float_interval(-1., 1.))))
174                        .with((
175                            "c",
176                            DataType::date_interval(
177                                chrono::NaiveDate::from_ymd_opt(1980, 12, 06).unwrap(),
178                                chrono::NaiveDate::from_ymd_opt(2023, 12, 06).unwrap(),
179                            ),
180                        ))
181                        .with(("d", DataType::integer_interval(0, 10))),
182                )
183                .build(),
184            TableBuilder::new()
185                .path(["table_2"])
186                .name("table_2")
187                .size(200)
188                .schema(
189                    Schema::empty()
190                        .with(("x", DataType::integer_interval(0, 100)))
191                        .with(("y", DataType::optional(DataType::text())))
192                        .with(("z", DataType::text_values(["Foo".into(), "Bar".into()]))),
193                )
194                .build(),
195            TableBuilder::new()
196                .path(["user_table"])
197                .name("users")
198                .size(100)
199                .schema(
200                    Schema::empty()
201                        .with(("id", DataType::integer_interval(0, 100)))
202                        .with(("name", DataType::text(), Constraint::Unique))
203                        .with((
204                            "age",
205                            DataType::optional(DataType::float_interval(0., 200.)),
206                        ))
207                        .with((
208                            "city",
209                            DataType::text_values(["Paris".into(), "New-York".into()]),
210                        )),
211                )
212                .build(),
213            TableBuilder::new()
214                .path(["order_table"])
215                .name("orders")
216                .size(200)
217                .schema(
218                    Schema::empty()
219                        .with(("id", DataType::integer_interval(0, 100)))
220                        .with(("user_id", DataType::integer_interval(0, 101)))
221                        .with(("description", DataType::text()))
222                        .with((
223                            "date",
224                            DataType::date_interval(
225                                chrono::NaiveDate::from_ymd_opt(2020, 12, 06).unwrap(),
226                                chrono::NaiveDate::from_ymd_opt(2023, 12, 06).unwrap(),
227                            ),
228                        )),
229                )
230                .build(),
231            TableBuilder::new()
232                .path(["item_table"])
233                .name("items")
234                .size(300)
235                .schema(
236                    Schema::empty()
237                        .with(("order_id", DataType::integer_interval(0, 100)))
238                        .with(("item", DataType::text()))
239                        .with(("price", DataType::float_interval(0., 50.))),
240                )
241                .build(),
242            TableBuilder::new()
243                .path(["large_user_table"])
244                .name("more_users")
245                .size(100000)
246                .schema(
247                    Schema::empty()
248                        .with(("id", DataType::integer_interval(0, 1000)))
249                        .with(("name", DataType::text()))
250                        .with((
251                            "age",
252                            DataType::optional(DataType::float_interval(0., 200.)),
253                        ))
254                        .with((
255                            "city",
256                            DataType::text_values([
257                                "Paris".into(),
258                                "New-York".into(),
259                                "Rome".into(),
260                            ]),
261                        ))
262                        .with(("income", DataType::float_interval(100.0, 200000.0))),
263                )
264                .build(),
265            TableBuilder::new()
266                .path(["MY SPECIAL TABLE"])
267                .name("my_table")
268                .size(100)
269                .schema(
270                    Schema::empty()
271                        .with(("Id", DataType::integer_interval(0, 1000)))
272                        .with(("Na.Me", DataType::text()))
273                        .with(("inc&ome", DataType::float_interval(100.0, 200000.0)))
274                        .with(("normal_col", DataType::text())),
275                )
276                .build(),
277        ]
278    }
279
280    /// Add a vec of tables
281    fn with_tables(self, tables: Vec<Table>) -> Result<Self> {
282        tables.into_iter().fold(Ok(self), |db, t| db?.with(t))
283    }
284
285    /// A basic test DB
286    fn with_test_tables(self) -> Result<Self> {
287        self.with_tables(Self::test_tables())
288    }
289}
290
291impl<D: Database> With<Table, Result<Self>> for D {
292    fn with(mut self, input: Table) -> Result<Self> {
293        self.create_table(&input)?;
294        self.insert_data(&input)?;
295        self.tables_mut().push(input);
296        Ok(self)
297    }
298}
299
300#[cfg(test)]
301mod tests {
302    use super::*;
303
304    #[test]
305    fn test_relation_hierarchy() -> Result<()> {
306        let database = postgresql::test_database();
307        println!("{}", database.relations());
308        Ok(())
309    }
310}
qrlew/io/mod.rs

qrlew/io/
mod.rs
