use std::collections::{HashMap, HashSet};
use std::cell::RefCell;
use indexmap::IndexMap;
use crate::parser::{Json, JsonObject};
use crate::compiler::{CompileResult, CompileError};
use crate::compiler::annotations::{Annotations, Ground};
use crate::compiler::dialects::{self, Dialect};
use crate::compiler::expr_translate::SubqueryTranslator;
use crate::compiler::rule_translate::{self, NamesAllocator, RuleStructure};
use crate::compiler::type_inference::{Type, TypesGraphBuilder, TypeInference};
#[derive(Debug, Clone)]
pub struct PredicateInfo {
pub embeddable: bool,
}
#[derive(Debug, Clone)]
pub struct GroundInfo {
pub table_name: String,
pub overwrite: bool,
pub copy_to_file: Option<String>,
}
#[derive(Debug, Clone, Default)]
pub struct Pagination {
pub limit: Option<u64>,
pub offset: Option<u64>,
}
pub fn format_sql(s: &str) -> String {
format!("{};", s)
}
pub fn indent2(s: &str) -> String {
s.split('\n')
.map(|l| format!(" {}", l))
.collect::<Vec<_>>()
.join("\n")
}
pub fn inject_structure(target: &mut RuleStructure, source: &RuleStructure) {
target.vars_map.extend(source.vars_map.clone());
target.inv_vars_map.extend(source.inv_vars_map.clone());
target.vars_unification.extend(source.vars_unification.clone());
target.unnestings.extend(source.unnestings.clone());
target.constraints.extend(source.constraints.clone());
target.synonym_log.extend(source.synonym_log.clone());
}
pub fn field_values_as_list(field_values: &HashMap<String, Json>) -> Option<Vec<Json>> {
let mut result = Vec::new();
let count = field_values.iter()
.filter(|(k, _)| k.as_str() != "__rule_text")
.count();
for i in 0..count {
let key = (i + 1).to_string();
match field_values.get(&key) {
Some(v) => result.push(v.clone()),
None => return None,
}
}
Some(result)
}
pub fn recursion_error_message() -> String {
"Recursion in this rule is too deep. It is running over the recursion limit. \
If this is intentional, consider using @Recursive annotation."
.to_string()
}
pub fn unquote_parenthesised(table: &str) -> String {
if table.len() > 4 && table.starts_with("`(") && table.ends_with(")`") {
table[2..table.len() - 2].to_string()
} else {
table.to_string()
}
}
pub struct Logica {
pub defines: Vec<String>,
pub export_statements: Vec<String>,
pub defines_and_exports: Vec<String>,
pub table_to_defined_table_map: HashMap<String, String>,
pub table_to_with_sql_map: HashMap<String, String>,
pub table_to_with_dependencies: HashMap<String, Vec<String>>,
pub with_compilation_done_for_parent: HashMap<String, HashSet<String>>,
pub dependency_edges: Vec<(String, String)>,
pub data_dependency_edges: Vec<(String, String)>,
pub table_to_export_map: HashMap<String, String>,
pub main_predicate_sql: Option<String>,
pub preamble: String,
pub workflow_predicates_stack: Vec<String>,
pub flags_comment: String,
pub compiling_udf: bool,
pub annotations: Option<Annotations>,
pub custom_udfs: IndexMap<String, String>,
pub custom_udf_definitions: IndexMap<String, String>,
pub custom_aggregation_semigroup: HashMap<String, String>,
pub main_predicate: Option<String>,
pub used_predicates: Vec<String>,
pub dependencies_of: HashMap<String, HashSet<String>>,
pub iterations: HashMap<String, IterationDef>,
pub dialect: Option<Box<dyn Dialect>>,
pub pending_with_compilations: Vec<(String, Option<String>, String)>,
}
#[derive(Debug, Clone)]
pub struct IterationDef {
pub predicates: Vec<String>,
pub repetitions: i64,
pub stop_signal: Option<String>,
}
impl Default for Logica {
fn default() -> Self {
Self::new()
}
}
impl Logica {
pub fn new() -> Self {
Logica {
defines: Vec::new(),
export_statements: Vec::new(),
defines_and_exports: Vec::new(),
table_to_defined_table_map: HashMap::new(),
table_to_with_sql_map: HashMap::new(),
table_to_with_dependencies: HashMap::new(),
with_compilation_done_for_parent: HashMap::new(),
dependency_edges: Vec::new(),
data_dependency_edges: Vec::new(),
table_to_export_map: HashMap::new(),
main_predicate_sql: None,
preamble: String::new(),
workflow_predicates_stack: Vec::new(),
flags_comment: String::new(),
compiling_udf: false,
annotations: None,
custom_udfs: IndexMap::new(),
custom_udf_definitions: IndexMap::new(),
custom_aggregation_semigroup: HashMap::new(),
main_predicate: None,
used_predicates: Vec::new(),
dependencies_of: HashMap::new(),
iterations: HashMap::new(),
dialect: None,
pending_with_compilations: Vec::new(),
}
}
pub fn add_define(&mut self, define: String) {
self.defines.push(define);
}
pub fn predicate_specific_preamble(&self, predicate_name: &str) -> String {
let deps = match self.dependencies_of.get(predicate_name) {
Some(d) => d,
None => return String::new(),
};
let mut needed_udfs: Vec<String> = deps
.iter()
.filter_map(|f| self.custom_udf_definitions.get(f).cloned())
.collect();
needed_udfs.sort();
let mut needed_semigroups = Vec::new();
for f in deps {
if let Some(sg) = self.custom_aggregation_semigroup.get(f) {
if let Some(sg_def) = self.custom_udf_definitions.get(sg) {
needed_semigroups.push(sg_def.clone());
needed_udfs.retain(|u| u != sg_def);
}
}
}
needed_semigroups.extend(needed_udfs);
needed_semigroups.join("\n")
}
pub fn needed_udf_definitions(&self) -> Vec<String> {
let mut needed_udfs: Vec<String> = self
.used_predicates
.iter()
.filter_map(|f| self.custom_udf_definitions.get(f).cloned())
.collect();
needed_udfs.sort();
let mut needed_semigroups = HashSet::new();
for f in &self.used_predicates {
if let Some(sg) = self.custom_aggregation_semigroup.get(f) {
if let Some(sg_def) = self.custom_udf_definitions.get(sg) {
needed_semigroups.insert(sg_def.clone());
needed_udfs.retain(|u| u != sg_def);
}
}
}
let mut result: Vec<String> = needed_semigroups.into_iter().collect();
result.extend(needed_udfs);
result
}
pub fn full_preamble(&self) -> String {
let mut parts = vec![self.flags_comment.clone(), self.preamble.clone()];
parts.extend(self.defines.clone());
parts.join("\n")
}
pub fn with_for(&self, predicate_name: &str) -> bool {
if self.compiling_udf {
return false;
}
self.annotations
.as_ref()
.map(|a| a.use_with(predicate_name))
.unwrap_or(true)
}
}
fn collect_record_types(node: &Json, out: &mut IndexMap<String, Type>) {
match node {
Json::Object(o) => {
for key in ["record", "the_record"] {
if let Some(rec) = o.get(key) {
if rec.is_object() && record_is_all_named(rec) {
let ty = crate::compiler::expr_translate::record_literal_type(rec);
register_record_type(&ty, out);
}
}
}
for v in o.values() {
collect_record_types(v, out);
}
}
Json::Array(items) => items.iter().for_each(|it| collect_record_types(it, out)),
_ => {}
}
}
fn record_is_all_named(rec: &Json) -> bool {
rec.as_object()
.get("field_value")
.map(|fvs| {
let arr = fvs.as_array();
!arr.is_empty()
&& arr.iter().all(|fv| {
fv.as_object()
.get("field")
.map(|f| f.is_string())
.unwrap_or(false)
})
})
.unwrap_or(false)
}
fn register_record_type(ty: &Type, out: &mut IndexMap<String, Type>) {
if let Type::Record { fields, .. } = ty {
if fields.is_empty() {
return;
}
for t in fields.values() {
register_record_type(t, out);
}
out.entry(dialects::record_type_name(ty))
.or_insert_with(|| ty.clone());
}
}
fn emit_record_type(
name: &str,
types: &IndexMap<String, Type>,
dialect: &dyn Dialect,
emitted: &mut HashSet<String>,
out: &mut String,
) {
if !emitted.insert(name.to_string()) {
return;
}
let Some(Type::Record { fields, .. }) = types.get(name) else {
return;
};
for t in fields.values() {
if matches!(t, Type::Record { .. }) {
emit_record_type(&dialects::record_type_name(t), types, dialect, emitted, out);
}
}
let mut items: Vec<(&String, &Type)> = fields.iter().collect();
items.sort_by(|a, b| a.0.cmp(b.0));
let cols: Vec<String> = items
.iter()
.map(|(k, t)| {
let col_type = match t {
Type::Record { .. } => dialects::record_type_name(t),
other => dialect.scalar_sql_type(other),
};
format!("\"{}\" {}", k, col_type)
})
.collect();
out.push_str(&format!(
"DO $$ BEGIN if not exists (select 1 from pg_type where typname = '{name}') \
then create type {name} as ({}); end if; END $$;\n",
cols.join(", ")
));
}
pub struct LogicaProgram {
pub raw_rules: Vec<Json>,
pub preparsed_rules: Vec<Json>,
pub rules: Vec<(String, Json)>,
pub defined_predicates: HashSet<String>,
pub library_predicates: HashSet<String>,
pub dollar_params: Vec<String>,
pub table_aliases: HashMap<String, String>,
pub annotations: Annotations,
pub flag_values: HashMap<String, String>,
pub custom_udfs: IndexMap<String, String>,
pub custom_udf_psql_type: HashMap<String, String>,
pub custom_aggregation_semigroup: HashMap<String, String>,
pub custom_udf_definitions: IndexMap<String, String>,
pub execution: RefCell<Option<Logica>>,
pub allocator: RefCell<NamesAllocator>,
pub user_flags: HashMap<String, String>,
pub functors_args_of: HashMap<String, HashSet<String>>,
pub typing_preamble: String,
pub predicate_signatures: HashMap<String, Json>,
pub predicate_types: HashMap<String, HashMap<String, Type>>,
pub required_type_definitions: HashMap<String, String>,
}
impl LogicaProgram {
pub fn new(
parsed: &Json,
table_aliases: HashMap<String, String>,
user_flags: HashMap<String, String>,
) -> CompileResult<Self> {
Self::new_with_engine(parsed, table_aliases, user_flags, None)
}
pub fn new_with_engine(
parsed: &Json,
table_aliases: HashMap<String, String>,
user_flags: HashMap<String, String>,
engine_override: Option<&str>,
) -> CompileResult<Self> {
let po = parsed.as_object();
let raw_rule_list: Vec<Json> = po["rule"].as_array().to_vec();
let temp_rules: Vec<(String, Json)> = raw_rule_list
.iter()
.map(|r| {
let name = r.as_object()["head"].as_object()["predicate_name"]
.as_str()
.to_string();
(name, r.clone())
})
.collect();
let temp_annotations = Annotations::extract(&temp_rules)?;
let engine = match engine_override {
Some(e) => e.to_string(),
None => temp_annotations.engine().to_string(),
};
let unfolded_rules =
crate::compiler::functors::unfold_recursion(&raw_rule_list, &engine)?;
let (extended_rules, functors_args_of) =
crate::compiler::functors::run_makes_with_deps(&unfolded_rules)?;
let dialect = dialects::get(&engine)?;
let lib_program = dialect.library_program();
let mut all_rules = extended_rules;
let mut library_predicates = HashSet::new();
if !lib_program.is_empty() {
if let Ok(lib_parsed) = crate::parser::parse_file(lib_program, None, &[]) {
for r in lib_parsed.as_object()["rule"].as_array() {
library_predicates.insert(
r.as_object()["head"].as_object()["predicate_name"]
.as_str()
.to_string(),
);
all_rules.push(r.clone());
}
}
}
let mut rules = Vec::with_capacity(all_rules.len());
for rule in &all_rules {
let head = &rule.as_object()["head"];
let name = head.as_object()["predicate_name"].as_str().to_string();
rules.push((name, rule.clone()));
}
let dollar_params = Self::extract_dollar_params_from_rules(&all_rules);
let mut annotations = Annotations::extract(&rules)?;
if let Some(e) = engine_override {
annotations.engine = e.to_string();
}
let mut flag_values = annotations.flag_values.clone();
flag_values.extend(user_flags.clone());
for param in &dollar_params {
if !flag_values.contains_key(param) {
return Err(CompileError::new(
format!("Parameter ${{{0}}} is undefined.", param),
format!("Undefined parameter: {}", param),
));
}
}
let defined_predicates: HashSet<String> = rules
.iter()
.filter(|(name, _)| !name.starts_with('@'))
.map(|(name, _)| name.clone())
.collect();
Self::check_distinct_consistency(&rules)?;
let (typing_preamble, predicate_signatures, predicate_types) = if annotations.should_typecheck() {
Self::run_typechecker(&rules)?
} else {
(String::new(), HashMap::new(), HashMap::new())
};
let custom_aggregation_semigroup: HashMap<String, String> = annotations
.bare_aggregation
.iter()
.map(|(pred, sg)| (pred.clone(), sg.clone()))
.collect();
Ok(LogicaProgram {
raw_rules: raw_rule_list,
preparsed_rules: unfolded_rules.clone(),
rules,
defined_predicates,
library_predicates,
dollar_params,
table_aliases,
annotations,
flag_values,
custom_udfs: IndexMap::new(),
custom_udf_psql_type: HashMap::new(),
custom_aggregation_semigroup,
custom_udf_definitions: IndexMap::new(),
execution: RefCell::new(None),
allocator: RefCell::new(NamesAllocator::new()),
user_flags,
functors_args_of,
typing_preamble,
predicate_signatures,
predicate_types,
required_type_definitions: HashMap::new(),
})
}
fn check_distinct_consistency(rules: &[(String, Json)]) -> CompileResult<()> {
let mut is_distinct: HashMap<String, bool> = HashMap::new();
for (p, r) in rules {
if p.starts_with('@') {
continue;
}
let distinct_here = r.as_object().contains_key("distinct_denoted");
if let Some(&distinct_before) = is_distinct.get(p) {
if distinct_before != distinct_here {
return Err(CompileError::new(
format!(
"Either all rules of a predicate must be distinct denoted \
or none. Predicate '{}' violates it.",
p
),
r.as_object()
.get("full_text")
.map(|v| v.as_str().to_string())
.unwrap_or_default(),
));
}
} else {
is_distinct.insert(p.clone(), distinct_here);
}
}
Ok(())
}
fn run_typechecker(
rules: &[(String, Json)],
) -> CompileResult<(
String,
HashMap<String, Json>,
HashMap<String, HashMap<String, Type>>,
)> {
let rule_array: Vec<Json> = rules.iter().map(|(_, r)| r.clone()).collect();
let parsed_program = crate::json_obj! {
"rule" => Json::Array(rule_array)
};
let mut builder = TypesGraphBuilder::new();
let graphs = builder.run(&parsed_program);
let mut inference = TypeInference::new(graphs);
if let Err(e) = inference.infer() {
return Err(CompileError::new(
format!("Type inference error: {}", e),
String::new(),
));
}
let mut predicate_types: HashMap<String, HashMap<String, Type>> = HashMap::new();
for (name, _) in rules {
if name.starts_with('@') {
continue;
}
let fields = inference.get_predicate_types(name);
if !fields.is_empty() {
predicate_types.entry(name.clone()).or_default().extend(fields);
}
}
let typing_preamble = String::new();
let predicate_signatures = HashMap::new();
Ok((typing_preamble, predicate_signatures, predicate_types))
}
pub fn combine_psql_type(&self, combine: &Json) -> Option<String> {
let head = jget(combine, "head")?;
let fvs = jget(head, "record").and_then(|r| jget(r, "field_value"))?;
let arr = match fvs {
Json::Array(a) => a,
_ => return None,
};
let body = jget(combine, "body");
for fv in arr {
let Some(value) = jget(fv, "value") else { continue };
let Some(agg) = jget(value, "aggregation") else { continue };
let call = jget(agg, "expression").and_then(|e| jget(e, "call"))?;
let agg_name = match jget(call, "predicate_name") {
Some(Json::Str(s)) => s.as_str(),
_ => return None,
};
let operand = jget(call, "record")
.and_then(|r| jget(r, "field_value"))
.and_then(|a| match a {
Json::Array(items) => items.first(),
_ => None,
})
.and_then(|f| jget(f, "value"))
.and_then(|v| jget(v, "expression"));
let ty = self.aggregation_result_type(agg_name, operand, body);
return Some(type_to_psql(&ty));
}
None
}
fn aggregation_result_type(
&self,
agg_name: &str,
operand: Option<&Json>,
body: Option<&Json>,
) -> Type {
let operand_ty = || operand.map(|o| self.operand_type(o, body)).unwrap_or(Type::Number);
match agg_name {
"Agg+" | "Avg" => Type::Number,
"StringAgg" => Type::String,
"Min" | "Max" => operand_ty(),
"List" | "Set" | "Array" => Type::List(Box::new(operand_ty())),
"Agg++" => operand_ty(),
_ => operand_ty(),
}
}
fn operand_type(&self, expr: &Json, body: Option<&Json>) -> Type {
if let Some(lit) = jget(expr, "literal") {
if jget(lit, "the_string").is_some() {
return Type::String;
}
if jget(lit, "the_bool").is_some() {
return Type::Bool;
}
if jget(lit, "the_number").is_some() {
return Type::Number;
}
}
if let Some(var) = jget(expr, "variable") {
if let Some(Json::Str(name)) = jget(var, "var_name") {
if let Some(body) = body {
if let Some((pred, field)) = find_var_binding(name, body) {
if let Some(t) = self
.predicate_types
.get(&pred)
.and_then(|fields| fields.get(&field))
{
return t.clone();
}
}
}
}
}
if let Some(call) = jget(expr, "call") {
if let Some(Json::Str(name)) = jget(call, "predicate_name") {
match name.as_str() {
"++" => return Type::String,
"+" | "-" | "*" | "/" | "^" | "%" | "Abs" | "Sqrt" | "Exp" | "Log" | "Sin"
| "Cos" | "Round" | "Floor" | "Ceiling" | "Pow" => return Type::Number,
"==" | "!=" | "<" | ">" | "<=" | ">=" | "&&" | "||" | "!" | "Like" | "in"
| "IsNull" => return Type::Bool,
_ => {}
}
}
}
Type::Number
}
fn extract_dollar_params_from_rules(rules: &[Json]) -> Vec<String> {
let mut params = HashSet::new();
for rule in rules {
Self::collect_dollar_params(rule, &mut params);
}
params.into_iter().collect()
}
fn collect_dollar_params(node: &Json, params: &mut HashSet<String>) {
let mut stack: Vec<&Json> = vec![node];
while let Some(current) = stack.pop() {
match current {
Json::Str(s) => {
Self::extract_dollar_params_from_string(s, params);
}
Json::Object(o) => {
for (_, v) in o.iter() {
stack.push(v);
}
}
Json::Array(a) => {
for v in a {
stack.push(v);
}
}
_ => {}
}
}
}
fn extract_dollar_params_from_string(s: &str, params: &mut HashSet<String>) {
use regex::Regex;
use std::sync::LazyLock;
static RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"\$\{(.*?)\}").unwrap());
for cap in RE.captures_iter(s) {
let p = &cap[1];
if !p.starts_with("YYYY") && p != "MM" && p != "DD" {
params.insert(p.to_string());
}
}
}
pub fn engine(&self) -> &str {
self.annotations.engine()
}
pub fn defined_predicates(&self) -> &HashSet<String> {
&self.defined_predicates
}
pub fn user_defined_predicates(&self) -> Vec<String> {
let mut names: Vec<String> = self
.defined_predicates
.iter()
.filter(|name| !self.library_predicates.contains(*name))
.cloned()
.collect();
names.sort();
names
}
pub fn new_names_allocator(&self) -> NamesAllocator {
let udfs: HashMap<String, String> = self.custom_udfs.iter()
.map(|(k, v)| (k.clone(), v.clone()))
.collect();
NamesAllocator::with_custom_udfs(udfs)
}
pub fn get_predicate_rules(&self, predicate_name: &str) -> Vec<Json> {
self.rules
.iter()
.filter(|(n, _)| n == predicate_name)
.map(|(_, r)| r.clone())
.collect()
}
fn initialize_execution(&self, main_predicate: &str) -> CompileResult<Logica> {
let mut exec = Logica::new();
exec.workflow_predicates_stack
.push(main_predicate.to_string());
exec.annotations = Some(self.annotations.clone());
exec.custom_udfs = self.custom_udfs.clone();
exec.custom_udf_definitions = self.custom_udf_definitions.clone();
exec.custom_aggregation_semigroup = self.custom_aggregation_semigroup.clone();
exec.main_predicate = Some(main_predicate.to_string());
exec.used_predicates = self
.functors_args_of
.get(main_predicate)
.map(|s| s.iter().cloned().collect())
.unwrap_or_default();
exec.dependencies_of = self
.functors_args_of
.iter()
.map(|(k, v)| (k.clone(), v.clone()))
.collect();
exec.dialect = Some(dialects::get(self.annotations.engine())?);
exec.preamble = self.annotations.preamble();
exec.iterations = self.annotations.iterations()?;
Ok(exec)
}
pub fn predicate_sql_ext(
&self,
name: &str,
allocator_is_none: bool,
) -> CompileResult<String> {
let rules = self.get_predicate_rules(name);
if rules.is_empty() {
return Err(CompileError::new(
format!(
"No rules are defining '{}', but compilation was requested.",
name
),
"",
));
}
if rules.len() == 1 {
if allocator_is_none {
*self.allocator.borrow_mut() = self.new_names_allocator();
}
let sql = self.single_rule_sql(&rules[0], None, false, true)?;
assert!(
!sql.starts_with("/* nil */"),
"Single rule is nil for predicate '{}'",
name
);
let order_by = self.annotations.order_by_clause(name);
let limit = self.annotations.limit_clause(name);
return Ok(format!("{}{}{}", sql, order_by, limit));
}
let mut rules_sql = Vec::new();
for rule in &rules {
if rule.as_object().contains_key("distinct_denoted") {
return Err(CompileError::new(
format!(
"For distinct denoted predicates multiple rules are not \
currently supported. Consider taking union of bodies manually."
),
rule.as_object()
.get("full_text")
.map(|v| v.as_str().to_string())
.unwrap_or_default(),
));
}
if allocator_is_none {
*self.allocator.borrow_mut() = self.new_names_allocator();
}
let single_sql = self.single_rule_sql(rule, None, false, false)?;
if !single_sql.starts_with("/* nil */") {
rules_sql.push(format!("\n{}\n", indent2(&single_sql)));
}
}
if rules_sql.is_empty() {
return Err(CompileError::new(
format!("All disjuncts are nil for predicate '{}'.", name),
"",
));
}
let rules_sql: Vec<String> = rules_sql
.iter()
.map(|r| {
r.split('\n')
.map(|l| format!(" {}", l))
.collect::<Vec<_>>()
.join("\n")
})
.collect();
let order_by = self.annotations.order_by_clause(name);
let limit = self.annotations.limit_clause(name);
Ok(format!(
"SELECT * FROM (\n{}\n) AS UNUSED_TABLE_NAME {} {}",
rules_sql.join(" UNION ALL\n"),
order_by,
limit,
))
}
pub fn predicate_sql(&self, name: &str) -> CompileResult<String> {
self.predicate_sql_ext(name, false)
}
pub fn single_rule_sql(
&self,
rule: &Json,
external_vocabulary: Option<&HashMap<String, String>>,
is_combine: bool,
must_not_be_nil: bool,
) -> CompileResult<String> {
let dialect = dialects::get(self.annotations.engine())?;
let r = if is_combine && dialect.decorate_combine_rule() {
let var = self.allocator.borrow_mut().alloc_var();
rule_translate::decorate_combine_rule(rule, &var)
} else {
rule.clone()
};
let taken = std::mem::take(&mut *self.allocator.borrow_mut());
let mut s = rule_translate::extract_rule_structure_with_vocabulary(
&r,
Some(taken),
external_vocabulary.cloned(),
)?;
self.run_injections(&mut s)?;
rule_translate::finalize_rule_structure(&mut s);
s.sort_unnestings()?;
fn contains_nil_predicate(json: &Json) -> bool {
match json {
Json::Object(o) => {
if let Some(call) = o.get("call") {
if let Some(pn) = call.as_object().get("predicate_name") {
if pn.is_string() && pn.as_str() == "nil" {
return true;
}
}
}
if let Some(pn) = o.get("predicate_name") {
if pn.is_string() && pn.as_str() == "nil" {
return true;
}
}
o.values().any(|v| contains_nil_predicate(v))
}
Json::Array(a) => a.iter().any(|v| contains_nil_predicate(v)),
_ => false,
}
}
let has_nil_table = s.tables.values().any(|v| v == "nil");
let has_nil_unnesting = s.unnestings.iter().any(|(_, list_expr)| {
contains_nil_predicate(list_expr)
});
let has_nil_select = s.select.values().any(|expr| contains_nil_predicate(expr));
if has_nil_table || has_nil_unnesting || has_nil_select {
*self.allocator.borrow_mut() = std::mem::take(&mut s.allocator);
if must_not_be_nil {
return Err(CompileError::new(
format!(
"Single rule is nil for predicate '{}'. Recursion unfolding failed.",
s.this_predicate_name
),
rule.as_object()
.get("full_text")
.map(|v| v.as_str().to_string())
.unwrap_or_default(),
));
} else {
return Ok(
"/* nil */ SELECT NULL FROM (SELECT 42 AS MONAD) AS NIRVANA WHERE MONAD = 0"
.to_string(),
);
}
}
*self.allocator.borrow_mut() = std::mem::take(&mut s.allocator);
let translator = UniverseSubqueryTranslator {
program: self,
};
s.as_sql(&translator, dialect.as_ref(), &self.flag_values)
}
pub fn run_injections(
&self,
s: &mut RuleStructure,
) -> CompileResult<()> {
let mut iterations = 0;
loop {
iterations += 1;
if iterations > 1000 {
return Err(CompileError::new(
recursion_error_message(),
&s.full_rule_text,
));
}
let mut new_tables = IndexMap::new();
let mut changed = false;
let old_tables: Vec<_> = s.tables.iter().map(|(k, v)| (k.clone(), v.clone())).collect();
for (table_name_rsql, table_predicate_rsql) in &old_tables {
let rules = self.get_predicate_rules(table_predicate_rsql);
let is_injectable = rules.len() == 1
&& !rules[0].as_object().contains_key("distinct_denoted")
&& self.annotations.ok_injection(table_predicate_rsql);
if is_injectable {
let taken_alloc = std::mem::take(&mut s.allocator);
let mut rs = rule_translate::extract_rule_structure(&rules[0], Some(taken_alloc))?;
rs.eliminate_internal_variables_no_unfold();
s.allocator = std::mem::take(&mut rs.allocator);
new_tables.extend(rs.tables.clone());
inject_structure(s, &rs);
let old_vars: Vec<_> = s
.vars_map
.iter()
.map(|(k, v)| (k.clone(), v.clone()))
.collect();
let mut new_vars_map = HashMap::new();
let mut new_inv_vars_map: HashMap<String, (String, String)> = s.inv_vars_map
.iter()
.filter(|(_, (tbl, _))| tbl.is_empty())
.map(|(k, v)| (k.clone(), v.clone()))
.collect();
for ((table_name, table_var), clause_var) in &old_vars {
if table_name != table_name_rsql {
new_vars_map
.insert((table_name.clone(), table_var.clone()), clause_var.clone());
new_inv_vars_map
.insert(clause_var.clone(), (table_name.clone(), table_var.clone()));
} else {
if let Some(select_expr) = rs.select.get(table_var.as_str()) {
s.vars_unification.push((
Json::Object({
let mut m = JsonObject::new();
m.insert(
"variable".into(),
Json::Object({
let mut vm = JsonObject::new();
vm.insert(
"var_name".into(),
Json::Str(clause_var.clone()),
);
vm
}),
);
m
}),
select_expr.clone(),
));
} else if rs.select.contains_key("*") {
let subscript = Json::Object({
let mut m = JsonObject::new();
m.insert(
"literal".into(),
Json::Object({
let mut lm = JsonObject::new();
lm.insert(
"the_symbol".into(),
Json::Object({
let mut sm = JsonObject::new();
sm.insert(
"symbol".into(),
Json::Str(table_var.clone()),
);
sm
}),
);
lm
}),
);
m
});
s.vars_unification.push((
Json::Object({
let mut m = JsonObject::new();
m.insert(
"variable".into(),
Json::Object({
let mut vm = JsonObject::new();
vm.insert(
"var_name".into(),
Json::Str(clause_var.clone()),
);
vm
}),
);
m
}),
Json::Object({
let mut m = JsonObject::new();
m.insert(
"subscript".into(),
Json::Object({
let mut sm = JsonObject::new();
sm.insert("subscript".into(), subscript);
sm.insert(
"record".into(),
rs.select["*"].clone(),
);
sm
}),
);
m
}),
));
} else if table_var == "*" {
s.vars_unification.push((
Json::Object({
let mut m = JsonObject::new();
m.insert(
"variable".into(),
Json::Object({
let mut vm = JsonObject::new();
vm.insert(
"var_name".into(),
Json::Str(clause_var.clone()),
);
vm
}),
);
m
}),
select_as_record(&rs.select),
));
} else {
return Err(CompileError::new(
format!(
"Predicate '{}' does not have an argument '{}', \
but this rule tries to access it.",
table_predicate_rsql, table_var
),
&s.full_rule_text,
));
}
}
}
s.vars_map = new_vars_map;
s.inv_vars_map = new_inv_vars_map;
changed = true;
} else {
new_tables.insert(table_name_rsql.clone(), table_predicate_rsql.clone());
}
}
if !changed || s.tables == new_tables {
break;
}
s.tables = new_tables;
}
Ok(())
}
pub fn formatted_predicate_sql(
&self,
name: &str,
) -> CompileResult<String> {
self.formatted_predicate_sql_impl(name, None, None)
}
fn record_type_definitions(&self, dialect: &dyn Dialect, body_sql: &str) -> String {
let mut types: IndexMap<String, Type> = IndexMap::new();
for (_, rule) in &self.rules {
collect_record_types(rule, &mut types);
}
let referenced: Vec<String> = types
.keys()
.filter(|n| body_sql.contains(&format!("::{}", n)))
.cloned()
.collect();
let mut emitted: HashSet<String> = HashSet::new();
let mut out = String::new();
for name in &referenced {
emit_record_type(name, &types, dialect, &mut emitted, &mut out);
}
out
}
fn formatted_predicate_sql_impl(
&self,
name: &str,
pagination: Option<&Pagination>,
search: Option<&str>,
) -> CompileResult<String> {
let exec = self.initialize_execution(name)?;
*self.execution.borrow_mut() = Some(exec);
let sql = self.predicate_sql_ext(name, true)?;
{
enum WorkItem {
Compile {
predicate: String,
cte_name: Option<String>,
},
AddDep {
predicate: String,
parent: String,
},
}
let mut stack: Vec<WorkItem> = Vec::new();
{
let mut exec_ref = self.execution.borrow_mut();
let exec_inner = exec_ref.as_mut().unwrap();
let pending = std::mem::take(&mut exec_inner.pending_with_compilations);
for (pred, cte, parent) in pending.into_iter().rev() {
stack.push(WorkItem::AddDep {
predicate: pred.clone(),
parent: parent.clone(),
});
stack.push(WorkItem::Compile {
predicate: pred,
cte_name: cte,
});
}
}
while let Some(item) = stack.pop() {
match item {
WorkItem::Compile { predicate, cte_name } => {
let result_sql = self.predicate_sql(&predicate)?;
if let Some(cn) = cte_name {
let mut exec_ref = self.execution.borrow_mut();
let exec_inner = exec_ref.as_mut().unwrap();
exec_inner.table_to_with_sql_map.insert(cn, result_sql);
}
let mut exec_ref = self.execution.borrow_mut();
let exec_inner = exec_ref.as_mut().unwrap();
let new_pending =
std::mem::take(&mut exec_inner.pending_with_compilations);
for (pred, cte, par) in new_pending.into_iter().rev() {
stack.push(WorkItem::AddDep {
predicate: pred.clone(),
parent: par.clone(),
});
stack.push(WorkItem::Compile {
predicate: pred,
cte_name: cte,
});
}
}
WorkItem::AddDep { predicate, parent } => {
let mut exec_ref = self.execution.borrow_mut();
let exec_inner = exec_ref.as_mut().unwrap();
let dep_list = exec_inner
.table_to_with_dependencies
.entry(parent)
.or_default();
if !dep_list.contains(&predicate) {
dep_list.push(predicate);
}
}
}
}
}
self.perform_iteration_closure()?;
let with_signature = self.generate_with_clauses(name);
let sql = if let Some(with_sig) = with_signature {
format!("{}\n{}", with_sig, sql)
} else {
sql
};
let sql = self.use_flags_as_parameters(&sql);
let sql = match search {
Some(pattern) => self.search_filter_query(name, sql, pattern)?,
None => sql,
};
let sql = match pagination {
Some(p) => self.paginate_query(name, sql, p),
None => sql,
};
let exec = self.execution.borrow();
let exec_ref = exec.as_ref().unwrap();
let mut result = String::new();
if !exec_ref.flags_comment.is_empty() {
result.push_str(&exec_ref.flags_comment);
}
if !exec_ref.preamble.is_empty() {
result.push_str(&self.use_flags_as_parameters(&exec_ref.preamble));
}
let engine = self.annotations.engine();
if engine == "psql" {
let dialect = dialects::get(engine)?;
result.push_str(&self.record_type_definitions(dialect.as_ref(), &sql));
} else if engine == "duckdb" && !self.typing_preamble.is_empty() {
result.push_str(&self.typing_preamble);
}
let udf_defs = exec_ref.needed_udf_definitions();
if !udf_defs.is_empty() {
result.push_str(&udf_defs.join("\n\n"));
result.push_str("\n\n");
}
if !exec_ref.defines_and_exports.is_empty() {
result.push_str(&exec_ref.defines_and_exports.join("\n\n"));
result.push_str("\n\n");
}
result.push_str(&format_sql(&sql));
Ok(self.use_flags_as_parameters(&result))
}
pub fn formatted_predicate_sql_with_pagination(
&self,
name: &str,
pagination: &Pagination,
) -> CompileResult<String> {
self.formatted_predicate_sql_impl(name, Some(pagination), None)
}
fn paginate_query(&self, name: &str, query: String, pagination: &Pagination) -> String {
let annotation_limit = self.annotations.limit_of(name);
let effective_limit = match (pagination.limit, annotation_limit) {
(Some(pl), Some(al)) => Some(pl.min(al as u64)),
(Some(pl), None) => Some(pl),
(None, _) => None,
};
let mut pagination_clause = String::new();
if let Some(limit) = effective_limit {
pagination_clause.push_str(&format!("\nLIMIT {}", limit));
}
if let Some(offset) = pagination.offset {
if offset > 0 {
pagination_clause.push_str(&format!("\nOFFSET {}", offset));
}
}
if pagination_clause.is_empty() {
query
} else {
format!(
"SELECT * FROM (\n{}\n) AS _paginated{};",
query.trim_end_matches(';'),
pagination_clause
)
}
}
pub fn predicate_columns(&self, name: &str) -> Vec<String> {
let rules = self.get_predicate_rules(name);
if rules.is_empty() {
return Vec::new();
}
let head = &rules[0].as_object()["head"];
let record = &head.as_object()["record"];
record.as_object()["field_value"]
.as_array()
.iter()
.map(|fv| fv.as_object()["field"].as_str().to_string())
.collect()
}
pub fn formatted_predicate_sql_with_search(
&self,
name: &str,
pattern: &str,
pagination: &Pagination,
) -> CompileResult<String> {
self.formatted_predicate_sql_impl(name, Some(pagination), Some(pattern))
}
fn search_filter_query(
&self,
name: &str,
query: String,
pattern: &str,
) -> CompileResult<String> {
let columns = self.predicate_columns(name);
if columns.is_empty() {
return Err(CompileError::new(
format!("No columns found for predicate '{}'.", name),
"",
));
}
let dialect = dialects::get(self.annotations.engine())?;
let where_clause = columns
.iter()
.map(|col| {
let cast_col = dialect.string_cast(col);
dialect.regex_match_condition(&cast_col, pattern)
})
.collect::<Vec<_>>()
.join(" OR ");
Ok(format!(
"SELECT * FROM (\n{}\n) AS _searched\nWHERE {}",
query.trim_end_matches(';'),
where_clause,
))
}
fn perform_iteration_closure(&self) -> CompileResult<()> {
let participating_predicates: Vec<String> = {
let exec = self.execution.borrow();
let exec_ref = exec.as_ref().unwrap();
exec_ref.table_to_defined_table_map.keys().cloned().collect()
};
let iterations: Vec<IterationDef> = {
let exec = self.execution.borrow();
let exec_ref = exec.as_ref().unwrap();
exec_ref.iterations.values().cloned().collect()
};
for iteration in &iterations {
let iteration_preds: HashSet<&str> = iteration.predicates.iter()
.map(|s| s.as_str()).collect();
for p in &participating_predicates {
if iteration_preds.contains(p.as_str()) {
for d in &iteration.predicates {
let _ = self.translate_table_in_context(d, None, false);
}
break;
}
}
}
Ok(())
}
fn generate_with_clauses(&self, predicate_name: &str) -> Option<String> {
let exec = self.execution.borrow();
let exec_ref = exec.as_ref()?;
let dependencies = exec_ref
.table_to_with_dependencies
.get(predicate_name)?;
if dependencies.is_empty() {
return None;
}
let mut with_bodies = Vec::new();
for dep in dependencies {
if let Some(table_name) = exec_ref.table_to_defined_table_map.get(dep) {
if let Some(sql) = exec_ref.table_to_with_sql_map.get(table_name) {
with_bodies.push(format!("{} AS ({})", table_name, sql));
}
}
}
if with_bodies.is_empty() {
return None;
}
Some(format!("WITH {}", with_bodies.join(",\n")))
}
pub fn use_flags_as_parameters(&self, sql: &str) -> String {
let mut result = sql.to_string();
let mut prev = String::new();
let mut num_subs = 0;
while result != prev {
num_subs += 1;
prev = result.clone();
if num_subs > 100 {
eprintln!("[WARNING] Recursive flag references detected, stopping substitution");
break;
}
for (flag, value) in &self.flag_values {
result = result.replace(&format!("${{{}}}", flag), value);
}
}
result
}
fn translate_withed_table(
&self,
table: &str,
) -> CompileResult<String> {
let parent_table = {
let exec = self.execution.borrow();
exec.as_ref()
.and_then(|e| e.workflow_predicates_stack.last().cloned())
.unwrap_or_default()
};
let already_defined = {
let exec = self.execution.borrow();
exec.as_ref()
.map(|e| e.table_to_defined_table_map.contains_key(table))
.unwrap_or(false)
};
if !already_defined {
let table_name = {
let mut alloc = self.allocator.borrow_mut();
alloc.alloc_table(Some(table))
};
{
let mut exec = self.execution.borrow_mut();
let exec_ref = exec.as_mut().unwrap();
exec_ref
.table_to_defined_table_map
.insert(table.to_string(), table_name.clone());
}
let result_sql = self.predicate_sql(table)?;
{
let mut exec = self.execution.borrow_mut();
let exec_ref = exec.as_mut().unwrap();
exec_ref
.table_to_with_sql_map
.insert(table_name, result_sql);
}
} else {
let already_done = {
let exec = self.execution.borrow();
exec.as_ref()
.map(|e| {
e.with_compilation_done_for_parent
.get(&parent_table)
.map(|s| s.contains(table))
.unwrap_or(false)
})
.unwrap_or(false)
};
if !already_done {
{
let mut exec = self.execution.borrow_mut();
let exec_ref = exec.as_mut().unwrap();
exec_ref
.with_compilation_done_for_parent
.entry(parent_table.clone())
.or_default()
.insert(table.to_string());
}
let _ = self.predicate_sql(table)?;
}
}
{
let mut exec = self.execution.borrow_mut();
let exec_ref = exec.as_mut().unwrap();
let dep_list = exec_ref
.table_to_with_dependencies
.entry(parent_table)
.or_default();
if !dep_list.contains(&table.to_string()) {
dep_list.push(table.to_string());
}
}
let exec = self.execution.borrow();
let exec_ref = exec.as_ref().unwrap();
Ok(exec_ref.table_to_defined_table_map[table].clone())
}
fn translate_table_attached_to_file(
&self,
table: &str,
ground: &Ground,
_allocator: &mut NamesAllocator,
edge_needed: bool,
) -> CompileResult<String> {
if edge_needed {
let mut exec = self.execution.borrow_mut();
if let Some(exec_ref) = exec.as_mut() {
let parent = exec_ref
.workflow_predicates_stack
.last()
.cloned()
.unwrap_or_default();
exec_ref
.dependency_edges
.push((table.to_string(), parent));
}
}
{
let exec = self.execution.borrow();
if let Some(exec_ref) = exec.as_ref() {
if let Some(name) = exec_ref.table_to_defined_table_map.get(table) {
return Ok(name.clone());
}
}
}
let table_name = ground.table_name.clone();
let define_statement = format!("-- Interacting with table {}", table_name);
{
let mut exec = self.execution.borrow_mut();
let exec_ref = exec.as_mut().unwrap();
exec_ref
.table_to_defined_table_map
.insert(table.to_string(), table_name.clone());
exec_ref.add_define(define_statement.clone());
}
let mut export_statement = None;
if self.defined_predicates.contains(table) {
{
let mut exec = self.execution.borrow_mut();
let exec_ref = exec.as_mut().unwrap();
exec_ref
.workflow_predicates_stack
.push(table.to_string());
}
let dependency_sql = self.predicate_sql(table)?;
let with_signature = self.generate_with_clauses(table);
let dependency_sql = if let Some(with_sig) = with_signature {
format!("{}\n{}", with_sig, dependency_sql)
} else {
dependency_sql
};
let dependency_sql = self.use_flags_as_parameters(&dependency_sql);
{
let mut exec = self.execution.borrow_mut();
let exec_ref = exec.as_mut().unwrap();
exec_ref.workflow_predicates_stack.pop();
let create_or_replace = ground.overwrite
&& exec_ref
.dialect
.as_ref()
.is_some_and(|d| d.supports_create_or_replace_table());
let maybe_drop = if ground.overwrite && !create_or_replace {
let cascade = exec_ref
.dialect
.as_ref()
.map(|d| d.cascading_deletion_word())
.unwrap_or("");
format!("DROP TABLE IF EXISTS {}{};\n", ground.table_name, cascade)
} else {
String::new()
};
let create_statement = format!(
"CREATE {}TABLE {} AS {}",
if create_or_replace { "OR REPLACE " } else { "" },
ground.table_name,
format_sql(&dependency_sql)
);
let stmt = format!("{}{}", maybe_drop, create_statement);
let stmt = self.use_flags_as_parameters(&stmt);
exec_ref
.table_to_export_map
.insert(table.to_string(), stmt.clone());
exec_ref.export_statements.push(stmt.clone());
export_statement = Some(stmt);
}
}
if let Some(ref copy_file) = ground.copy_to_file {
let copy_stmt = format!(
"COPY {} TO '{}' (FORMAT 'json', ARRAY true);",
ground.table_name, copy_file
);
let mut exec = self.execution.borrow_mut();
let exec_ref = exec.as_mut().unwrap();
exec_ref.defines_and_exports.push(copy_stmt);
}
{
let mut exec = self.execution.borrow_mut();
let exec_ref = exec.as_mut().unwrap();
if let Some(ref es) = export_statement {
exec_ref.defines_and_exports.push(es.clone());
}
exec_ref.defines_and_exports.push(define_statement);
}
Ok(table_name)
}
fn translate_table_in_context(
&self,
table: &str,
_external_vocabulary: Option<&HashMap<String, String>>,
edge_needed: bool,
) -> CompileResult<String> {
if table == "Today" || table == "Now" {
let dialect = dialects::get(self.annotations.engine())?;
return Ok(if table == "Today" {
dialect.today_relation_sql()
} else {
dialect.now_relation_sql()
});
}
if let Some(alias) = self.table_aliases.get(table) {
return Ok(alias.clone());
}
if let Some(ground) = self.annotations.ground(table) {
let mut alloc = std::mem::take(&mut *self.allocator.borrow_mut());
let result = self.translate_table_attached_to_file(
table,
&ground,
&mut alloc,
edge_needed,
);
*self.allocator.borrow_mut() = alloc;
return result;
}
if self.defined_predicates.contains(table) {
let use_with = {
let exec = self.execution.borrow();
exec.as_ref()
.map(|e| e.with_for(table))
.unwrap_or(self.annotations.use_with(table))
};
if use_with {
return self.translate_withed_table(table);
}
let sql = self.predicate_sql(table)?;
return Ok(format!("({})", sql));
}
if edge_needed {
let mut exec = self.execution.borrow_mut();
if let Some(exec_ref) = exec.as_mut() {
let parent = exec_ref
.workflow_predicates_stack
.last()
.cloned()
.unwrap_or_default();
exec_ref
.data_dependency_edges
.push((table.to_string(), parent));
}
}
Ok(unquote_parenthesised(table))
}
}
pub struct UniverseSubqueryTranslator<'a> {
pub program: &'a LogicaProgram,
}
impl<'a> SubqueryTranslator for UniverseSubqueryTranslator<'a> {
fn translate_table(
&self,
predicate: &str,
external_vocabulary: Option<&HashMap<String, String>>,
) -> CompileResult<String> {
self.program
.translate_table_in_context(predicate, external_vocabulary, true)
}
fn translate_rule(
&self,
rule: &Json,
external_vocabulary: &HashMap<String, String>,
is_combine: bool,
) -> CompileResult<String> {
self.program
.single_rule_sql(rule, Some(external_vocabulary), is_combine, false)
}
fn combine_psql_type(&self, combine: &Json) -> Option<String> {
self.program.combine_psql_type(combine)
}
}
fn jget<'a>(node: &'a Json, key: &str) -> Option<&'a Json> {
match node {
Json::Object(o) => o.get(key),
_ => None,
}
}
fn type_to_psql(t: &Type) -> String {
match t {
Type::Number => "numeric".to_string(),
Type::String => "text".to_string(),
Type::Bool => "bool".to_string(),
Type::List(element) => format!("{}[]", type_to_psql(element)),
_ => "numeric".to_string(),
}
}
fn find_var_binding(var_name: &str, body: &Json) -> Option<(String, String)> {
let conjuncts = jget(body, "conjunction").and_then(|c| jget(c, "conjunct"))?;
let arr = match conjuncts {
Json::Array(a) => a,
_ => return None,
};
for conj in arr {
let Some(pred) = jget(conj, "predicate") else { continue };
let pname = match jget(pred, "predicate_name") {
Some(Json::Str(s)) => s.clone(),
_ => continue,
};
let Some(Json::Array(items)) =
jget(pred, "record").and_then(|r| jget(r, "field_value"))
else {
continue;
};
for item in items {
let field = match jget(item, "field") {
Some(Json::Str(s)) => s.clone(),
Some(Json::Int(n)) => format!("col{}", n),
_ => continue,
};
let bound = jget(item, "value")
.and_then(|v| jget(v, "expression"))
.and_then(|e| jget(e, "variable"))
.and_then(|vv| jget(vv, "var_name"));
if let Some(Json::Str(vn)) = bound {
if vn.as_str() == var_name {
return Some((pname, field));
}
}
}
}
None
}
fn select_as_record(select: &IndexMap<String, Json>) -> Json {
let mut field_values = Vec::new();
for (field, expr) in select {
if field == "logica_value" || field == "synalog_value" {
continue;
}
field_values.push(Json::Object({
let mut m = JsonObject::new();
m.insert("field".into(), Json::Str(field.clone()));
m.insert(
"value".into(),
Json::Object({
let mut vm = JsonObject::new();
vm.insert("expression".into(), expr.clone());
vm
}),
);
m
}));
}
Json::Object({
let mut m = JsonObject::new();
m.insert(
"record".into(),
Json::Object({
let mut rm = JsonObject::new();
rm.insert("field_value".into(), Json::Array(field_values));
rm
}),
);
m
})
}
#[cfg(test)]
#[path = "universe_test.rs"]
mod universe_test;