shifty_engine/
compiled.rs

1//! Shapes-document ownership and admission for reusable evaluation sessions.
2
3use crate::frozen::SourceStorage;
4use crate::sparql::FunctionDef;
5use crate::validate::NonStratifiable;
6use shifty_algebra::{Schema, ShapeId};
7use shifty_opt::{
8    AccessCatalog, PhysicalPlan, RuleDependencies, Stratification, analyze,
9    normalize_with_mapping_and_analysis, plan, rule_dependencies, rule_guard_dependencies,
10};
11use shifty_parse::{Diagnostic, Loaded, ParseError, parse_loaded};
12use spargebra::Query;
13use std::collections::HashMap;
14use std::fmt;
15use std::sync::{Arc, OnceLock};
16
17pub(crate) type ParsedQueries = HashMap<String, Query>;
18
19/// A shapes document compiled once for use with many data snapshots.
20#[derive(Clone)]
21pub struct CompiledShapes {
22    pub(crate) inner: Arc<CompiledShapesInner>,
23}
24
25pub(crate) struct CompiledShapesInner {
26    pub(crate) source: Arc<Loaded>,
27    pub(crate) authored: Arc<Schema>,
28    pub(crate) normalized: Arc<Schema>,
29    statement_map: Arc<Vec<usize>>,
30    pub(crate) shape_map: Arc<Vec<Option<ShapeId>>>,
31    pub(crate) raw_by_normalized: Arc<Vec<Vec<usize>>>,
32    pub(crate) diagnostics: Vec<Diagnostic>,
33    pub(crate) functions: Vec<FunctionDef>,
34    pub(crate) rules: Vec<CompiledRuleSchedule>,
35    pub(crate) access: AccessCatalog,
36    pub(crate) parsed_queries: Arc<ParsedQueries>,
37    physical: OnceLock<PhysicalPlan>,
38    source_storage: OnceLock<Arc<SourceStorage>>,
39}
40
41pub(crate) struct CompiledRuleSchedule {
42    pub(crate) index: usize,
43    pub(crate) order: i64,
44    pub(crate) dependencies: RuleDependencies,
45    pub(crate) guard_dependencies: RuleDependencies,
46}
47
48#[derive(Debug)]
49pub enum CompileError {
50    Invalid(ParseError),
51    /// The offending cycle, plus the message naming its shapes as the document
52    /// named them. The error outlives the schema it was derived from, and
53    /// [`NonStratifiable`]'s own `Display` can only print arena slot ids, so
54    /// the readable form is rendered at the point where the schema is still in
55    /// hand.
56    NonStratifiable(NonStratifiable, String),
57}
58
59impl CompileError {
60    /// The cycle itself, for a caller that wants the shape ids rather than prose.
61    pub fn non_stratifiable(&self) -> Option<&NonStratifiable> {
62        match self {
63            Self::NonStratifiable(error, _) => Some(error),
64            Self::Invalid(_) => None,
65        }
66    }
67}
68
69impl fmt::Display for CompileError {
70    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
71        match self {
72            Self::Invalid(error) => write!(f, "{error}"),
73            Self::NonStratifiable(_, described) => write!(f, "{described}"),
74        }
75    }
76}
77
78impl std::error::Error for CompileError {}
79
80fn check_strata(schema: &Schema) -> Result<Stratification, CompileError> {
81    let analysis = analyze(&schema.arena);
82    if analysis.stratifiable {
83        return Ok(analysis);
84    }
85    let error = NonStratifiable {
86        components: analysis
87            .strata
88            .into_iter()
89            .filter(|stratum| !stratum.stratifiable)
90            .map(|stratum| stratum.shapes)
91            .collect(),
92    };
93    let described = error.describe(schema);
94    Err(CompileError::NonStratifiable(error, described))
95}
96
97impl CompiledShapes {
98    pub fn compile(source: Loaded) -> Result<Self, CompileError> {
99        let parsed = parse_loaded(&source);
100        parsed.require_valid().map_err(CompileError::Invalid)?;
101        let authored_analysis = check_strata(&parsed.schema)?;
102        let normalized = normalize_with_mapping_and_analysis(&parsed.schema, &authored_analysis);
103        check_strata(&normalized.schema)?;
104        let mut raw_by_normalized = vec![Vec::new(); normalized.schema.statements.len()];
105        for (authored, normalized_id) in normalized.statement_map.iter().copied().enumerate() {
106            raw_by_normalized[normalized_id].push(authored);
107        }
108        let functions = shifty_parse::collect_functions(&source);
109        let access_started = crate::profile::is_enabled().then(web_time::Instant::now);
110        let access = AccessCatalog::compile(&parsed.schema, &functions);
111        if let Some(access_started) = access_started {
112            crate::profile::record_access_catalog_time(access_started.elapsed().as_micros() as u64);
113        }
114        let parsed_queries = Arc::new(
115            access
116                .queries
117                .iter()
118                .filter_map(|query| {
119                    query
120                        .parsed
121                        .clone()
122                        .map(|parsed| (query.text.clone(), parsed))
123                })
124                .collect(),
125        );
126        let mut rules: Vec<_> = normalized
127            .schema
128            .rules
129            .iter()
130            .enumerate()
131            .filter(|(_, rule)| !rule.deactivated)
132            .map(|(index, rule)| CompiledRuleSchedule {
133                index,
134                order: rule.order.unwrap_or(0),
135                dependencies: rule_dependencies(rule, &normalized.schema.arena),
136                guard_dependencies: rule_guard_dependencies(rule, &normalized.schema.arena),
137            })
138            .collect();
139        rules.sort_by_key(|rule| (rule.order, rule.index));
140        Ok(Self {
141            inner: Arc::new(CompiledShapesInner {
142                source: Arc::new(source),
143                authored: Arc::new(parsed.schema),
144                normalized: Arc::new(normalized.schema),
145                statement_map: Arc::new(normalized.statement_map),
146                shape_map: Arc::new(normalized.shape_map),
147                raw_by_normalized: Arc::new(raw_by_normalized),
148                diagnostics: parsed.diagnostics,
149                functions,
150                rules,
151                access,
152                parsed_queries,
153                physical: OnceLock::new(),
154                source_storage: OnceLock::new(),
155            }),
156        })
157    }
158
159    pub fn diagnostics(&self) -> &[Diagnostic] {
160        &self.inner.diagnostics
161    }
162
163    /// Data-independent graph reads associated with authored consumers.
164    pub fn access_catalog(&self) -> &AccessCatalog {
165        &self.inner.access
166    }
167
168    pub fn source(&self) -> &Loaded {
169        &self.inner.source
170    }
171
172    /// Share the authored RDF document without copying its graph.
173    pub fn source_shared(&self) -> Arc<Loaded> {
174        Arc::clone(&self.inner.source)
175    }
176
177    pub(crate) fn source_storage(&self) -> Arc<SourceStorage> {
178        Arc::clone(self.inner.source_storage.get_or_init(|| {
179            SourceStorage::encode_with_demand(&self.inner.source.graph, &self.inner.access)
180        }))
181    }
182
183    pub(crate) fn parsed_queries(&self) -> Arc<ParsedQueries> {
184        Arc::clone(&self.inner.parsed_queries)
185    }
186
187    /// Mapping from authored statement IDs to normalized execution IDs.
188    pub fn statement_map(&self) -> &[usize] {
189        &self.inner.statement_map
190    }
191
192    /// Share the authored-to-normalized statement mapping.
193    pub fn statement_map_shared(&self) -> Arc<Vec<usize>> {
194        Arc::clone(&self.inner.statement_map)
195    }
196
197    /// The authored schema retained for source-oriented inspection.
198    pub fn authored_schema(&self) -> &Schema {
199        &self.inner.authored
200    }
201
202    /// Share the authored schema with a result or adapter.
203    pub fn authored_schema_shared(&self) -> Arc<Schema> {
204        Arc::clone(&self.inner.authored)
205    }
206
207    /// The canonical schema used by sessions.
208    pub fn normalized_schema(&self) -> &Schema {
209        &self.inner.normalized
210    }
211
212    /// Share the executable schema with a result or adapter.
213    pub fn normalized_schema_shared(&self) -> Arc<Schema> {
214        Arc::clone(&self.inner.normalized)
215    }
216
217    /// Build the physical validation plan only when explicitly requested.
218    pub fn physical_plan(&self) -> &PhysicalPlan {
219        self.inner
220            .physical
221            .get_or_init(|| plan(&self.inner.normalized))
222    }
223}
224
225// Compilation contains only owned RDF, IR, and immutable planning data.
226const _: fn() = || {
227    fn assert_send_sync<T: Send + Sync>() {}
228    assert_send_sync::<CompiledShapes>();
229};
230
231#[cfg(test)]
232mod tests {
233    use super::*;
234    use crate::{FindingOptions, SessionData, SessionOptions};
235
236    #[test]
237    fn source_encoding_is_lazy_and_shared_by_clones() {
238        let source =
239            shifty_parse::load_turtle(br#"@prefix ex: <http://ex/> . ex:a ex:p ex:b ."#, None)
240                .unwrap();
241        let compiled = CompiledShapes::compile(source).unwrap();
242        assert!(compiled.inner.source_storage.get().is_none());
243        crate::profile::enable();
244        let first = compiled.source_storage();
245        let second = compiled.clone().source_storage();
246        assert!(Arc::ptr_eq(&first, &second));
247        let storage = crate::profile::take().unwrap().storage().clone();
248        assert_eq!(storage.source_builds, 1);
249        assert_eq!(storage.source_rows, 1);
250    }
251
252    #[test]
253    fn canonical_query_templates_are_parsed_once_and_shared() {
254        let source = shifty_parse::load_turtle(
255            br#"
256            @prefix sh: <http://www.w3.org/ns/shacl#> .
257            @prefix ex: <http://ex/> .
258            ex:S a sh:NodeShape ; sh:targetNode ex:a ;
259                sh:sparql [ sh:select "SELECT $this WHERE { $this ex:p ?o }" ] .
260            "#,
261            None,
262        )
263        .unwrap();
264        let compiled = CompiledShapes::compile(source).unwrap();
265        assert_eq!(compiled.inner.parsed_queries.len(), 1);
266        assert!(Arc::ptr_eq(
267            &compiled.parsed_queries(),
268            &compiled.clone().parsed_queries()
269        ));
270    }
271
272    #[test]
273    fn sessions_share_compilation_and_plan_only_when_validation_uses_it() {
274        let source = shifty_parse::load_turtle(
275            br#"
276            @prefix sh: <http://www.w3.org/ns/shacl#> .
277            @prefix ex: <http://ex/> .
278            ex:S a sh:NodeShape ; sh:targetNode ex:a ;
279                sh:property [ sh:path ex:p ; sh:minCount 1 ] .
280            "#,
281            None,
282        )
283        .unwrap();
284        let compiled = CompiledShapes::compile(source).unwrap();
285        let clone = compiled.clone();
286        assert!(Arc::ptr_eq(&compiled.inner, &clone.inner));
287        assert!(compiled.inner.physical.get().is_none());
288
289        crate::profile::enable();
290        let session = compiled
291            .session(
292                SessionData::Embedded,
293                SessionOptions {
294                    inference: true,
295                    ..SessionOptions::default()
296                },
297            )
298            .unwrap();
299        assert_eq!(
300            crate::profile::take().unwrap().storage().dataset_builds,
301            0,
302            "a compilation with no executable rules needs no inference dataset",
303        );
304        assert!(
305            compiled.inner.source_storage.get().is_none(),
306            "source encoding should wait until a graph reader needs it",
307        );
308        assert!(compiled.inner.physical.get().is_none());
309        assert!(!session.has_prepared_dataset());
310        session.validate(&FindingOptions::default());
311        assert!(compiled.inner.source_storage.get().is_some());
312        assert!(compiled.inner.physical.get().is_some());
313        assert!(session.has_prepared_dataset());
314        assert!(std::ptr::eq(
315            compiled.physical_plan(),
316            clone.physical_plan()
317        ));
318
319        crate::profile::enable();
320        let second = clone
321            .session(SessionData::Embedded, SessionOptions::default())
322            .unwrap();
323        second.validate(&FindingOptions::default());
324        let storage = crate::profile::take().unwrap().storage().clone();
325        assert_eq!(
326            storage.source_builds, 0,
327            "source was encoded by the first session"
328        );
329        assert_eq!(storage.dataset_builds, 1);
330    }
331}
shifty_engine/compiled.rs

shifty_engine/
compiled.rs
