1#![allow(clippy::result_large_err)]
3
4pub use shape_value::AlignedVec;
15
16pub mod alerts;
17pub mod annotation_context;
18pub mod arrow_c;
19pub mod ast_extensions;
20pub mod binary_reader;
21pub mod blob_prefetch;
22pub mod blob_store;
23pub mod blob_wire_format;
24pub mod builtin_metadata;
25pub mod chart_detect;
26pub mod closure;
27pub mod code_search;
28pub mod columnar_aggregations;
29pub mod const_eval;
30pub mod content_builders;
31pub mod content_dispatch;
32pub mod content_methods;
33pub mod content_renderer;
34pub mod context;
35pub mod crypto;
36pub mod data;
37pub mod dependency_resolver;
38pub mod distributed_gc;
39pub mod doc_extract;
40pub mod engine;
41pub mod event_queue;
42pub mod execution_proof;
43pub mod extension_context;
44pub mod extensions;
45pub mod extensions_config;
46pub mod frontmatter;
47pub mod fuzzy;
48pub mod fuzzy_property;
49pub mod hashing;
50pub mod intrinsics;
51pub mod json_value;
52pub mod marshal;
53pub mod leakage;
54pub mod lookahead_guard;
55pub mod metadata;
56pub mod module_bindings;
57pub mod module_exports;
58pub mod typed_module_exports;
59pub mod module_loader;
60pub mod module_manifest;
61pub mod multi_table;
62pub mod multiple_testing;
63pub mod native_resolution;
64pub mod output_adapter;
65pub mod print_result;
66pub mod package_bundle;
67pub mod package_lock;
68pub mod pattern_library;
69pub mod plugins;
70pub mod progress;
71pub mod project;
72#[cfg(all(test, feature = "deep-tests"))]
73mod project_deep_tests;
74pub mod provider_registry;
75pub mod query_builder;
76pub mod query_executor;
77pub mod query_result;
78pub mod renderers;
79pub mod schema_cache;
80pub mod schema_inference;
81pub mod simd_comparisons;
82pub mod simd_forward_fill;
83pub mod simd_i64;
84pub mod simd_rolling;
85pub mod simd_statistics;
86pub mod snapshot;
87pub mod statistics;
92pub mod stdlib;
93pub mod stdlib_io;
94pub mod stdlib_metadata;
95pub mod stdlib_time;
96pub mod sync_bridge;
97pub mod time_window;
98pub mod timeframe_utils;
99pub mod type_mapping;
100pub mod type_methods;
101pub mod type_schema;
102pub mod type_system;
103pub mod visitor;
104pub mod window_manager;
105pub mod wire_conversion;
106
107pub use alerts::{Alert, AlertRouter, AlertSeverity, AlertSink};
109pub use context::{DataLoadMode, ExecutionContext as Context};
110pub use data::DataFrame;
111pub use data::OwnedDataRow as RowValue;
112pub use event_queue::{
113 EventQueue, MemoryEventQueue, QueuedEvent, SharedEventQueue, SuspensionState, WaitCondition,
114 create_event_queue,
115};
116pub use extensions::{
117 ExtensionCapability, ExtensionDataSource, ExtensionLoader, ExtensionOutputSink,
118 LoadedExtension, ParsedOutputField, ParsedOutputSchema, ParsedQueryParam, ParsedQuerySchema,
119};
120pub use extensions_config::{
121 ExtensionEntry as GlobalExtensionEntry, ExtensionsConfig as GlobalExtensionsConfig,
122 load_extensions_config, load_extensions_config_from,
123};
124pub use hashing::{HashDigest, combine_hashes, hash_bytes, hash_file, hash_string};
125pub use intrinsics::{IntrinsicFn, IntrinsicsRegistry};
126pub use leakage::{LeakageDetector, LeakageReport, LeakageSeverity, LeakageType, LeakageWarning};
127pub use module_bindings::ModuleBindingRegistry;
128pub use module_exports::{
129 FrameInfo, ModuleContext, ModuleExportRegistry, ModuleExports, ModuleFn, VmStateAccessor,
130};
131pub use multiple_testing::{MultipleTestingGuard, MultipleTestingStats, WarningLevel};
132pub use progress::{
133 LoadPhase, ProgressEvent, ProgressGranularity, ProgressHandle, ProgressRegistry,
134};
135pub use query_result::{AlertResult, QueryResult, QueryType};
136pub use sync_bridge::{
137 SyncDataProvider, block_on_shared, get_runtime_handle, initialize_shared_runtime,
138};
139pub use type_schema::{
140 FieldDef, FieldType, SchemaId, TypeSchema, TypeSchemaBuilder, TypeSchemaRegistry,
141};
142pub use wire_conversion::{
143 slot_extract_content, slot_to_envelope, slot_to_wire, wire_to_slot,
144};
145
146use self::type_methods::TypeMethodRegistry;
147pub use error::{Result, ShapeError, SourceLocation};
148use shape_ast::ast::Program;
149pub use shape_ast::error;
150use shape_wire::WireValue;
151use std::any::Any;
152use std::collections::HashMap;
153use std::path::PathBuf;
154use std::sync::Arc;
155use std::sync::OnceLock;
156use std::sync::RwLock;
157use std::time::Duration;
158
159pub struct Runtime {
161 module_loader: module_loader::ModuleLoader,
163 persistent_context: Option<context::ExecutionContext>,
165 type_method_registry: Arc<TypeMethodRegistry>,
167 annotation_registry: Arc<RwLock<annotation_context::AnnotationRegistry>>,
169 module_binding_registry: Arc<RwLock<module_bindings::ModuleBindingRegistry>>,
171 pub schema_registry: Arc<type_schema::TypeSchemaRegistry>,
181 debug_mode: bool,
183 execution_timeout: Option<Duration>,
185 memory_limit: Option<usize>,
187 last_runtime_error: Option<WireValue>,
192 blob_store: Option<Arc<dyn crate::blob_store::BlobStore>>,
194 keychain: Option<crate::crypto::keychain::Keychain>,
196 extension_module_schemas: Arc<extension_context::ExtensionModuleSchemaCache>,
204 core_stdlib_cache: OnceLock<Arc<dyn Any + Send + Sync>>,
213}
214
215impl Default for Runtime {
216 fn default() -> Self {
217 Self::new()
218 }
219}
220
221impl Runtime {
222 pub fn new() -> Self {
224 Self::new_internal(true)
225 }
226
227 pub(crate) fn new_without_stdlib() -> Self {
228 Self::new_internal(false)
229 }
230
231 fn new_internal(_load_stdlib: bool) -> Self {
232 let module_loader = module_loader::ModuleLoader::new();
233 let module_binding_registry =
234 Arc::new(RwLock::new(module_bindings::ModuleBindingRegistry::new()));
235
236 Self {
237 module_loader,
238 persistent_context: None,
239 type_method_registry: Arc::new(TypeMethodRegistry::new()),
240 annotation_registry: Arc::new(RwLock::new(
241 annotation_context::AnnotationRegistry::new(),
242 )),
243 module_binding_registry,
244 schema_registry: Arc::new(type_schema::TypeSchemaRegistry::new_with_stdlib()),
245 debug_mode: false,
246 execution_timeout: None,
247 memory_limit: None,
248 last_runtime_error: None,
249 blob_store: None,
250 keychain: None,
251 extension_module_schemas: Arc::new(
252 extension_context::ExtensionModuleSchemaCache::new(),
253 ),
254 core_stdlib_cache: OnceLock::new(),
255 }
256 }
257
258 pub fn extension_module_schemas(
260 &self,
261 ) -> &Arc<extension_context::ExtensionModuleSchemaCache> {
262 &self.extension_module_schemas
263 }
264
265 pub fn annotation_registry(&self) -> Arc<RwLock<annotation_context::AnnotationRegistry>> {
266 self.annotation_registry.clone()
267 }
268
269 pub fn enable_persistent_context(&mut self, data: &DataFrame) {
270 self.persistent_context = Some(context::ExecutionContext::new_with_registry(
271 data,
272 self.type_method_registry.clone(),
273 ));
274 }
275
276 pub fn enable_persistent_context_without_data(&mut self) {
277 self.persistent_context = Some(context::ExecutionContext::new_empty_with_registry(
278 self.type_method_registry.clone(),
279 ));
280 }
281
282 pub fn set_persistent_context(&mut self, ctx: context::ExecutionContext) {
283 self.persistent_context = Some(ctx);
284 }
285
286 pub fn persistent_context(&self) -> Option<&context::ExecutionContext> {
287 self.persistent_context.as_ref()
288 }
289
290 pub fn persistent_context_mut(&mut self) -> Option<&mut context::ExecutionContext> {
291 self.persistent_context.as_mut()
292 }
293
294 pub fn set_last_runtime_error(&mut self, payload: Option<WireValue>) {
296 self.last_runtime_error = payload;
297 }
298
299 pub fn clear_last_runtime_error(&mut self) {
301 self.last_runtime_error = None;
302 }
303
304 pub fn last_runtime_error(&self) -> Option<&WireValue> {
306 self.last_runtime_error.as_ref()
307 }
308
309 pub fn take_last_runtime_error(&mut self) -> Option<WireValue> {
311 self.last_runtime_error.take()
312 }
313
314 pub fn type_method_registry(&self) -> &Arc<TypeMethodRegistry> {
315 &self.type_method_registry
316 }
317
318 pub fn schema_registry(&self) -> &type_schema::TypeSchemaRegistry {
325 &self.schema_registry
326 }
327
328 pub fn schema_registry_mut(&mut self) -> &mut type_schema::TypeSchemaRegistry {
336 Arc::make_mut(&mut self.schema_registry)
337 }
338
339 pub fn schema_registry_arc(&self) -> Arc<type_schema::TypeSchemaRegistry> {
342 self.schema_registry.clone()
343 }
344
345 pub fn enter_schema_scope(&self) -> type_schema::SyncRegistryScope {
354 type_schema::SyncRegistryScope::enter(self.schema_registry_arc())
355 }
356
357 pub fn module_binding_registry(&self) -> Arc<RwLock<module_bindings::ModuleBindingRegistry>> {
359 self.module_binding_registry.clone()
360 }
361
362 pub fn get_or_init_core_stdlib_cache<T, F>(&self, init: F) -> Arc<T>
372 where
373 T: Any + Send + Sync + 'static,
374 F: FnOnce() -> Arc<T>,
375 {
376 let entry = self
377 .core_stdlib_cache
378 .get_or_init(|| init() as Arc<dyn Any + Send + Sync>);
379 entry
380 .clone()
381 .downcast::<T>()
382 .expect("core_stdlib_cache type mismatch: already initialized with a different type")
383 }
384
385 pub fn add_module_path(&mut self, path: PathBuf) {
390 self.module_loader.add_module_path(path);
391 }
392
393 pub fn set_keychain(&mut self, keychain: crate::crypto::keychain::Keychain) {
398 self.keychain = Some(keychain.clone());
399 self.module_loader.set_keychain(keychain);
400 }
401
402 pub fn set_blob_store(&mut self, store: Arc<dyn crate::blob_store::BlobStore>) {
407 self.blob_store = Some(store.clone());
408 self.module_loader.set_blob_store(store);
409 }
410
411 pub fn set_project_root(&mut self, root: &std::path::Path, extra_paths: &[PathBuf]) {
413 self.module_loader.set_project_root(root, extra_paths);
414 }
415
416 pub fn set_dependency_paths(
418 &mut self,
419 deps: std::collections::HashMap<String, std::path::PathBuf>,
420 ) {
421 self.module_loader.set_dependency_paths(deps);
422 }
423
424 pub fn get_dependency_paths(&self) -> &std::collections::HashMap<String, std::path::PathBuf> {
426 self.module_loader.get_dependency_paths()
427 }
428
429 pub fn register_extension_module_artifacts(
431 &mut self,
432 modules: &[crate::extensions::ParsedModuleSchema],
433 ) {
434 for module in modules {
435 for artifact in &module.artifacts {
436 let code = match (&artifact.source, &artifact.compiled) {
437 (Some(source), Some(compiled)) => module_loader::ModuleCode::Both {
438 source: Arc::from(source.as_str()),
439 compiled: Arc::from(compiled.clone()),
440 },
441 (Some(source), None) => {
442 module_loader::ModuleCode::Source(Arc::from(source.as_str()))
443 }
444 (None, Some(compiled)) => {
445 module_loader::ModuleCode::Compiled(Arc::from(compiled.clone()))
446 }
447 (None, None) => continue,
448 };
449 self.module_loader
450 .register_extension_module(artifact.module_path.clone(), code);
451 }
452 }
453 }
454
455 pub fn configured_module_loader(&self) -> module_loader::ModuleLoader {
460 let mut loader = self.module_loader.clone_without_cache();
461 if let Some(ref store) = self.blob_store {
462 loader.set_blob_store(store.clone());
463 }
464 if let Some(ref kc) = self.keychain {
465 loader.set_keychain(kc.clone());
466 }
467 loader
468 }
469
470 pub fn load_core_stdlib_into_context(&mut self, data: &DataFrame) -> Result<()> {
474 let module_paths = self.module_loader.list_core_stdlib_module_imports()?;
475
476 for module_path in module_paths {
477 if module_path == "std::core::prelude" {
481 continue;
482 }
483
484 let resolved = self.module_loader.resolve_module_path(&module_path).ok();
485 let context_dir = resolved
486 .as_ref()
487 .and_then(|path| path.parent().map(|p| p.to_path_buf()));
488 let module = self.module_loader.load_module(&module_path)?;
489 self.load_program_with_context(&module.ast, data, context_dir.as_ref())?;
490 }
491
492 Ok(())
493 }
494
495 pub fn load_program(&mut self, program: &Program, data: &DataFrame) -> Result<()> {
496 self.load_program_with_context(program, data, None)
497 }
498
499 pub(crate) fn load_program_with_context(
500 &mut self,
501 program: &Program,
502 data: &DataFrame,
503 context_dir: Option<&PathBuf>,
504 ) -> Result<()> {
505 let mut persistent_ctx = self.persistent_context.take();
506
507 let result = if let Some(ref mut ctx) = persistent_ctx {
508 if data.row_count() > 0 {
509 ctx.update_data(data);
510 }
511 self.process_program_items(program, ctx, context_dir)
512 } else {
513 let mut ctx = context::ExecutionContext::new_with_registry(
514 data,
515 self.type_method_registry.clone(),
516 );
517 self.process_program_items(program, &mut ctx, context_dir)
518 };
519
520 self.persistent_context = persistent_ctx;
521 result
522 }
523
524 fn process_program_items(
525 &mut self,
526 program: &Program,
527 ctx: &mut context::ExecutionContext,
528 context_dir: Option<&PathBuf>,
529 ) -> Result<()> {
530 for item in &program.items {
531 match item {
532 shape_ast::ast::Item::Import(import, _) => {
533 let module = self
534 .module_loader
535 .load_module_with_context(&import.from, context_dir)?;
536
537 match &import.items {
538 shape_ast::ast::ImportItems::Named(imports) => {
539 for import_spec in imports {
540 if let Some(export) = module.exports.get(&import_spec.name) {
541 if import_spec.is_annotation {
542 continue;
543 }
544 let var_name =
545 import_spec.alias.as_ref().unwrap_or(&import_spec.name);
546 match export {
547 module_loader::Export::Function(_) => {
548 }
550 module_loader::Export::Value(value) => {
551 if ctx.get_variable(var_name)?.is_none() {
552 ctx.set_variable(var_name, value.clone())?;
553 }
554 self.module_binding_registry
555 .write()
556 .unwrap()
557 .register_const(var_name, value.clone())?;
558 }
559 _ => {}
560 }
561 } else {
562 return Err(ShapeError::ModuleError {
563 message: format!(
564 "Export '{}' not found in module '{}'",
565 import_spec.name, import.from
566 ),
567 module_path: Some(import.from.clone().into()),
568 });
569 }
570 }
571 }
572 shape_ast::ast::ImportItems::Namespace { .. } => {
573 }
575 }
576 }
577 shape_ast::ast::Item::Export(export, _) => {
578 match &export.item {
579 shape_ast::ast::ExportItem::Function(_) => {
580 }
582 shape_ast::ast::ExportItem::BuiltinFunction(_)
583 | shape_ast::ast::ExportItem::BuiltinType(_)
584 | shape_ast::ast::ExportItem::Annotation(_) => {}
585 shape_ast::ast::ExportItem::Named(specs) => {
586 if export.source_decl.is_none() {
594 for spec in specs {
595 if let Ok(value) = ctx.get_variable(&spec.name) {
596 if value.is_none() {
597 return Err(ShapeError::RuntimeError {
598 message: format!(
599 "Cannot export undefined variable '{}'",
600 spec.name
601 ),
602 location: None,
603 });
604 }
605 }
606 }
607 }
608 }
609 shape_ast::ast::ExportItem::TypeAlias(alias_def) => {
610 let overrides = HashMap::new();
611 if let Some(ref overrides_ast) = alias_def.meta_param_overrides {
612 for (_key, _expr) in overrides_ast {
613 }
615 }
616
617 let base_type = match &alias_def.type_annotation {
618 shape_ast::ast::TypeAnnotation::Basic(n) => n.clone(),
619 shape_ast::ast::TypeAnnotation::Reference(n) => n.to_string(),
620 _ => "any".to_string(),
621 };
622
623 ctx.register_type_alias(&alias_def.name, &base_type, Some(overrides));
624 }
625 shape_ast::ast::ExportItem::Enum(enum_def) => {
626 ctx.register_enum(enum_def.clone());
627 }
628 shape_ast::ast::ExportItem::Struct(struct_def) => {
629 ctx.register_struct_type(struct_def.clone());
630 }
631 shape_ast::ast::ExportItem::Trait(_) => {
632 }
634 shape_ast::ast::ExportItem::ForeignFunction(_) => {
635 }
637 }
638 }
639 shape_ast::ast::Item::Function(_function, _) => {
640 }
642 shape_ast::ast::Item::TypeAlias(alias_def, _) => {
643 let overrides = HashMap::new();
644 if let Some(ref overrides_ast) = alias_def.meta_param_overrides {
645 for (_key, _expr) in overrides_ast {
646 }
648 }
649
650 let base_type = match &alias_def.type_annotation {
651 shape_ast::ast::TypeAnnotation::Basic(n) => n.clone(),
652 shape_ast::ast::TypeAnnotation::Reference(n) => n.to_string(),
653 _ => "any".to_string(),
654 };
655
656 ctx.register_type_alias(&alias_def.name, &base_type, Some(overrides));
657 }
658 shape_ast::ast::Item::Trait(_, _) => {}
659 shape_ast::ast::Item::Impl(_, _) => {}
660 shape_ast::ast::Item::Enum(enum_def, _) => {
661 ctx.register_enum(enum_def.clone());
662 }
663 shape_ast::ast::Item::StructType(struct_def, _) => {
664 ctx.register_struct_type(struct_def.clone());
665 }
666 shape_ast::ast::Item::Extend(extend_stmt, _) => {
667 let registry = ctx.type_method_registry();
668 for method in &extend_stmt.methods {
669 registry.register_method(&extend_stmt.type_name, method.clone());
670 }
671 }
672 shape_ast::ast::Item::AnnotationDef(ann_def, _) => {
673 self.annotation_registry
674 .write()
675 .unwrap()
676 .register(ann_def.clone());
677 }
678 _ => {}
679 }
680 }
681 Ok(())
682 }
683
684 pub fn execute_query(
693 &mut self,
694 query: &shape_ast::ast::Item,
695 data: &DataFrame,
696 ) -> Result<QueryResult> {
697 let _ = (query, data);
698 let id = self
699 .persistent_context
700 .as_ref()
701 .and_then(|ctx| ctx.get_current_id().ok())
702 .unwrap_or_default();
703 let timeframe = self
704 .persistent_context
705 .as_ref()
706 .and_then(|ctx| ctx.get_current_timeframe().ok())
707 .map(|t| t.to_string())
708 .unwrap_or_default();
709 Ok(QueryResult::new(QueryType::Value, id, timeframe))
710 }
711
712 pub fn format_value(
716 &mut self,
717 _value: WireValue,
718 type_name: &str,
719 format_name: Option<&str>,
720 _param_overrides: std::collections::HashMap<String, WireValue>,
721 ) -> Result<String> {
722 if let Some(name) = format_name {
723 Ok(format!("<formatted {} as {}>", type_name, name))
724 } else {
725 Ok(format!("<formatted {}>", type_name))
726 }
727 }
728
729 pub fn set_debug_mode(&mut self, enabled: bool) {
734 self.debug_mode = enabled;
735 if enabled {
736 tracing::debug!("Shape runtime debug mode enabled");
737 }
738 }
739
740 pub fn debug_mode(&self) -> bool {
742 self.debug_mode
743 }
744
745 pub fn set_execution_timeout(&mut self, timeout: Duration) {
750 self.execution_timeout = Some(timeout);
751 }
752
753 pub fn execution_timeout(&self) -> Option<Duration> {
755 self.execution_timeout
756 }
757
758 pub fn set_memory_limit(&mut self, limit: usize) {
763 self.memory_limit = Some(limit);
764 }
765
766 pub fn memory_limit(&self) -> Option<usize> {
768 self.memory_limit
769 }
770}
771
772#[cfg(test)]
773mod runtime_tests {
774 use super::*;
775 use crate::type_schema::FieldType;
776
777 #[test]
781 fn b1_2_runtime_schema_registries_are_independent() {
782 let mut r1 = Runtime::new();
783 let mut r2 = Runtime::new();
784
785 assert!(r1.schema_registry().has_type("Row"));
787 assert!(r2.schema_registry().has_type("Row"));
788 assert!(r1.schema_registry().has_type("Option"));
789 assert!(r2.schema_registry().has_type("Result"));
790
791 let id_a = r1
792 .schema_registry_mut()
793 .register_type_scoped("UserA", vec![("x".to_string(), FieldType::F64)]);
794 let id_b = r2
795 .schema_registry_mut()
796 .register_type_scoped("UserA", vec![("x".to_string(), FieldType::F64)]);
797
798 assert_eq!(r1.schema_registry().get("UserA").unwrap().id, id_a);
802 assert_eq!(r2.schema_registry().get("UserA").unwrap().id, id_b);
803 assert_eq!(id_a, id_b);
804 }
805}