1#![allow(clippy::result_large_err)]
3
4pub use shape_value::AlignedVec;
15
16pub mod alerts;
17pub mod annotation_context;
18pub mod arrow_c;
19pub mod ast_extensions;
20pub mod binary_reader;
21pub mod blob_prefetch;
22pub mod blob_store;
23pub mod blob_wire_format;
24pub mod builtin_metadata;
25pub mod closure;
26pub mod code_search;
27pub mod columnar_aggregations;
28pub mod const_eval;
29pub mod content_builders;
30pub mod content_dispatch;
31pub mod content_methods;
32pub mod content_renderer;
33pub mod context;
34pub mod crypto;
35pub mod data;
36pub mod dependency_resolver;
37pub mod distributed_gc;
38pub mod engine;
39pub mod event_queue;
40pub mod execution_proof;
41pub mod extension_context;
42pub mod extensions;
43pub mod extensions_config;
44pub mod frontmatter;
45pub mod fuzzy;
46pub mod fuzzy_property;
47pub mod hashing;
48pub mod intrinsics;
49pub mod join_executor;
50pub mod leakage;
51pub mod lookahead_guard;
52pub mod metadata;
53pub mod module_bindings;
54pub mod module_exports;
55pub mod module_loader;
56pub mod module_manifest;
57pub mod multi_table;
58pub mod multiple_testing;
59pub mod output_adapter;
60pub mod package_bundle;
61pub mod package_lock;
62pub mod pattern_library;
63pub mod pattern_state_machine;
64pub mod plugins;
65pub mod progress;
66pub mod project;
67#[cfg(test)]
68mod project_deep_tests;
69pub mod provider_registry;
70pub mod query_builder;
71pub mod query_executor;
72pub mod query_result;
73pub mod renderers;
74pub mod schema_cache;
75pub mod schema_inference;
76pub mod semantic;
77pub mod simd_comparisons;
78pub mod simd_forward_fill;
79pub mod simd_i64;
80pub mod simd_rolling;
81pub mod simd_statistics;
82pub mod simulation;
83pub mod snapshot;
84pub mod state_diff;
85pub mod statistics;
86pub mod stdlib;
87pub mod stdlib_io;
88pub mod stdlib_metadata;
89pub mod stdlib_time;
90pub mod stream_executor;
91pub mod sync_bridge;
92pub mod time_window;
93pub mod timeframe_utils;
94pub mod type_mapping;
95pub mod type_methods;
96pub mod type_schema;
97pub mod type_system;
98pub mod visitor;
99pub mod window_executor;
100pub mod window_manager;
101pub mod wire_conversion;
102
103pub use alerts::{Alert, AlertRouter, AlertSeverity, AlertSink};
105pub use context::{DataLoadMode, ExecutionContext as Context};
106pub use data::DataFrame;
107pub use data::OwnedDataRow as RowValue;
108pub use event_queue::{
109 EventQueue, MemoryEventQueue, QueuedEvent, SharedEventQueue, SuspensionState, WaitCondition,
110 create_event_queue,
111};
112pub use extensions::{
113 ExtensionCapability, ExtensionDataSource, ExtensionLoader, ExtensionOutputSink,
114 LoadedExtension, ParsedOutputField, ParsedOutputSchema, ParsedQueryParam, ParsedQuerySchema,
115};
116pub use extensions_config::{
117 ExtensionEntry as GlobalExtensionEntry, ExtensionsConfig as GlobalExtensionsConfig,
118 load_extensions_config, load_extensions_config_from,
119};
120pub use hashing::{HashDigest, combine_hashes, hash_bytes, hash_file, hash_string};
121pub use intrinsics::{IntrinsicFn, IntrinsicsRegistry};
122pub use join_executor::JoinExecutor;
123pub use leakage::{LeakageDetector, LeakageReport, LeakageSeverity, LeakageType, LeakageWarning};
124pub use module_bindings::ModuleBindingRegistry;
125pub use module_exports::{
126 FrameInfo, ModuleContext, ModuleExportRegistry, ModuleExports, ModuleFn, VmStateAccessor,
127};
128pub use multiple_testing::{MultipleTestingGuard, MultipleTestingStats, WarningLevel};
129pub use progress::{
130 LoadPhase, ProgressEvent, ProgressGranularity, ProgressHandle, ProgressRegistry,
131};
132pub use query_result::{AlertResult, QueryResult, QueryType};
133use shape_value::ValueWord;
134pub use shape_value::ValueWord as Value;
135pub use stream_executor::{StreamEvent, StreamExecutor, StreamState};
136pub use sync_bridge::{SyncDataProvider, block_on_shared, initialize_shared_runtime};
137pub use type_schema::{
138 FieldDef, FieldType, SchemaId, TypeSchema, TypeSchemaBuilder, TypeSchemaRegistry,
139};
140pub use window_executor::WindowExecutor;
141pub use wire_conversion::{
142 nb_extract_typed_value, nb_to_envelope, nb_to_wire, nb_typed_value_to_envelope, wire_to_nb,
143};
144
145use self::type_methods::TypeMethodRegistry;
146pub use error::{Result, ShapeError, SourceLocation};
147use shape_ast::ast::{Program, Query};
148pub use shape_ast::error;
149use shape_wire::WireValue;
150use std::collections::HashMap;
151use std::path::PathBuf;
152use std::sync::Arc;
153use std::sync::RwLock;
154use std::time::Duration;
155
156pub struct Runtime {
158 module_loader: module_loader::ModuleLoader,
160 persistent_context: Option<context::ExecutionContext>,
162 type_method_registry: Arc<TypeMethodRegistry>,
164 annotation_registry: Arc<RwLock<annotation_context::AnnotationRegistry>>,
166 module_binding_registry: Arc<RwLock<module_bindings::ModuleBindingRegistry>>,
168 debug_mode: bool,
170 execution_timeout: Option<Duration>,
172 memory_limit: Option<usize>,
174 last_runtime_error: Option<WireValue>,
179 blob_store: Option<Arc<dyn crate::blob_store::BlobStore>>,
181 keychain: Option<crate::crypto::keychain::Keychain>,
183}
184
185impl Default for Runtime {
186 fn default() -> Self {
187 Self::new()
188 }
189}
190
191impl Runtime {
192 pub fn new() -> Self {
194 Self::new_internal(true)
195 }
196
197 pub(crate) fn new_without_stdlib() -> Self {
198 Self::new_internal(false)
199 }
200
201 fn new_internal(_load_stdlib: bool) -> Self {
202 let module_loader = module_loader::ModuleLoader::new();
203 let module_binding_registry =
204 Arc::new(RwLock::new(module_bindings::ModuleBindingRegistry::new()));
205
206 Self {
207 module_loader,
208 persistent_context: None,
209 type_method_registry: Arc::new(TypeMethodRegistry::new()),
210 annotation_registry: Arc::new(RwLock::new(
211 annotation_context::AnnotationRegistry::new(),
212 )),
213 module_binding_registry,
214 debug_mode: false,
215 execution_timeout: None,
216 memory_limit: None,
217 last_runtime_error: None,
218 blob_store: None,
219 keychain: None,
220 }
221 }
222
223 pub fn annotation_registry(&self) -> Arc<RwLock<annotation_context::AnnotationRegistry>> {
224 self.annotation_registry.clone()
225 }
226
227 pub fn enable_persistent_context(&mut self, data: &DataFrame) {
228 self.persistent_context = Some(context::ExecutionContext::new_with_registry(
229 data,
230 self.type_method_registry.clone(),
231 ));
232 }
233
234 pub fn enable_persistent_context_without_data(&mut self) {
235 self.persistent_context = Some(context::ExecutionContext::new_empty_with_registry(
236 self.type_method_registry.clone(),
237 ));
238 }
239
240 pub fn set_persistent_context(&mut self, ctx: context::ExecutionContext) {
241 self.persistent_context = Some(ctx);
242 }
243
244 pub fn persistent_context(&self) -> Option<&context::ExecutionContext> {
245 self.persistent_context.as_ref()
246 }
247
248 pub fn persistent_context_mut(&mut self) -> Option<&mut context::ExecutionContext> {
249 self.persistent_context.as_mut()
250 }
251
252 pub fn set_last_runtime_error(&mut self, payload: Option<WireValue>) {
254 self.last_runtime_error = payload;
255 }
256
257 pub fn clear_last_runtime_error(&mut self) {
259 self.last_runtime_error = None;
260 }
261
262 pub fn last_runtime_error(&self) -> Option<&WireValue> {
264 self.last_runtime_error.as_ref()
265 }
266
267 pub fn take_last_runtime_error(&mut self) -> Option<WireValue> {
269 self.last_runtime_error.take()
270 }
271
272 pub fn type_method_registry(&self) -> &Arc<TypeMethodRegistry> {
273 &self.type_method_registry
274 }
275
276 pub fn module_binding_registry(&self) -> Arc<RwLock<module_bindings::ModuleBindingRegistry>> {
278 self.module_binding_registry.clone()
279 }
280
281 pub fn add_module_path(&mut self, path: PathBuf) {
286 self.module_loader.add_module_path(path);
287 }
288
289 pub fn set_keychain(&mut self, keychain: crate::crypto::keychain::Keychain) {
294 self.keychain = Some(keychain.clone());
295 self.module_loader.set_keychain(keychain);
296 }
297
298 pub fn set_blob_store(&mut self, store: Arc<dyn crate::blob_store::BlobStore>) {
303 self.blob_store = Some(store.clone());
304 self.module_loader.set_blob_store(store);
305 }
306
307 pub fn set_project_root(&mut self, root: &std::path::Path, extra_paths: &[PathBuf]) {
309 self.module_loader.set_project_root(root, extra_paths);
310 }
311
312 pub fn set_dependency_paths(
314 &mut self,
315 deps: std::collections::HashMap<String, std::path::PathBuf>,
316 ) {
317 self.module_loader.set_dependency_paths(deps);
318 }
319
320 pub fn get_dependency_paths(&self) -> &std::collections::HashMap<String, std::path::PathBuf> {
322 self.module_loader.get_dependency_paths()
323 }
324
325 pub fn register_extension_module_artifacts(
327 &mut self,
328 modules: &[crate::extensions::ParsedModuleSchema],
329 ) {
330 for module in modules {
331 for artifact in &module.artifacts {
332 let code = match (&artifact.source, &artifact.compiled) {
333 (Some(source), Some(compiled)) => module_loader::ModuleCode::Both {
334 source: Arc::from(source.as_str()),
335 compiled: Arc::from(compiled.clone()),
336 },
337 (Some(source), None) => {
338 module_loader::ModuleCode::Source(Arc::from(source.as_str()))
339 }
340 (None, Some(compiled)) => {
341 module_loader::ModuleCode::Compiled(Arc::from(compiled.clone()))
342 }
343 (None, None) => continue,
344 };
345 self.module_loader
346 .register_extension_module(artifact.module_path.clone(), code);
347 }
348 }
349 }
350
351 pub fn configured_module_loader(&self) -> module_loader::ModuleLoader {
356 let mut loader = self.module_loader.clone_without_cache();
357 if let Some(ref store) = self.blob_store {
358 loader.set_blob_store(store.clone());
359 }
360 if let Some(ref kc) = self.keychain {
361 loader.set_keychain(kc.clone());
362 }
363 loader
364 }
365
366 pub fn load_core_stdlib_into_context(&mut self, data: &DataFrame) -> Result<()> {
370 let module_paths = self.module_loader.list_core_stdlib_module_imports()?;
371
372 for module_path in module_paths {
373 if module_path == "std::core::prelude" {
377 continue;
378 }
379
380 let resolved = self.module_loader.resolve_module_path(&module_path).ok();
381 let context_dir = resolved
382 .as_ref()
383 .and_then(|path| path.parent().map(|p| p.to_path_buf()));
384 let module = self.module_loader.load_module(&module_path)?;
385 self.load_program_with_context(&module.ast, data, context_dir.as_ref())?;
386 }
387
388 Ok(())
389 }
390
391 pub fn load_program(&mut self, program: &Program, data: &DataFrame) -> Result<()> {
392 self.load_program_with_context(program, data, None)
393 }
394
395 pub(crate) fn load_program_with_context(
396 &mut self,
397 program: &Program,
398 data: &DataFrame,
399 context_dir: Option<&PathBuf>,
400 ) -> Result<()> {
401 let mut persistent_ctx = self.persistent_context.take();
402
403 let result = if let Some(ref mut ctx) = persistent_ctx {
404 if data.row_count() > 0 {
405 ctx.update_data(data);
406 }
407 self.process_program_items(program, ctx, context_dir)
408 } else {
409 let mut ctx = context::ExecutionContext::new_with_registry(
410 data,
411 self.type_method_registry.clone(),
412 );
413 self.process_program_items(program, &mut ctx, context_dir)
414 };
415
416 self.persistent_context = persistent_ctx;
417 result
418 }
419
420 fn process_program_items(
421 &mut self,
422 program: &Program,
423 ctx: &mut context::ExecutionContext,
424 context_dir: Option<&PathBuf>,
425 ) -> Result<()> {
426 for item in &program.items {
427 match item {
428 shape_ast::ast::Item::Import(import, _) => {
429 let module = self
430 .module_loader
431 .load_module_with_context(&import.from, context_dir)?;
432
433 match &import.items {
434 shape_ast::ast::ImportItems::Named(imports) => {
435 for import_spec in imports {
436 if let Some(export) = module.exports.get(&import_spec.name) {
437 let var_name =
438 import_spec.alias.as_ref().unwrap_or(&import_spec.name);
439 match export {
440 module_loader::Export::Function(_) => {
441 }
443 module_loader::Export::Value(value) => {
444 if ctx.get_variable(var_name)?.is_none() {
445 ctx.set_variable(var_name, value.clone())?;
446 }
447 self.module_binding_registry
448 .write()
449 .unwrap()
450 .register_const(var_name, value.clone())?;
451 }
452 _ => {}
453 }
454 } else {
455 return Err(ShapeError::ModuleError {
456 message: format!(
457 "Export '{}' not found in module '{}'",
458 import_spec.name, import.from
459 ),
460 module_path: Some(import.from.clone().into()),
461 });
462 }
463 }
464 }
465 shape_ast::ast::ImportItems::Namespace { .. } => {
466 }
468 }
469 }
470 shape_ast::ast::Item::Export(export, _) => {
471 match &export.item {
472 shape_ast::ast::ExportItem::Function(_) => {
473 }
475 shape_ast::ast::ExportItem::Named(specs) => {
476 for spec in specs {
477 if let Ok(value) = ctx.get_variable(&spec.name) {
478 if value.is_none() {
479 return Err(ShapeError::RuntimeError {
480 message: format!(
481 "Cannot export undefined variable '{}'",
482 spec.name
483 ),
484 location: None,
485 });
486 }
487 }
488 }
489 }
490 shape_ast::ast::ExportItem::TypeAlias(alias_def) => {
491 let overrides = HashMap::new();
492 if let Some(ref overrides_ast) = alias_def.meta_param_overrides {
493 for (_key, _expr) in overrides_ast {
494 }
496 }
497
498 let base_type = match &alias_def.type_annotation {
499 shape_ast::ast::TypeAnnotation::Basic(n)
500 | shape_ast::ast::TypeAnnotation::Reference(n) => n.clone(),
501 _ => "any".to_string(),
502 };
503
504 ctx.register_type_alias(&alias_def.name, &base_type, Some(overrides));
505 }
506 shape_ast::ast::ExportItem::Enum(_)
507 | shape_ast::ast::ExportItem::Struct(_)
508 | shape_ast::ast::ExportItem::Interface(_)
509 | shape_ast::ast::ExportItem::Trait(_) => {
510 }
512 shape_ast::ast::ExportItem::ForeignFunction(_) => {
513 }
515 }
516 }
517 shape_ast::ast::Item::Function(_function, _) => {
518 }
520 shape_ast::ast::Item::TypeAlias(alias_def, _) => {
521 let overrides = HashMap::new();
522 if let Some(ref overrides_ast) = alias_def.meta_param_overrides {
523 for (_key, _expr) in overrides_ast {
524 }
526 }
527
528 let base_type = match &alias_def.type_annotation {
529 shape_ast::ast::TypeAnnotation::Basic(n)
530 | shape_ast::ast::TypeAnnotation::Reference(n) => n.clone(),
531 _ => "any".to_string(),
532 };
533
534 ctx.register_type_alias(&alias_def.name, &base_type, Some(overrides));
535 }
536 shape_ast::ast::Item::Interface(_, _) => {}
537 shape_ast::ast::Item::Trait(_, _) => {}
538 shape_ast::ast::Item::Impl(_, _) => {}
539 shape_ast::ast::Item::Enum(enum_def, _) => {
540 ctx.register_enum(enum_def.clone());
541 }
542 shape_ast::ast::Item::Extend(extend_stmt, _) => {
543 let registry = ctx.type_method_registry();
544 for method in &extend_stmt.methods {
545 registry.register_method(&extend_stmt.type_name, method.clone());
546 }
547 }
548 shape_ast::ast::Item::AnnotationDef(ann_def, _) => {
549 self.annotation_registry
550 .write()
551 .unwrap()
552 .register(ann_def.clone());
553 }
554 _ => {}
555 }
556 }
557 Ok(())
558 }
559
560 pub fn execute_query(
561 &mut self,
562 query: &shape_ast::ast::Item,
563 data: &DataFrame,
564 ) -> Result<QueryResult> {
565 let mut persistent_ctx = self.persistent_context.take();
566 let result = if let Some(ref mut ctx) = persistent_ctx {
567 ctx.update_data(data);
568 self.execute_query_with_context(query, ctx)
569 } else {
570 let mut ctx = context::ExecutionContext::new_with_registry(
571 data,
572 self.type_method_registry.clone(),
573 );
574 self.execute_query_with_context(query, &mut ctx)
575 };
576 self.persistent_context = persistent_ctx;
577 result
578 }
579
580 fn execute_query_with_context(
581 &mut self,
582 query: &shape_ast::ast::Item,
583 ctx: &mut context::ExecutionContext,
584 ) -> Result<QueryResult> {
585 let id = ctx.get_current_id().unwrap_or_default();
586 let timeframe = ctx
587 .get_current_timeframe()
588 .map(|t| t.to_string())
589 .unwrap_or_default();
590
591 match query {
592 shape_ast::ast::Item::Query(q, _) => match q {
593 Query::Backtest(_) => Err(ShapeError::RuntimeError {
594 message: "Backtesting not supported in core runtime.".to_string(),
595 location: None,
596 }),
597 Query::Alert(_alert_query) => {
598 let alert_id = format!("alert_{}", chrono::Utc::now().timestamp_micros());
599 Ok(
600 QueryResult::new(QueryType::Alert, id, timeframe).with_alert(AlertResult {
601 id: alert_id,
602 active: false,
603 message: "Alert triggered".to_string(),
604 level: "info".to_string(),
605 timestamp: chrono::Utc::now(),
606 }),
607 )
608 }
609 Query::With(with_query) => {
610 for cte in &with_query.ctes {
611 let cte_result = self.execute_query_with_context(
612 &shape_ast::ast::Item::Query(
613 (*cte.query).clone(),
614 shape_ast::ast::Span::DUMMY,
615 ),
616 ctx,
617 )?;
618 let value = cte_result.value.unwrap_or(ValueWord::none());
619 ctx.set_variable_nb(&cte.name, value)?;
620 }
621 self.execute_query_with_context(
622 &shape_ast::ast::Item::Query(
623 (*with_query.query).clone(),
624 shape_ast::ast::Span::DUMMY,
625 ),
626 ctx,
627 )
628 }
629 },
630 shape_ast::ast::Item::Expression(_, _) => {
631 Ok(QueryResult::new(QueryType::Value, id, timeframe).with_value(ValueWord::none()))
632 }
633 shape_ast::ast::Item::VariableDecl(var_decl, _) => {
634 ctx.declare_pattern(&var_decl.pattern, var_decl.kind, ValueWord::none())?;
635 Ok(QueryResult::new(QueryType::Value, id, timeframe).with_value(ValueWord::none()))
636 }
637 shape_ast::ast::Item::Assignment(assignment, _) => {
638 ctx.set_pattern(&assignment.pattern, ValueWord::none())?;
639 Ok(QueryResult::new(QueryType::Value, id, timeframe).with_value(ValueWord::none()))
640 }
641 shape_ast::ast::Item::Statement(_, _) => {
642 Ok(QueryResult::new(QueryType::Value, id, timeframe).with_value(ValueWord::none()))
643 }
644 _ => Err(ShapeError::RuntimeError {
645 message: format!("Unsupported item for query execution: {:?}", query),
646 location: None,
647 }),
648 }
649 }
650
651 pub fn execute_without_data(&mut self, item: &shape_ast::ast::Item) -> Result<QueryResult> {
652 let mut persistent_ctx = self.persistent_context.take();
653
654 let result = if let Some(ref mut ctx) = persistent_ctx {
655 match item {
656 shape_ast::ast::Item::Expression(_, _) => {
657 Ok(
658 QueryResult::new(QueryType::Value, "".to_string(), "".to_string())
659 .with_value(ValueWord::none()),
660 )
661 }
662 shape_ast::ast::Item::Statement(_, _) => {
663 Ok(
664 QueryResult::new(QueryType::Value, "".to_string(), "".to_string())
665 .with_value(ValueWord::none()),
666 )
667 }
668 shape_ast::ast::Item::VariableDecl(var_decl, _) => {
669 ctx.declare_pattern(&var_decl.pattern, var_decl.kind, ValueWord::none())?;
670 Ok(
671 QueryResult::new(QueryType::Value, "".to_string(), "".to_string())
672 .with_value(ValueWord::none()),
673 )
674 }
675 shape_ast::ast::Item::Assignment(assignment, _) => {
676 ctx.set_pattern(&assignment.pattern, ValueWord::none())?;
677 Ok(
678 QueryResult::new(QueryType::Value, "".to_string(), "".to_string())
679 .with_value(ValueWord::none()),
680 )
681 }
682 shape_ast::ast::Item::TypeAlias(_, _) => {
683 self.process_program_items(
684 &Program {
685 items: vec![item.clone()],
686 },
687 ctx,
688 None,
689 )?;
690 Ok(
691 QueryResult::new(QueryType::Value, "".to_string(), "".to_string())
692 .with_value(ValueWord::none()),
693 )
694 }
695 _ => Err(ShapeError::RuntimeError {
696 message: format!("Operation requires context: {:?}", item),
697 location: None,
698 }),
699 }
700 } else {
701 let mut ctx = context::ExecutionContext::new_empty_with_registry(
702 self.type_method_registry.clone(),
703 );
704 self.process_program_items(
705 &Program {
706 items: vec![item.clone()],
707 },
708 &mut ctx,
709 None,
710 )?;
711 persistent_ctx = Some(ctx);
712 Ok(
713 QueryResult::new(QueryType::Value, "".to_string(), "".to_string())
714 .with_value(ValueWord::none()),
715 )
716 };
717
718 self.persistent_context = persistent_ctx;
719 result
720 }
721
722 pub fn format_value(
726 &mut self,
727 _value: Value,
728 type_name: &str,
729 format_name: Option<&str>,
730 _param_overrides: std::collections::HashMap<String, Value>,
731 ) -> Result<String> {
732 if let Some(name) = format_name {
733 Ok(format!("<formatted {} as {}>", type_name, name))
734 } else {
735 Ok(format!("<formatted {}>", type_name))
736 }
737 }
738
739 pub fn set_debug_mode(&mut self, enabled: bool) {
744 self.debug_mode = enabled;
745 if enabled {
746 tracing::debug!("Shape runtime debug mode enabled");
747 }
748 }
749
750 pub fn debug_mode(&self) -> bool {
752 self.debug_mode
753 }
754
755 pub fn set_execution_timeout(&mut self, timeout: Duration) {
760 self.execution_timeout = Some(timeout);
761 }
762
763 pub fn execution_timeout(&self) -> Option<Duration> {
765 self.execution_timeout
766 }
767
768 pub fn set_memory_limit(&mut self, limit: usize) {
773 self.memory_limit = Some(limit);
774 }
775
776 pub fn memory_limit(&self) -> Option<usize> {
778 self.memory_limit
779 }
780}