1#![allow(clippy::result_large_err)]
3
4pub use shape_value::AlignedVec;
15
16pub mod alerts;
17pub mod annotation_context;
18pub mod arrow_c;
19pub mod ast_extensions;
20pub mod binary_reader;
21pub mod blob_prefetch;
22pub mod blob_store;
23pub mod blob_wire_format;
24pub mod builtin_metadata;
25pub mod closure;
26pub mod code_search;
27pub mod columnar_aggregations;
28pub mod chart_detect;
29pub mod const_eval;
30pub mod content_builders;
31pub mod content_dispatch;
32pub mod content_methods;
33pub mod content_renderer;
34pub mod context;
35pub mod crypto;
36pub mod data;
37pub mod dependency_resolver;
38pub mod doc_extract;
39pub mod distributed_gc;
40pub mod engine;
41pub mod event_queue;
42pub mod execution_proof;
43pub mod extension_context;
44pub mod extensions;
45pub mod extensions_config;
46pub mod frontmatter;
47pub mod fuzzy;
48pub mod fuzzy_property;
49pub mod hashing;
50pub mod intrinsics;
51pub mod join_executor;
52pub mod leakage;
53pub mod lookahead_guard;
54pub mod metadata;
55pub mod native_resolution;
56pub mod module_bindings;
57pub mod module_exports;
58pub mod module_loader;
59pub mod module_manifest;
60pub mod multi_table;
61pub mod multiple_testing;
62pub mod output_adapter;
63pub mod package_bundle;
64pub mod package_lock;
65pub mod pattern_library;
66pub mod pattern_state_machine;
67pub mod plugins;
68pub mod progress;
69pub mod project;
70#[cfg(all(test, feature = "deep-tests"))]
71mod project_deep_tests;
72pub mod provider_registry;
73pub mod query_builder;
74pub mod query_executor;
75pub mod query_result;
76pub mod renderers;
77pub mod schema_cache;
78pub mod schema_inference;
79pub mod simd_comparisons;
80pub mod simd_forward_fill;
81pub mod simd_i64;
82pub mod simd_rolling;
83pub mod simd_statistics;
84pub mod simulation;
85pub mod snapshot;
86pub mod state_diff;
87pub mod statistics;
88pub mod stdlib;
89pub mod stdlib_io;
90pub mod stdlib_metadata;
91pub mod stdlib_time;
92pub mod stream_executor;
93pub mod sync_bridge;
94pub mod time_window;
95pub mod timeframe_utils;
96pub mod type_mapping;
97pub mod type_methods;
98pub mod type_schema;
99pub mod type_system;
100pub mod visitor;
101pub mod window_executor;
102pub mod window_manager;
103pub mod wire_conversion;
104
105pub use alerts::{Alert, AlertRouter, AlertSeverity, AlertSink};
107pub use context::{DataLoadMode, ExecutionContext as Context};
108pub use data::DataFrame;
109pub use data::OwnedDataRow as RowValue;
110pub use event_queue::{
111 EventQueue, MemoryEventQueue, QueuedEvent, SharedEventQueue, SuspensionState, WaitCondition,
112 create_event_queue,
113};
114pub use extensions::{
115 ExtensionCapability, ExtensionDataSource, ExtensionLoader, ExtensionOutputSink,
116 LoadedExtension, ParsedOutputField, ParsedOutputSchema, ParsedQueryParam, ParsedQuerySchema,
117};
118pub use extensions_config::{
119 ExtensionEntry as GlobalExtensionEntry, ExtensionsConfig as GlobalExtensionsConfig,
120 load_extensions_config, load_extensions_config_from,
121};
122pub use hashing::{HashDigest, combine_hashes, hash_bytes, hash_file, hash_string};
123pub use intrinsics::{IntrinsicFn, IntrinsicsRegistry};
124pub use join_executor::JoinExecutor;
125pub use leakage::{LeakageDetector, LeakageReport, LeakageSeverity, LeakageType, LeakageWarning};
126pub use module_bindings::ModuleBindingRegistry;
127pub use module_exports::{
128 FrameInfo, ModuleContext, ModuleExportRegistry, ModuleExports, ModuleFn, VmStateAccessor,
129};
130pub use multiple_testing::{MultipleTestingGuard, MultipleTestingStats, WarningLevel};
131pub use progress::{
132 LoadPhase, ProgressEvent, ProgressGranularity, ProgressHandle, ProgressRegistry,
133};
134pub use query_result::{AlertResult, QueryResult, QueryType};
135use shape_value::ValueWord;
136pub use shape_value::ValueWord as Value;
137pub use stream_executor::{StreamEvent, StreamExecutor, StreamState};
138pub use sync_bridge::{SyncDataProvider, block_on_shared, get_runtime_handle, initialize_shared_runtime};
139pub use type_schema::{
140 FieldDef, FieldType, SchemaId, TypeSchema, TypeSchemaBuilder, TypeSchemaRegistry,
141};
142pub use window_executor::WindowExecutor;
143pub use wire_conversion::{
144 nb_extract_typed_value, nb_to_envelope, nb_to_wire, nb_typed_value_to_envelope, wire_to_nb,
145};
146
147use self::type_methods::TypeMethodRegistry;
148pub use error::{Result, ShapeError, SourceLocation};
149use shape_ast::ast::{Program, Query};
150pub use shape_ast::error;
151use shape_wire::WireValue;
152use std::collections::HashMap;
153use std::path::PathBuf;
154use std::sync::Arc;
155use std::sync::RwLock;
156use std::time::Duration;
157
158pub struct Runtime {
160 module_loader: module_loader::ModuleLoader,
162 persistent_context: Option<context::ExecutionContext>,
164 type_method_registry: Arc<TypeMethodRegistry>,
166 annotation_registry: Arc<RwLock<annotation_context::AnnotationRegistry>>,
168 module_binding_registry: Arc<RwLock<module_bindings::ModuleBindingRegistry>>,
170 debug_mode: bool,
172 execution_timeout: Option<Duration>,
174 memory_limit: Option<usize>,
176 last_runtime_error: Option<WireValue>,
181 blob_store: Option<Arc<dyn crate::blob_store::BlobStore>>,
183 keychain: Option<crate::crypto::keychain::Keychain>,
185}
186
187impl Default for Runtime {
188 fn default() -> Self {
189 Self::new()
190 }
191}
192
193impl Runtime {
194 pub fn new() -> Self {
196 Self::new_internal(true)
197 }
198
199 pub(crate) fn new_without_stdlib() -> Self {
200 Self::new_internal(false)
201 }
202
203 fn new_internal(_load_stdlib: bool) -> Self {
204 let module_loader = module_loader::ModuleLoader::new();
205 let module_binding_registry =
206 Arc::new(RwLock::new(module_bindings::ModuleBindingRegistry::new()));
207
208 Self {
209 module_loader,
210 persistent_context: None,
211 type_method_registry: Arc::new(TypeMethodRegistry::new()),
212 annotation_registry: Arc::new(RwLock::new(
213 annotation_context::AnnotationRegistry::new(),
214 )),
215 module_binding_registry,
216 debug_mode: false,
217 execution_timeout: None,
218 memory_limit: None,
219 last_runtime_error: None,
220 blob_store: None,
221 keychain: None,
222 }
223 }
224
225 pub fn annotation_registry(&self) -> Arc<RwLock<annotation_context::AnnotationRegistry>> {
226 self.annotation_registry.clone()
227 }
228
229 pub fn enable_persistent_context(&mut self, data: &DataFrame) {
230 self.persistent_context = Some(context::ExecutionContext::new_with_registry(
231 data,
232 self.type_method_registry.clone(),
233 ));
234 }
235
236 pub fn enable_persistent_context_without_data(&mut self) {
237 self.persistent_context = Some(context::ExecutionContext::new_empty_with_registry(
238 self.type_method_registry.clone(),
239 ));
240 }
241
242 pub fn set_persistent_context(&mut self, ctx: context::ExecutionContext) {
243 self.persistent_context = Some(ctx);
244 }
245
246 pub fn persistent_context(&self) -> Option<&context::ExecutionContext> {
247 self.persistent_context.as_ref()
248 }
249
250 pub fn persistent_context_mut(&mut self) -> Option<&mut context::ExecutionContext> {
251 self.persistent_context.as_mut()
252 }
253
254 pub fn set_last_runtime_error(&mut self, payload: Option<WireValue>) {
256 self.last_runtime_error = payload;
257 }
258
259 pub fn clear_last_runtime_error(&mut self) {
261 self.last_runtime_error = None;
262 }
263
264 pub fn last_runtime_error(&self) -> Option<&WireValue> {
266 self.last_runtime_error.as_ref()
267 }
268
269 pub fn take_last_runtime_error(&mut self) -> Option<WireValue> {
271 self.last_runtime_error.take()
272 }
273
274 pub fn type_method_registry(&self) -> &Arc<TypeMethodRegistry> {
275 &self.type_method_registry
276 }
277
278 pub fn module_binding_registry(&self) -> Arc<RwLock<module_bindings::ModuleBindingRegistry>> {
280 self.module_binding_registry.clone()
281 }
282
283 pub fn add_module_path(&mut self, path: PathBuf) {
288 self.module_loader.add_module_path(path);
289 }
290
291 pub fn set_keychain(&mut self, keychain: crate::crypto::keychain::Keychain) {
296 self.keychain = Some(keychain.clone());
297 self.module_loader.set_keychain(keychain);
298 }
299
300 pub fn set_blob_store(&mut self, store: Arc<dyn crate::blob_store::BlobStore>) {
305 self.blob_store = Some(store.clone());
306 self.module_loader.set_blob_store(store);
307 }
308
309 pub fn set_project_root(&mut self, root: &std::path::Path, extra_paths: &[PathBuf]) {
311 self.module_loader.set_project_root(root, extra_paths);
312 }
313
314 pub fn set_dependency_paths(
316 &mut self,
317 deps: std::collections::HashMap<String, std::path::PathBuf>,
318 ) {
319 self.module_loader.set_dependency_paths(deps);
320 }
321
322 pub fn get_dependency_paths(&self) -> &std::collections::HashMap<String, std::path::PathBuf> {
324 self.module_loader.get_dependency_paths()
325 }
326
327 pub fn register_extension_module_artifacts(
329 &mut self,
330 modules: &[crate::extensions::ParsedModuleSchema],
331 ) {
332 for module in modules {
333 for artifact in &module.artifacts {
334 let code = match (&artifact.source, &artifact.compiled) {
335 (Some(source), Some(compiled)) => module_loader::ModuleCode::Both {
336 source: Arc::from(source.as_str()),
337 compiled: Arc::from(compiled.clone()),
338 },
339 (Some(source), None) => {
340 module_loader::ModuleCode::Source(Arc::from(source.as_str()))
341 }
342 (None, Some(compiled)) => {
343 module_loader::ModuleCode::Compiled(Arc::from(compiled.clone()))
344 }
345 (None, None) => continue,
346 };
347 self.module_loader
348 .register_extension_module(artifact.module_path.clone(), code);
349 }
350 }
351 }
352
353 pub fn configured_module_loader(&self) -> module_loader::ModuleLoader {
358 let mut loader = self.module_loader.clone_without_cache();
359 if let Some(ref store) = self.blob_store {
360 loader.set_blob_store(store.clone());
361 }
362 if let Some(ref kc) = self.keychain {
363 loader.set_keychain(kc.clone());
364 }
365 loader
366 }
367
368 pub fn load_core_stdlib_into_context(&mut self, data: &DataFrame) -> Result<()> {
372 let module_paths = self.module_loader.list_core_stdlib_module_imports()?;
373
374 for module_path in module_paths {
375 if module_path == "std::core::prelude" {
379 continue;
380 }
381
382 let resolved = self.module_loader.resolve_module_path(&module_path).ok();
383 let context_dir = resolved
384 .as_ref()
385 .and_then(|path| path.parent().map(|p| p.to_path_buf()));
386 let module = self.module_loader.load_module(&module_path)?;
387 self.load_program_with_context(&module.ast, data, context_dir.as_ref())?;
388 }
389
390 Ok(())
391 }
392
393 pub fn load_program(&mut self, program: &Program, data: &DataFrame) -> Result<()> {
394 self.load_program_with_context(program, data, None)
395 }
396
397 pub(crate) fn load_program_with_context(
398 &mut self,
399 program: &Program,
400 data: &DataFrame,
401 context_dir: Option<&PathBuf>,
402 ) -> Result<()> {
403 let mut persistent_ctx = self.persistent_context.take();
404
405 let result = if let Some(ref mut ctx) = persistent_ctx {
406 if data.row_count() > 0 {
407 ctx.update_data(data);
408 }
409 self.process_program_items(program, ctx, context_dir)
410 } else {
411 let mut ctx = context::ExecutionContext::new_with_registry(
412 data,
413 self.type_method_registry.clone(),
414 );
415 self.process_program_items(program, &mut ctx, context_dir)
416 };
417
418 self.persistent_context = persistent_ctx;
419 result
420 }
421
422 fn process_program_items(
423 &mut self,
424 program: &Program,
425 ctx: &mut context::ExecutionContext,
426 context_dir: Option<&PathBuf>,
427 ) -> Result<()> {
428 for item in &program.items {
429 match item {
430 shape_ast::ast::Item::Import(import, _) => {
431 let module = self
432 .module_loader
433 .load_module_with_context(&import.from, context_dir)?;
434
435 match &import.items {
436 shape_ast::ast::ImportItems::Named(imports) => {
437 for import_spec in imports {
438 if let Some(export) = module.exports.get(&import_spec.name) {
439 let var_name =
440 import_spec.alias.as_ref().unwrap_or(&import_spec.name);
441 match export {
442 module_loader::Export::Function(_) => {
443 }
445 module_loader::Export::Value(value) => {
446 if ctx.get_variable(var_name)?.is_none() {
447 ctx.set_variable(var_name, value.clone())?;
448 }
449 self.module_binding_registry
450 .write()
451 .unwrap()
452 .register_const(var_name, value.clone())?;
453 }
454 _ => {}
455 }
456 } else {
457 return Err(ShapeError::ModuleError {
458 message: format!(
459 "Export '{}' not found in module '{}'",
460 import_spec.name, import.from
461 ),
462 module_path: Some(import.from.clone().into()),
463 });
464 }
465 }
466 }
467 shape_ast::ast::ImportItems::Namespace { .. } => {
468 }
470 }
471 }
472 shape_ast::ast::Item::Export(export, _) => {
473 match &export.item {
474 shape_ast::ast::ExportItem::Function(_) => {
475 }
477 shape_ast::ast::ExportItem::Named(specs) => {
478 for spec in specs {
479 if let Ok(value) = ctx.get_variable(&spec.name) {
480 if value.is_none() {
481 return Err(ShapeError::RuntimeError {
482 message: format!(
483 "Cannot export undefined variable '{}'",
484 spec.name
485 ),
486 location: None,
487 });
488 }
489 }
490 }
491 }
492 shape_ast::ast::ExportItem::TypeAlias(alias_def) => {
493 let overrides = HashMap::new();
494 if let Some(ref overrides_ast) = alias_def.meta_param_overrides {
495 for (_key, _expr) in overrides_ast {
496 }
498 }
499
500 let base_type = match &alias_def.type_annotation {
501 shape_ast::ast::TypeAnnotation::Basic(n)
502 | shape_ast::ast::TypeAnnotation::Reference(n) => n.clone(),
503 _ => "any".to_string(),
504 };
505
506 ctx.register_type_alias(&alias_def.name, &base_type, Some(overrides));
507 }
508 shape_ast::ast::ExportItem::Enum(_)
509 | shape_ast::ast::ExportItem::Struct(_)
510 | shape_ast::ast::ExportItem::Interface(_)
511 | shape_ast::ast::ExportItem::Trait(_) => {
512 }
514 shape_ast::ast::ExportItem::ForeignFunction(_) => {
515 }
517 }
518 }
519 shape_ast::ast::Item::Function(_function, _) => {
520 }
522 shape_ast::ast::Item::TypeAlias(alias_def, _) => {
523 let overrides = HashMap::new();
524 if let Some(ref overrides_ast) = alias_def.meta_param_overrides {
525 for (_key, _expr) in overrides_ast {
526 }
528 }
529
530 let base_type = match &alias_def.type_annotation {
531 shape_ast::ast::TypeAnnotation::Basic(n)
532 | shape_ast::ast::TypeAnnotation::Reference(n) => n.clone(),
533 _ => "any".to_string(),
534 };
535
536 ctx.register_type_alias(&alias_def.name, &base_type, Some(overrides));
537 }
538 shape_ast::ast::Item::Interface(_, _) => {}
539 shape_ast::ast::Item::Trait(_, _) => {}
540 shape_ast::ast::Item::Impl(_, _) => {}
541 shape_ast::ast::Item::Enum(enum_def, _) => {
542 ctx.register_enum(enum_def.clone());
543 }
544 shape_ast::ast::Item::Extend(extend_stmt, _) => {
545 let registry = ctx.type_method_registry();
546 for method in &extend_stmt.methods {
547 registry.register_method(&extend_stmt.type_name, method.clone());
548 }
549 }
550 shape_ast::ast::Item::AnnotationDef(ann_def, _) => {
551 self.annotation_registry
552 .write()
553 .unwrap()
554 .register(ann_def.clone());
555 }
556 _ => {}
557 }
558 }
559 Ok(())
560 }
561
562 pub fn execute_query(
563 &mut self,
564 query: &shape_ast::ast::Item,
565 data: &DataFrame,
566 ) -> Result<QueryResult> {
567 let mut persistent_ctx = self.persistent_context.take();
568 let result = if let Some(ref mut ctx) = persistent_ctx {
569 ctx.update_data(data);
570 self.execute_query_with_context(query, ctx)
571 } else {
572 let mut ctx = context::ExecutionContext::new_with_registry(
573 data,
574 self.type_method_registry.clone(),
575 );
576 self.execute_query_with_context(query, &mut ctx)
577 };
578 self.persistent_context = persistent_ctx;
579 result
580 }
581
582 fn execute_query_with_context(
583 &mut self,
584 query: &shape_ast::ast::Item,
585 ctx: &mut context::ExecutionContext,
586 ) -> Result<QueryResult> {
587 let id = ctx.get_current_id().unwrap_or_default();
588 let timeframe = ctx
589 .get_current_timeframe()
590 .map(|t| t.to_string())
591 .unwrap_or_default();
592
593 match query {
594 shape_ast::ast::Item::Query(q, _) => match q {
595 Query::Backtest(_) => Err(ShapeError::RuntimeError {
596 message: "Backtesting not supported in core runtime.".to_string(),
597 location: None,
598 }),
599 Query::Alert(_alert_query) => {
600 let alert_id = format!("alert_{}", chrono::Utc::now().timestamp_micros());
601 Ok(
602 QueryResult::new(QueryType::Alert, id, timeframe).with_alert(AlertResult {
603 id: alert_id,
604 active: false,
605 message: "Alert triggered".to_string(),
606 level: "info".to_string(),
607 timestamp: chrono::Utc::now(),
608 }),
609 )
610 }
611 Query::With(with_query) => {
612 for cte in &with_query.ctes {
613 let cte_result = self.execute_query_with_context(
614 &shape_ast::ast::Item::Query(
615 (*cte.query).clone(),
616 shape_ast::ast::Span::DUMMY,
617 ),
618 ctx,
619 )?;
620 let value = cte_result.value.unwrap_or(ValueWord::none());
621 ctx.set_variable_nb(&cte.name, value)?;
622 }
623 self.execute_query_with_context(
624 &shape_ast::ast::Item::Query(
625 (*with_query.query).clone(),
626 shape_ast::ast::Span::DUMMY,
627 ),
628 ctx,
629 )
630 }
631 },
632 shape_ast::ast::Item::Expression(_, _) => {
633 Ok(QueryResult::new(QueryType::Value, id, timeframe).with_value(ValueWord::none()))
634 }
635 shape_ast::ast::Item::VariableDecl(var_decl, _) => {
636 ctx.declare_pattern(&var_decl.pattern, var_decl.kind, ValueWord::none())?;
637 Ok(QueryResult::new(QueryType::Value, id, timeframe).with_value(ValueWord::none()))
638 }
639 shape_ast::ast::Item::Assignment(assignment, _) => {
640 ctx.set_pattern(&assignment.pattern, ValueWord::none())?;
641 Ok(QueryResult::new(QueryType::Value, id, timeframe).with_value(ValueWord::none()))
642 }
643 shape_ast::ast::Item::Statement(_, _) => {
644 Ok(QueryResult::new(QueryType::Value, id, timeframe).with_value(ValueWord::none()))
645 }
646 _ => Err(ShapeError::RuntimeError {
647 message: format!("Unsupported item for query execution: {:?}", query),
648 location: None,
649 }),
650 }
651 }
652
653 pub fn execute_without_data(&mut self, item: &shape_ast::ast::Item) -> Result<QueryResult> {
654 let mut persistent_ctx = self.persistent_context.take();
655
656 let result = if let Some(ref mut ctx) = persistent_ctx {
657 match item {
658 shape_ast::ast::Item::Expression(_, _) => {
659 Ok(
660 QueryResult::new(QueryType::Value, "".to_string(), "".to_string())
661 .with_value(ValueWord::none()),
662 )
663 }
664 shape_ast::ast::Item::Statement(_, _) => {
665 Ok(
666 QueryResult::new(QueryType::Value, "".to_string(), "".to_string())
667 .with_value(ValueWord::none()),
668 )
669 }
670 shape_ast::ast::Item::VariableDecl(var_decl, _) => {
671 ctx.declare_pattern(&var_decl.pattern, var_decl.kind, ValueWord::none())?;
672 Ok(
673 QueryResult::new(QueryType::Value, "".to_string(), "".to_string())
674 .with_value(ValueWord::none()),
675 )
676 }
677 shape_ast::ast::Item::Assignment(assignment, _) => {
678 ctx.set_pattern(&assignment.pattern, ValueWord::none())?;
679 Ok(
680 QueryResult::new(QueryType::Value, "".to_string(), "".to_string())
681 .with_value(ValueWord::none()),
682 )
683 }
684 shape_ast::ast::Item::TypeAlias(_, _) => {
685 self.process_program_items(
686 &Program {
687 items: vec![item.clone()],
688 docs: shape_ast::ast::ProgramDocs::default(),
689 },
690 ctx,
691 None,
692 )?;
693 Ok(
694 QueryResult::new(QueryType::Value, "".to_string(), "".to_string())
695 .with_value(ValueWord::none()),
696 )
697 }
698 _ => Err(ShapeError::RuntimeError {
699 message: format!("Operation requires context: {:?}", item),
700 location: None,
701 }),
702 }
703 } else {
704 let mut ctx = context::ExecutionContext::new_empty_with_registry(
705 self.type_method_registry.clone(),
706 );
707 self.process_program_items(
708 &Program {
709 items: vec![item.clone()],
710 docs: shape_ast::ast::ProgramDocs::default(),
711 },
712 &mut ctx,
713 None,
714 )?;
715 persistent_ctx = Some(ctx);
716 Ok(
717 QueryResult::new(QueryType::Value, "".to_string(), "".to_string())
718 .with_value(ValueWord::none()),
719 )
720 };
721
722 self.persistent_context = persistent_ctx;
723 result
724 }
725
726 pub fn format_value(
730 &mut self,
731 _value: Value,
732 type_name: &str,
733 format_name: Option<&str>,
734 _param_overrides: std::collections::HashMap<String, Value>,
735 ) -> Result<String> {
736 if let Some(name) = format_name {
737 Ok(format!("<formatted {} as {}>", type_name, name))
738 } else {
739 Ok(format!("<formatted {}>", type_name))
740 }
741 }
742
743 pub fn set_debug_mode(&mut self, enabled: bool) {
748 self.debug_mode = enabled;
749 if enabled {
750 tracing::debug!("Shape runtime debug mode enabled");
751 }
752 }
753
754 pub fn debug_mode(&self) -> bool {
756 self.debug_mode
757 }
758
759 pub fn set_execution_timeout(&mut self, timeout: Duration) {
764 self.execution_timeout = Some(timeout);
765 }
766
767 pub fn execution_timeout(&self) -> Option<Duration> {
769 self.execution_timeout
770 }
771
772 pub fn set_memory_limit(&mut self, limit: usize) {
777 self.memory_limit = Some(limit);
778 }
779
780 pub fn memory_limit(&self) -> Option<usize> {
782 self.memory_limit
783 }
784}