1#![allow(clippy::result_large_err)]
3
4pub use shape_value::AlignedVec;
15
16pub mod alerts;
17pub mod annotation_context;
18pub mod arrow_c;
19pub mod ast_extensions;
20pub mod binary_reader;
21pub mod blob_prefetch;
22pub mod blob_store;
23pub mod blob_wire_format;
24pub mod builtin_metadata;
25pub mod closure;
26pub mod code_search;
27pub mod columnar_aggregations;
28pub mod chart_detect;
29pub mod const_eval;
30pub mod content_builders;
31pub mod content_dispatch;
32pub mod content_methods;
33pub mod content_renderer;
34pub mod context;
35pub mod crypto;
36pub mod data;
37pub mod dependency_resolver;
38pub mod distributed_gc;
39pub mod engine;
40pub mod event_queue;
41pub mod execution_proof;
42pub mod extension_context;
43pub mod extensions;
44pub mod extensions_config;
45pub mod frontmatter;
46pub mod fuzzy;
47pub mod fuzzy_property;
48pub mod hashing;
49pub mod intrinsics;
50pub mod join_executor;
51pub mod leakage;
52pub mod lookahead_guard;
53pub mod metadata;
54pub mod module_bindings;
55pub mod module_exports;
56pub mod module_loader;
57pub mod module_manifest;
58pub mod multi_table;
59pub mod multiple_testing;
60pub mod output_adapter;
61pub mod package_bundle;
62pub mod package_lock;
63pub mod pattern_library;
64pub mod pattern_state_machine;
65pub mod plugins;
66pub mod progress;
67pub mod project;
68#[cfg(test)]
69mod project_deep_tests;
70pub mod provider_registry;
71pub mod query_builder;
72pub mod query_executor;
73pub mod query_result;
74pub mod renderers;
75pub mod schema_cache;
76pub mod schema_inference;
77pub mod semantic;
78pub mod simd_comparisons;
79pub mod simd_forward_fill;
80pub mod simd_i64;
81pub mod simd_rolling;
82pub mod simd_statistics;
83pub mod simulation;
84pub mod snapshot;
85pub mod state_diff;
86pub mod statistics;
87pub mod stdlib;
88pub mod stdlib_io;
89pub mod stdlib_metadata;
90pub mod stdlib_time;
91pub mod stream_executor;
92pub mod sync_bridge;
93pub mod time_window;
94pub mod timeframe_utils;
95pub mod type_mapping;
96pub mod type_methods;
97pub mod type_schema;
98pub mod type_system;
99pub mod visitor;
100pub mod window_executor;
101pub mod window_manager;
102pub mod wire_conversion;
103
104pub use alerts::{Alert, AlertRouter, AlertSeverity, AlertSink};
106pub use context::{DataLoadMode, ExecutionContext as Context};
107pub use data::DataFrame;
108pub use data::OwnedDataRow as RowValue;
109pub use event_queue::{
110 EventQueue, MemoryEventQueue, QueuedEvent, SharedEventQueue, SuspensionState, WaitCondition,
111 create_event_queue,
112};
113pub use extensions::{
114 ExtensionCapability, ExtensionDataSource, ExtensionLoader, ExtensionOutputSink,
115 LoadedExtension, ParsedOutputField, ParsedOutputSchema, ParsedQueryParam, ParsedQuerySchema,
116};
117pub use extensions_config::{
118 ExtensionEntry as GlobalExtensionEntry, ExtensionsConfig as GlobalExtensionsConfig,
119 load_extensions_config, load_extensions_config_from,
120};
121pub use hashing::{HashDigest, combine_hashes, hash_bytes, hash_file, hash_string};
122pub use intrinsics::{IntrinsicFn, IntrinsicsRegistry};
123pub use join_executor::JoinExecutor;
124pub use leakage::{LeakageDetector, LeakageReport, LeakageSeverity, LeakageType, LeakageWarning};
125pub use module_bindings::ModuleBindingRegistry;
126pub use module_exports::{
127 FrameInfo, ModuleContext, ModuleExportRegistry, ModuleExports, ModuleFn, VmStateAccessor,
128};
129pub use multiple_testing::{MultipleTestingGuard, MultipleTestingStats, WarningLevel};
130pub use progress::{
131 LoadPhase, ProgressEvent, ProgressGranularity, ProgressHandle, ProgressRegistry,
132};
133pub use query_result::{AlertResult, QueryResult, QueryType};
134use shape_value::ValueWord;
135pub use shape_value::ValueWord as Value;
136pub use stream_executor::{StreamEvent, StreamExecutor, StreamState};
137pub use sync_bridge::{SyncDataProvider, block_on_shared, initialize_shared_runtime};
138pub use type_schema::{
139 FieldDef, FieldType, SchemaId, TypeSchema, TypeSchemaBuilder, TypeSchemaRegistry,
140};
141pub use window_executor::WindowExecutor;
142pub use wire_conversion::{
143 nb_extract_typed_value, nb_to_envelope, nb_to_wire, nb_typed_value_to_envelope, wire_to_nb,
144};
145
146use self::type_methods::TypeMethodRegistry;
147pub use error::{Result, ShapeError, SourceLocation};
148use shape_ast::ast::{Program, Query};
149pub use shape_ast::error;
150use shape_wire::WireValue;
151use std::collections::HashMap;
152use std::path::PathBuf;
153use std::sync::Arc;
154use std::sync::RwLock;
155use std::time::Duration;
156
157pub struct Runtime {
159 module_loader: module_loader::ModuleLoader,
161 persistent_context: Option<context::ExecutionContext>,
163 type_method_registry: Arc<TypeMethodRegistry>,
165 annotation_registry: Arc<RwLock<annotation_context::AnnotationRegistry>>,
167 module_binding_registry: Arc<RwLock<module_bindings::ModuleBindingRegistry>>,
169 debug_mode: bool,
171 execution_timeout: Option<Duration>,
173 memory_limit: Option<usize>,
175 last_runtime_error: Option<WireValue>,
180 blob_store: Option<Arc<dyn crate::blob_store::BlobStore>>,
182 keychain: Option<crate::crypto::keychain::Keychain>,
184}
185
186impl Default for Runtime {
187 fn default() -> Self {
188 Self::new()
189 }
190}
191
192impl Runtime {
193 pub fn new() -> Self {
195 Self::new_internal(true)
196 }
197
198 pub(crate) fn new_without_stdlib() -> Self {
199 Self::new_internal(false)
200 }
201
202 fn new_internal(_load_stdlib: bool) -> Self {
203 let module_loader = module_loader::ModuleLoader::new();
204 let module_binding_registry =
205 Arc::new(RwLock::new(module_bindings::ModuleBindingRegistry::new()));
206
207 Self {
208 module_loader,
209 persistent_context: None,
210 type_method_registry: Arc::new(TypeMethodRegistry::new()),
211 annotation_registry: Arc::new(RwLock::new(
212 annotation_context::AnnotationRegistry::new(),
213 )),
214 module_binding_registry,
215 debug_mode: false,
216 execution_timeout: None,
217 memory_limit: None,
218 last_runtime_error: None,
219 blob_store: None,
220 keychain: None,
221 }
222 }
223
224 pub fn annotation_registry(&self) -> Arc<RwLock<annotation_context::AnnotationRegistry>> {
225 self.annotation_registry.clone()
226 }
227
228 pub fn enable_persistent_context(&mut self, data: &DataFrame) {
229 self.persistent_context = Some(context::ExecutionContext::new_with_registry(
230 data,
231 self.type_method_registry.clone(),
232 ));
233 }
234
235 pub fn enable_persistent_context_without_data(&mut self) {
236 self.persistent_context = Some(context::ExecutionContext::new_empty_with_registry(
237 self.type_method_registry.clone(),
238 ));
239 }
240
241 pub fn set_persistent_context(&mut self, ctx: context::ExecutionContext) {
242 self.persistent_context = Some(ctx);
243 }
244
245 pub fn persistent_context(&self) -> Option<&context::ExecutionContext> {
246 self.persistent_context.as_ref()
247 }
248
249 pub fn persistent_context_mut(&mut self) -> Option<&mut context::ExecutionContext> {
250 self.persistent_context.as_mut()
251 }
252
253 pub fn set_last_runtime_error(&mut self, payload: Option<WireValue>) {
255 self.last_runtime_error = payload;
256 }
257
258 pub fn clear_last_runtime_error(&mut self) {
260 self.last_runtime_error = None;
261 }
262
263 pub fn last_runtime_error(&self) -> Option<&WireValue> {
265 self.last_runtime_error.as_ref()
266 }
267
268 pub fn take_last_runtime_error(&mut self) -> Option<WireValue> {
270 self.last_runtime_error.take()
271 }
272
273 pub fn type_method_registry(&self) -> &Arc<TypeMethodRegistry> {
274 &self.type_method_registry
275 }
276
277 pub fn module_binding_registry(&self) -> Arc<RwLock<module_bindings::ModuleBindingRegistry>> {
279 self.module_binding_registry.clone()
280 }
281
282 pub fn add_module_path(&mut self, path: PathBuf) {
287 self.module_loader.add_module_path(path);
288 }
289
290 pub fn set_keychain(&mut self, keychain: crate::crypto::keychain::Keychain) {
295 self.keychain = Some(keychain.clone());
296 self.module_loader.set_keychain(keychain);
297 }
298
299 pub fn set_blob_store(&mut self, store: Arc<dyn crate::blob_store::BlobStore>) {
304 self.blob_store = Some(store.clone());
305 self.module_loader.set_blob_store(store);
306 }
307
308 pub fn set_project_root(&mut self, root: &std::path::Path, extra_paths: &[PathBuf]) {
310 self.module_loader.set_project_root(root, extra_paths);
311 }
312
313 pub fn set_dependency_paths(
315 &mut self,
316 deps: std::collections::HashMap<String, std::path::PathBuf>,
317 ) {
318 self.module_loader.set_dependency_paths(deps);
319 }
320
321 pub fn get_dependency_paths(&self) -> &std::collections::HashMap<String, std::path::PathBuf> {
323 self.module_loader.get_dependency_paths()
324 }
325
326 pub fn register_extension_module_artifacts(
328 &mut self,
329 modules: &[crate::extensions::ParsedModuleSchema],
330 ) {
331 for module in modules {
332 for artifact in &module.artifacts {
333 let code = match (&artifact.source, &artifact.compiled) {
334 (Some(source), Some(compiled)) => module_loader::ModuleCode::Both {
335 source: Arc::from(source.as_str()),
336 compiled: Arc::from(compiled.clone()),
337 },
338 (Some(source), None) => {
339 module_loader::ModuleCode::Source(Arc::from(source.as_str()))
340 }
341 (None, Some(compiled)) => {
342 module_loader::ModuleCode::Compiled(Arc::from(compiled.clone()))
343 }
344 (None, None) => continue,
345 };
346 self.module_loader
347 .register_extension_module(artifact.module_path.clone(), code);
348 }
349 }
350 }
351
352 pub fn configured_module_loader(&self) -> module_loader::ModuleLoader {
357 let mut loader = self.module_loader.clone_without_cache();
358 if let Some(ref store) = self.blob_store {
359 loader.set_blob_store(store.clone());
360 }
361 if let Some(ref kc) = self.keychain {
362 loader.set_keychain(kc.clone());
363 }
364 loader
365 }
366
367 pub fn load_core_stdlib_into_context(&mut self, data: &DataFrame) -> Result<()> {
371 let module_paths = self.module_loader.list_core_stdlib_module_imports()?;
372
373 for module_path in module_paths {
374 if module_path == "std::core::prelude" {
378 continue;
379 }
380
381 let resolved = self.module_loader.resolve_module_path(&module_path).ok();
382 let context_dir = resolved
383 .as_ref()
384 .and_then(|path| path.parent().map(|p| p.to_path_buf()));
385 let module = self.module_loader.load_module(&module_path)?;
386 self.load_program_with_context(&module.ast, data, context_dir.as_ref())?;
387 }
388
389 Ok(())
390 }
391
392 pub fn load_program(&mut self, program: &Program, data: &DataFrame) -> Result<()> {
393 self.load_program_with_context(program, data, None)
394 }
395
396 pub(crate) fn load_program_with_context(
397 &mut self,
398 program: &Program,
399 data: &DataFrame,
400 context_dir: Option<&PathBuf>,
401 ) -> Result<()> {
402 let mut persistent_ctx = self.persistent_context.take();
403
404 let result = if let Some(ref mut ctx) = persistent_ctx {
405 if data.row_count() > 0 {
406 ctx.update_data(data);
407 }
408 self.process_program_items(program, ctx, context_dir)
409 } else {
410 let mut ctx = context::ExecutionContext::new_with_registry(
411 data,
412 self.type_method_registry.clone(),
413 );
414 self.process_program_items(program, &mut ctx, context_dir)
415 };
416
417 self.persistent_context = persistent_ctx;
418 result
419 }
420
421 fn process_program_items(
422 &mut self,
423 program: &Program,
424 ctx: &mut context::ExecutionContext,
425 context_dir: Option<&PathBuf>,
426 ) -> Result<()> {
427 for item in &program.items {
428 match item {
429 shape_ast::ast::Item::Import(import, _) => {
430 let module = self
431 .module_loader
432 .load_module_with_context(&import.from, context_dir)?;
433
434 match &import.items {
435 shape_ast::ast::ImportItems::Named(imports) => {
436 for import_spec in imports {
437 if let Some(export) = module.exports.get(&import_spec.name) {
438 let var_name =
439 import_spec.alias.as_ref().unwrap_or(&import_spec.name);
440 match export {
441 module_loader::Export::Function(_) => {
442 }
444 module_loader::Export::Value(value) => {
445 if ctx.get_variable(var_name)?.is_none() {
446 ctx.set_variable(var_name, value.clone())?;
447 }
448 self.module_binding_registry
449 .write()
450 .unwrap()
451 .register_const(var_name, value.clone())?;
452 }
453 _ => {}
454 }
455 } else {
456 return Err(ShapeError::ModuleError {
457 message: format!(
458 "Export '{}' not found in module '{}'",
459 import_spec.name, import.from
460 ),
461 module_path: Some(import.from.clone().into()),
462 });
463 }
464 }
465 }
466 shape_ast::ast::ImportItems::Namespace { .. } => {
467 }
469 }
470 }
471 shape_ast::ast::Item::Export(export, _) => {
472 match &export.item {
473 shape_ast::ast::ExportItem::Function(_) => {
474 }
476 shape_ast::ast::ExportItem::Named(specs) => {
477 for spec in specs {
478 if let Ok(value) = ctx.get_variable(&spec.name) {
479 if value.is_none() {
480 return Err(ShapeError::RuntimeError {
481 message: format!(
482 "Cannot export undefined variable '{}'",
483 spec.name
484 ),
485 location: None,
486 });
487 }
488 }
489 }
490 }
491 shape_ast::ast::ExportItem::TypeAlias(alias_def) => {
492 let overrides = HashMap::new();
493 if let Some(ref overrides_ast) = alias_def.meta_param_overrides {
494 for (_key, _expr) in overrides_ast {
495 }
497 }
498
499 let base_type = match &alias_def.type_annotation {
500 shape_ast::ast::TypeAnnotation::Basic(n)
501 | shape_ast::ast::TypeAnnotation::Reference(n) => n.clone(),
502 _ => "any".to_string(),
503 };
504
505 ctx.register_type_alias(&alias_def.name, &base_type, Some(overrides));
506 }
507 shape_ast::ast::ExportItem::Enum(_)
508 | shape_ast::ast::ExportItem::Struct(_)
509 | shape_ast::ast::ExportItem::Interface(_)
510 | shape_ast::ast::ExportItem::Trait(_) => {
511 }
513 shape_ast::ast::ExportItem::ForeignFunction(_) => {
514 }
516 }
517 }
518 shape_ast::ast::Item::Function(_function, _) => {
519 }
521 shape_ast::ast::Item::TypeAlias(alias_def, _) => {
522 let overrides = HashMap::new();
523 if let Some(ref overrides_ast) = alias_def.meta_param_overrides {
524 for (_key, _expr) in overrides_ast {
525 }
527 }
528
529 let base_type = match &alias_def.type_annotation {
530 shape_ast::ast::TypeAnnotation::Basic(n)
531 | shape_ast::ast::TypeAnnotation::Reference(n) => n.clone(),
532 _ => "any".to_string(),
533 };
534
535 ctx.register_type_alias(&alias_def.name, &base_type, Some(overrides));
536 }
537 shape_ast::ast::Item::Interface(_, _) => {}
538 shape_ast::ast::Item::Trait(_, _) => {}
539 shape_ast::ast::Item::Impl(_, _) => {}
540 shape_ast::ast::Item::Enum(enum_def, _) => {
541 ctx.register_enum(enum_def.clone());
542 }
543 shape_ast::ast::Item::Extend(extend_stmt, _) => {
544 let registry = ctx.type_method_registry();
545 for method in &extend_stmt.methods {
546 registry.register_method(&extend_stmt.type_name, method.clone());
547 }
548 }
549 shape_ast::ast::Item::AnnotationDef(ann_def, _) => {
550 self.annotation_registry
551 .write()
552 .unwrap()
553 .register(ann_def.clone());
554 }
555 _ => {}
556 }
557 }
558 Ok(())
559 }
560
561 pub fn execute_query(
562 &mut self,
563 query: &shape_ast::ast::Item,
564 data: &DataFrame,
565 ) -> Result<QueryResult> {
566 let mut persistent_ctx = self.persistent_context.take();
567 let result = if let Some(ref mut ctx) = persistent_ctx {
568 ctx.update_data(data);
569 self.execute_query_with_context(query, ctx)
570 } else {
571 let mut ctx = context::ExecutionContext::new_with_registry(
572 data,
573 self.type_method_registry.clone(),
574 );
575 self.execute_query_with_context(query, &mut ctx)
576 };
577 self.persistent_context = persistent_ctx;
578 result
579 }
580
581 fn execute_query_with_context(
582 &mut self,
583 query: &shape_ast::ast::Item,
584 ctx: &mut context::ExecutionContext,
585 ) -> Result<QueryResult> {
586 let id = ctx.get_current_id().unwrap_or_default();
587 let timeframe = ctx
588 .get_current_timeframe()
589 .map(|t| t.to_string())
590 .unwrap_or_default();
591
592 match query {
593 shape_ast::ast::Item::Query(q, _) => match q {
594 Query::Backtest(_) => Err(ShapeError::RuntimeError {
595 message: "Backtesting not supported in core runtime.".to_string(),
596 location: None,
597 }),
598 Query::Alert(_alert_query) => {
599 let alert_id = format!("alert_{}", chrono::Utc::now().timestamp_micros());
600 Ok(
601 QueryResult::new(QueryType::Alert, id, timeframe).with_alert(AlertResult {
602 id: alert_id,
603 active: false,
604 message: "Alert triggered".to_string(),
605 level: "info".to_string(),
606 timestamp: chrono::Utc::now(),
607 }),
608 )
609 }
610 Query::With(with_query) => {
611 for cte in &with_query.ctes {
612 let cte_result = self.execute_query_with_context(
613 &shape_ast::ast::Item::Query(
614 (*cte.query).clone(),
615 shape_ast::ast::Span::DUMMY,
616 ),
617 ctx,
618 )?;
619 let value = cte_result.value.unwrap_or(ValueWord::none());
620 ctx.set_variable_nb(&cte.name, value)?;
621 }
622 self.execute_query_with_context(
623 &shape_ast::ast::Item::Query(
624 (*with_query.query).clone(),
625 shape_ast::ast::Span::DUMMY,
626 ),
627 ctx,
628 )
629 }
630 },
631 shape_ast::ast::Item::Expression(_, _) => {
632 Ok(QueryResult::new(QueryType::Value, id, timeframe).with_value(ValueWord::none()))
633 }
634 shape_ast::ast::Item::VariableDecl(var_decl, _) => {
635 ctx.declare_pattern(&var_decl.pattern, var_decl.kind, ValueWord::none())?;
636 Ok(QueryResult::new(QueryType::Value, id, timeframe).with_value(ValueWord::none()))
637 }
638 shape_ast::ast::Item::Assignment(assignment, _) => {
639 ctx.set_pattern(&assignment.pattern, ValueWord::none())?;
640 Ok(QueryResult::new(QueryType::Value, id, timeframe).with_value(ValueWord::none()))
641 }
642 shape_ast::ast::Item::Statement(_, _) => {
643 Ok(QueryResult::new(QueryType::Value, id, timeframe).with_value(ValueWord::none()))
644 }
645 _ => Err(ShapeError::RuntimeError {
646 message: format!("Unsupported item for query execution: {:?}", query),
647 location: None,
648 }),
649 }
650 }
651
652 pub fn execute_without_data(&mut self, item: &shape_ast::ast::Item) -> Result<QueryResult> {
653 let mut persistent_ctx = self.persistent_context.take();
654
655 let result = if let Some(ref mut ctx) = persistent_ctx {
656 match item {
657 shape_ast::ast::Item::Expression(_, _) => {
658 Ok(
659 QueryResult::new(QueryType::Value, "".to_string(), "".to_string())
660 .with_value(ValueWord::none()),
661 )
662 }
663 shape_ast::ast::Item::Statement(_, _) => {
664 Ok(
665 QueryResult::new(QueryType::Value, "".to_string(), "".to_string())
666 .with_value(ValueWord::none()),
667 )
668 }
669 shape_ast::ast::Item::VariableDecl(var_decl, _) => {
670 ctx.declare_pattern(&var_decl.pattern, var_decl.kind, ValueWord::none())?;
671 Ok(
672 QueryResult::new(QueryType::Value, "".to_string(), "".to_string())
673 .with_value(ValueWord::none()),
674 )
675 }
676 shape_ast::ast::Item::Assignment(assignment, _) => {
677 ctx.set_pattern(&assignment.pattern, ValueWord::none())?;
678 Ok(
679 QueryResult::new(QueryType::Value, "".to_string(), "".to_string())
680 .with_value(ValueWord::none()),
681 )
682 }
683 shape_ast::ast::Item::TypeAlias(_, _) => {
684 self.process_program_items(
685 &Program {
686 items: vec![item.clone()],
687 },
688 ctx,
689 None,
690 )?;
691 Ok(
692 QueryResult::new(QueryType::Value, "".to_string(), "".to_string())
693 .with_value(ValueWord::none()),
694 )
695 }
696 _ => Err(ShapeError::RuntimeError {
697 message: format!("Operation requires context: {:?}", item),
698 location: None,
699 }),
700 }
701 } else {
702 let mut ctx = context::ExecutionContext::new_empty_with_registry(
703 self.type_method_registry.clone(),
704 );
705 self.process_program_items(
706 &Program {
707 items: vec![item.clone()],
708 },
709 &mut ctx,
710 None,
711 )?;
712 persistent_ctx = Some(ctx);
713 Ok(
714 QueryResult::new(QueryType::Value, "".to_string(), "".to_string())
715 .with_value(ValueWord::none()),
716 )
717 };
718
719 self.persistent_context = persistent_ctx;
720 result
721 }
722
723 pub fn format_value(
727 &mut self,
728 _value: Value,
729 type_name: &str,
730 format_name: Option<&str>,
731 _param_overrides: std::collections::HashMap<String, Value>,
732 ) -> Result<String> {
733 if let Some(name) = format_name {
734 Ok(format!("<formatted {} as {}>", type_name, name))
735 } else {
736 Ok(format!("<formatted {}>", type_name))
737 }
738 }
739
740 pub fn set_debug_mode(&mut self, enabled: bool) {
745 self.debug_mode = enabled;
746 if enabled {
747 tracing::debug!("Shape runtime debug mode enabled");
748 }
749 }
750
751 pub fn debug_mode(&self) -> bool {
753 self.debug_mode
754 }
755
756 pub fn set_execution_timeout(&mut self, timeout: Duration) {
761 self.execution_timeout = Some(timeout);
762 }
763
764 pub fn execution_timeout(&self) -> Option<Duration> {
766 self.execution_timeout
767 }
768
769 pub fn set_memory_limit(&mut self, limit: usize) {
774 self.memory_limit = Some(limit);
775 }
776
777 pub fn memory_limit(&self) -> Option<usize> {
779 self.memory_limit
780 }
781}