graphix_compiler/
lib.rs

1#[macro_use]
2extern crate netidx_core;
3#[macro_use]
4extern crate combine;
5#[macro_use]
6extern crate serde_derive;
7
8pub mod env;
9pub mod expr;
10pub mod node;
11pub mod typ;
12
13use crate::{
14    env::Env,
15    expr::{ExprId, ModPath},
16    typ::{FnType, Type},
17};
18use anyhow::{bail, Result};
19use arcstr::ArcStr;
20use enumflags2::{bitflags, BitFlags};
21use expr::Expr;
22use fxhash::{FxHashMap, FxHashSet};
23use log::info;
24use netidx::{
25    path::Path,
26    publisher::{Id, Val, WriteRequest},
27    subscriber::{self, Dval, SubId, UpdatesFlags, Value},
28};
29use netidx_protocols::rpc::server::{ArgSpec, RpcCall};
30use node::compiler;
31use parking_lot::RwLock;
32use poolshark::local::LPooled;
33use std::{
34    any::Any,
35    cell::Cell,
36    collections::{hash_map::Entry, HashMap},
37    fmt::Debug,
38    mem,
39    sync::{
40        self,
41        atomic::{AtomicBool, Ordering},
42        LazyLock,
43    },
44    time::Duration,
45};
46use tokio::time::Instant;
47use triomphe::Arc;
48
49#[derive(Debug, Clone, Copy)]
50#[bitflags]
51#[repr(u64)]
52pub enum CFlag {
53    WarnUnhandled,
54    WarnUnhandledArith,
55    WarnUnused,
56    WarningsAreErrors,
57}
58
59#[allow(dead_code)]
60static TRACE: AtomicBool = AtomicBool::new(false);
61
62#[allow(dead_code)]
63fn set_trace(b: bool) {
64    TRACE.store(b, Ordering::Relaxed)
65}
66
67#[allow(dead_code)]
68fn with_trace<F: FnOnce() -> Result<R>, R>(enable: bool, spec: &Expr, f: F) -> Result<R> {
69    let set = if enable {
70        eprintln!("trace enabled at {}, spec: {}", spec.pos, spec);
71        let prev = trace();
72        set_trace(true);
73        !prev
74    } else {
75        false
76    };
77    let r = match f() {
78        Err(e) => {
79            eprintln!("traced at {} failed with {e:?}", spec.pos);
80            Err(e)
81        }
82        r => r,
83    };
84    if set {
85        eprintln!("trace disabled at {}", spec.pos);
86        set_trace(false)
87    }
88    r
89}
90
91#[allow(dead_code)]
92fn trace() -> bool {
93    TRACE.load(Ordering::Relaxed)
94}
95
96#[macro_export]
97macro_rules! tdbg {
98    ($e:expr) => {
99        if $crate::trace() {
100            dbg!($e)
101        } else {
102            $e
103        }
104    };
105}
106
107#[macro_export]
108macro_rules! err {
109    ($tag:expr, $err:literal) => {{
110        let e: Value = ($tag.clone(), literal!($err)).into();
111        Value::Error(triomphe::Arc::new(e))
112    }};
113}
114
115#[macro_export]
116macro_rules! errf {
117    ($tag:expr, $fmt:expr, $args:tt) => {{
118        let msg: ArcStr = compact_str::format_compact!($fmt, $args).as_str().into();
119        let e: Value = ($tag.clone(), msg).into();
120        Value::Error(triomphe::Arc::new(e))
121    }};
122    ($tag:expr, $fmt:expr) => {{
123        let msg: ArcStr = compact_str::format_compact!($fmt).as_str().into();
124        let e: Value = ($tag.clone(), msg).into();
125        Value::Error(triomphe::Arc::new(e))
126    }};
127}
128
129#[macro_export]
130macro_rules! defetyp {
131    ($name:ident, $tag_name:ident, $tag:literal, $typ:expr) => {
132        static $tag_name: ArcStr = literal!($tag);
133        static $name: ::std::sync::LazyLock<$crate::typ::Type> =
134            ::std::sync::LazyLock::new(|| {
135                let scope = $crate::expr::ModPath::root();
136                $crate::expr::parser::parse_type(&format!($typ, $tag))
137                    .expect("failed to parse type")
138                    .scope_refs(&scope)
139            });
140    };
141}
142
143atomic_id!(LambdaId);
144
145impl From<u64> for LambdaId {
146    fn from(v: u64) -> Self {
147        LambdaId(v)
148    }
149}
150
151atomic_id!(BindId);
152
153impl From<u64> for BindId {
154    fn from(v: u64) -> Self {
155        BindId(v)
156    }
157}
158
159impl TryFrom<Value> for BindId {
160    type Error = anyhow::Error;
161
162    fn try_from(value: Value) -> Result<Self> {
163        match value {
164            Value::U64(id) => Ok(BindId(id)),
165            v => bail!("invalid bind id {v}"),
166        }
167    }
168}
169
170pub trait UserEvent: Clone + Debug + Any {
171    fn clear(&mut self);
172}
173
174#[derive(Debug, Clone)]
175pub struct NoUserEvent;
176
177impl UserEvent for NoUserEvent {
178    fn clear(&mut self) {}
179}
180
181#[derive(Debug, Clone, Copy)]
182#[bitflags]
183#[repr(u64)]
184pub enum PrintFlag {
185    /// Dereference type variables and print both the tvar name and the bound
186    /// type or "unbound".
187    DerefTVars,
188    /// Replace common primitives with shorter type names as defined
189    /// in core. e.g. Any, instead of the set of every primitive type.
190    ReplacePrims,
191    /// When formatting an Origin don't print the source, just the location
192    NoSource,
193    /// When formatting an Origin don't print the origin's parents
194    NoParents,
195}
196
197thread_local! {
198    static PRINT_FLAGS: Cell<BitFlags<PrintFlag>> = Cell::new(PrintFlag::ReplacePrims | PrintFlag::NoSource);
199}
200
201/// For the duration of the closure F change the way type variables
202/// are formatted (on this thread only) according to the specified
203/// flags.
204pub fn format_with_flags<G: Into<BitFlags<PrintFlag>>, R, F: FnOnce() -> R>(
205    flags: G,
206    f: F,
207) -> R {
208    let prev = PRINT_FLAGS.replace(flags.into());
209    let res = f();
210    PRINT_FLAGS.set(prev);
211    res
212}
213
214/// Event represents all the things that happened simultaneously in a
215/// given execution cycle. Event may contain only one update for each
216/// variable and netidx subscription in a given cycle, if more updates
217/// happen simultaneously they must be queued and deferred to later
218/// cycles.
219#[derive(Debug)]
220pub struct Event<E: UserEvent> {
221    pub init: bool,
222    pub variables: FxHashMap<BindId, Value>,
223    pub netidx: FxHashMap<SubId, subscriber::Event>,
224    pub writes: FxHashMap<Id, WriteRequest>,
225    pub rpc_calls: FxHashMap<BindId, RpcCall>,
226    pub user: E,
227}
228
229impl<E: UserEvent> Event<E> {
230    pub fn new(user: E) -> Self {
231        Event {
232            init: false,
233            variables: HashMap::default(),
234            netidx: HashMap::default(),
235            writes: HashMap::default(),
236            rpc_calls: HashMap::default(),
237            user,
238        }
239    }
240
241    pub fn clear(&mut self) {
242        let Self { init, variables, netidx, rpc_calls, writes, user } = self;
243        *init = false;
244        variables.clear();
245        netidx.clear();
246        rpc_calls.clear();
247        writes.clear();
248        user.clear();
249    }
250}
251
252#[derive(Debug, Clone, Default)]
253pub struct Refs {
254    refed: LPooled<FxHashSet<BindId>>,
255    bound: LPooled<FxHashSet<BindId>>,
256}
257
258impl Refs {
259    pub fn clear(&mut self) {
260        self.refed.clear();
261        self.bound.clear();
262    }
263
264    pub fn with_external_refs(&self, mut f: impl FnMut(BindId)) {
265        for id in &*self.refed {
266            if !self.bound.contains(id) {
267                f(*id);
268            }
269        }
270    }
271}
272
273pub type Node<R, E> = Box<dyn Update<R, E>>;
274
275pub type BuiltInInitFn<R, E> = sync::Arc<
276    dyn for<'a, 'b, 'c> Fn(
277            &'a mut ExecCtx<R, E>,
278            &'a FnType,
279            &'b Scope,
280            &'c [Node<R, E>],
281            ExprId,
282        ) -> Result<Box<dyn Apply<R, E>>>
283        + Send
284        + Sync
285        + 'static,
286>;
287
288pub type InitFn<R, E> = sync::Arc<
289    dyn for<'a, 'b, 'c> Fn(
290            &'a Scope,
291            &'b mut ExecCtx<R, E>,
292            &'c [Node<R, E>],
293            ExprId,
294        ) -> Result<Box<dyn Apply<R, E>>>
295        + Send
296        + Sync
297        + 'static,
298>;
299
300/// Apply is a kind of node that represents a function application. It
301/// does not hold ownership of it's arguments, instead those are held
302/// by a CallSite node. This allows us to change the function called
303/// at runtime without recompiling the arguments.
304pub trait Apply<R: Rt, E: UserEvent>: Debug + Send + Sync + Any {
305    fn update(
306        &mut self,
307        ctx: &mut ExecCtx<R, E>,
308        from: &mut [Node<R, E>],
309        event: &mut Event<E>,
310    ) -> Option<Value>;
311
312    /// delete any internally generated nodes, only needed for
313    /// builtins that dynamically generate code at runtime
314    fn delete(&mut self, _ctx: &mut ExecCtx<R, E>) {
315        ()
316    }
317
318    /// apply custom typechecking to the lambda, only needed for
319    /// builtins that take lambdas as arguments
320    fn typecheck(
321        &mut self,
322        _ctx: &mut ExecCtx<R, E>,
323        _from: &mut [Node<R, E>],
324    ) -> Result<()> {
325        Ok(())
326    }
327
328    /// return the lambdas type, builtins do not need to implement
329    /// this, it is implemented by the BuiltIn wrapper
330    fn typ(&self) -> Arc<FnType> {
331        static EMPTY: LazyLock<Arc<FnType>> = LazyLock::new(|| {
332            Arc::new(FnType {
333                args: Arc::from_iter([]),
334                constraints: Arc::new(RwLock::new(LPooled::take())),
335                rtype: Type::Bottom,
336                throws: Type::Bottom,
337                vargs: None,
338            })
339        });
340        Arc::clone(&*EMPTY)
341    }
342
343    /// Populate the Refs structure with all the ids bound and refed by this
344    /// node. It is only necessary for builtins to implement this if they create
345    /// nodes, such as call sites.
346    fn refs<'a>(&self, _refs: &mut Refs) {}
347
348    /// put the node to sleep, used in conditions like select for branches that
349    /// are not selected. Any cached values should be cleared on sleep.
350    fn sleep(&mut self, _ctx: &mut ExecCtx<R, E>);
351}
352
353/// Update represents a regular graph node, as opposed to a function
354/// application represented by Apply. Regular graph nodes are used for
355/// every built in node except for builtin functions.
356pub trait Update<R: Rt, E: UserEvent>: Debug + Send + Sync + Any + 'static {
357    /// update the node with the specified event and return any output
358    /// it might generate
359    fn update(&mut self, ctx: &mut ExecCtx<R, E>, event: &mut Event<E>) -> Option<Value>;
360
361    /// delete the node and it's children from the specified context
362    fn delete(&mut self, ctx: &mut ExecCtx<R, E>);
363
364    /// type check the node and it's children
365    fn typecheck(&mut self, ctx: &mut ExecCtx<R, E>) -> Result<()>;
366
367    /// return the node type
368    fn typ(&self) -> &Type;
369
370    /// Populate the Refs structure with all the bind ids either refed or bound
371    /// by the node and it's children
372    fn refs(&self, refs: &mut Refs);
373
374    /// return the original expression used to compile this node
375    fn spec(&self) -> &Expr;
376
377    /// put the node to sleep, called on unselected branches
378    fn sleep(&mut self, ctx: &mut ExecCtx<R, E>);
379}
380
381pub trait BuiltIn<R: Rt, E: UserEvent> {
382    const NAME: &str;
383    const TYP: LazyLock<FnType>;
384
385    fn init(ctx: &mut ExecCtx<R, E>) -> BuiltInInitFn<R, E>;
386}
387
388pub trait Rt: Debug + 'static {
389    fn clear(&mut self);
390
391    /// Subscribe to the specified netidx path. When the subscription
392    /// updates you are expected to deliver Netidx events to the
393    /// expression specified by ref_by.
394    fn subscribe(&mut self, flags: UpdatesFlags, path: Path, ref_by: ExprId) -> Dval;
395
396    /// Called when a subscription is no longer needed
397    fn unsubscribe(&mut self, path: Path, dv: Dval, ref_by: ExprId);
398
399    /// List the netidx path, return Value::Null if the path did not
400    /// change. When the path did update you should send the output
401    /// back as a properly formatted struct with two fields, rows and
402    /// columns both containing string arrays.
403    fn list(&mut self, id: BindId, path: Path);
404
405    /// List the table at path, return Value::Null if the path did not
406    /// change
407    fn list_table(&mut self, id: BindId, path: Path);
408
409    /// list or table will no longer be called on this BindId, and
410    /// related resources can be cleaned up.
411    fn stop_list(&mut self, id: BindId);
412
413    /// Publish the specified value, returning it's Id, which must be
414    /// used to update the value and unpublish it. If the path is
415    /// already published, return an error.
416    fn publish(&mut self, path: Path, value: Value, ref_by: ExprId) -> Result<Val>;
417
418    /// Update the specified value
419    fn update(&mut self, id: &Val, value: Value);
420
421    /// Stop publishing the specified id
422    fn unpublish(&mut self, id: Val, ref_by: ExprId);
423
424    /// This will be called by the compiler whenever a bound variable
425    /// is referenced. The ref_by is the toplevel expression that
426    /// contains the variable reference. When a variable event
427    /// happens, you should update all the toplevel expressions that
428    /// ref that variable.
429    ///
430    /// ref_var will also be called when a bound lambda expression is
431    /// referenced, in that case the ref_by id will be the toplevel
432    /// expression containing the call site.
433    fn ref_var(&mut self, id: BindId, ref_by: ExprId);
434    fn unref_var(&mut self, id: BindId, ref_by: ExprId);
435
436    /// Called by the ExecCtx when set_var is called on it.
437    ///
438    /// All expressions that ref the id should be updated when this happens. The
439    /// runtime must deliver all set_vars in a single event except that set_vars
440    /// for the same variable in the same cycle must be queued and deferred to
441    /// the next cycle.
442    ///
443    /// The runtime MUST NOT change event while a cycle is in
444    /// progress. set_var must be queued until the cycle ends and then
445    /// presented as a new batch.
446    fn set_var(&mut self, id: BindId, value: Value);
447
448    /// Notify the RT that a top level variable has been set internally
449    ///
450    /// This is called when the compiler has determined that it's safe to set a
451    /// variable without waiting a cycle. When the updated variable is a
452    /// toplevel node this method is called to notify the runtime that needs to
453    /// update any dependent toplevel nodes.
454    fn notify_set(&mut self, id: BindId);
455
456    /// This must return results from the same path in the call order.
457    ///
458    /// when the rpc returns you are expected to deliver a Variable
459    /// event with the specified id to the expression specified by
460    /// ref_by.
461    fn call_rpc(&mut self, name: Path, args: Vec<(ArcStr, Value)>, id: BindId);
462
463    /// Publish an rpc at the specified path with the specified
464    /// procedure level doc and arg spec.
465    ///
466    /// When the RPC is called the rpc table in event will be
467    /// populated under the specified bind id.
468    ///
469    /// If the procedure is already published an error will be
470    /// returned
471    fn publish_rpc(
472        &mut self,
473        name: Path,
474        doc: Value,
475        spec: Vec<ArgSpec>,
476        id: BindId,
477    ) -> Result<()>;
478
479    /// unpublish the rpc identified by the bind id.
480    fn unpublish_rpc(&mut self, name: Path);
481
482    /// arrange to have a Timer event delivered after timeout. When
483    /// the timer expires you are expected to deliver a Variable event
484    /// for the id, containing the current time.
485    fn set_timer(&mut self, id: BindId, timeout: Duration);
486}
487
488pub struct ExecCtx<R: Rt, E: UserEvent> {
489    builtins: FxHashMap<&'static str, (FnType, BuiltInInitFn<R, E>)>,
490    builtins_allowed: bool,
491    tags: FxHashSet<ArcStr>,
492    pub env: Env<R, E>,
493    pub cached: FxHashMap<BindId, Value>,
494    pub rt: R,
495}
496
497impl<R: Rt, E: UserEvent> ExecCtx<R, E> {
498    pub fn clear(&mut self) {
499        self.env.clear();
500        self.rt.clear();
501    }
502
503    /// Build a new execution context.
504    ///
505    /// This is a very low level interface that you can use to build a
506    /// custom runtime with deep integration to your code. It is very
507    /// difficult to use, and if you don't implement everything
508    /// correctly the semantics of the language can be wrong.
509    ///
510    /// Most likely you want to use the `rt` module instead.
511    pub fn new(user: R) -> Self {
512        Self {
513            env: Env::new(),
514            builtins: FxHashMap::default(),
515            builtins_allowed: true,
516            tags: FxHashSet::default(),
517            cached: HashMap::default(),
518            rt: user,
519        }
520    }
521
522    pub fn register_builtin<T: BuiltIn<R, E>>(&mut self) -> Result<()> {
523        let f = T::init(self);
524        match self.builtins.entry(T::NAME) {
525            Entry::Vacant(e) => {
526                e.insert((T::TYP.clone(), f));
527            }
528            Entry::Occupied(_) => bail!("builtin {} is already registered", T::NAME),
529        }
530        Ok(())
531    }
532
533    /// Built in functions should call this when variables are set
534    /// unless they are sure the variable does not need to be
535    /// cached. This will also call the user ctx set_var.
536    pub fn set_var(&mut self, id: BindId, v: Value) {
537        self.cached.insert(id, v.clone());
538        self.rt.set_var(id, v)
539    }
540
541    fn tag(&mut self, s: &ArcStr) -> ArcStr {
542        match self.tags.get(s) {
543            Some(s) => s.clone(),
544            None => {
545                self.tags.insert(s.clone());
546                s.clone()
547            }
548        }
549    }
550
551    /// Restore the lexical environment to the snapshot `env` for the
552    /// duration of `f` restoring it to it's original value
553    /// afterwords. `by_id` and `lambdas` defined by the closure will
554    /// be retained.
555    pub fn with_restored<T, F: FnOnce(&mut Self) -> T>(
556        &mut self,
557        env: Env<R, E>,
558        f: F,
559    ) -> T {
560        let snap = self.env.restore_lexical_env(env);
561        let orig = mem::replace(&mut self.env, snap);
562        let r = f(self);
563        self.env = self.env.restore_lexical_env(orig);
564        r
565    }
566}
567
568#[derive(Debug, Clone)]
569pub struct Scope {
570    pub lexical: ModPath,
571    pub dynamic: ModPath,
572}
573
574impl Scope {
575    pub fn append<S: AsRef<str> + ?Sized>(&self, s: &S) -> Self {
576        Self {
577            lexical: ModPath(self.lexical.append(s)),
578            dynamic: ModPath(self.dynamic.append(s)),
579        }
580    }
581
582    pub fn root() -> Self {
583        Self { lexical: ModPath::root(), dynamic: ModPath::root() }
584    }
585}
586
587/// compile the expression into a node graph in the specified context
588/// and scope, return the root node or an error if compilation failed.
589pub fn compile<R: Rt, E: UserEvent>(
590    ctx: &mut ExecCtx<R, E>,
591    flags: BitFlags<CFlag>,
592    scope: &Scope,
593    spec: Expr,
594) -> Result<Node<R, E>> {
595    let top_id = spec.id;
596    let env = ctx.env.clone();
597    let st = Instant::now();
598    let mut node = match compiler::compile(ctx, flags, spec, scope, top_id) {
599        Ok(n) => n,
600        Err(e) => {
601            ctx.env = env;
602            return Err(e);
603        }
604    };
605    info!("compile time {:?}", st.elapsed());
606    let st = Instant::now();
607    if let Err(e) = node.typecheck(ctx) {
608        ctx.env = env;
609        return Err(e);
610    }
611    info!("typecheck time {:?}", st.elapsed());
612    Ok(node)
613}
graphix_compiler/lib.rs

graphix_compiler/
lib.rs
