graphix_compiler/expr/
resolver.rs

1use crate::expr::SourceOrigin;
2
3use super::{
4    parser, Bind, Expr, ExprId, ExprKind, Lambda, ModPath, ModuleKind, Origin, Pattern,
5};
6use anyhow::{anyhow, bail, Context, Result};
7use arcstr::ArcStr;
8use combine::stream::position::SourcePosition;
9use futures::future::try_join_all;
10use fxhash::FxHashMap;
11use log::info;
12use netidx::{
13    path::Path,
14    subscriber::{Event, Subscriber},
15    utils::{self, Either},
16};
17use netidx_value::Value;
18use std::{path::PathBuf, pin::Pin, str::FromStr, time::Duration};
19use tokio::{task, time::Instant, try_join};
20use triomphe::Arc;
21
22#[derive(Debug, Clone)]
23pub enum ModuleResolver {
24    VFS(FxHashMap<Path, ArcStr>),
25    Files(PathBuf),
26    Netidx { subscriber: Subscriber, base: Path, timeout: Option<Duration> },
27}
28
29impl ModuleResolver {
30    /// Parse a comma separated list of module resolvers. Netidx
31    /// resolvers are of the form, netidx:/path/in/netidx, and
32    /// filesystem resolvers are of the form file:/path/in/fs
33    ///
34    /// This format is intended to be used in an environment variable,
35    /// for example.
36    pub fn parse_env(
37        subscriber: Subscriber,
38        timeout: Option<Duration>,
39        s: &str,
40    ) -> Result<Vec<Self>> {
41        let mut res = vec![];
42        for l in utils::split_escaped(s, '\\', ',') {
43            let l = l.trim();
44            if let Some(s) = l.strip_prefix("netidx:") {
45                let base = Path::from_str(s);
46                let r = Self::Netidx { subscriber: subscriber.clone(), timeout, base };
47                res.push(r);
48            } else if let Some(s) = l.strip_prefix("file:") {
49                let base = PathBuf::from_str(s)?;
50                let r = Self::Files(base);
51                res.push(r);
52            } else {
53                bail!("expected netidx: or file:")
54            }
55        }
56        Ok(res)
57    }
58}
59
60impl Expr {
61    pub fn has_unresolved_modules(&self) -> bool {
62        self.fold(false, &mut |acc, e| {
63            acc || match &e.kind {
64                ExprKind::Module { value: ModuleKind::Unresolved, .. } => true,
65                _ => false,
66            }
67        })
68    }
69
70    /// Resolve external modules referenced in the expression using
71    /// the resolvers list. Each resolver will be tried in order,
72    /// until one succeeds. If no resolver succeeds then an error will
73    /// be returned.
74    pub async fn resolve_modules<'a>(
75        &'a self,
76        resolvers: &'a Arc<[ModuleResolver]>,
77    ) -> Result<Expr> {
78        self.resolve_modules_int(&ModPath::root(), &None, resolvers).await
79    }
80
81    async fn resolve_modules_int<'a>(
82        &'a self,
83        scope: &ModPath,
84        prepend: &'a Option<Arc<ModuleResolver>>,
85        resolvers: &'a Arc<[ModuleResolver]>,
86    ) -> Result<Expr> {
87        if self.has_unresolved_modules() {
88            self.resolve_modules_inner(scope, prepend, resolvers).await
89        } else {
90            Ok(self.clone())
91        }
92    }
93
94    fn resolve_modules_inner<'a>(
95        &'a self,
96        scope: &'a ModPath,
97        prepend: &'a Option<Arc<ModuleResolver>>,
98        resolvers: &'a Arc<[ModuleResolver]>,
99    ) -> Pin<Box<dyn Future<Output = Result<Expr>> + Send + Sync + 'a>> {
100        macro_rules! subexprs {
101            ($args:expr) => {{
102                try_join_all($args.iter().map(|e| async {
103                    e.resolve_modules_int(scope, prepend, resolvers).await
104                }))
105                .await?
106            }};
107        }
108        macro_rules! subtuples {
109            ($args:expr) => {{
110                try_join_all($args.iter().map(|(k, e)| async {
111                    Ok::<_, anyhow::Error>((
112                        k.clone(),
113                        e.resolve_modules_int(scope, prepend, resolvers).await?,
114                    ))
115                }))
116                .await?
117            }};
118        }
119        macro_rules! only_args {
120            ($kind:ident, $args:expr) => {
121                Box::pin(async move {
122                    let args = Arc::from(subexprs!($args));
123                    Ok(Expr {
124                        id: self.id,
125                        pos: self.pos,
126                        kind: ExprKind::$kind { args },
127                    })
128                })
129            };
130        }
131        macro_rules! bin_op {
132            ($kind:ident, $lhs:expr, $rhs:expr) => {
133                Box::pin(async move {
134                    let (lhs, rhs) = try_join!(
135                        $lhs.resolve_modules_int(scope, prepend, resolvers),
136                        $rhs.resolve_modules_int(scope, prepend, resolvers)
137                    )?;
138                    Ok(Expr {
139                        id: self.id,
140                        pos: self.pos,
141                        kind: ExprKind::$kind {
142                            lhs: Arc::from(lhs),
143                            rhs: Arc::from(rhs),
144                        },
145                    })
146                })
147            };
148        }
149        async fn resolve(
150            scope: ModPath,
151            prepend: Option<Arc<ModuleResolver>>,
152            resolvers: Arc<[ModuleResolver]>,
153            id: ExprId,
154            pos: SourcePosition,
155            export: bool,
156            name: ArcStr,
157        ) -> Result<Expr> {
158            let jh = task::spawn(async move {
159                let ts = Instant::now();
160                let name_rel = name.trim_start_matches(Path::SEP);
161                let name_mod = Path::from(name.clone()).append("mod.gx");
162                let name_mod = name_mod.trim_start_matches(Path::SEP);
163                let mut errors = vec![];
164                for r in prepend.iter().map(|r| r.as_ref()).chain(resolvers.iter()) {
165                    let (origin, s) = match r {
166                        ModuleResolver::VFS(vfs) => {
167                            let scoped = scope.append(&*name);
168                            match vfs.get(&scoped) {
169                                Some(s) => {
170                                    let origin = SourceOrigin::Internal(name.clone());
171                                    (origin, s.clone())
172                                }
173                                None => continue,
174                            }
175                        }
176                        ModuleResolver::Files(base) => {
177                            let full_path = base
178                                .join(name_rel)
179                                .with_extension("gx")
180                                .canonicalize()?;
181                            match tokio::fs::read_to_string(&full_path).await {
182                                Ok(s) => (SourceOrigin::File(full_path), ArcStr::from(s)),
183                                Err(_) => {
184                                    let full_path = base.join(name_mod).canonicalize()?;
185                                    match tokio::fs::read_to_string(&full_path).await {
186                                        Ok(s) => (
187                                            SourceOrigin::File(full_path),
188                                            ArcStr::from(s),
189                                        ),
190                                        Err(e) => {
191                                            errors.push(anyhow::Error::from(e));
192                                            continue;
193                                        }
194                                    }
195                                }
196                            }
197                        }
198                        ModuleResolver::Netidx { subscriber, base, timeout } => {
199                            let full_path = base.append(name_rel);
200                            let origin = SourceOrigin::Netidx(full_path.clone());
201                            let sub = subscriber
202                                .subscribe_nondurable_one(full_path, *timeout)
203                                .await;
204                            match sub {
205                                Err(e) => {
206                                    errors.push(e);
207                                    continue;
208                                }
209                                Ok(v) => match v.last() {
210                                    Event::Update(Value::String(s)) => (origin, s),
211                                    Event::Unsubscribed | Event::Update(_) => {
212                                        errors.push(anyhow!("expected string"));
213                                        continue;
214                                    }
215                                },
216                            }
217                        }
218                    };
219                    let value = ModuleKind::Resolved(
220                        parser::parse(origin.clone(), s)
221                            .with_context(|| format!("parsing file {origin:?}"))?,
222                    );
223                    let kind = ExprKind::Module { name, export, value };
224                    info!("load and parse {origin:?} {:?}", ts.elapsed());
225                    return Ok(Expr { id, pos, kind });
226                }
227                bail!("module {name} could not be found {errors:?}")
228            });
229            jh.await?
230        }
231        if !self.has_unresolved_modules() {
232            return Box::pin(async { Ok(self.clone()) });
233        }
234        match self.kind.clone() {
235            ExprKind::Module { value: ModuleKind::Unresolved, export, name } => {
236                let (id, pos, prepend, resolvers) =
237                    (self.id, self.pos, prepend.clone(), Arc::clone(resolvers));
238                Box::pin(async move {
239                    let e = resolve(
240                        scope.clone(),
241                        prepend.clone(),
242                        resolvers.clone(),
243                        id,
244                        pos,
245                        export,
246                        name.clone(),
247                    )
248                    .await?;
249                    let scope = ModPath(scope.append(&*name));
250                    e.resolve_modules_int(&scope, &prepend, &resolvers).await
251                })
252            }
253            ExprKind::Constant(_)
254            | ExprKind::Use { .. }
255            | ExprKind::Ref { .. }
256            | ExprKind::StructRef { .. }
257            | ExprKind::TupleRef { .. }
258            | ExprKind::TypeDef { .. } => Box::pin(async move { Ok(self.clone()) }),
259            ExprKind::Module { value: ModuleKind::Inline(exprs), export, name } => {
260                Box::pin(async move {
261                    let scope = ModPath(scope.append(&*name));
262                    let exprs = try_join_all(exprs.iter().map(|e| async {
263                        e.resolve_modules_int(&scope, prepend, resolvers).await
264                    }))
265                    .await?;
266                    Ok(Expr {
267                        id: self.id,
268                        pos: self.pos,
269                        kind: ExprKind::Module {
270                            value: ModuleKind::Inline(Arc::from(exprs)),
271                            name,
272                            export,
273                        },
274                    })
275                })
276            }
277            ExprKind::Module { value: ModuleKind::Resolved(o), export, name } => {
278                Box::pin(async move {
279                    let prepend = match &o.origin {
280                        SourceOrigin::Unspecified | SourceOrigin::Internal(_) => None,
281                        SourceOrigin::File(p) => {
282                            p.parent().map(|p| Arc::new(ModuleResolver::Files(p.into())))
283                        }
284                        SourceOrigin::Netidx(p) => {
285                            resolvers.iter().find_map(|m| match m {
286                                ModuleResolver::Netidx {
287                                    subscriber, timeout, ..
288                                } => Some(Arc::new(ModuleResolver::Netidx {
289                                    subscriber: subscriber.clone(),
290                                    base: p.clone(),
291                                    timeout: *timeout,
292                                })),
293                                ModuleResolver::Files(_) | ModuleResolver::VFS(_) => None,
294                            })
295                        }
296                    };
297                    let exprs = try_join_all(o.exprs.iter().map(|e| async {
298                        e.resolve_modules_int(&scope, &prepend, resolvers).await
299                    }))
300                    .await?;
301                    Ok(Expr {
302                        id: self.id,
303                        pos: self.pos,
304                        kind: ExprKind::Module {
305                            value: ModuleKind::Resolved(Origin {
306                                exprs: Arc::from(exprs),
307                                ..o.clone()
308                            }),
309                            name,
310                            export,
311                        },
312                    })
313                })
314            }
315            ExprKind::Do { exprs } => Box::pin(async move {
316                let exprs = Arc::from(subexprs!(exprs));
317                Ok(Expr { id: self.id, pos: self.pos, kind: ExprKind::Do { exprs } })
318            }),
319            ExprKind::Bind(b) => Box::pin(async move {
320                let Bind { doc, pattern, typ, export, value } = &*b;
321                let value = value.resolve_modules_int(scope, prepend, resolvers).await?;
322                Ok(Expr {
323                    id: self.id,
324                    pos: self.pos,
325                    kind: ExprKind::Bind(Arc::new(Bind {
326                        doc: doc.clone(),
327                        pattern: pattern.clone(),
328                        typ: typ.clone(),
329                        export: *export,
330                        value,
331                    })),
332                })
333            }),
334            ExprKind::StructWith { source, replace } => Box::pin(async move {
335                Ok(Expr {
336                    id: self.id,
337                    pos: self.pos,
338                    kind: ExprKind::StructWith {
339                        source: Arc::new(
340                            source.resolve_modules_int(scope, prepend, resolvers).await?,
341                        ),
342                        replace: Arc::from(subtuples!(replace)),
343                    },
344                })
345            }),
346            ExprKind::Connect { name, value, deref } => Box::pin(async move {
347                let value = value.resolve_modules_int(scope, prepend, resolvers).await?;
348                Ok(Expr {
349                    id: self.id,
350                    pos: self.pos,
351                    kind: ExprKind::Connect { name, value: Arc::new(value), deref },
352                })
353            }),
354            ExprKind::Lambda(l) => Box::pin(async move {
355                let Lambda { args, vargs, rtype, constraints, body } = &*l;
356                let body = match body {
357                    Either::Right(s) => Either::Right(s.clone()),
358                    Either::Left(e) => Either::Left(
359                        e.resolve_modules_int(scope, prepend, resolvers).await?,
360                    ),
361                };
362                let l = Lambda {
363                    args: args.clone(),
364                    vargs: vargs.clone(),
365                    rtype: rtype.clone(),
366                    constraints: constraints.clone(),
367                    body,
368                };
369                let kind = ExprKind::Lambda(Arc::new(l));
370                Ok(Expr { id: self.id, pos: self.pos, kind })
371            }),
372            ExprKind::TypeCast { expr, typ } => Box::pin(async move {
373                let expr = expr.resolve_modules_int(scope, prepend, resolvers).await?;
374                Ok(Expr {
375                    id: self.id,
376                    pos: self.pos,
377                    kind: ExprKind::TypeCast { expr: Arc::new(expr), typ },
378                })
379            }),
380            ExprKind::Apply { args, function } => Box::pin(async move {
381                Ok(Expr {
382                    id: self.id,
383                    pos: self.pos,
384                    kind: ExprKind::Apply { args: Arc::from(subtuples!(args)), function },
385                })
386            }),
387            ExprKind::Any { args } => only_args!(Any, args),
388            ExprKind::Array { args } => only_args!(Array, args),
389            ExprKind::Tuple { args } => only_args!(Tuple, args),
390            ExprKind::StringInterpolate { args } => only_args!(StringInterpolate, args),
391            ExprKind::Struct { args } => Box::pin(async move {
392                let args = Arc::from(subtuples!(args));
393                Ok(Expr { id: self.id, pos: self.pos, kind: ExprKind::Struct { args } })
394            }),
395            ExprKind::ArrayRef { source, i } => Box::pin(async move {
396                let source = Arc::new(
397                    source.resolve_modules_int(scope, prepend, resolvers).await?,
398                );
399                let i = Arc::new(i.resolve_modules_int(scope, prepend, resolvers).await?);
400                Ok(Expr {
401                    id: self.id,
402                    pos: self.pos,
403                    kind: ExprKind::ArrayRef { source, i },
404                })
405            }),
406            ExprKind::ArraySlice { source, start, end } => Box::pin(async move {
407                let source = Arc::new(
408                    source.resolve_modules_int(scope, prepend, resolvers).await?,
409                );
410                let start = match start {
411                    None => None,
412                    Some(e) => Some(Arc::new(
413                        e.resolve_modules_int(scope, prepend, resolvers).await?,
414                    )),
415                };
416                let end = match end {
417                    None => None,
418                    Some(e) => Some(Arc::new(
419                        e.resolve_modules_int(scope, prepend, resolvers).await?,
420                    )),
421                };
422                Ok(Expr {
423                    id: self.id,
424                    pos: self.pos,
425                    kind: ExprKind::ArraySlice { source, start, end },
426                })
427            }),
428            ExprKind::Variant { tag, args } => Box::pin(async move {
429                let args = Arc::from(subexprs!(args));
430                Ok(Expr {
431                    id: self.id,
432                    pos: self.pos,
433                    kind: ExprKind::Variant { tag, args },
434                })
435            }),
436            ExprKind::Select { arg, arms } => Box::pin(async move {
437                let arg =
438                    Arc::new(arg.resolve_modules_int(scope, prepend, resolvers).await?);
439                let arms = try_join_all(arms.iter().map(|(p, e)| async {
440                    let p = match &p.guard {
441                        None => p.clone(),
442                        Some(e) => {
443                            let e =
444                                e.resolve_modules_int(scope, prepend, resolvers).await?;
445                            Pattern {
446                                guard: Some(e),
447                                type_predicate: p.type_predicate.clone(),
448                                structure_predicate: p.structure_predicate.clone(),
449                            }
450                        }
451                    };
452                    let e = e.resolve_modules_int(scope, prepend, resolvers).await?;
453                    Ok::<_, anyhow::Error>((p, e))
454                }))
455                .await?;
456                Ok(Expr {
457                    id: self.id,
458                    pos: self.pos,
459                    kind: ExprKind::Select { arg, arms: Arc::from(arms) },
460                })
461            }),
462            ExprKind::Qop(e) => Box::pin(async move {
463                let e = e.resolve_modules_int(scope, prepend, resolvers).await?;
464                Ok(Expr { id: self.id, pos: self.pos, kind: ExprKind::Qop(Arc::new(e)) })
465            }),
466            ExprKind::ByRef(e) => Box::pin(async move {
467                let e = e.resolve_modules_int(scope, prepend, resolvers).await?;
468                Ok(Expr {
469                    id: self.id,
470                    pos: self.pos,
471                    kind: ExprKind::ByRef(Arc::new(e)),
472                })
473            }),
474            ExprKind::Deref(e) => Box::pin(async move {
475                let e = e.resolve_modules_int(scope, prepend, resolvers).await?;
476                Ok(Expr {
477                    id: self.id,
478                    pos: self.pos,
479                    kind: ExprKind::Deref(Arc::new(e)),
480                })
481            }),
482            ExprKind::Not { expr: e } => Box::pin(async move {
483                let e = e.resolve_modules_int(scope, prepend, resolvers).await?;
484                Ok(Expr {
485                    id: self.id,
486                    pos: self.pos,
487                    kind: ExprKind::Not { expr: Arc::new(e) },
488                })
489            }),
490            ExprKind::Add { lhs, rhs } => bin_op!(Add, lhs, rhs),
491            ExprKind::Sub { lhs, rhs } => bin_op!(Sub, lhs, rhs),
492            ExprKind::Mul { lhs, rhs } => bin_op!(Mul, lhs, rhs),
493            ExprKind::Div { lhs, rhs } => bin_op!(Div, lhs, rhs),
494            ExprKind::Mod { lhs, rhs } => bin_op!(Mul, lhs, rhs),
495            ExprKind::And { lhs, rhs } => bin_op!(And, lhs, rhs),
496            ExprKind::Or { lhs, rhs } => bin_op!(Or, lhs, rhs),
497            ExprKind::Eq { lhs, rhs } => bin_op!(Eq, lhs, rhs),
498            ExprKind::Ne { lhs, rhs } => bin_op!(Ne, lhs, rhs),
499            ExprKind::Gt { lhs, rhs } => bin_op!(Gt, lhs, rhs),
500            ExprKind::Lt { lhs, rhs } => bin_op!(Lt, lhs, rhs),
501            ExprKind::Gte { lhs, rhs } => bin_op!(Gte, lhs, rhs),
502            ExprKind::Lte { lhs, rhs } => bin_op!(Lte, lhs, rhs),
503            ExprKind::Sample { lhs, rhs } => bin_op!(Sample, lhs, rhs),
504        }
505    }
506}
graphix_compiler/expr/resolver.rs

graphix_compiler/expr/
resolver.rs
