graphix_compiler/expr/
resolver.rs

1use crate::expr::{
2    parser, Bind, Expr, ExprId, ExprKind, Lambda, ModPath, ModuleKind, Origin, Pattern,
3    Source,
4};
5use anyhow::{anyhow, bail, Context, Result};
6use arcstr::ArcStr;
7use combine::stream::position::SourcePosition;
8use futures::future::try_join_all;
9use fxhash::FxHashMap;
10use log::info;
11use netidx::{
12    path::Path,
13    subscriber::{Event, Subscriber},
14    utils::{self, Either},
15};
16use netidx_value::Value;
17use std::{path::PathBuf, pin::Pin, str::FromStr, time::Duration};
18use tokio::{task, time::Instant, try_join};
19use triomphe::Arc;
20
21#[derive(Debug, Clone)]
22pub enum ModuleResolver {
23    VFS(FxHashMap<Path, ArcStr>),
24    Files(PathBuf),
25    Netidx { subscriber: Subscriber, base: Path, timeout: Option<Duration> },
26}
27
28impl ModuleResolver {
29    /// Parse a comma separated list of module resolvers. Netidx
30    /// resolvers are of the form, netidx:/path/in/netidx, and
31    /// filesystem resolvers are of the form file:/path/in/fs
32    ///
33    /// This format is intended to be used in an environment variable,
34    /// for example.
35    pub fn parse_env(
36        subscriber: Subscriber,
37        timeout: Option<Duration>,
38        s: &str,
39    ) -> Result<Vec<Self>> {
40        let mut res = vec![];
41        for l in utils::split_escaped(s, '\\', ',') {
42            let l = l.trim();
43            if let Some(s) = l.strip_prefix("netidx:") {
44                let base = Path::from_str(s);
45                let r = Self::Netidx { subscriber: subscriber.clone(), timeout, base };
46                res.push(r);
47            } else if let Some(s) = l.strip_prefix("file:") {
48                let base = PathBuf::from_str(s)?;
49                let r = Self::Files(base);
50                res.push(r);
51            } else {
52                bail!("expected netidx: or file:")
53            }
54        }
55        Ok(res)
56    }
57}
58
59impl Expr {
60    pub fn has_unresolved_modules(&self) -> bool {
61        self.fold(false, &mut |acc, e| {
62            acc || match &e.kind {
63                ExprKind::Module { value: ModuleKind::Unresolved, .. } => true,
64                _ => false,
65            }
66        })
67    }
68
69    /// Resolve external modules referenced in the expression using
70    /// the resolvers list. Each resolver will be tried in order,
71    /// until one succeeds. If no resolver succeeds then an error will
72    /// be returned.
73    pub async fn resolve_modules<'a>(
74        &'a self,
75        resolvers: &'a Arc<[ModuleResolver]>,
76    ) -> Result<Expr> {
77        self.resolve_modules_int(&ModPath::root(), &None, resolvers).await
78    }
79
80    async fn resolve_modules_int<'a>(
81        &'a self,
82        scope: &ModPath,
83        prepend: &'a Option<Arc<ModuleResolver>>,
84        resolvers: &'a Arc<[ModuleResolver]>,
85    ) -> Result<Expr> {
86        if self.has_unresolved_modules() {
87            self.resolve_modules_inner(scope, prepend, resolvers).await
88        } else {
89            Ok(self.clone())
90        }
91    }
92
93    fn resolve_modules_inner<'a>(
94        &'a self,
95        scope: &'a ModPath,
96        prepend: &'a Option<Arc<ModuleResolver>>,
97        resolvers: &'a Arc<[ModuleResolver]>,
98    ) -> Pin<Box<dyn Future<Output = Result<Expr>> + Send + Sync + 'a>> {
99        macro_rules! subexprs {
100            ($args:expr) => {{
101                try_join_all($args.iter().map(|e| async {
102                    e.resolve_modules_int(scope, prepend, resolvers).await
103                }))
104                .await?
105            }};
106        }
107        macro_rules! subtuples {
108            ($args:expr) => {{
109                try_join_all($args.iter().map(|(k, e)| async {
110                    Ok::<_, anyhow::Error>((
111                        k.clone(),
112                        e.resolve_modules_int(scope, prepend, resolvers).await?,
113                    ))
114                }))
115                .await?
116            }};
117        }
118        macro_rules! only_args {
119            ($kind:ident, $args:expr) => {
120                Box::pin(async move {
121                    let args = Arc::from(subexprs!($args));
122                    Ok(Expr {
123                        id: self.id,
124                        ori: self.ori.clone(),
125                        pos: self.pos,
126                        kind: ExprKind::$kind { args },
127                    })
128                })
129            };
130        }
131        macro_rules! bin_op {
132            ($kind:ident, $lhs:expr, $rhs:expr) => {
133                Box::pin(async move {
134                    let (lhs, rhs) = try_join!(
135                        $lhs.resolve_modules_int(scope, prepend, resolvers),
136                        $rhs.resolve_modules_int(scope, prepend, resolvers)
137                    )?;
138                    Ok(Expr {
139                        id: self.id,
140                        ori: self.ori.clone(),
141                        pos: self.pos,
142                        kind: ExprKind::$kind {
143                            lhs: Arc::from(lhs),
144                            rhs: Arc::from(rhs),
145                        },
146                    })
147                })
148            };
149        }
150        async fn resolve(
151            scope: ModPath,
152            prepend: Option<Arc<ModuleResolver>>,
153            resolvers: Arc<[ModuleResolver]>,
154            id: ExprId,
155            parent: Arc<Origin>,
156            pos: SourcePosition,
157            export: bool,
158            name: ArcStr,
159        ) -> Result<Expr> {
160            let jh = task::spawn(async move {
161                let ts = Instant::now();
162                let name_rel = name.trim_start_matches(Path::SEP);
163                let name_mod = Path::from(name.clone()).append("mod.gx");
164                let name_mod = name_mod.trim_start_matches(Path::SEP);
165                let mut errors = vec![];
166                for r in prepend.iter().map(|r| r.as_ref()).chain(resolvers.iter()) {
167                    let ori = match r {
168                        ModuleResolver::VFS(vfs) => {
169                            let scoped = scope.append(&*name);
170                            match vfs.get(&scoped) {
171                                Some(s) => Origin {
172                                    parent: Some(parent.clone()),
173                                    source: Source::Internal(name.clone()),
174                                    text: s.clone(),
175                                },
176                                None => continue,
177                            }
178                        }
179                        ModuleResolver::Files(base) => {
180                            let full_path = base
181                                .join(name_rel)
182                                .with_extension("gx")
183                                .canonicalize()?;
184                            match tokio::fs::read_to_string(&full_path).await {
185                                Ok(s) => Origin {
186                                    parent: Some(parent.clone()),
187                                    source: Source::File(full_path),
188                                    text: ArcStr::from(s),
189                                },
190                                Err(_) => {
191                                    let full_path = base.join(name_mod).canonicalize()?;
192                                    match tokio::fs::read_to_string(&full_path).await {
193                                        Ok(s) => Origin {
194                                            parent: Some(parent.clone()),
195                                            source: Source::File(full_path),
196                                            text: ArcStr::from(s),
197                                        },
198                                        Err(e) => {
199                                            errors.push(anyhow::Error::from(e));
200                                            continue;
201                                        }
202                                    }
203                                }
204                            }
205                        }
206                        ModuleResolver::Netidx { subscriber, base, timeout } => {
207                            let full_path = base.append(name_rel);
208                            let source = Source::Netidx(full_path.clone());
209                            let sub = subscriber
210                                .subscribe_nondurable_one(full_path, *timeout)
211                                .await;
212                            match sub {
213                                Err(e) => {
214                                    errors.push(e);
215                                    continue;
216                                }
217                                Ok(v) => match v.last() {
218                                    Event::Update(Value::String(text)) => Origin {
219                                        parent: Some(parent.clone()),
220                                        source,
221                                        text,
222                                    },
223                                    Event::Unsubscribed | Event::Update(_) => {
224                                        errors.push(anyhow!("expected string"));
225                                        continue;
226                                    }
227                                },
228                            }
229                        }
230                    };
231                    let value = ModuleKind::Resolved(
232                        parser::parse(ori.clone())
233                            .with_context(|| format!("parsing file {ori:?}"))?,
234                    );
235                    let kind = ExprKind::Module { name, export, value };
236                    info!("load and parse {ori:?} {:?}", ts.elapsed());
237                    return Ok(Expr { id, ori: Arc::new(ori), pos, kind });
238                }
239                bail!("module {name} could not be found {errors:?}")
240            });
241            jh.await?
242        }
243        if !self.has_unresolved_modules() {
244            return Box::pin(async { Ok(self.clone()) });
245        }
246        match self.kind.clone() {
247            ExprKind::Module { value: ModuleKind::Unresolved, export, name } => {
248                let (id, pos, prepend, resolvers) =
249                    (self.id, self.pos, prepend.clone(), Arc::clone(resolvers));
250                Box::pin(async move {
251                    let e = resolve(
252                        scope.clone(),
253                        prepend.clone(),
254                        resolvers.clone(),
255                        id,
256                        self.ori.clone(),
257                        pos,
258                        export,
259                        name.clone(),
260                    )
261                    .await?;
262                    let scope = ModPath(scope.append(&*name));
263                    e.resolve_modules_int(&scope, &prepend, &resolvers).await
264                })
265            }
266            ExprKind::Constant(_)
267            | ExprKind::Use { .. }
268            | ExprKind::Ref { .. }
269            | ExprKind::StructRef { .. }
270            | ExprKind::TupleRef { .. }
271            | ExprKind::TypeDef { .. } => Box::pin(async move { Ok(self.clone()) }),
272            ExprKind::Module { value: ModuleKind::Inline(exprs), export, name } => {
273                Box::pin(async move {
274                    let scope = ModPath(scope.append(&*name));
275                    let exprs = try_join_all(exprs.iter().map(|e| async {
276                        e.resolve_modules_int(&scope, prepend, resolvers).await
277                    }))
278                    .await?;
279                    Ok(Expr {
280                        id: self.id,
281                        ori: self.ori.clone(),
282                        pos: self.pos,
283                        kind: ExprKind::Module {
284                            value: ModuleKind::Inline(Arc::from(exprs)),
285                            name,
286                            export,
287                        },
288                    })
289                })
290            }
291            ExprKind::Module { value: ModuleKind::Resolved(exprs), export, name } => {
292                Box::pin(async move {
293                    let prepend = match &self.ori.source {
294                        Source::Unspecified | Source::Internal(_) => None,
295                        Source::File(p) => {
296                            p.parent().map(|p| Arc::new(ModuleResolver::Files(p.into())))
297                        }
298                        Source::Netidx(p) => resolvers.iter().find_map(|m| match m {
299                            ModuleResolver::Netidx { subscriber, timeout, .. } => {
300                                Some(Arc::new(ModuleResolver::Netidx {
301                                    subscriber: subscriber.clone(),
302                                    base: p.clone(),
303                                    timeout: *timeout,
304                                }))
305                            }
306                            ModuleResolver::Files(_) | ModuleResolver::VFS(_) => None,
307                        }),
308                    };
309                    let exprs = try_join_all(exprs.iter().map(|e| async {
310                        e.resolve_modules_int(&scope, &prepend, resolvers).await
311                    }))
312                    .await?;
313                    Ok(Expr {
314                        id: self.id,
315                        ori: self.ori.clone(),
316                        pos: self.pos,
317                        kind: ExprKind::Module {
318                            value: ModuleKind::Resolved(Arc::from(exprs)),
319                            name,
320                            export,
321                        },
322                    })
323                })
324            }
325            ExprKind::Do { exprs } => Box::pin(async move {
326                let exprs = Arc::from(subexprs!(exprs));
327                Ok(Expr {
328                    id: self.id,
329                    ori: self.ori.clone(),
330                    pos: self.pos,
331                    kind: ExprKind::Do { exprs },
332                })
333            }),
334            ExprKind::Bind(b) => Box::pin(async move {
335                let Bind { doc, pattern, typ, export, value } = &*b;
336                let value = value.resolve_modules_int(scope, prepend, resolvers).await?;
337                Ok(Expr {
338                    id: self.id,
339                    ori: self.ori.clone(),
340                    pos: self.pos,
341                    kind: ExprKind::Bind(Arc::new(Bind {
342                        doc: doc.clone(),
343                        pattern: pattern.clone(),
344                        typ: typ.clone(),
345                        export: *export,
346                        value,
347                    })),
348                })
349            }),
350            ExprKind::StructWith { source, replace } => Box::pin(async move {
351                Ok(Expr {
352                    id: self.id,
353                    ori: self.ori.clone(),
354                    pos: self.pos,
355                    kind: ExprKind::StructWith {
356                        source: Arc::new(
357                            source.resolve_modules_int(scope, prepend, resolvers).await?,
358                        ),
359                        replace: Arc::from(subtuples!(replace)),
360                    },
361                })
362            }),
363            ExprKind::Connect { name, value, deref } => Box::pin(async move {
364                let value = value.resolve_modules_int(scope, prepend, resolvers).await?;
365                Ok(Expr {
366                    id: self.id,
367                    ori: self.ori.clone(),
368                    pos: self.pos,
369                    kind: ExprKind::Connect { name, value: Arc::new(value), deref },
370                })
371            }),
372            ExprKind::Lambda(l) => Box::pin(async move {
373                let Lambda { args, vargs, rtype, constraints, body } = &*l;
374                let body = match body {
375                    Either::Right(s) => Either::Right(s.clone()),
376                    Either::Left(e) => Either::Left(
377                        e.resolve_modules_int(scope, prepend, resolvers).await?,
378                    ),
379                };
380                let l = Lambda {
381                    args: args.clone(),
382                    vargs: vargs.clone(),
383                    rtype: rtype.clone(),
384                    constraints: constraints.clone(),
385                    body,
386                };
387                let kind = ExprKind::Lambda(Arc::new(l));
388                Ok(Expr { id: self.id, ori: self.ori.clone(), pos: self.pos, kind })
389            }),
390            ExprKind::TypeCast { expr, typ } => Box::pin(async move {
391                let expr = expr.resolve_modules_int(scope, prepend, resolvers).await?;
392                Ok(Expr {
393                    id: self.id,
394                    ori: self.ori.clone(),
395                    pos: self.pos,
396                    kind: ExprKind::TypeCast { expr: Arc::new(expr), typ },
397                })
398            }),
399            ExprKind::Apply { args, function } => Box::pin(async move {
400                Ok(Expr {
401                    id: self.id,
402                    ori: self.ori.clone(),
403                    pos: self.pos,
404                    kind: ExprKind::Apply { args: Arc::from(subtuples!(args)), function },
405                })
406            }),
407            ExprKind::Any { args } => only_args!(Any, args),
408            ExprKind::Array { args } => only_args!(Array, args),
409            ExprKind::Tuple { args } => only_args!(Tuple, args),
410            ExprKind::StringInterpolate { args } => only_args!(StringInterpolate, args),
411            ExprKind::Struct { args } => Box::pin(async move {
412                let args = Arc::from(subtuples!(args));
413                Ok(Expr {
414                    id: self.id,
415                    ori: self.ori.clone(),
416                    pos: self.pos,
417                    kind: ExprKind::Struct { args },
418                })
419            }),
420            ExprKind::ArrayRef { source, i } => Box::pin(async move {
421                let source = Arc::new(
422                    source.resolve_modules_int(scope, prepend, resolvers).await?,
423                );
424                let i = Arc::new(i.resolve_modules_int(scope, prepend, resolvers).await?);
425                Ok(Expr {
426                    id: self.id,
427                    ori: self.ori.clone(),
428                    pos: self.pos,
429                    kind: ExprKind::ArrayRef { source, i },
430                })
431            }),
432            ExprKind::ArraySlice { source, start, end } => Box::pin(async move {
433                let source = Arc::new(
434                    source.resolve_modules_int(scope, prepend, resolvers).await?,
435                );
436                let start = match start {
437                    None => None,
438                    Some(e) => Some(Arc::new(
439                        e.resolve_modules_int(scope, prepend, resolvers).await?,
440                    )),
441                };
442                let end = match end {
443                    None => None,
444                    Some(e) => Some(Arc::new(
445                        e.resolve_modules_int(scope, prepend, resolvers).await?,
446                    )),
447                };
448                Ok(Expr {
449                    id: self.id,
450                    ori: self.ori.clone(),
451                    pos: self.pos,
452                    kind: ExprKind::ArraySlice { source, start, end },
453                })
454            }),
455            ExprKind::Variant { tag, args } => Box::pin(async move {
456                let args = Arc::from(subexprs!(args));
457                Ok(Expr {
458                    id: self.id,
459                    ori: self.ori.clone(),
460                    pos: self.pos,
461                    kind: ExprKind::Variant { tag, args },
462                })
463            }),
464            ExprKind::Select { arg, arms } => Box::pin(async move {
465                let arg =
466                    Arc::new(arg.resolve_modules_int(scope, prepend, resolvers).await?);
467                let arms = try_join_all(arms.iter().map(|(p, e)| async {
468                    let p = match &p.guard {
469                        None => p.clone(),
470                        Some(e) => {
471                            let e =
472                                e.resolve_modules_int(scope, prepend, resolvers).await?;
473                            Pattern {
474                                guard: Some(e),
475                                type_predicate: p.type_predicate.clone(),
476                                structure_predicate: p.structure_predicate.clone(),
477                            }
478                        }
479                    };
480                    let e = e.resolve_modules_int(scope, prepend, resolvers).await?;
481                    Ok::<_, anyhow::Error>((p, e))
482                }))
483                .await?;
484                Ok(Expr {
485                    id: self.id,
486                    ori: self.ori.clone(),
487                    pos: self.pos,
488                    kind: ExprKind::Select { arg, arms: Arc::from(arms) },
489                })
490            }),
491            ExprKind::Qop(e) => Box::pin(async move {
492                let e = e.resolve_modules_int(scope, prepend, resolvers).await?;
493                Ok(Expr {
494                    id: self.id,
495                    ori: self.ori.clone(),
496                    pos: self.pos,
497                    kind: ExprKind::Qop(Arc::new(e)),
498                })
499            }),
500            ExprKind::ByRef(e) => Box::pin(async move {
501                let e = e.resolve_modules_int(scope, prepend, resolvers).await?;
502                Ok(Expr {
503                    id: self.id,
504                    ori: self.ori.clone(),
505                    pos: self.pos,
506                    kind: ExprKind::ByRef(Arc::new(e)),
507                })
508            }),
509            ExprKind::Deref(e) => Box::pin(async move {
510                let e = e.resolve_modules_int(scope, prepend, resolvers).await?;
511                Ok(Expr {
512                    id: self.id,
513                    ori: self.ori.clone(),
514                    pos: self.pos,
515                    kind: ExprKind::Deref(Arc::new(e)),
516                })
517            }),
518            ExprKind::Not { expr: e } => Box::pin(async move {
519                let e = e.resolve_modules_int(scope, prepend, resolvers).await?;
520                Ok(Expr {
521                    id: self.id,
522                    ori: self.ori.clone(),
523                    pos: self.pos,
524                    kind: ExprKind::Not { expr: Arc::new(e) },
525                })
526            }),
527            ExprKind::Add { lhs, rhs } => bin_op!(Add, lhs, rhs),
528            ExprKind::Sub { lhs, rhs } => bin_op!(Sub, lhs, rhs),
529            ExprKind::Mul { lhs, rhs } => bin_op!(Mul, lhs, rhs),
530            ExprKind::Div { lhs, rhs } => bin_op!(Div, lhs, rhs),
531            ExprKind::Mod { lhs, rhs } => bin_op!(Mul, lhs, rhs),
532            ExprKind::And { lhs, rhs } => bin_op!(And, lhs, rhs),
533            ExprKind::Or { lhs, rhs } => bin_op!(Or, lhs, rhs),
534            ExprKind::Eq { lhs, rhs } => bin_op!(Eq, lhs, rhs),
535            ExprKind::Ne { lhs, rhs } => bin_op!(Ne, lhs, rhs),
536            ExprKind::Gt { lhs, rhs } => bin_op!(Gt, lhs, rhs),
537            ExprKind::Lt { lhs, rhs } => bin_op!(Lt, lhs, rhs),
538            ExprKind::Gte { lhs, rhs } => bin_op!(Gte, lhs, rhs),
539            ExprKind::Lte { lhs, rhs } => bin_op!(Lte, lhs, rhs),
540            ExprKind::Sample { lhs, rhs } => bin_op!(Sample, lhs, rhs),
541        }
542    }
543}
graphix_compiler/expr/resolver.rs

graphix_compiler/expr/
resolver.rs
