1use crate::{Gcx, hir::SourceId, ty::GcxMut};
2use rayon::prelude::*;
3use solar_ast::{self as ast, Span};
4use solar_data_structures::{
5 index::{Idx, IndexVec, index_vec},
6 map::{FxHashMap, FxHashSet},
7 sync::Mutex,
8};
9use solar_interface::{
10 Result, Session,
11 config::CompilerStage,
12 diagnostics::{DiagCtxt, ErrorGuaranteed},
13 source_map::{FileName, FileResolver, ResolveError, SourceFile},
14};
15use solar_parse::{Lexer, Parser, unescape};
16use std::{fmt, path::Path, sync::Arc};
17use thread_local::ThreadLocal;
18
19#[must_use = "`ParsingContext::parse` must be called to parse the sources"]
35pub struct ParsingContext<'gcx> {
36 pub sess: &'gcx Session,
38 pub file_resolver: FileResolver<'gcx>,
40 pub(crate) sources: &'gcx mut Sources<'gcx>,
42 pub(crate) arenas: &'gcx ThreadLocal<ast::Arena>,
44 resolve_imports: bool,
46 parsed: bool,
48 gcx: Gcx<'gcx>,
49}
50
51impl<'gcx> ParsingContext<'gcx> {
52 pub(crate) fn new(mut gcx_: GcxMut<'gcx>) -> Self {
54 let gcx = gcx_.get_mut();
55 let sess = gcx.sess;
56 let mut file_resolver = FileResolver::new(sess.source_map());
57 file_resolver.configure_from_sess(sess);
58 Self {
59 sess,
60 file_resolver,
61 sources: &mut gcx.sources,
62 arenas: &gcx.ast_arenas,
63 resolve_imports: !sess.opts.unstable.no_resolve_imports,
64 parsed: false,
65 gcx: gcx_.get(),
66 }
67 }
68
69 #[inline]
71 pub fn dcx(&self) -> &'gcx DiagCtxt {
72 &self.sess.dcx
73 }
74
75 pub fn set_resolve_imports(&mut self, resolve_imports: bool) {
79 self.resolve_imports = resolve_imports;
80 }
81
82 pub fn resolve_file(&self, path: impl AsRef<Path>) -> Result<Arc<SourceFile>> {
84 self.file_resolver.resolve_file(path.as_ref(), None).map_err(self.map_resolve_error())
85 }
86
87 pub fn resolve_files(
89 &self,
90 paths: impl IntoIterator<Item = impl AsRef<Path>>,
91 ) -> impl Iterator<Item = Result<Arc<SourceFile>>> {
92 paths.into_iter().map(|path| self.resolve_file(path))
93 }
94
95 pub fn par_resolve_files(
97 &self,
98 paths: impl IntoParallelIterator<Item = impl AsRef<Path>>,
99 ) -> impl ParallelIterator<Item = Result<Arc<SourceFile>>> {
100 paths.into_par_iter().map(|path| self.resolve_file(path))
101 }
102
103 #[instrument(level = "debug", skip_all)]
105 pub fn load_stdin(&mut self) -> Result<()> {
106 let file = self.file_resolver.load_stdin().map_err(self.map_resolve_error())?;
107 self.add_file(file);
108 Ok(())
109 }
110
111 #[instrument(level = "debug", skip_all)]
113 pub fn load_files(&mut self, paths: impl IntoIterator<Item = impl AsRef<Path>>) -> Result<()> {
114 for path in paths {
115 self.load_file(path.as_ref())?;
116 }
117 Ok(())
118 }
119
120 pub fn par_load_files(
122 &mut self,
123 paths: impl IntoParallelIterator<Item = impl AsRef<Path>>,
124 ) -> Result<()> {
125 let resolved = self.par_resolve_files(paths).collect::<Result<Vec<_>>>()?;
126 self.add_files(resolved);
127 Ok(())
128 }
129
130 pub fn par_load_files_with_contents<I, N, S>(&mut self, files: I) -> Result<()>
132 where
133 I: IntoParallelIterator<Item = (N, S)>,
134 N: Into<FileName> + Send,
135 S: Into<String> + Send,
136 {
137 let source_map = self.sess.source_map();
138 let dcx = self.dcx();
139 let files = files
140 .into_par_iter()
141 .map(|(name, src)| {
142 source_map
143 .new_source_file(name, src)
144 .map_err(|e| dcx.err(format!("failed to load source: {e}")).emit())
145 })
146 .collect::<Result<Vec<_>>>()?;
147 self.add_files(files);
148 Ok(())
149 }
150
151 #[instrument(level = "debug", skip_all)]
153 pub fn load_file(&mut self, path: &Path) -> Result<()> {
154 let file = self.resolve_file(path)?;
155 self.add_file(file);
156 Ok(())
157 }
158
159 pub fn add_files(&mut self, files: impl IntoIterator<Item = Arc<SourceFile>>) {
161 for file in files {
162 self.add_file(file);
163 }
164 }
165
166 pub fn add_file(&mut self, file: Arc<SourceFile>) {
168 self.sources.get_or_insert_file(file);
169 }
170
171 pub fn force_resolve_all_imports(mut self) {
173 let mut sources = std::mem::take(self.sources);
174 let mut any_new = false;
175 for id in sources.indices() {
176 let source = &mut sources[id];
177 let ast = source.ast.take();
178 for (import_item_id, import_file) in
179 self.resolve_imports(&source.file.clone(), ast.as_ref())
180 {
181 let (_import_id, is_new) =
182 sources.add_import(id, import_item_id, import_file, true);
183 if is_new {
184 any_new = true;
185 }
186 }
187 sources[id].ast = ast;
188 }
189 *self.sources = sources;
190
191 self.parsed = true;
192 if any_new {
193 self.parse_inner();
194 }
195 }
196
197 pub fn parse(mut self) {
201 self.parse_inner();
202 }
203
204 #[instrument(name = "parse", level = "debug", skip_all)]
205 fn parse_inner(&mut self) {
206 self.parsed = true;
207 let _ = self.gcx.advance_stage(CompilerStage::Parsing);
208
209 let mut sources = std::mem::take(self.sources);
210 if !sources.is_empty() {
211 let dbg = enabled!(tracing::Level::DEBUG);
212 let len_before = sources.len();
213 let sources_parsed_before = if dbg { sources.count_parsed() } else { 0 };
214
215 if self.sess.is_sequential() || (sources.len() == 1 && !self.resolve_imports) {
216 self.parse_sequential(&mut sources, self.arenas.get_or_default());
217 } else {
218 self.parse_parallel(&mut sources, self.arenas);
219 }
220
221 if dbg {
222 let len_after = sources.len();
223 let sources_added =
224 len_after.checked_sub(len_before).expect("parsing removed sources?");
225
226 let sources_parsed_after = sources.count_parsed();
227 let solidity_sources_parsed = sources_parsed_after
228 .checked_sub(sources_parsed_before)
229 .expect("parsing removed parsed sources?");
230
231 if sources_added > 0 || solidity_sources_parsed > 0 {
232 debug!(
233 sources_added,
234 solidity_sources_parsed,
235 num_sources = len_after,
236 num_contracts = sources.iter().map(|s| s.count_contracts()).sum::<usize>(),
237 total_bytes = %crate::fmt_bytes(sources.iter().map(|s| s.file.src.len()).sum::<usize>()),
238 total_lines = sources.iter().map(|s| s.file.count_lines()).sum::<usize>(),
239 "parsed",
240 );
241 }
242 }
243 }
244
245 sources.assert_unique();
246 *self.sources = sources;
247 }
248
249 fn parse_sequential<'ast>(&self, sources: &mut Sources<'ast>, arena: &'ast ast::Arena) {
250 for i in 0.. {
251 let id = SourceId::from_usize(i);
252 let Some(source) = sources.get(id) else { break };
253 if source.ast.is_some() {
254 continue;
255 }
256
257 let file = source.file.clone();
258 let parent = parent_path(&file);
259 let imports_len = sources[id].imports.len();
260 let ast = self.parse_one(&file, arena, |item_id, _, import| {
261 let _guard = debug_span!("resolve_import").entered();
262 let Some(import_file) = self.resolve_import_directive(import, parent) else {
263 return;
264 };
265 sources.add_import(id, item_id, import_file, false);
266 });
267 if ast.is_none() {
268 sources[id].imports.truncate(imports_len);
269 }
270 sources[id].ast = ast;
271 }
272 }
273
274 fn parse_parallel<'ast>(
275 &self,
276 sources: &mut Sources<'ast>,
277 arenas: &'ast ThreadLocal<ast::Arena>,
278 ) {
279 let lock = Mutex::new(std::mem::take(sources));
280 rayon::scope(|scope| {
281 let sources = &*lock.lock();
282 for (id, source) in sources.iter_enumerated() {
283 if source.ast.is_some() {
284 continue;
285 }
286 let file = source.file.clone();
287 self.spawn_parse_job(&lock, id, file, arenas, scope);
288 }
289 });
290 *sources = lock.into_inner();
291 }
292
293 fn spawn_parse_job<'ast, 'scope>(
294 &'scope self,
295 lock: &'scope Mutex<Sources<'ast>>,
296 id: SourceId,
297 file: Arc<SourceFile>,
298 arenas: &'ast ThreadLocal<ast::Arena>,
299 scope: &rayon::Scope<'scope>,
300 ) {
301 scope.spawn(move |scope| self.parse_job(lock, id, file, arenas, scope));
302 }
303
304 #[instrument(level = "debug", skip_all)]
305 fn parse_job<'ast, 'scope>(
306 &'scope self,
307 lock: &'scope Mutex<Sources<'ast>>,
308 id: SourceId,
309 file: Arc<SourceFile>,
310 arenas: &'ast ThreadLocal<ast::Arena>,
311 scope: &rayon::Scope<'scope>,
312 ) {
313 let mut imports = Vec::new();
314 let parent = parent_path(&file);
315 let ast = self.parse_one(&file, arenas.get_or_default(), |item_id, _, import| {
316 let _guard = debug_span!("resolve_import").entered();
317 let Some(import_file) = self.resolve_import_directive(import, parent) else {
318 return;
319 };
320 imports.push((item_id, import_file.clone()));
321
322 let (import_id, is_new) = {
323 let sources = &mut *lock.lock();
324 sources.get_or_insert_file(import_file.clone())
325 };
326 if is_new {
327 self.spawn_parse_job(lock, import_id, import_file, arenas, scope);
328 }
329 });
330
331 let _guard = debug_span!("add_imports").entered();
333 let sources = &mut *lock.lock();
334 assert!(sources[id].ast.is_none());
335 sources[id].ast = ast;
336 if sources[id].ast.is_some() {
337 for (import_item_id, import_file) in imports {
338 sources.add_import(id, import_item_id, import_file, false);
339 }
340 }
341 }
342
343 #[instrument(level = "debug", skip_all, fields(file = %file.name.display()))]
345 fn parse_one<'ast>(
346 &self,
347 file: &SourceFile,
348 arena: &'ast ast::Arena,
349 import_callback: impl FnMut(ast::ItemId, Span, &ast::ImportDirective<'ast>),
350 ) -> Option<ast::SourceUnit<'ast>> {
351 let lexer = Lexer::from_source_file(self.sess, file);
352 let mut parser = Parser::from_lexer(arena, lexer);
353 if self.resolve_imports {
354 parser.set_import_callback(import_callback);
355 }
356 if self.sess.opts.language.is_yul() {
357 let _file = parser.parse_yul_file_object().map_err(|e| e.emit());
358 None
359 } else {
360 parser.parse_file().map_err(|e| e.emit()).ok()
361 }
362 }
363
364 fn resolve_imports(
367 &self,
368 file: &SourceFile,
369 ast: Option<&ast::SourceUnit<'_>>,
370 ) -> impl Iterator<Item = (ast::ItemId, Arc<SourceFile>)> {
371 let parent = parent_path(file);
372 let items =
373 ast.filter(|_| self.resolve_imports).map(|ast| &ast.items[..]).unwrap_or_default();
374 items
375 .iter_enumerated()
376 .filter_map(move |(id, item)| self.resolve_import(item, parent).map(|file| (id, file)))
377 }
378
379 fn resolve_import(
380 &self,
381 item: &ast::Item<'_>,
382 parent: Option<&Path>,
383 ) -> Option<Arc<SourceFile>> {
384 let ast::ItemKind::Import(import) = &item.kind else { return None };
385 self.resolve_import_directive(import, parent)
386 }
387
388 fn resolve_import_directive(
389 &self,
390 import: &ast::ImportDirective<'_>,
391 parent: Option<&Path>,
392 ) -> Option<Arc<SourceFile>> {
393 self.resolve_import_path(&import.path, parent)
394 }
395
396 fn resolve_import_path(
397 &self,
398 import_path: &ast::StrLit,
399 parent: Option<&Path>,
400 ) -> Option<Arc<SourceFile>> {
401 let span = import_path.span;
402 let path_str = import_path.value.as_str();
403 let (path_bytes, any_error) =
404 unescape::parse_string_literal(path_str, unescape::StrKind::Str, span, self.sess);
405 if any_error {
406 return None;
407 }
408 let Some(path) = path_from_bytes(&path_bytes[..]) else {
409 self.dcx().emit_err(span, "import path is not a valid UTF-8 string");
410 return None;
411 };
412 self.file_resolver
413 .resolve_file(path, parent)
414 .map_err(self.map_resolve_error_with(Some(span)))
415 .ok()
416 }
417
418 fn map_resolve_error(&self) -> impl FnOnce(ResolveError) -> ErrorGuaranteed {
419 self.map_resolve_error_with(None)
420 }
421
422 fn map_resolve_error_with(
423 &self,
424 span: Option<Span>,
425 ) -> impl FnOnce(ResolveError) -> ErrorGuaranteed {
426 move |e| {
427 let mut err = self.dcx().err(e.to_string());
428 if let Some(span) = span {
429 err = err.span(span);
430 }
431 err.emit()
432 }
433 }
434}
435
436impl Drop for ParsingContext<'_> {
437 fn drop(&mut self) {
438 if self.parsed || self.dcx().err_count() > 0 {
439 return;
440 }
441 warn!("`ParsingContext::parse` not called");
444 }
445}
446
447fn parent_path(file: &SourceFile) -> Option<&Path> {
448 match &file.name {
449 FileName::Real(path) => Some(path.as_path()),
450 FileName::Stdin | FileName::Custom(_) => None,
451 }
452}
453
454#[cfg(unix)]
455fn path_from_bytes(bytes: &[u8]) -> Option<&Path> {
456 use std::os::unix::ffi::OsStrExt;
457 Some(Path::new(std::ffi::OsStr::from_bytes(bytes)))
458}
459
460#[cfg(not(unix))]
461fn path_from_bytes(bytes: &[u8]) -> Option<&Path> {
462 std::str::from_utf8(bytes).ok().map(Path::new)
463}
464
465#[derive(Default)]
467pub struct Sources<'ast> {
468 sources: IndexVec<SourceId, Source<'ast>>,
469 file_to_id: FxHashMap<Arc<SourceFile>, SourceId>,
470}
471
472impl fmt::Debug for Sources<'_> {
473 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
474 f.write_str("ParsedSources")?;
475 self.sources.fmt(f)
476 }
477}
478
479impl<'ast> Sources<'ast> {
480 pub fn new() -> Self {
482 Self::default()
483 }
484
485 #[inline]
487 pub fn get(&self, id: SourceId) -> Option<&Source<'ast>> {
488 self.sources.get(id)
489 }
490
491 #[inline]
493 pub fn get_mut(&mut self, id: SourceId) -> Option<&mut Source<'ast>> {
494 self.sources.get_mut(id)
495 }
496
497 pub fn get_file(&self, file: &Arc<SourceFile>) -> Option<(SourceId, &Source<'ast>)> {
499 self.file_to_id.get(file).map(|&id| {
500 debug_assert_eq!(self.sources[id].file, *file, "file_to_id is inconsistent");
501 (id, &self.sources[id])
502 })
503 }
504
505 pub fn get_file_mut(
507 &mut self,
508 file: &Arc<SourceFile>,
509 ) -> Option<(SourceId, &mut Source<'ast>)> {
510 self.file_to_id.get(file).map(|&id| {
511 debug_assert_eq!(self.sources[id].file, *file, "file_to_id is inconsistent");
512 (id, &mut self.sources[id])
513 })
514 }
515
516 #[instrument(level = "debug", skip_all)]
520 pub fn get_or_insert_file(&mut self, file: Arc<SourceFile>) -> (SourceId, bool) {
521 let mut new = false;
522 let id = *self.file_to_id.entry(file).or_insert_with_key(|file| {
523 new = true;
524 self.sources.push(Source::new(file.clone()))
525 });
526 (id, new)
527 }
528
529 pub fn remove_file(&mut self, file: &Arc<SourceFile>) -> Option<Source<'ast>> {
531 self.file_to_id.remove(file).map(|id| self.sources.remove(id))
532 }
533
534 pub fn asts(&self) -> impl DoubleEndedIterator<Item = &ast::SourceUnit<'ast>> {
536 self.sources.iter().filter_map(|source| source.ast.as_ref())
537 }
538
539 pub fn par_asts(&self) -> impl ParallelIterator<Item = &ast::SourceUnit<'ast>> {
541 self.sources.as_raw_slice().par_iter().filter_map(|source| source.ast.as_ref())
542 }
543
544 fn count_parsed(&self) -> usize {
545 self.sources.iter().filter(|s| s.ast.is_some()).count()
546 }
547
548 fn add_import(
550 &mut self,
551 current: SourceId,
552 import_item_id: ast::ItemId,
553 import: Arc<SourceFile>,
554 check_dup: bool,
555 ) -> (SourceId, bool) {
556 let ret = self.get_or_insert_file(import);
557 let (import_id, new) = ret;
558
559 let current = &mut self.sources[current].imports;
560 let value = (import_item_id, import_id);
561 if check_dup && current.contains(&value) {
562 assert!(!new, "duplicate import but source is new?");
563 return ret;
564 }
565 current.push(value);
566
567 ret
568 }
569
570 fn assert_unique(&self) {
572 if self.sources.len() <= 1 {
573 return;
574 }
575
576 debug_assert_eq!(
577 self.sources.iter().map(|s| &*s.file).collect::<FxHashSet<_>>().len(),
578 self.sources.len(),
579 "parsing produced duplicate source files"
580 );
581 }
582
583 #[instrument(level = "debug", skip_all)]
587 pub fn topo_sort(&mut self) {
588 let len = self.len();
589 if len <= 1 {
590 return;
591 }
592
593 let mut order = IndexVec::with_capacity(len);
594 let mut map = index_vec![SourceId::MAX; len];
595 let mut seen = FxHashSet::with_capacity_and_hasher(len, Default::default());
596 debug_span!("topo_order").in_scope(|| {
597 for id in self.sources.indices() {
598 self.topo_order(id, &mut order, &mut map, &mut seen);
599 }
600 });
601 debug_assert!(
602 order.len() == len && !map.contains(&SourceId::MAX) && seen.len() == len,
603 "topo_order did not visit all sources"
604 );
605
606 debug_span!("remap_state").in_scope(|| {
607 for source in &mut self.sources {
608 for (_, import) in &mut source.imports {
609 *import = map[*import];
610 }
611 }
612
613 for id in self.file_to_id.values_mut() {
614 *id = map[*id];
615 }
616 });
617
618 debug_span!("sort_by_indices").in_scope(|| {
619 sort_by_indices(&mut self.sources, order);
620 });
621 }
622
623 fn topo_order(
624 &self,
625 id: SourceId,
626 order: &mut IndexVec<SourceId, SourceId>,
627 map: &mut IndexVec<SourceId, SourceId>,
628 seen: &mut FxHashSet<SourceId>,
629 ) {
630 if !seen.insert(id) {
631 return;
632 }
633 for &(_, import_id) in &self.sources[id].imports {
634 self.topo_order(import_id, order, map, seen);
635 }
636 map[id] = order.push(id);
637 }
638}
639
640impl<'ast> std::ops::Deref for Sources<'ast> {
641 type Target = IndexVec<SourceId, Source<'ast>>;
642
643 #[inline]
644 fn deref(&self) -> &Self::Target {
645 &self.sources
646 }
647}
648
649impl std::ops::DerefMut for Sources<'_> {
650 #[inline]
651 fn deref_mut(&mut self) -> &mut Self::Target {
652 &mut self.sources
653 }
654}
655
656pub struct Source<'ast> {
658 pub file: Arc<SourceFile>,
660 pub imports: Vec<(ast::ItemId, SourceId)>,
665 pub ast: Option<ast::SourceUnit<'ast>>,
673}
674
675impl fmt::Debug for Source<'_> {
676 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
677 f.debug_struct("Source")
678 .field("file", &self.file.name)
679 .field("imports", &self.imports)
680 .field("ast", &self.ast)
681 .finish()
682 }
683}
684
685impl Source<'_> {
686 pub fn new(file: Arc<SourceFile>) -> Self {
688 Self { file, ast: None, imports: Vec::new() }
689 }
690
691 fn count_contracts(&self) -> usize {
692 self.ast.as_ref().map(|ast| ast.count_contracts()).unwrap_or(0)
693 }
694}
695
696fn sort_by_indices<I: Idx, T>(data: &mut IndexVec<I, T>, mut indices: IndexVec<I, I>) {
700 assert_eq!(data.len(), indices.len());
701 for idx in data.indices() {
702 if indices[idx] != idx {
703 let mut current_idx = idx;
704 loop {
705 let target_idx = indices[current_idx];
706 indices[current_idx] = current_idx;
707 if indices[target_idx] == target_idx {
708 break;
709 }
710 data.swap(current_idx, target_idx);
711 current_idx = target_idx;
712 }
713 }
714 }
715}
716
717#[cfg(test)]
718mod tests {
719 use std::path::PathBuf;
720
721 use solar_ast::ItemId;
722
723 use super::*;
724
725 #[test]
726 fn sources_consistency() {
727 let sess = Session::builder().with_test_emitter().build();
728 sess.enter_sequential(|| {
729 let mut sources = Sources::new();
730
731 let (aid, new) = sources.get_or_insert_file(
732 sess.source_map().new_source_file(PathBuf::from("a.sol"), "abcd").unwrap(),
733 );
734 assert!(new);
735
736 let (bid, new) = sources.get_or_insert_file(
737 sess.source_map().new_source_file(PathBuf::from("b.sol"), "aaaaa").unwrap(),
738 );
739 assert!(new);
740
741 let (cid, new) = sources.get_or_insert_file(
742 sess.source_map().new_source_file(PathBuf::from("c.sol"), "cccccc").unwrap(),
743 );
744 assert!(new);
745
746 let files = vec![
747 (aid, PathBuf::from("a.sol")),
748 (bid, PathBuf::from("b.sol")),
749 (cid, PathBuf::from("c.sol")),
750 ];
751
752 sources[aid].imports.push((ItemId::new(0), cid));
753
754 for (id, path) in &files {
755 assert_eq!(sources[*id].file.name, FileName::Real(path.clone()));
756 }
757
758 let assert_maps = |sources: &mut Sources<'_>| {
759 for (_, path) in &files {
760 let file = sess.source_map().get_file(path).unwrap();
761 let id = sources.get_file(&file).unwrap().0;
762 assert_eq!(sources[id].file, file);
763 }
764 };
765
766 assert_maps(&mut sources);
767 sources.topo_sort();
768 assert_maps(&mut sources);
769 });
770 }
771}