1use std::ffi::{OsStr, OsString};
4use std::io::{self, BufWriter, IsTerminal, Read, Write};
5use std::path::{Path, PathBuf};
6use std::process::ExitCode;
7
8use std::sync::atomic::{AtomicUsize, Ordering};
9use std::sync::{Arc, Mutex};
10
11use jsonvet::{Dialect, Problem, Report, Severity};
12
13mod args;
14mod editorconfig;
15mod format;
16mod frame;
17mod glob;
18mod json;
19mod messages;
20mod output;
21mod rules;
22mod sarif;
23mod schema;
24mod settings;
25mod write;
26
27use args::{Command, STDIN};
28use output::{Format, Look, Printer};
29use schema::Schema;
30use schema::select::{Job, Schemas, check_own_schema, jobs, with_schema};
31use settings::{Resolver, Settings};
32use write::write_atomically;
33
34const STACK: usize = 64 << 20;
39
40const CHUNK_BYTES: usize = 64 << 20;
42
43const SCHEMA_BYTES: usize = 20;
47
48const SPLIT_BYTES: usize = 1 << 20;
51
52pub fn run() -> ExitCode {
54 std::panic::set_hook(Box::new(|info| {
57 eprintln!(
58 "jsonvet: internal error: {info}\n\nThis is a bug in jsonvet. Please report it at \
59 https://github.com/simonvanlierde/jsonvet/issues with the command you ran and, if \
60 you can share it, the file that caused it."
61 );
62 std::process::exit(101);
63 }));
64 on_big_stack(run_here)
65}
66
67fn on_big_stack<T: Send + 'static>(f: fn() -> T) -> T {
70 let thread = std::thread::Builder::new().name("jsonvet".into());
71 match thread.stack_size(STACK).spawn(f) {
72 Ok(thread) => thread
73 .join()
74 .unwrap_or_else(|panic| std::panic::resume_unwind(panic)),
75 Err(_) => f(),
76 }
77}
78
79fn run_here() -> ExitCode {
80 #[cfg(debug_assertions)]
83 assert!(
84 std::env::var_os("JSONVET_TEST_PANIC").is_none(),
85 "test panic"
86 );
87 let annotate = std::env::var_os("GITHUB_ACTIONS").is_some_and(|v| v == "true");
88 let stdout = io::stdout().lock();
89 let set = |name| std::env::var_os(name).is_some_and(|v| !v.is_empty());
90 let color = if set("NO_COLOR") {
93 Some(false)
94 } else if set("FORCE_COLOR") {
95 Some(std::env::var_os("FORCE_COLOR").is_none_or(|v| v != "0" && v != "false"))
96 } else {
97 None
98 };
99 let terminal = Terminal {
100 tty: stdout.is_terminal(),
101 color,
102 };
103 match run_with(
104 std::env::args_os().skip(1),
105 annotate,
106 terminal,
107 BufWriter::new(stdout),
108 ) {
109 Ok(code) => ExitCode::from(code),
110 Err(message) => {
111 eprintln!("jsonvet: {message}");
112 ExitCode::from(2)
113 }
114 }
115}
116
117fn print_summary(
120 printer: &mut Printer<impl Write>,
121 line: Option<&str>,
122 text: bool,
123 found: bool,
124) -> io::Result<()> {
125 if let Some(line) = line {
126 if text {
127 printer.text_line(line)?;
128 }
129 if found {
130 printer.summary_annotation(line)?;
131 }
132 }
133 printer.finish()
134}
135
136#[doc(hidden)]
139#[derive(Clone, Copy, Debug, Default)]
140pub struct Terminal {
141 pub tty: bool,
144 pub color: Option<bool>,
147}
148
149#[doc(hidden)]
154pub fn run_with(
155 args: impl IntoIterator<Item = OsString>,
156 annotate: bool,
157 terminal: Terminal,
158 out: impl Write,
159) -> Result<u8, String> {
160 let args = match args::parse(args)
162 .map_err(|e| messages::fill("error.usage", &[("error", &output::printable(&e))]))?
163 {
164 Command::Help => return print(out, messages::text("help")),
165 Command::Version => return print(out, concat!("jsonvet ", env!("CARGO_PKG_VERSION"))),
166 Command::ListRules => return print(out, &rules::list()),
167 Command::Explain(text) => return print(out, &text),
168 Command::Check(args) => *args,
169 };
170 messages::english_only(args.format != Format::Text);
171 let resolver = Arc::new(Mutex::new(Resolver::new(&args.config, args.settings)?));
172 let settings_for = |path: &Path| lock(&resolver).for_file(path);
173 let threads = std::thread::available_parallelism().map_or(1, std::num::NonZero::get);
174 let files = expand(args.files, &resolver, threads)?;
175 if let Some(check_only) = args.fmt {
176 return format::run(&files, check_only, args.stdin_filename, &resolver, out);
177 }
178 let schemas = Schemas::default();
179 let load = |path: &Path, quiet| {
180 schemas.get(path, quiet, |admit| {
181 Schema::load(path, &*settings_for(path)?, quiet, admit)
182 })
183 };
184 let root = |path: &Path| lock(&resolver).root(path);
185
186 let stdin_name = args.stdin_filename.unwrap_or_else(|| STDIN.into());
188 let shown = |file: &OsStr| -> OsString {
189 if file == STDIN {
190 stdin_name.clone()
191 } else {
192 file.to_owned()
193 }
194 };
195 let files = jobs(files, &shown, &settings_for, args.schema.as_deref(), &load)?;
198 let look = look(args.format, args.color, terminal);
199 let stdin_fix = args.fix && files.iter().any(|(_, file)| file == STDIN);
201 let mut out = out;
202 let sink = Diagnostics::new(stdin_fix, &mut out);
203 let mut printer = Printer::new(sink, args.format, annotate, look);
204 let mut fixed = FixRun::default();
205 let mut counts = Counts::default();
206 let mut rest = &files[..];
216 let shares = schema_shares(&files, threads);
217 let mut ahead = Vec::new();
218 let written = loop {
219 if rest.is_empty() {
220 let line = (!files.is_empty()).then(|| counts.summary(files.len()));
221 let found = counts.errors + counts.warnings > 0;
222 break print_summary(&mut printer, line.as_deref(), terminal.tty, found);
223 }
224 let shares = shares.get(files.len() - rest.len()..);
225 let with_schema = |job: &Job, text: &str| with_schema(job, &shown(&job.1), text);
226 let mut texts = take_chunk(&mut ahead, rest, shares, threads, &with_schema);
227 let chunk;
228 (chunk, rest) = rest.split_at(texts.len());
229 let known = if args.fix {
230 fix_chunk(
231 chunk,
232 &mut texts,
233 &shown,
234 args.unsafe_fixes,
235 threads,
236 &mut fixed,
237 )
238 } else {
239 vec![None; texts.len()]
240 };
241 let jobs: Vec<_> = chunk.iter().zip(&texts).zip(&known).collect();
242 let parts = (threads / jobs.len()).max(1);
243 let checked = parallel(
244 &jobs,
245 threads,
246 || false,
247 |((((settings, schema), file), (text, with)), known)| {
248 let name = shown(file);
249 let name = name.to_string_lossy();
250 text.as_deref().map(|text| match schema {
251 Some(schema) => check(&name, text, settings, Some(schema), parts, None),
252 None if *with => {
253 check_own_schema(&name, text, settings, &schemas, &root, &load, parts)
254 }
255 None => check(&name, text, settings, None, parts, known.as_deref()),
256 })
257 },
258 );
259 schemas.over().map_err(|e| output::printable(&e))?;
260 let outcome = print_checked(chunk, checked, &shown, &mut printer, &mut counts);
261 if outcome.is_err() {
262 break outcome;
263 }
264 };
265 let count = !stdin_fix || args.format == Format::Text;
267 let written = written.and_then(|()| fixed.finish(count, &mut out));
268 conclude(written, &schemas, &counts, args.max_warnings)
269}
270
271type Checked<'t> = (Report<'t>, Vec<(usize, String, bool)>);
273
274fn print_checked(
277 chunk: &[Job],
278 checked: Vec<Result<Checked<'_>, &String>>,
279 shown: &impl Fn(&OsStr) -> OsString,
280 printer: &mut Printer<impl Write>,
281 counts: &mut Counts,
282) -> io::Result<()> {
283 chunk
284 .iter()
285 .zip(checked)
286 .try_for_each(|((_, file), checked)| match checked {
287 Ok((report, schema_errors)) => {
288 print_file(&shown(file), &report, &schema_errors, printer, counts)
289 }
290 Err(message) => {
291 counts.errors += 1;
292 counts.files += 1;
293 printer.unreadable(&shown(file), message)
294 }
295 })
296}
297
298fn conclude(
301 written: io::Result<()>,
302 schemas: &Schemas,
303 counts: &Counts,
304 max_warnings: Option<usize>,
305) -> Result<u8, String> {
306 match written {
308 Err(e) if e.kind() != io::ErrorKind::BrokenPipe => {
309 return Err(messages::fill("error.write", &[("error", &e)]));
310 }
311 _ => {}
312 }
313 schemas.note_missing();
314 let too_many_warnings = max_warnings.is_some_and(|max| counts.warnings > max);
315 Ok(u8::from(counts.errors > 0 || too_many_warnings))
316}
317
318fn read_chunk<S: Sync>(
322 files: &[(S, OsString)],
323 threads: usize,
324 with_schema: &(impl Fn(&(S, OsString), &str) -> bool + Sync),
325) -> Vec<(Result<String, String>, bool)> {
326 let bytes = AtomicUsize::new(0);
327 let full = || bytes.load(Ordering::Relaxed) >= CHUNK_BYTES;
328 parallel(files, threads, full, |file| {
329 let text = if file.1 == STDIN {
330 read_stdin()
331 } else {
332 read(Path::new(&file.1))
333 };
334 let with = text.as_ref().is_ok_and(|text| with_schema(file, text));
335 bytes.fetch_add(weight(&text, with), Ordering::Relaxed);
336 (text, with)
337 })
338}
339
340fn take_chunk(
345 ahead: &mut Vec<(Result<String, String>, bool)>,
346 rest: &[Job],
347 shares: Option<&[usize]>,
348 threads: usize,
349 with_schema: &(impl Fn(&Job, &str) -> bool + Sync),
350) -> Vec<(Result<String, String>, bool)> {
351 if ahead.is_empty() {
352 let offered = match shares {
353 Some(shares) if !shares.is_empty() => &rest[..chunk_len(shares)],
354 _ => rest,
355 };
356 *ahead = read_chunk(offered, threads, with_schema);
357 }
358 let weights: Vec<usize> = ahead
359 .iter()
360 .map(|(text, with)| weight(text, *with))
361 .collect();
362 ahead.drain(..chunk_len(&weights)).collect()
363}
364
365fn weight(text: &Result<String, String>, with_schema: bool) -> usize {
368 let size = text.as_ref().map_or(0, String::len);
369 if with_schema {
370 size.saturating_mul(SCHEMA_BYTES).min(CHUNK_BYTES)
371 } else {
372 size
373 }
374}
375
376fn schema_share(file: &OsStr) -> usize {
381 if file == STDIN {
382 return CHUNK_BYTES;
383 }
384 let size = std::fs::metadata(file).map_or(0, |meta| meta.len());
385 usize::try_from(size)
386 .unwrap_or(usize::MAX)
387 .saturating_mul(SCHEMA_BYTES)
388 .min(CHUNK_BYTES)
389}
390
391fn schema_shares(files: &[Job], threads: usize) -> Vec<usize> {
393 if files.iter().all(|((_, schema), _)| schema.is_none()) {
394 return Vec::new();
395 }
396 parallel(
397 files,
398 threads,
399 || false,
400 |((_, schema), file)| schema.as_ref().map_or(0, |_| schema_share(file)),
401 )
402}
403
404fn chunk_len(shares: &[usize]) -> usize {
407 let mut total = 0;
408 shares
409 .iter()
410 .position(|share| {
411 total += share;
412 total >= CHUNK_BYTES
413 })
414 .map_or(shares.len(), |last| last + 1)
415}
416
417fn expand(
422 args: Vec<OsString>,
423 resolver: &Arc<Mutex<Resolver>>,
424 threads: usize,
425) -> Result<Vec<OsString>, String> {
426 let mut files = Vec::with_capacity(args.len());
427 let dirs = parallel(
429 &args,
430 threads,
431 || false,
432 |arg| arg != STDIN && Path::new(arg).is_dir(),
433 );
434 let mut roots = Vec::new();
436 let mut distinct = true;
437 for (arg, dir) in args.into_iter().zip(dirs) {
438 if !dir {
439 distinct = false;
440 files.push(arg);
441 continue;
442 }
443 let start = files.len();
444 let above = std::fs::canonicalize(&arg).ok();
447 match &above {
448 Some(root) => roots.push(root.clone()),
449 None => distinct = false,
450 }
451 if let Some(problem) = std::iter::once(Path::new(&arg))
452 .chain(above.iter().flat_map(|dir| dir.ancestors()))
453 .find_map(unsafe_ignore_file)
454 {
455 return Err(problem);
456 }
457 let skipped = Arc::new(Mutex::new(None));
458 let skip = Arc::clone(&skipped);
459 let resolver = Arc::clone(resolver);
461 let walk = ignore::WalkBuilder::new(&arg)
464 .filter_entry(move |entry| {
465 if entry.file_type().is_some_and(|t| t.is_dir())
467 && let Some(problem) = unsafe_ignore_file(entry.path())
468 {
469 skip.lock()
471 .unwrap_or_else(std::sync::PoisonError::into_inner)
472 .get_or_insert(problem);
473 return false;
474 }
475 let settings = lock(&resolver).for_file(entry.path());
476 !settings.is_ok_and(|settings| settings.excluded(entry.path()))
477 })
478 .build();
479 for entry in walk {
480 let entry = entry
481 .map_err(|e| output::printable(&messages::fill("error.walk", &[("error", &e)])))?;
482 if entry.file_type().is_some_and(|t| t.is_file())
483 && Dialect::for_path(entry.path()).is_some()
484 {
485 files.push(entry.into_path().into_os_string());
486 }
487 }
488 if let Some(problem) = skipped
489 .lock()
490 .unwrap_or_else(std::sync::PoisonError::into_inner)
491 .take()
492 {
493 return Err(problem);
494 }
495 files[start..].sort_unstable();
496 }
497 let overlap = |i: usize, root: &PathBuf| {
499 roots[..i]
500 .iter()
501 .any(|r| r.starts_with(root) || root.starts_with(r))
502 };
503 if distinct && !roots.iter().enumerate().any(|(i, root)| overlap(i, root)) {
504 return Ok(files);
505 }
506 Ok(unique(files, threads))
507}
508
509#[derive(PartialEq, Eq, Hash)]
511enum Identity {
512 #[cfg(unix)]
513 Inode(u64, u64),
514 Path(PathBuf),
515}
516
517fn identity(file: &OsStr) -> Identity {
518 #[cfg(unix)]
519 if let Ok(meta) = std::fs::metadata(file) {
520 use std::os::unix::fs::MetadataExt;
521 return Identity::Inode(meta.dev(), meta.ino());
522 }
523 Identity::Path(std::fs::canonicalize(file).unwrap_or_else(|_| file.into()))
525}
526
527fn unique(files: Vec<OsString>, threads: usize) -> Vec<OsString> {
530 let ids = parallel(&files, threads, || false, |file| identity(file));
531 let mut seen = std::collections::HashSet::with_capacity(files.len());
532 files
533 .into_iter()
534 .zip(ids)
535 .filter_map(|(file, id)| seen.insert(id).then_some(file))
536 .collect()
537}
538
539fn lock(resolver: &Mutex<Resolver>) -> std::sync::MutexGuard<'_, Resolver> {
541 resolver
542 .lock()
543 .unwrap_or_else(std::sync::PoisonError::into_inner)
544}
545
546enum Diagnostics<'w, W: Write> {
549 Out(&'w mut W),
550 Stderr(io::Stderr),
551}
552
553impl<'w, W: Write> Diagnostics<'w, W> {
554 fn new(to_stderr: bool, out: &'w mut W) -> Self {
555 if to_stderr {
556 Diagnostics::Stderr(io::stderr())
557 } else {
558 Diagnostics::Out(out)
559 }
560 }
561}
562
563impl<W: Write> Write for Diagnostics<'_, W> {
564 fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
565 match self {
566 Diagnostics::Out(out) => out.write(buf),
567 Diagnostics::Stderr(err) => err.write(buf),
568 }
569 }
570
571 fn flush(&mut self) -> io::Result<()> {
572 match self {
573 Diagnostics::Out(out) => out.flush(),
574 Diagnostics::Stderr(err) => err.flush(),
575 }
576 }
577}
578
579#[derive(Default)]
581struct FixRun {
582 problems: usize,
583 files: usize,
584 stdin: Option<String>,
586}
587
588impl FixRun {
589 fn finish(&self, count: bool, out: &mut impl Write) -> io::Result<()> {
592 if count && let Some(summary) = self.summary() {
593 eprintln!("{summary}");
594 }
595 match &self.stdin {
596 Some(text) => out.write_all(text.as_bytes()).and_then(|()| out.flush()),
597 None => Ok(()),
598 }
599 }
600
601 fn summary(&self) -> Option<String> {
603 (self.problems > 0).then(|| {
604 messages::fill(
605 "summary.fixed",
606 &[
607 (
608 "problems",
609 &messages::count("summary.problems", self.problems),
610 ),
611 ("files", &messages::count("summary.files", self.files)),
612 ],
613 )
614 })
615 }
616}
617
618fn fix_chunk(
621 chunk: &[Job],
622 texts: &mut [(Result<String, String>, bool)],
623 shown: &(impl Fn(&OsStr) -> OsString + Sync),
624 unsafe_fixes: bool,
625 threads: usize,
626 run: &mut FixRun,
627) -> Vec<Option<Vec<Problem>>> {
628 let jobs: Vec<_> = chunk.iter().zip(texts.iter()).collect();
629 let done = parallel(
630 &jobs,
631 threads,
632 || false,
633 |(((settings, _), file), (text, _))| {
634 let text = text.as_ref().ok()?;
635 let name = shown(file);
636 let config = settings.config_for(&name.to_string_lossy());
637 let fixed = match jsonvet::try_fix(text, &config, unsafe_fixes) {
638 Ok(fixed) => fixed,
639 Err(problems) => return Some(Err(problems)),
640 };
641 let saved = if file == STDIN {
642 Ok(())
643 } else {
644 write_atomically(Path::new(file), &fixed.text)
645 };
646 Some(Ok((fixed, saved)))
647 },
648 );
649 drop(jobs);
650 let mut known = Vec::with_capacity(done.len());
651 for (((_, file), entry), done) in chunk.iter().zip(texts.iter_mut()).zip(done) {
652 known.push(match &done {
654 Some(Err(problems)) => problems.clone(),
655 _ => None,
656 });
657 let done = done.and_then(Result::ok);
658 match done {
659 Some((fixed, Ok(()))) => {
660 run.problems += fixed.fixes.len();
661 run.files += 1;
662 entry.0 = Ok(fixed.text);
663 }
664 Some((_, Err(error))) => {
665 entry.0 = Err(error);
666 }
667 None => {}
668 }
669 if file == STDIN
670 && let Ok(text) = &entry.0
671 {
672 run.stdin = Some(text.clone());
673 }
674 }
675 known
676}
677
678#[derive(Default)]
679struct Counts {
680 errors: usize,
681 warnings: usize,
682 files: usize,
684}
685
686impl Counts {
687 fn summary(&self, checked: usize) -> String {
689 let files = messages::count("summary.files", checked);
690 if self.errors + self.warnings == 0 {
691 return messages::fill("summary.none", &[("files", &files)]);
692 }
693 let errors = || messages::count("summary.errors", self.errors);
694 let warnings = || messages::count("summary.warnings", self.warnings);
695 let problems = match (self.errors, self.warnings) {
696 (_, 0) => errors(),
697 (0, _) => warnings(),
698 _ => messages::fill(
699 "summary.both",
700 &[("errors", &errors()), ("warnings", &warnings())],
701 ),
702 };
703 messages::fill(
704 "summary.some",
705 &[
706 ("problems", &problems),
707 ("failed", &self.files),
708 ("files", &files),
709 ],
710 )
711 }
712}
713
714fn parallel<T: Sync, U: Send>(
718 items: &[T],
719 threads: usize,
720 stop: impl Fn() -> bool + Sync,
721 f: impl Fn(&T) -> U + Sync,
722) -> Vec<U> {
723 let next = AtomicUsize::new(0);
724 let work = || {
727 let mut done = Vec::new();
728 loop {
729 if stop() && next.load(Ordering::Relaxed) > 0 {
730 break done;
731 }
732 let i = next.fetch_add(1, Ordering::Relaxed);
733 let Some(item) = items.get(i) else { break done };
734 done.push((i, f(item)));
735 }
736 };
737 let mut done = std::thread::scope(|scope| {
738 let helpers: Vec<_> = (1..threads.min(items.len()))
740 .filter_map(|_| {
741 std::thread::Builder::new()
742 .stack_size(STACK)
743 .spawn_scoped(scope, work)
744 .ok()
745 })
746 .collect();
747 let mut done = work();
748 for helper in helpers {
749 done.extend(
750 helper
751 .join()
752 .unwrap_or_else(|panic| std::panic::resume_unwind(panic)),
753 );
754 }
755 done
756 });
757 done.sort_unstable_by_key(|&(i, _)| i);
758 done.into_iter().map(|(_, result)| result).collect()
759}
760
761fn split(text: &str, dialect: Dialect, parts: usize) -> Vec<std::ops::Range<usize>> {
764 let split = dialect == Dialect::JsonLines && parts > 1 && text.len() >= SPLIT_BYTES;
765 jsonvet::line_parts(text, if split { parts } else { 1 })
766}
767
768fn check<'t>(
771 name: &str,
772 text: &'t str,
773 settings: &Settings,
774 schema: Option<&Schema>,
775 parts: usize,
776 known: Option<&[Problem]>,
777) -> (Report<'t>, Vec<(usize, String, bool)>) {
778 let config = settings.config_for(name);
779 let ranges = split(text, config.dialect(), parts);
780 if ranges.len() > 1 {
781 let (lines, errors): (Vec<_>, Vec<_>) = parallel(
784 &ranges,
785 parts,
786 || false,
787 |lines| match schema {
788 Some(schema) => schema.check_lines(text, lines.clone(), &config),
789 None => (
790 jsonvet::parse_lines(text, lines.clone(), &config, None),
791 Vec::new(),
792 ),
793 },
794 )
795 .into_iter()
796 .unzip();
797 let (report, syntax_error) = jsonvet::join_lines(text, lines);
798 let errors = if syntax_error {
800 Vec::new()
801 } else {
802 errors.into_iter().flatten().collect()
803 };
804 return (report, errors);
805 }
806 match schema {
807 Some(schema) => schema.check(text, &config),
809 None => match known {
811 Some(problems) => (
812 jsonvet::report(text, problems.to_vec(), &config),
813 Vec::new(),
814 ),
815 None => (jsonvet::check(text, &config), Vec::new()),
816 },
817 }
818}
819
820fn look(format: Format, color: Option<bool>, terminal: Terminal) -> Look {
823 let text = format == Format::Text;
824 Look {
825 frames: text && terminal.tty,
826 color: text && color.or(terminal.color).unwrap_or(terminal.tty),
827 }
828}
829
830fn print_file(
834 file: &OsStr,
835 report: &Report<'_>,
836 schema_errors: &[(usize, String, bool)],
837 printer: &mut Printer<impl Write>,
838 counts: &mut Counts,
839) -> io::Result<()> {
840 let before = counts.errors + counts.warnings;
841 for problem in report.problems() {
842 match problem.severity() {
843 Severity::Error => counts.errors += 1,
844 Severity::Warning => counts.warnings += 1,
845 _ => {}
846 }
847 printer.problem(file, report, problem)?;
848 }
849 for (offset, message, summary) in schema_errors {
850 counts.errors += usize::from(!summary);
851 printer.schema_error(file, report, *offset, message, *summary)?;
852 }
853 counts.files += usize::from(counts.errors + counts.warnings > before);
854 Ok(())
855}
856
857fn read(path: &Path) -> Result<String, String> {
861 let read = std::fs::metadata(path).and_then(|meta| {
862 if meta.is_file() {
863 std::fs::read(path).map(Some)
864 } else {
865 Ok(None)
866 }
867 });
868 match read {
869 Ok(Some(bytes)) => decode(bytes),
870 Ok(None) => Err(messages::text("read.not-regular").to_owned()),
871 Err(e) if e.kind() == io::ErrorKind::NotFound => {
872 Err(messages::text("read.missing").to_owned())
873 }
874 Err(e) => Err(messages::fill("read.error", &[("error", &e)])),
875 }
876}
877
878fn read_stdin() -> Result<String, String> {
880 let mut bytes = Vec::new();
881 match io::stdin().lock().read_to_end(&mut bytes) {
882 Ok(_) => decode(bytes),
883 Err(e) => Err(messages::fill("read.error", &[("error", &e)])),
884 }
885}
886
887fn decode(bytes: Vec<u8>) -> Result<String, String> {
891 let opens_string = |b: u8| b == b'"' || b == b'\'';
895 let encoding = match *bytes.as_slice() {
897 [0xFF, 0xFE, 0, 0, ..] => Some("UTF-32 (LE)"),
898 [0, 0, 0xFE, 0xFF, ..] => Some("UTF-32 (BE)"),
899 [0xFF, 0xFE, ..] => Some("UTF-16 (LE)"),
900 [0xFE, 0xFF, ..] => Some("UTF-16 (BE)"),
901 [0, 0, 0, a, ..] if a != 0 => Some("UTF-32 (BE)"),
902 [0, a, 0, b, ..] if a != 0 && b != 0 => Some("UTF-16 (BE)"),
903 [a, 0, 0, 0, ..] if a != 0 && !opens_string(a) => Some("UTF-32 (LE)"),
904 [a, 0, b, 0, ..] if a != 0 && b != 0 && !opens_string(a) => Some("UTF-16 (LE)"),
905 _ => None,
906 };
907 if let Some(encoding) = encoding {
908 return Err(messages::fill("read.encoding", &[("encoding", &encoding)]));
909 }
910 String::from_utf8(bytes).map_err(|e| {
911 let byte = e.utf8_error().valid_up_to();
912 messages::fill("read.not-utf8", &[("byte", &byte)])
913 })
914}
915
916pub(crate) const MAX_CONFIG_BYTES: u64 = 16 << 20;
918
919pub(crate) const MAX_SCHEMA_BYTES: u64 = 64 << 20;
922
923pub(crate) fn read_small(path: &Path, max: u64) -> io::Result<String> {
927 if !std::fs::metadata(path)?.is_file() {
928 return Err(io::Error::other(messages::text("read.not-regular")));
929 }
930 let mut text = String::new();
931 std::fs::File::open(path)?
933 .take(max + 1)
934 .read_to_string(&mut text)?;
935 if text.len() as u64 > max {
936 let mb = max >> 20;
937 return Err(io::Error::other(messages::fill(
938 "read.too-large",
939 &[("mb", &mb)],
940 )));
941 }
942 Ok(text)
943}
944
945fn unsafe_ignore_file(dir: &Path) -> Option<String> {
950 [".gitignore", ".ignore"].into_iter().find_map(|name| {
951 let path = dir.join(name);
952 let meta = std::fs::metadata(&path).ok()?;
953 let problem = if !meta.is_file() {
954 messages::text("read.not-regular").to_owned()
955 } else if meta.len() > MAX_CONFIG_BYTES {
956 messages::fill("read.too-large", &[("mb", &(MAX_CONFIG_BYTES >> 20))])
957 } else {
958 return None;
959 };
960 let error = format!("{}: {problem}", path.display());
961 Some(output::printable(&messages::fill(
962 "error.walk",
963 &[("error", &error)],
964 )))
965 })
966}
967
968fn print(mut out: impl Write, text: &str) -> Result<u8, String> {
969 match writeln!(out, "{text}").and_then(|()| out.flush()) {
970 Err(e) if e.kind() != io::ErrorKind::BrokenPipe => {
971 Err(messages::fill("error.write", &[("error", &e)]))
972 }
973 _ => Ok(0),
974 }
975}
976
977#[cfg(test)]
978mod tests {
979 use std::fmt::Write as _;
980 use std::sync::atomic::{AtomicUsize, Ordering};
981
982 use jsonvet::Dialect;
983
984 use super::{
985 CHUNK_BYTES, MAX_CONFIG_BYTES, SCHEMA_BYTES, SPLIT_BYTES, STDIN, Schema, Settings, check,
986 chunk_len, parallel, read_small, schema_share, schema_shares, split, take_chunk,
987 unsafe_ignore_file,
988 };
989 use crate::schema::select::Job;
990 use std::sync::Arc;
991
992 fn any_schema(test: &str) -> (std::path::PathBuf, Schema) {
995 let dir = std::env::temp_dir().join(format!("jsonvet-{test}-{}", std::process::id()));
996 std::fs::create_dir_all(dir.join("x")).expect("a temporary directory");
997 std::fs::write(dir.join("s.json"), "{}").expect("written");
998 let schema = Schema::load(&dir.join("s.json"), &Settings::default(), false, |_| Ok(()));
999 (dir, schema.expect("a schema"))
1000 }
1001
1002 #[test]
1003 fn only_files_with_a_schema_take_a_share() {
1004 let (dir, schema) = any_schema("shares");
1005 let _ = std::fs::remove_dir_all(&dir);
1006 let schema = Arc::new(schema);
1007 let settings = Arc::new(Settings::default());
1008 let job =
1009 |schema: Option<&Arc<Schema>>| ((Arc::clone(&settings), schema.cloned()), STDIN.into());
1010 assert_eq!(
1011 schema_shares(&[job(None), job(None)], 2),
1012 Vec::<usize>::new()
1013 );
1014 assert_eq!(
1015 schema_shares(&[job(None), job(Some(&schema)), job(None)], 2),
1016 [0, CHUNK_BYTES, 0]
1017 );
1018 }
1019
1020 #[test]
1021 fn small_files_are_read_up_to_their_limit_and_no_further() {
1022 let dir = std::env::temp_dir().join(format!("jsonvet-small-{}", std::process::id()));
1023 std::fs::create_dir_all(&dir).expect("a temporary directory");
1024 let path = dir.join("f");
1025 std::fs::write(&path, "0123456789").expect("written");
1026 assert_eq!(read_small(&path, 10).expect("ten bytes"), "0123456789");
1027 assert!(read_small(&path, 9).is_err());
1028 let sized = |len: u64| {
1030 std::fs::File::create(dir.join(".gitignore"))
1031 .and_then(|file| file.set_len(len))
1032 .expect("a large file");
1033 };
1034 sized(MAX_CONFIG_BYTES);
1035 assert_eq!(unsafe_ignore_file(&dir), None);
1036 sized(MAX_CONFIG_BYTES + 1);
1037 assert!(unsafe_ignore_file(&dir).is_some_and(|e| e.ends_with("larger than 16 MB")));
1038 let _ = std::fs::remove_dir_all(&dir);
1039 }
1040
1041 #[test]
1042 fn only_json_lines_of_the_split_size_is_split_one_part_per_thread() {
1043 let line = "[1]\n".repeat(SPLIT_BYTES / 4);
1044 assert_eq!(split(&line, Dialect::JsonLines, 8).len(), 8);
1045 assert_eq!(split(&line, Dialect::JsonLines, 1).len(), 1);
1046 assert_eq!(split(&line, Dialect::Json, 8).len(), 1);
1047 assert_eq!(split(&line[4..], Dialect::JsonLines, 8).len(), 1);
1048 }
1049
1050 #[test]
1051 fn a_large_json_lines_file_checked_in_parts_reports_what_one_pass_does() {
1052 let pad = "x".repeat(500);
1056 let wide: Vec<String> = (0..150).map(|k| format!("\"k{k}\": \"s\"")).collect();
1057 let wide = format!("{{{}}}", wide.join(", "));
1058 let mut text = String::from("\u{FEFF}");
1059 for i in 0..4_000 {
1060 let _ = match i % 7 {
1061 _ if i % 1_000 == 999 => writeln!(text, "{wide}"),
1062 _ if i % 1_000 == 500 => writeln!(text, "{{\"a\": 1e400}}"),
1063 _ if i % 1_000 == 501 => writeln!(text, r#"{{"\ud800": 1, "\ud801": 2}}"#),
1064 0 => writeln!(text, "{{\"a\": {i}, \"a\": 1}}"),
1065 1 => write!(text, "{{\"a\": \"{i}\", \"p\": \"{pad}\"}}\r\n"),
1066 _ => writeln!(text, "{{\"a\": {i}, \"p\": \"{pad}\"}}"),
1067 };
1068 }
1069 assert!(text.len() > SPLIT_BYTES);
1070 let deep = format!("{}{}\n", "[".repeat(600), "]".repeat(600));
1071 let texts = [
1072 text.clone(),
1073 format!("{text}{{\"a\": 1\n{text}"),
1074 format!("{text}{deep}{text}"),
1075 ];
1076 let dir = std::env::temp_dir().join(format!("jsonvet-split-{}", std::process::id()));
1077 std::fs::create_dir_all(&dir).expect("a temporary directory");
1078 let path = dir.join("schema.json");
1079 let schema = concat!(
1080 r#"{"properties": {"a": {"type": "integer"}}, "#,
1081 r#""patternProperties": {"^k": {"type": "integer"}}}"#,
1082 );
1083 std::fs::write(&path, schema).expect("written");
1084 let settings = Settings::default();
1085 let schema = Schema::load(&path, &settings, false, |_| Ok(())).expect("a schema");
1086 let _ = std::fs::remove_dir_all(&dir);
1087 for text in &texts {
1088 for schema in [None, Some(&schema)] {
1089 let (whole, whole_errors) = check("x.jsonl", text, &settings, schema, 1, None);
1090 let (split, split_errors) = check("x.jsonl", text, &settings, schema, 8, None);
1091 assert_ne!(whole.problems(), &[]);
1092 assert_eq!(split.problems(), whole.problems());
1093 assert_eq!(split_errors, whole_errors);
1094 }
1095 }
1096 let (_, errors) = check("x.jsonl", &texts[0], &settings, Some(&schema), 8, None);
1097 assert_eq!(errors.len(), 570 + 4 * 101 + 8);
1099 let unchecked = errors.iter().filter(|(_, m, _)| m.contains("not checked"));
1100 assert_eq!(unchecked.count(), 8);
1101 }
1102
1103 #[test]
1104 fn parallel_keeps_the_order_and_stops_after_a_prefix() {
1105 let items: Vec<usize> = (0..1000).collect();
1106 for threads in [1, 3, 16] {
1107 let squares = parallel(&items, threads, || false, |i| i * i);
1108 assert_eq!(squares, items.iter().map(|i| i * i).collect::<Vec<_>>());
1109 let stopped = parallel(&items, threads, || true, |&i| i);
1112 assert!((1..=threads).contains(&stopped.len()), "{stopped:?}");
1113 assert_eq!(stopped, (0..stopped.len()).collect::<Vec<_>>());
1114 let done = AtomicUsize::new(0);
1115 let some = parallel(
1116 &items,
1117 threads,
1118 || done.load(Ordering::Relaxed) >= 100,
1119 |&i| {
1120 done.fetch_add(1, Ordering::Relaxed);
1121 i
1122 },
1123 );
1124 assert!((100..100 + threads).contains(&some.len()), "{}", some.len());
1126 assert_eq!(some, (0..some.len()).collect::<Vec<_>>());
1127 }
1128 assert_eq!(
1129 parallel(&[] as &[u8], 4, || false, |&b| b),
1130 Vec::<u8>::new()
1131 );
1132 }
1133
1134 #[test]
1135 fn a_chunk_ends_with_the_file_that_fills_it() {
1136 assert_eq!(chunk_len(&[]), 0);
1137 assert_eq!(chunk_len(&[1, 2, 3]), 3);
1138 assert_eq!(chunk_len(&[CHUNK_BYTES, 1]), 1);
1139 assert_eq!(chunk_len(&[CHUNK_BYTES - 1, 1, 1]), 2);
1140 assert_eq!(chunk_len(&[CHUNK_BYTES - 2, 1, 1]), 3);
1141 assert_eq!(chunk_len(&[0, CHUNK_BYTES - 1, 0, 2, 5]), 4);
1142 }
1143
1144 #[test]
1145 fn take_chunk_respects_shares() {
1146 let dir = std::env::temp_dir().join(format!("jsonvet-chunk-{}", std::process::id()));
1147 std::fs::create_dir_all(&dir).expect("dir");
1148 let f1 = dir.join("1.json");
1149 let f2 = dir.join("2.json");
1150 std::fs::write(&f1, "{}").expect("write");
1151 std::fs::write(&f2, "{}").expect("write");
1152 let settings = Arc::new(Settings::default());
1153 let job1: Job = ((settings.clone(), None), f1.into_os_string());
1154 let job2: Job = ((settings.clone(), None), f2.into_os_string());
1155 let rest = [job1, job2];
1156 let mut ahead = Vec::new();
1157 let shares = [CHUNK_BYTES, CHUNK_BYTES];
1158 let taken = take_chunk(&mut ahead, &rest, Some(&shares), 1, &|_, _| false);
1159 assert_eq!(taken.len(), 1);
1160 assert_eq!(ahead, []);
1161 let _ = std::fs::remove_dir_all(&dir);
1162 }
1163
1164 #[test]
1165 fn a_file_counts_its_size_twenty_times_with_a_schema() {
1166 let dir = std::env::temp_dir().join(format!("jsonvet-share-{}", std::process::id()));
1167 std::fs::create_dir_all(&dir).expect("a temp dir");
1168 let path = dir.join("a.json");
1169 std::fs::write(&path, " ".repeat(1000)).expect("write");
1170 let path = path.as_os_str();
1171 assert_eq!(schema_share(path), 1000 * SCHEMA_BYTES);
1172 std::fs::write(path, " ".repeat(CHUNK_BYTES / SCHEMA_BYTES + 1)).expect("write");
1173 assert_eq!(schema_share(path), CHUNK_BYTES, "capped");
1174 assert_eq!(schema_share(dir.join("gone.json").as_os_str()), 0);
1175 assert_eq!(schema_share(std::ffi::OsStr::new("-")), CHUNK_BYTES);
1176 let _ = std::fs::remove_dir_all(&dir);
1177 }
1178
1179 #[test]
1180 fn frames_show_on_a_terminal_and_color_follows_the_option_then_the_environment() {
1181 use super::{Format, Look, Terminal, look};
1182 let terminal = |tty, color| Terminal { tty, color };
1183 let frames = |color| Look {
1186 frames: true,
1187 color,
1188 };
1189 let lines = Look {
1190 frames: false,
1191 color: false,
1192 };
1193 let cases = [
1194 (Format::Text, None, terminal(true, None), frames(true)),
1195 (
1196 Format::Text,
1197 None,
1198 terminal(true, Some(false)),
1199 frames(false),
1200 ),
1201 (
1202 Format::Text,
1203 Some(false),
1204 terminal(true, Some(true)),
1205 frames(false),
1206 ),
1207 (Format::Sarif, Some(true), terminal(true, None), lines),
1208 (Format::Json, Some(true), terminal(true, None), lines),
1209 ];
1210 for (format, color, terminal, want) in cases {
1211 assert_eq!(
1212 look(format, color, terminal),
1213 want,
1214 "{format:?} {color:?} {terminal:?}"
1215 );
1216 }
1217 }
1218
1219 #[test]
1220 fn the_summary_counts_problems_and_files() {
1221 let dir = std::env::temp_dir().join(format!("jsonvet-summary-{}", std::process::id()));
1222 std::fs::create_dir_all(&dir).expect("a temp dir");
1223 let dup = dir.join("dup.json");
1224 let ok = dir.join("ok.json");
1225 std::fs::write(&dup, "{\"a\": 1, \"a\": 2}").expect("write");
1226 std::fs::write(&ok, "{}").expect("write");
1227 let run = |summary: bool, files: &[&std::path::Path]| {
1228 let mut out = Vec::new();
1229 let args = ["--no-config".into()]
1230 .into_iter()
1231 .chain(files.iter().map(|f| f.as_os_str().to_owned()));
1232 let terminal = super::Terminal {
1233 tty: summary,
1234 color: Some(false),
1235 };
1236 let code = super::run_with(args, false, terminal, &mut out).expect("a check");
1237 (code, String::from_utf8(out).expect("UTF-8"))
1238 };
1239 let (code, out) = run(true, &[&dup, &ok]);
1240 assert_eq!(code, 1);
1241 assert!(out.ends_with("\n1 error in 1 of 2 files\n"), "{out}");
1242 let gone = dir.join("gone.json");
1244 let dup2 = dir.join("dup2.json");
1246 std::fs::write(&dup2, "{\"a\": 1, \"a\": 2}").expect("write");
1247 let (_, out) = run(true, &[&dup, &gone, &dup2, &ok]);
1248 assert!(out.ends_with("\n3 errors in 3 of 4 files\n"), "{out}");
1249 assert_eq!(run(true, &[&ok]).1, "no problems in 1 file\n");
1250 assert!(!run(false, &[&dup]).1.contains(" in 1 of 1 file"));
1252 assert_eq!(run(true, &[]).1, "");
1253 for format in ["sarif", "json"] {
1255 let args = ["--no-config", "--format", format]
1256 .map(std::ffi::OsString::from)
1257 .into_iter()
1258 .chain([dup.clone().into_os_string()]);
1259 let mut out = Vec::new();
1260 let terminal = super::Terminal {
1261 tty: true,
1262 color: None,
1263 };
1264 super::run_with(args, false, terminal, &mut out).expect("a check");
1265 let out = String::from_utf8(out).expect("UTF-8");
1266 assert!(
1267 serde_json::from_str::<serde_json::Value>(&out).is_ok(),
1268 "{out}"
1269 );
1270 }
1271 let _ = std::fs::remove_dir_all(&dir);
1272 }
1273
1274 #[test]
1276 fn a_translation_reaches_text_output_only() {
1277 let dir = std::env::temp_dir().join(format!("jsonvet-locale-{}", std::process::id()));
1278 std::fs::create_dir_all(&dir).expect("a temp dir");
1279 let dup = dir.join("dup.json");
1280 std::fs::write(&dup, "{\"a\": 1, \"a\": 2}").expect("write");
1281 crate::messages::use_translation(
1282 "\"duplicate-key\" = \"XX {path}\"\n\"related.first-at\" = \"XX {line}:{column}\"\n",
1283 );
1284 let run = |format: &str| {
1285 let args = ["--no-config", "--format", format]
1286 .map(std::ffi::OsString::from)
1287 .into_iter()
1288 .chain([dup.clone().into_os_string()]);
1289 let mut out = Vec::new();
1290 super::run_with(args, false, super::Terminal::default(), &mut out).expect("a check");
1291 String::from_utf8(out).expect("UTF-8")
1292 };
1293 let text = run("text");
1294 assert!(text.contains("XX $.a (XX 1:2)"), "{text}");
1295 for format in ["json", "sarif"] {
1296 let out = run(format);
1297 assert!(out.contains("duplicate key $.a (first at 1:2)"), "{out}");
1298 assert!(!out.contains("XX"), "{out}");
1299 }
1300 let _ = std::fs::remove_dir_all(&dir);
1301 }
1302
1303 #[test]
1306 fn json_lines_and_schema_errors_get_code_frames_on_a_terminal() {
1307 let dir = std::env::temp_dir().join(format!("jsonvet-frames-{}", std::process::id()));
1308 std::fs::create_dir_all(&dir).expect("a temp dir");
1309 let lines = dir.join("l.jsonl");
1310 let schema = dir.join("s.json");
1311 let data = dir.join("d.json");
1312 std::fs::write(&lines, "{\"a\": 1}\n{\"a\": True}\n").expect("write");
1313 std::fs::write(&schema, r#"{"properties": {"a": {"type": "string"}}}"#).expect("write");
1314 std::fs::write(&data, "{\n \"a\": 1\n}\n").expect("write");
1315 let run = |args: &[&std::ffi::OsStr]| {
1316 let args = ["--no-config".into()]
1317 .into_iter()
1318 .chain(args.iter().map(|a| (*a).to_owned()));
1319 let terminal = super::Terminal {
1320 tty: true,
1321 color: Some(false),
1322 };
1323 let mut out = Vec::new();
1324 let code = super::run_with(args, false, terminal, &mut out).expect("a check");
1325 let out = String::from_utf8(out).expect("UTF-8");
1326 (
1327 code,
1328 out.replace(
1329 &format!("{}{}", dir.display(), std::path::MAIN_SEPARATOR),
1330 "",
1331 ),
1332 )
1333 };
1334 assert_eq!(
1335 run(&[lines.as_os_str()]),
1336 (
1337 1,
1338 "error[syntax]: expected a value, found `True`; did you mean `true`?\n \
1339 --> l.jsonl:2:7\n |\n2 | {\"a\": True}\n | ^^^^\n |\n = help: https://github.com/simonvanlierde/jsonvet/blob/main/docs/usage.md#syntax\n\n\
1340 1 error in 1 of 1 file\n"
1341 .to_owned()
1342 )
1343 );
1344 assert_eq!(
1345 run(&["--schema".as_ref(), schema.as_os_str(), data.as_os_str()]),
1346 (
1347 1,
1348 "error[schema]: 1 is not of type \"string\" (at $.a)\n \
1349 --> d.json:2:8\n |\n2 | \"a\": 1\n | ^\n |\n = help: https://github.com/simonvanlierde/jsonvet/blob/main/docs/usage.md#schema\n\n\
1350 1 error in 1 of 1 file\n"
1351 .to_owned()
1352 )
1353 );
1354 let _ = std::fs::remove_dir_all(&dir);
1355 }
1356
1357 #[test]
1358 fn the_summary_names_errors_and_warnings_together() {
1359 let counts = |errors, warnings| super::Counts {
1360 errors,
1361 warnings,
1362 files: 1,
1363 };
1364 assert_eq!(
1365 counts(1, 1).summary(2),
1366 "1 error and 1 warning in 1 of 2 files"
1367 );
1368 assert_eq!(counts(0, 2).summary(1), "2 warnings in 1 of 1 file");
1369 }
1370
1371 #[test]
1372 fn a_line_separator_ends_a_frame_line_only_in_a_json5_file() {
1373 let dir = std::env::temp_dir().join(format!("jsonvet-u2028-{}", std::process::id()));
1374 std::fs::create_dir_all(&dir).expect("a temp dir");
1375 let frame = |name: &str| {
1376 let file = dir.join(name);
1377 std::fs::write(&file, "{\"a\": \"x\u{2028}y\", \"a\": 2}\n").expect("write");
1378 let mut out = Vec::new();
1379 let args = ["--no-config".into(), file.into_os_string()];
1380 let terminal = super::Terminal {
1381 tty: true,
1382 color: Some(false),
1383 };
1384 super::run_with(args, false, terminal, &mut out).expect("a check");
1385 String::from_utf8(out).expect("UTF-8")
1386 };
1387 let five = frame("a.json5");
1388 assert!(five.contains("\n2 | y\", \"a\": 2}\n"), "{five}");
1390 assert!(five.contains("\n1 | {\"a\": \"x\n"), "{five}");
1391 let other = frame("a.jsonc");
1392 assert!(
1394 other
1395 .lines()
1396 .any(|l| l.starts_with("1 | {\"a\": \"x\u{FFFD}y\", \"a\": 2}")),
1397 "{other}"
1398 );
1399 let _ = std::fs::remove_dir_all(&dir);
1400 }
1401
1402 #[test]
1403 fn a_panic_on_the_big_stack_carries_on_in_the_caller() {
1404 static RUNS: AtomicUsize = AtomicUsize::new(0);
1405 let panic = std::panic::catch_unwind(|| {
1406 super::on_big_stack(|| -> u8 {
1407 RUNS.fetch_add(1, Ordering::Relaxed);
1408 panic!("on the big stack")
1409 })
1410 })
1411 .expect_err("the panic carries on");
1412 assert_eq!(panic.downcast_ref::<&str>(), Some(&"on the big stack"));
1413 assert_eq!(RUNS.load(Ordering::Relaxed), 1);
1414 }
1415}