Skip to main content

jsonvet_cli/
lib.rs

1//! The jsonvet command line, shared by the `jsonvet` and `check-json5` binaries.
2
3use std::ffi::{OsStr, OsString};
4use std::io::{self, BufWriter, IsTerminal, Read, Write};
5use std::path::Path;
6use std::process::ExitCode;
7
8use std::sync::atomic::{AtomicUsize, Ordering};
9use std::sync::{Arc, Mutex};
10
11use jsonvet::{Dialect, Report, Severity};
12
13mod args;
14mod frame;
15mod glob;
16mod json;
17mod output;
18mod rules;
19mod sarif;
20mod schema;
21mod settings;
22
23use args::{Command, STDIN};
24use output::{Format, Look, Printer};
25use schema::Schema;
26use schema::select::{Job, Schemas, check_own_schema, jobs, with_schema};
27use settings::{Resolver, Settings};
28
29/// The stack every run uses, for `--schema`: jsonschema validates recursively, and building,
30/// validating and dropping a value [`schema::SCHEMA_MAX_DEPTH`] levels deep against a recursive schema
31/// took up to 2.2 MB in a debug build, more than Windows' 1 MB main thread. Only the pages used
32/// are committed, so a plain check costs no more memory.
33const STACK: usize = 64 << 20;
34
35/// How much memory the files read at once may take before they are checked and printed.
36const CHUNK_BYTES: usize = 64 << 20;
37
38/// How many bytes of memory a byte of file text takes with a schema, which holds the file's value
39/// one document at a time: 4 to 22 per byte of that document (docs/usage.md, "JSON Schema
40/// limits"). 20 is near the top of that range: only many small records take more.
41const SCHEMA_BYTES: usize = 20;
42
43/// From this size, a JSON Lines file is split at newlines and its parts checked on the cores
44/// its chunk leaves free.
45const SPLIT_BYTES: usize = 1 << 20;
46
47/// Run the command with the process's arguments and environment, and return its exit code.
48pub fn run() -> ExitCode {
49    on_big_stack(run_here)
50}
51
52/// Run `f` on a thread with a [`STACK`]-byte stack, or here if no thread can start. A panic in
53/// `f` carries on here.
54fn on_big_stack<T: Send + 'static>(f: fn() -> T) -> T {
55    let thread = std::thread::Builder::new().name("jsonvet".into());
56    match thread.stack_size(STACK).spawn(f) {
57        Ok(thread) => thread
58            .join()
59            .unwrap_or_else(|panic| std::panic::resume_unwind(panic)),
60        Err(_) => f(),
61    }
62}
63
64fn run_here() -> ExitCode {
65    let annotate = std::env::var_os("GITHUB_ACTIONS").is_some_and(|v| v == "true");
66    let stdout = io::stdout().lock();
67    let set = |name| std::env::var_os(name).is_some_and(|v| !v.is_empty());
68    // NO_COLOR wins over FORCE_COLOR: asking for no color is the safer wish to honour.
69    // FORCE_COLOR=0 and =false ask for no color, as Node and chalk read them.
70    let color = if set("NO_COLOR") {
71        Some(false)
72    } else if set("FORCE_COLOR") {
73        Some(std::env::var_os("FORCE_COLOR").is_none_or(|v| v != "0" && v != "false"))
74    } else {
75        None
76    };
77    let terminal = Terminal {
78        tty: stdout.is_terminal(),
79        color,
80    };
81    match run_with(
82        std::env::args_os().skip(1),
83        annotate,
84        terminal,
85        BufWriter::new(stdout),
86    ) {
87        Ok(code) => ExitCode::from(code),
88        Err(message) => {
89            eprintln!("jsonvet: {message}");
90            ExitCode::from(2)
91        }
92    }
93}
94
95/// Where output goes, as far as text output's look depends on it. Public only for
96/// `benches/codspeed.rs`; not part of the crate's API.
97#[doc(hidden)]
98#[derive(Clone, Copy, Debug, Default)]
99pub struct Terminal {
100    /// Standard output is a terminal. A summary and code frames are for people: hooks and CI
101    /// read standard output from a pipe and see one line per problem.
102    pub tty: bool,
103    /// What the environment asks for when `--color` is `auto`: `Some(false)` for `NO_COLOR`,
104    /// `Some(true)` for `FORCE_COLOR`.
105    pub color: Option<bool>,
106}
107
108/// The command, with its inputs passed in: 0 when no file has an error, 1 when one does.
109/// On a terminal, text output ends with a count of the problems. A usage error comes back as
110/// `Err`, which `run` turns into exit status 2. Public only for `benches/codspeed.rs`, which
111/// measures `--schema` in-process; not part of the crate's API.
112#[doc(hidden)]
113pub fn run_with(
114    args: impl IntoIterator<Item = OsString>,
115    annotate: bool,
116    terminal: Terminal,
117    out: impl Write,
118) -> Result<u8, String> {
119    // Usage errors quote arguments and the schema file, which can hold a newline or an escape.
120    let args = match args::parse(args).map_err(|e| {
121        format!(
122            "{}\n\nRun `jsonvet --help` for usage.",
123            output::printable(&e)
124        )
125    })? {
126        Command::Help => return print(out, args::HELP),
127        Command::Version => return print(out, concat!("jsonvet ", env!("CARGO_PKG_VERSION"))),
128        Command::ListRules => return print(out, &rules::list()),
129        Command::Explain(text) => return print(out, &text),
130        Command::Check(args) => args,
131    };
132    let resolver = Arc::new(Mutex::new(Resolver::new(&args.config, args.settings)?));
133    let settings_for = |path: &Path| lock(&resolver).for_file(path);
134    let threads = std::thread::available_parallelism().map_or(1, std::num::NonZero::get);
135    let files = expand(args.files, &resolver, threads)?;
136    let schemas = Schemas::default();
137    let load = |path: &Path, quiet| {
138        schemas.get(path, quiet, |admit| {
139            Schema::load(path, &*settings_for(path)?, quiet, admit)
140        })
141    };
142    let root = |path: &Path| lock(&resolver).root(path);
143
144    // Standard input goes by its `--stdin-filename`, for settings and in the output.
145    let stdin_name = args.stdin_filename.unwrap_or_else(|| STDIN.into());
146    let shown = |file: &OsStr| -> OsString {
147        if file == STDIN {
148            stdin_name.clone()
149        } else {
150            file.to_owned()
151        }
152    };
153    // Every config file, and every schema one names, is read before any file is checked, so a
154    // bad one stops the run at once.
155    let files = jobs(files, &shown, &settings_for, args.schema.as_deref(), &load)?;
156    let summary = terminal.tty;
157    let look = look(args.format, args.color, terminal);
158    let mut printer = Printer::new(out, args.format, annotate, look);
159    let mut counts = Counts::default();
160    // Files are read and checked on every core, a chunk at a time, then printed here in the
161    // order given. A chunk ends once its files hold CHUNK_BYTES, so memory stays near that. With
162    // a schema a file takes SCHEMA_BYTES times its size, so the chunk is cut from the sizes
163    // before any file is read: otherwise every core would start on a file at once. Without
164    // a schema the files are not sized first, which made a walk of 10,000 files 7 ms slower.
165    // A file whose own `$schema` may name a schema is known only once read: it counts the same,
166    // and the texts read past the end of the chunk are kept, in `ahead`, for the next one.
167    // NOTE: each chunk waits for its slowest file before the next starts; hand files out
168    // one at a time across chunks if one big file among many small ones holds a run up.
169    let mut rest = &files[..];
170    let shares = schema_shares(&files, threads);
171    let mut ahead = Vec::new();
172    let written = loop {
173        if rest.is_empty() {
174            let line = (summary && !files.is_empty()).then(|| counts.summary(files.len()));
175            break line
176                .map_or(Ok(()), |line| printer.text_line(&line))
177                .and_then(|()| printer.finish());
178        }
179        let shares = shares.get(files.len() - rest.len()..);
180        let with_schema = |job: &Job, text: &str| with_schema(job, &shown(&job.1), text);
181        let texts = take_chunk(&mut ahead, rest, shares, threads, &with_schema);
182        let chunk;
183        (chunk, rest) = rest.split_at(texts.len());
184        let jobs: Vec<_> = chunk.iter().zip(&texts).collect();
185        let parts = (threads / jobs.len()).max(1);
186        let checked = parallel(
187            &jobs,
188            threads,
189            || false,
190            |(((settings, schema), file), (text, with))| {
191                let name = shown(file);
192                let name = name.to_string_lossy();
193                text.as_deref().map(|text| match schema {
194                    Some(schema) => check(&name, text, settings, Some(schema), parts),
195                    None if *with => {
196                        check_own_schema(&name, text, settings, &schemas, &root, &load, parts)
197                    }
198                    None => check(&name, text, settings, None, parts),
199                })
200            },
201        );
202        schemas.over().map_err(|e| output::printable(&e))?;
203        let outcome =
204            chunk
205                .iter()
206                .zip(checked)
207                .try_for_each(|((_, file), checked)| match checked {
208                    Ok((report, schema_errors)) => print_file(
209                        &shown(file),
210                        &report,
211                        &schema_errors,
212                        &mut printer,
213                        &mut counts,
214                    ),
215                    Err(message) => {
216                        counts.errors += 1;
217                        counts.files += 1;
218                        printer.unreadable(&shown(file), message)
219                    }
220                });
221        if outcome.is_err() {
222            break outcome;
223        }
224    };
225    // A reader that has gone, as with `| head`, wants no more output: the verdict so far stands.
226    match written {
227        Err(e) if e.kind() != io::ErrorKind::BrokenPipe => {
228            return Err(format!("cannot write output: {e}"));
229        }
230        _ => {}
231    }
232    let too_many_warnings = args.max_warnings.is_some_and(|max| counts.warnings > max);
233    Ok(u8::from(counts.errors > 0 || too_many_warnings))
234}
235
236/// The texts of the first of `files`, read on up to `threads` threads until their [`weight`]s
237/// reach [`CHUNK_BYTES`], and always the first, each with whether `with_schema` says it is
238/// checked against a schema.
239fn read_chunk<S: Sync>(
240    files: &[(S, OsString)],
241    threads: usize,
242    with_schema: &(impl Fn(&(S, OsString), &str) -> bool + Sync),
243) -> Vec<(Result<String, String>, bool)> {
244    let bytes = AtomicUsize::new(0);
245    let full = || bytes.load(Ordering::Relaxed) >= CHUNK_BYTES;
246    parallel(files, threads, full, |file| {
247        let text = if file.1 == STDIN {
248            read_stdin()
249        } else {
250            read(Path::new(&file.1))
251        };
252        let with = text.as_ref().is_ok_and(|text| with_schema(file, text));
253        bytes.fetch_add(weight(&text, with), Ordering::Relaxed);
254        (text, with)
255    })
256}
257
258/// The texts of the next chunk of `rest`, the files left, each with whether it is checked
259/// against a schema: up to and including the one whose [`weight`] fills [`CHUNK_BYTES`], from
260/// `ahead`, the texts of the first of them read already, or if none are, from those read now.
261/// `shares` are the files' [`schema_share`]s, if any.
262fn take_chunk(
263    ahead: &mut Vec<(Result<String, String>, bool)>,
264    rest: &[Job],
265    shares: Option<&[usize]>,
266    threads: usize,
267    with_schema: &(impl Fn(&Job, &str) -> bool + Sync),
268) -> Vec<(Result<String, String>, bool)> {
269    if ahead.is_empty() {
270        let offered = match shares {
271            Some(shares) if !shares.is_empty() => &rest[..chunk_len(shares)],
272            _ => rest,
273        };
274        *ahead = read_chunk(offered, threads, with_schema);
275    }
276    let weights: Vec<usize> = ahead
277        .iter()
278        .map(|(text, with)| weight(text, *with))
279        .collect();
280    ahead.drain(..chunk_len(&weights)).collect()
281}
282
283/// The memory a text read takes while it is checked, as far as a chunk counts it: its size, or
284/// with a schema, [`SCHEMA_BYTES`] times that, capped at [`CHUNK_BYTES`].
285fn weight(text: &Result<String, String>, with_schema: bool) -> usize {
286    let size = text.as_ref().map_or(0, String::len);
287    if with_schema {
288        size.saturating_mul(SCHEMA_BYTES).min(CHUNK_BYTES)
289    } else {
290        size
291    }
292}
293
294/// The memory a file takes while a schema checks it, as far as a chunk counts it:
295/// [`SCHEMA_BYTES`] times its size, capped at [`CHUNK_BYTES`]. Standard input's size is known
296/// only once it is read, so it takes a chunk's worth. A file that cannot be read takes nothing:
297/// `read` reports why.
298fn schema_share(file: &OsStr) -> usize {
299    if file == STDIN {
300        return CHUNK_BYTES;
301    }
302    let size = std::fs::metadata(file).map_or(0, |meta| meta.len());
303    usize::try_from(size)
304        .unwrap_or(usize::MAX)
305        .saturating_mul(SCHEMA_BYTES)
306        .min(CHUNK_BYTES)
307}
308
309/// Each file's [`schema_share`], or 0 for a file with no schema; nothing if no file has one.
310fn schema_shares(files: &[Job], threads: usize) -> Vec<usize> {
311    if files.iter().all(|((_, schema), _)| schema.is_none()) {
312        return Vec::new();
313    }
314    parallel(
315        files,
316        threads,
317        || false,
318        |((_, schema), file)| schema.as_ref().map_or(0, |_| schema_share(file)),
319    )
320}
321
322/// How many of the files with these shares the next chunk takes: up to and including the one
323/// that fills [`CHUNK_BYTES`], so always at least one.
324fn chunk_len(shares: &[usize]) -> usize {
325    let mut total = 0;
326    shares
327        .iter()
328        .position(|share| {
329            total += share;
330            total >= CHUNK_BYTES
331        })
332        .map_or(shares.len(), |last| last + 1)
333}
334
335/// The files to check: each argument as given, except that a directory stands for the files under
336/// it that jsonvet checks by default, in sorted order. The walk skips hidden files and what
337/// `.gitignore`, `.ignore` and the `exclude` setting of each folder's settings name, as ripgrep
338/// would.
339fn expand(
340    args: Vec<OsString>,
341    resolver: &Arc<Mutex<Resolver>>,
342    threads: usize,
343) -> Result<Vec<OsString>, String> {
344    let mut files = Vec::with_capacity(args.len());
345    // Each argument costs a call to the system, so thousands of them are asked about in parallel.
346    let dirs = parallel(
347        &args,
348        threads,
349        || false,
350        |arg| arg != STDIN && Path::new(arg).is_dir(),
351    );
352    for (arg, dir) in args.into_iter().zip(dirs) {
353        if !dir {
354            files.push(arg);
355            continue;
356        }
357        let start = files.len();
358        // The walk reads the ignore files of the folder, and of every folder above it when it
359        // can find them, before the filter sees anything.
360        let above = std::fs::canonicalize(&arg).ok();
361        if let Some(problem) = std::iter::once(Path::new(&arg))
362            .chain(above.iter().flat_map(|dir| dir.ancestors()))
363            .find_map(unsafe_ignore_file)
364        {
365            return Err(problem);
366        }
367        let skipped = Arc::new(Mutex::new(None));
368        let skip = Arc::clone(&skipped);
369        // The filter must own what it reads.
370        let resolver = Arc::clone(resolver);
371        // A config file that cannot be read excludes nothing here: the run reports it when it
372        // looks up the settings of the files under it.
373        let walk = ignore::WalkBuilder::new(&arg)
374            .filter_entry(move |entry| {
375                // The walk reads a folder's ignore files once the filter lets the folder in.
376                if entry.file_type().is_some_and(|t| t.is_dir())
377                    && let Some(problem) = unsafe_ignore_file(entry.path())
378                {
379                    // A poisoned lock still holds the message.
380                    skip.lock()
381                        .unwrap_or_else(std::sync::PoisonError::into_inner)
382                        .get_or_insert(problem);
383                    return false;
384                }
385                let settings = lock(&resolver).for_file(entry.path());
386                !settings.is_ok_and(|settings| settings.excluded(entry.path()))
387            })
388            .build();
389        for entry in walk {
390            let entry = entry.map_err(|e| output::printable(&format!("cannot walk: {e}")))?;
391            if entry.file_type().is_some_and(|t| t.is_file())
392                && Dialect::for_path(entry.path()).is_some()
393            {
394                files.push(entry.into_path().into_os_string());
395            }
396        }
397        if let Some(problem) = skipped
398            .lock()
399            .unwrap_or_else(std::sync::PoisonError::into_inner)
400            .take()
401        {
402            return Err(problem);
403        }
404        files[start..].sort_unstable();
405    }
406    Ok(files)
407}
408
409/// The resolver, even if a panic elsewhere left its lock poisoned: it holds only a cache.
410fn lock(resolver: &Mutex<Resolver>) -> std::sync::MutexGuard<'_, Resolver> {
411    resolver
412        .lock()
413        .unwrap_or_else(std::sync::PoisonError::into_inner)
414}
415
416#[derive(Default)]
417struct Counts {
418    errors: usize,
419    warnings: usize,
420    /// Files with at least one problem, or that could not be read.
421    files: usize,
422}
423
424impl Counts {
425    /// Such as `3 errors and 1 warning in 2 of 5 files`.
426    fn summary(&self, checked: usize) -> String {
427        let n = |count: usize, what: &str| {
428            format!("{count} {what}{}", if count == 1 { "" } else { "s" })
429        };
430        if self.errors + self.warnings == 0 {
431            return format!("no problems in {}", n(checked, "file"));
432        }
433        let problems = match (self.errors, self.warnings) {
434            (e, 0) => n(e, "error"),
435            (0, w) => n(w, "warning"),
436            (e, w) => format!("{} and {}", n(e, "error"), n(w, "warning")),
437        };
438        format!("{problems} in {} of {}", self.files, n(checked, "file"))
439    }
440}
441
442/// `f` on the items, on up to `threads` threads counting this one, each with a [`STACK`]-byte
443/// stack, until `stop` says so before an item is taken: the results for the items done, in
444/// order, always at least the first. A panic in `f` carries on here.
445fn parallel<T: Sync, U: Send>(
446    items: &[T],
447    threads: usize,
448    stop: impl Fn() -> bool + Sync,
449    f: impl Fn(&T) -> U + Sync,
450) -> Vec<U> {
451    let next = AtomicUsize::new(0);
452    // Each thread takes the next item in turn, so the items taken are always the first ones,
453    // and the results are tagged to put them back in order.
454    let work = || {
455        let mut done = Vec::new();
456        loop {
457            if stop() && next.load(Ordering::Relaxed) > 0 {
458                break done;
459            }
460            let i = next.fetch_add(1, Ordering::Relaxed);
461            let Some(item) = items.get(i) else { break done };
462            done.push((i, f(item)));
463        }
464    };
465    let mut done = std::thread::scope(|scope| {
466        // A thread that cannot start leaves its share to the others.
467        let helpers: Vec<_> = (1..threads.min(items.len()))
468            .filter_map(|_| {
469                std::thread::Builder::new()
470                    .stack_size(STACK)
471                    .spawn_scoped(scope, work)
472                    .ok()
473            })
474            .collect();
475        let mut done = work();
476        for helper in helpers {
477            done.extend(
478                helper
479                    .join()
480                    .unwrap_or_else(|panic| std::panic::resume_unwind(panic)),
481            );
482        }
483        done
484    });
485    done.sort_unstable_by_key(|&(i, _)| i);
486    done.into_iter().map(|(_, result)| result).collect()
487}
488
489/// The parts to check a text in `dialect` in, on up to `parts` threads: more than one only for
490/// JSON Lines of [`SPLIT_BYTES`] or more.
491fn split(text: &str, dialect: Dialect, parts: usize) -> Vec<std::ops::Range<usize>> {
492    let split = dialect == Dialect::JsonLines && parts > 1 && text.len() >= SPLIT_BYTES;
493    jsonvet::line_parts(text, if split { parts } else { 1 })
494}
495
496/// Check a file's text, named by its lossy UTF-8 form for glob matching, and against `schema`
497/// if there is one, on up to `parts` threads: the report and the schema errors.
498fn check<'t>(
499    name: &str,
500    text: &'t str,
501    settings: &Settings,
502    schema: Option<&Schema>,
503    parts: usize,
504) -> (Report<'t>, Vec<(usize, String, bool)>) {
505    let config = settings.config_for(name);
506    let ranges = split(text, config.dialect(), parts);
507    if ranges.len() > 1 {
508        // Each line is a document of its own, and the error cap counts per document, so each
509        // part's schema errors are the ones a single pass finds in its lines.
510        let (lines, errors): (Vec<_>, Vec<_>) = parallel(
511            &ranges,
512            parts,
513            || false,
514            |lines| match schema {
515                Some(schema) => schema.check_lines(text, lines.clone(), &config),
516                None => (
517                    jsonvet::parse_lines(text, lines.clone(), &config, None),
518                    Vec::new(),
519                ),
520            },
521        )
522        .into_iter()
523        .unzip();
524        let (report, syntax_error) = jsonvet::join_lines(text, lines);
525        // A file with a syntax error in any part gets no schema errors.
526        let errors = if syntax_error {
527            Vec::new()
528        } else {
529            errors.into_iter().flatten().collect()
530        };
531        return (report, errors);
532    }
533    match schema {
534        // A file with a syntax error gets no schema errors.
535        Some(schema) => schema.check(text, &config),
536        None => (jsonvet::check(text, &config), Vec::new()),
537    }
538}
539
540/// How text output looks: code frames on a terminal and lines otherwise, so tools reading a pipe
541/// always get `file:line:col`, and in color as `--color` and the environment ask.
542fn look(format: Format, color: Option<bool>, terminal: Terminal) -> Look {
543    let text = format == Format::Text;
544    Look {
545        frames: text && terminal.tty,
546        color: text && color.or(terminal.color).unwrap_or(terminal.tty),
547    }
548}
549
550/// Print one file's problems and schema errors, and add them to `counts`, each before it is
551/// printed, so a failed write still leaves the verdict. The file count is added last: only the
552/// summary reads it, and a failed write ends the run before the summary.
553fn print_file(
554    file: &OsStr,
555    report: &Report<'_>,
556    schema_errors: &[(usize, String, bool)],
557    printer: &mut Printer<impl Write>,
558    counts: &mut Counts,
559) -> io::Result<()> {
560    let before = counts.errors + counts.warnings;
561    for problem in report.problems() {
562        match problem.severity() {
563            Severity::Error => counts.errors += 1,
564            Severity::Warning => counts.warnings += 1,
565            _ => {}
566        }
567        printer.problem(file, report, problem)?;
568    }
569    for (offset, message, summary) in schema_errors {
570        counts.errors += usize::from(!summary);
571        printer.schema_error(file, report, *offset, message, *summary)?;
572    }
573    counts.files += usize::from(counts.errors + counts.warnings > before);
574    Ok(())
575}
576
577/// A file's text, or why it has none. Only regular files are read, so a FIFO cannot hang the
578/// run and a device such as `/dev/zero` cannot fill memory. A symlink to a regular file is
579/// followed, as `cat` would.
580fn read(path: &Path) -> Result<String, String> {
581    let read = std::fs::metadata(path).and_then(|meta| {
582        if meta.is_file() {
583            std::fs::read(path).map(Some)
584        } else {
585            Ok(None)
586        }
587    });
588    match read {
589        Ok(Some(bytes)) => decode(bytes),
590        Ok(None) => Err("not a regular file".to_owned()),
591        Err(e) if e.kind() == io::ErrorKind::NotFound => Err("no such file".to_owned()),
592        Err(e) => Err(format!("cannot read: {e}")),
593    }
594}
595
596/// Standard input's text, or why it has none. Empty input is an empty file.
597fn read_stdin() -> Result<String, String> {
598    let mut bytes = Vec::new();
599    match io::stdin().lock().read_to_end(&mut bytes) {
600        Ok(_) => decode(bytes),
601        Err(e) => Err(format!("cannot read: {e}")),
602    }
603}
604
605/// A file's bytes as UTF-8 text. A UTF-16 or UTF-32 file says so, rather than that its first
606/// byte is not UTF-8: by its byte order mark, or without one by where the zero bytes of its first
607/// two characters fall, as RFC 4627 section 3 tells the encodings apart.
608fn decode(bytes: Vec<u8>) -> Result<String, String> {
609    // Without a mark, a text these patterns match would fail the check anyway: a zero byte is
610    // U+0000, and outside a string no dialect allows it. JSON5 does allow it raw inside one, so
611    // a first byte that opens a string is left alone.
612    let opens_string = |b: u8| b == b'"' || b == b'\'';
613    // UTF-32 LE first: its mark starts with UTF-16 LE's.
614    let encoding = match *bytes.as_slice() {
615        [0xFF, 0xFE, 0, 0, ..] => Some("UTF-32 (LE)"),
616        [0, 0, 0xFE, 0xFF, ..] => Some("UTF-32 (BE)"),
617        [0xFF, 0xFE, ..] => Some("UTF-16 (LE)"),
618        [0xFE, 0xFF, ..] => Some("UTF-16 (BE)"),
619        [0, 0, 0, a, ..] if a != 0 => Some("UTF-32 (BE)"),
620        [0, a, 0, b, ..] if a != 0 && b != 0 => Some("UTF-16 (BE)"),
621        [a, 0, 0, 0, ..] if a != 0 && !opens_string(a) => Some("UTF-32 (LE)"),
622        [a, 0, b, 0, ..] if a != 0 && b != 0 && !opens_string(a) => Some("UTF-16 (LE)"),
623        _ => None,
624    };
625    if let Some(encoding) = encoding {
626        return Err(format!("file is {encoding}; jsonvet reads UTF-8"));
627    }
628    String::from_utf8(bytes)
629        .map_err(|e| format!("not valid UTF-8 (byte {})", e.utf8_error().valid_up_to()))
630}
631
632/// The most a config file or an ignore file may hold: real ones take a few kilobytes.
633pub(crate) const MAX_CONFIG_BYTES: u64 = 16 << 20;
634
635/// The most a `--schema` file may hold: 40 times the largest schema on schemastore.org
636/// (1.6 MB), and room for the 5,000,000 objects and arrays a schema may hold at 2 to 3 bytes each.
637pub(crate) const MAX_SCHEMA_BYTES: u64 = 64 << 20;
638
639/// The text of a config or schema file of at most `max` bytes. As in [`read`], only a regular
640/// file, or a symlink to one, is opened, so a FIFO cannot hang the run and `/dev/zero` cannot
641/// fill memory.
642pub(crate) fn read_small(path: &Path, max: u64) -> io::Result<String> {
643    if !std::fs::metadata(path)?.is_file() {
644        return Err(io::Error::other("not a regular file"));
645    }
646    let mut text = String::new();
647    // The cap holds even for a file that grows after the check.
648    std::fs::File::open(path)?
649        .take(max + 1)
650        .read_to_string(&mut text)?;
651    if text.len() as u64 > max {
652        return Err(io::Error::other(format!("larger than {} MB", max >> 20)));
653    }
654    Ok(text)
655}
656
657/// Why a folder walk must not read the `.gitignore` or `.ignore` in `dir`, if it must not. The
658/// `ignore` crate reads them to the end with no check of what they are, so a link to
659/// `/dev/zero` takes memory until the run is killed and a FIFO hangs it. A file that cannot be
660/// looked at is no problem: the crate skips it too.
661fn unsafe_ignore_file(dir: &Path) -> Option<String> {
662    [".gitignore", ".ignore"].into_iter().find_map(|name| {
663        let path = dir.join(name);
664        let meta = std::fs::metadata(&path).ok()?;
665        let problem = if !meta.is_file() {
666            "not a regular file".to_owned()
667        } else if meta.len() > MAX_CONFIG_BYTES {
668            format!("larger than {} MB", MAX_CONFIG_BYTES >> 20)
669        } else {
670            return None;
671        };
672        Some(output::printable(&format!(
673            "cannot walk: {}: {problem}",
674            path.display()
675        )))
676    })
677}
678
679fn print(mut out: impl Write, text: &str) -> Result<u8, String> {
680    match writeln!(out, "{text}").and_then(|()| out.flush()) {
681        Err(e) if e.kind() != io::ErrorKind::BrokenPipe => Err(format!("cannot write output: {e}")),
682        _ => Ok(0),
683    }
684}
685
686#[cfg(test)]
687mod tests {
688    use std::fmt::Write as _;
689    use std::sync::atomic::{AtomicUsize, Ordering};
690
691    use jsonvet::Dialect;
692
693    use super::{
694        CHUNK_BYTES, MAX_CONFIG_BYTES, SCHEMA_BYTES, SPLIT_BYTES, STDIN, Schema, Settings, check,
695        chunk_len, parallel, read_small, schema_share, schema_shares, split, unsafe_ignore_file,
696    };
697    use std::sync::Arc;
698
699    /// A schema that takes any value, in a fresh folder named for `test`, which also holds an
700    /// empty folder `x`.
701    fn any_schema(test: &str) -> (std::path::PathBuf, Schema) {
702        let dir = std::env::temp_dir().join(format!("jsonvet-{test}-{}", std::process::id()));
703        std::fs::create_dir_all(dir.join("x")).expect("a temporary directory");
704        std::fs::write(dir.join("s.json"), "{}").expect("written");
705        let schema = Schema::load(&dir.join("s.json"), &Settings::default(), false, |_| Ok(()));
706        (dir, schema.expect("a schema"))
707    }
708
709    #[test]
710    fn only_files_with_a_schema_take_a_share() {
711        let (dir, schema) = any_schema("shares");
712        let _ = std::fs::remove_dir_all(&dir);
713        let schema = Arc::new(schema);
714        let settings = Arc::new(Settings::default());
715        let job =
716            |schema: Option<&Arc<Schema>>| ((Arc::clone(&settings), schema.cloned()), STDIN.into());
717        assert_eq!(
718            schema_shares(&[job(None), job(None)], 2),
719            Vec::<usize>::new()
720        );
721        assert_eq!(
722            schema_shares(&[job(None), job(Some(&schema)), job(None)], 2),
723            [0, CHUNK_BYTES, 0]
724        );
725    }
726
727    #[test]
728    fn small_files_are_read_up_to_their_limit_and_no_further() {
729        let dir = std::env::temp_dir().join(format!("jsonvet-small-{}", std::process::id()));
730        std::fs::create_dir_all(&dir).expect("a temporary directory");
731        let path = dir.join("f");
732        std::fs::write(&path, "0123456789").expect("written");
733        assert_eq!(read_small(&path, 10).expect("ten bytes"), "0123456789");
734        assert!(read_small(&path, 9).is_err());
735        // An ignore file is judged by its size alone; sparse files, so nothing is written.
736        let sized = |len: u64| {
737            std::fs::File::create(dir.join(".gitignore"))
738                .and_then(|file| file.set_len(len))
739                .expect("a large file");
740        };
741        sized(MAX_CONFIG_BYTES);
742        assert_eq!(unsafe_ignore_file(&dir), None);
743        sized(MAX_CONFIG_BYTES + 1);
744        assert!(unsafe_ignore_file(&dir).is_some_and(|e| e.ends_with("larger than 16 MB")));
745        let _ = std::fs::remove_dir_all(&dir);
746    }
747
748    #[test]
749    fn only_json_lines_of_the_split_size_is_split_one_part_per_thread() {
750        let line = "[1]\n".repeat(SPLIT_BYTES / 4);
751        assert_eq!(split(&line, Dialect::JsonLines, 8).len(), 8);
752        assert_eq!(split(&line, Dialect::JsonLines, 1).len(), 1);
753        assert_eq!(split(&line, Dialect::Json, 8).len(), 1);
754        assert_eq!(split(&line[4..], Dialect::JsonLines, 8).len(), 1);
755    }
756
757    #[test]
758    fn a_large_json_lines_file_checked_in_parts_reports_what_one_pass_does() {
759        // Problems in every part: duplicate keys, and strings the schema wants as integers. Every
760        // thousandth line has more schema errors than a document reports, and two in each
761        // thousand have values the schema cannot check.
762        let pad = "x".repeat(500);
763        let wide: Vec<String> = (0..150).map(|k| format!("\"k{k}\": \"s\"")).collect();
764        let wide = format!("{{{}}}", wide.join(", "));
765        let mut text = String::from("\u{FEFF}");
766        for i in 0..4_000 {
767            let _ = match i % 7 {
768                _ if i % 1_000 == 999 => writeln!(text, "{wide}"),
769                _ if i % 1_000 == 500 => writeln!(text, "{{\"a\": 1e400}}"),
770                _ if i % 1_000 == 501 => writeln!(text, r#"{{"\ud800": 1, "\ud801": 2}}"#),
771                0 => writeln!(text, "{{\"a\": {i}, \"a\": 1}}"),
772                1 => write!(text, "{{\"a\": \"{i}\", \"p\": \"{pad}\"}}\r\n"),
773                _ => writeln!(text, "{{\"a\": {i}, \"p\": \"{pad}\"}}"),
774            };
775        }
776        assert!(text.len() > SPLIT_BYTES);
777        let deep = format!("{}{}\n", "[".repeat(600), "]".repeat(600));
778        let texts = [
779            text.clone(),
780            format!("{text}{{\"a\": 1\n{text}"),
781            format!("{text}{deep}{text}"),
782        ];
783        let dir = std::env::temp_dir().join(format!("jsonvet-split-{}", std::process::id()));
784        std::fs::create_dir_all(&dir).expect("a temporary directory");
785        let path = dir.join("schema.json");
786        let schema = concat!(
787            r#"{"properties": {"a": {"type": "integer"}}, "#,
788            r#""patternProperties": {"^k": {"type": "integer"}}}"#,
789        );
790        std::fs::write(&path, schema).expect("written");
791        let settings = Settings::default();
792        let schema = Schema::load(&path, &settings, false, |_| Ok(())).expect("a schema");
793        let _ = std::fs::remove_dir_all(&dir);
794        for text in &texts {
795            for schema in [None, Some(&schema)] {
796                let (whole, whole_errors) = check("x.jsonl", text, &settings, schema, 1);
797                let (split, split_errors) = check("x.jsonl", text, &settings, schema, 8);
798                assert_ne!(whole.problems(), &[]);
799                assert_eq!(split.problems(), whole.problems());
800                assert_eq!(split_errors, whole_errors);
801            }
802        }
803        let (_, errors) = check("x.jsonl", &texts[0], &settings, Some(&schema), 8);
804        // 570 strings, 100 and a summary for each wide line, and one for each line not checked.
805        assert_eq!(errors.len(), 570 + 4 * 101 + 8);
806        let unchecked = errors.iter().filter(|(_, m, _)| m.contains("not checked"));
807        assert_eq!(unchecked.count(), 8);
808    }
809
810    #[test]
811    fn parallel_keeps_the_order_and_stops_after_a_prefix() {
812        let items: Vec<usize> = (0..1000).collect();
813        for threads in [1, 3, 16] {
814            let squares = parallel(&items, threads, || false, |i| i * i);
815            assert_eq!(squares, items.iter().map(|i| i * i).collect::<Vec<_>>());
816            // Stopped from the start, it still does the first item.
817            assert_eq!(parallel(&items, threads, || true, |&i| i), [0]);
818            let done = AtomicUsize::new(0);
819            let some = parallel(
820                &items,
821                threads,
822                || done.load(Ordering::Relaxed) >= 100,
823                |&i| {
824                    done.fetch_add(1, Ordering::Relaxed);
825                    i
826                },
827            );
828            // At least 100, plus at most one per thread already under way.
829            assert!((100..100 + threads).contains(&some.len()), "{}", some.len());
830            assert_eq!(some, (0..some.len()).collect::<Vec<_>>());
831        }
832        assert_eq!(
833            parallel(&[] as &[u8], 4, || false, |&b| b),
834            Vec::<u8>::new()
835        );
836    }
837
838    #[test]
839    fn a_chunk_ends_with_the_file_that_fills_it() {
840        assert_eq!(chunk_len(&[]), 0);
841        assert_eq!(chunk_len(&[1, 2, 3]), 3);
842        assert_eq!(chunk_len(&[CHUNK_BYTES, 1]), 1);
843        assert_eq!(chunk_len(&[CHUNK_BYTES - 1, 1, 1]), 2);
844        assert_eq!(chunk_len(&[CHUNK_BYTES - 2, 1, 1]), 3);
845        assert_eq!(chunk_len(&[0, CHUNK_BYTES - 1, 0, 2, 5]), 4);
846    }
847
848    #[test]
849    fn a_file_counts_its_size_twenty_times_with_a_schema() {
850        let dir = std::env::temp_dir().join(format!("jsonvet-share-{}", std::process::id()));
851        std::fs::create_dir_all(&dir).expect("a temp dir");
852        let path = dir.join("a.json");
853        std::fs::write(&path, " ".repeat(1000)).expect("write");
854        let path = path.as_os_str();
855        assert_eq!(schema_share(path), 1000 * SCHEMA_BYTES);
856        std::fs::write(path, " ".repeat(CHUNK_BYTES / SCHEMA_BYTES + 1)).expect("write");
857        assert_eq!(schema_share(path), CHUNK_BYTES, "capped");
858        assert_eq!(schema_share(dir.join("gone.json").as_os_str()), 0);
859        assert_eq!(schema_share(std::ffi::OsStr::new("-")), CHUNK_BYTES);
860        let _ = std::fs::remove_dir_all(&dir);
861    }
862
863    #[test]
864    fn frames_show_on_a_terminal_and_color_follows_the_option_then_the_environment() {
865        use super::{Format, Look, Terminal, look};
866        let terminal = |tty, color| Terminal { tty, color };
867        // Off a terminal, `color_forced_off_a_terminal_keeps_one_line_per_problem` in
868        // tests/cli.rs covers each case end to end.
869        let frames = |color| Look {
870            frames: true,
871            color,
872        };
873        let lines = Look {
874            frames: false,
875            color: false,
876        };
877        let cases = [
878            (Format::Text, None, terminal(true, None), frames(true)),
879            (
880                Format::Text,
881                None,
882                terminal(true, Some(false)),
883                frames(false),
884            ),
885            (
886                Format::Text,
887                Some(false),
888                terminal(true, Some(true)),
889                frames(false),
890            ),
891            (Format::Sarif, Some(true), terminal(true, None), lines),
892            (Format::Json, Some(true), terminal(true, None), lines),
893        ];
894        for (format, color, terminal, want) in cases {
895            assert_eq!(
896                look(format, color, terminal),
897                want,
898                "{format:?} {color:?} {terminal:?}"
899            );
900        }
901    }
902
903    #[test]
904    fn the_summary_counts_problems_and_files() {
905        let dir = std::env::temp_dir().join(format!("jsonvet-summary-{}", std::process::id()));
906        std::fs::create_dir_all(&dir).expect("a temp dir");
907        let dup = dir.join("dup.json");
908        let ok = dir.join("ok.json");
909        std::fs::write(&dup, "{\"a\": 1, \"a\": 2}").expect("write");
910        std::fs::write(&ok, "{}").expect("write");
911        let run = |summary: bool, files: &[&std::path::Path]| {
912            let mut out = Vec::new();
913            let args = ["--no-config".into()]
914                .into_iter()
915                .chain(files.iter().map(|f| f.as_os_str().to_owned()));
916            let terminal = super::Terminal {
917                tty: summary,
918                color: Some(false),
919            };
920            let code = super::run_with(args, false, terminal, &mut out).expect("a check");
921            (code, String::from_utf8(out).expect("UTF-8"))
922        };
923        let (code, out) = run(true, &[&dup, &ok]);
924        assert_eq!(code, 1);
925        assert!(out.ends_with("\n1 error in 1 of 2 files\n"), "{out}");
926        // A file that cannot be read counts as a file with an error.
927        let gone = dir.join("gone.json");
928        // The clean file last, so counting it instead of a file with problems shows.
929        let (_, out) = run(true, &[&dup, &gone, &dup, &ok]);
930        assert!(out.ends_with("\n3 errors in 3 of 4 files\n"), "{out}");
931        assert_eq!(run(true, &[&ok]).1, "no problems in 1 file\n");
932        // No summary off a terminal, nor for no files.
933        assert!(!run(false, &[&dup]).1.contains(" in 1 of 1 file"));
934        assert_eq!(run(true, &[]).1, "");
935        // SARIF and JSON stay one document on a terminal too.
936        for format in ["sarif", "json"] {
937            let args = ["--no-config", "--format", format]
938                .map(std::ffi::OsString::from)
939                .into_iter()
940                .chain([dup.clone().into_os_string()]);
941            let mut out = Vec::new();
942            let terminal = super::Terminal {
943                tty: true,
944                color: None,
945            };
946            super::run_with(args, false, terminal, &mut out).expect("a check");
947            let out = String::from_utf8(out).expect("UTF-8");
948            assert!(
949                serde_json::from_str::<serde_json::Value>(&out).is_ok(),
950                "{out}"
951            );
952        }
953        let _ = std::fs::remove_dir_all(&dir);
954    }
955
956    /// A JSON Lines file and a `--schema` error get the same code frames on a terminal as any
957    /// other problem, at the line they are on.
958    #[test]
959    fn json_lines_and_schema_errors_get_code_frames_on_a_terminal() {
960        let dir = std::env::temp_dir().join(format!("jsonvet-frames-{}", std::process::id()));
961        std::fs::create_dir_all(&dir).expect("a temp dir");
962        let lines = dir.join("l.jsonl");
963        let schema = dir.join("s.json");
964        let data = dir.join("d.json");
965        std::fs::write(&lines, "{\"a\": 1}\n{\"a\": True}\n").expect("write");
966        std::fs::write(&schema, r#"{"properties": {"a": {"type": "string"}}}"#).expect("write");
967        std::fs::write(&data, "{\n  \"a\": 1\n}\n").expect("write");
968        let run = |args: &[&std::ffi::OsStr]| {
969            let args = ["--no-config".into()]
970                .into_iter()
971                .chain(args.iter().map(|a| (*a).to_owned()));
972            let terminal = super::Terminal {
973                tty: true,
974                color: Some(false),
975            };
976            let mut out = Vec::new();
977            let code = super::run_with(args, false, terminal, &mut out).expect("a check");
978            let out = String::from_utf8(out).expect("UTF-8");
979            (
980                code,
981                out.replace(
982                    &format!("{}{}", dir.display(), std::path::MAIN_SEPARATOR),
983                    "",
984                ),
985            )
986        };
987        assert_eq!(
988            run(&[lines.as_os_str()]),
989            (
990                1,
991                "error[syntax]: expected a value, found `True`; did you mean `true`?\n \
992                 --> l.jsonl:2:7\n  |\n2 | {\"a\": True}\n  |       ^^^^\n\n\
993                 1 error in 1 of 1 file\n"
994                    .to_owned()
995            )
996        );
997        assert_eq!(
998            run(&["--schema".as_ref(), schema.as_os_str(), data.as_os_str()]),
999            (
1000                1,
1001                "error[schema]: 1 is not of type \"string\" (at $.a)\n \
1002                 --> d.json:2:8\n  |\n2 |   \"a\": 1\n  |        ^\n\n\
1003                 1 error in 1 of 1 file\n"
1004                    .to_owned()
1005            )
1006        );
1007        let _ = std::fs::remove_dir_all(&dir);
1008    }
1009
1010    #[test]
1011    fn the_summary_names_errors_and_warnings_together() {
1012        let counts = |errors, warnings| super::Counts {
1013            errors,
1014            warnings,
1015            files: 1,
1016        };
1017        assert_eq!(
1018            counts(1, 1).summary(2),
1019            "1 error and 1 warning in 1 of 2 files"
1020        );
1021        assert_eq!(counts(0, 2).summary(1), "2 warnings in 1 of 1 file");
1022    }
1023
1024    #[test]
1025    fn a_line_separator_ends_a_frame_line_only_in_a_json5_file() {
1026        let dir = std::env::temp_dir().join(format!("jsonvet-u2028-{}", std::process::id()));
1027        std::fs::create_dir_all(&dir).expect("a temp dir");
1028        let frame = |name: &str| {
1029            let file = dir.join(name);
1030            std::fs::write(&file, "{\"a\": \"x\u{2028}y\", \"a\": 2}\n").expect("write");
1031            let mut out = Vec::new();
1032            let args = ["--no-config".into(), file.into_os_string()];
1033            let terminal = super::Terminal {
1034                tty: true,
1035                color: Some(false),
1036            };
1037            super::run_with(args, false, terminal, &mut out).expect("a check");
1038            String::from_utf8(out).expect("UTF-8")
1039        };
1040        let five = frame("a.json5");
1041        // Both the duplicate and the first copy, whose line ends at the separator.
1042        assert!(five.contains("\n2 | y\", \"a\": 2}\n"), "{five}");
1043        assert!(five.contains("\n1 | {\"a\": \"x\n"), "{five}");
1044        let other = frame("a.jsonc");
1045        // The frame shows U+2028 as a replacement character, on the one line.
1046        assert!(
1047            other
1048                .lines()
1049                .any(|l| l.starts_with("1 | {\"a\": \"x\u{FFFD}y\", \"a\": 2}")),
1050            "{other}"
1051        );
1052        let _ = std::fs::remove_dir_all(&dir);
1053    }
1054
1055    #[test]
1056    fn a_panic_on_the_big_stack_carries_on_in_the_caller() {
1057        static RUNS: AtomicUsize = AtomicUsize::new(0);
1058        let panic = std::panic::catch_unwind(|| {
1059            super::on_big_stack(|| -> u8 {
1060                RUNS.fetch_add(1, Ordering::Relaxed);
1061                panic!("on the big stack")
1062            })
1063        })
1064        .expect_err("the panic carries on");
1065        assert_eq!(panic.downcast_ref::<&str>(), Some(&"on the big stack"));
1066        assert_eq!(RUNS.load(Ordering::Relaxed), 1);
1067    }
1068}