Skip to main content

jsonvet_cli/
lib.rs

1//! The jsonvet command line, shared by the `jsonvet` and `check-json5` binaries.
2
3use std::ffi::{OsStr, OsString};
4use std::io::{self, BufWriter, IsTerminal, Read, Write};
5use std::path::{Path, PathBuf};
6use std::process::ExitCode;
7
8use std::sync::atomic::{AtomicUsize, Ordering};
9use std::sync::{Arc, Mutex};
10
11use jsonvet::{Dialect, Problem, Report, Severity};
12
13mod args;
14mod catalog;
15mod editorconfig;
16mod format;
17mod frame;
18mod glob;
19mod json;
20mod lsp;
21mod messages;
22mod output;
23mod rules;
24mod sarif;
25mod schema;
26mod settings;
27mod write;
28
29use args::{Command, STDIN};
30use output::{Format, Look, Printer};
31use schema::Schema;
32use schema::select::{Job, Schemas, check_own_schema, jobs, with_schema};
33use settings::{Resolver, Settings};
34use write::write_atomically;
35
36/// The stack every run uses, for `--schema`: jsonschema validates recursively, and building,
37/// validating and dropping a value [`schema::SCHEMA_MAX_DEPTH`] levels deep against a recursive schema
38/// took up to 2.2 MB in a debug build, more than Windows' 1 MB main thread. Only the pages used
39/// are committed, so a plain check costs no more memory.
40const STACK: usize = 64 << 20;
41
42/// How much memory the files read at once may take before they are checked and printed.
43const CHUNK_BYTES: usize = 64 << 20;
44
45/// How many bytes of memory a byte of file text takes with a schema, which holds the file's value
46/// one document at a time. Kept above the most that docs/usage.md ("JSON Schema limits") gives,
47/// so a chunk never overcommits.
48const SCHEMA_BYTES: usize = 20;
49
50/// From this size, a JSON Lines file is split at newlines and its parts checked on the cores
51/// its chunk leaves free.
52const SPLIT_BYTES: usize = 1 << 20;
53
54/// Run the command with the process's arguments and environment, and return its exit code.
55pub fn run() -> ExitCode {
56    // The release profile aborts on a panic, which ends the process with a signal and no way to
57    // report it. A hook runs first, so a bug exits 101 with a pointer to the issue tracker.
58    std::panic::set_hook(Box::new(|info| {
59        eprintln!(
60            "jsonvet: internal error: {info}\n\nThis is a bug in jsonvet. Please report it at \
61             https://github.com/simonvanlierde/jsonvet/issues with the command you ran and, if \
62             you can share it, the file that caused it."
63        );
64        std::process::exit(101);
65    }));
66    on_big_stack(run_here)
67}
68
69/// Run `f` on a thread with a [`STACK`]-byte stack, or here if no thread can start. A panic in
70/// `f` carries on here.
71fn on_big_stack<T: Send + 'static>(f: fn() -> T) -> T {
72    let thread = std::thread::Builder::new().name("jsonvet".into());
73    match thread.stack_size(STACK).spawn(f) {
74        Ok(thread) => thread
75            .join()
76            .unwrap_or_else(|panic| std::panic::resume_unwind(panic)),
77        Err(_) => f(),
78    }
79}
80
81fn run_here() -> ExitCode {
82    // NOTE: a hidden trigger for tests/cli.rs, so the panic hook runs without a real bug. Debug
83    // builds only, so release binaries cannot be made to panic on purpose.
84    #[cfg(debug_assertions)]
85    assert!(
86        std::env::var_os("JSONVET_TEST_PANIC").is_none(),
87        "test panic"
88    );
89    let annotate = std::env::var_os("GITHUB_ACTIONS").is_some_and(|v| v == "true");
90    // Not locked: `jsonvet lsp` writes to it from a thread of its own.
91    let stdout = io::stdout();
92    let set = |name| std::env::var_os(name).is_some_and(|v| !v.is_empty());
93    // NO_COLOR wins over FORCE_COLOR: asking for no color is the safer wish to honour.
94    // FORCE_COLOR=0 and =false ask for no color, as Node and chalk read them.
95    let color = if set("NO_COLOR") {
96        Some(false)
97    } else if set("FORCE_COLOR") {
98        Some(std::env::var_os("FORCE_COLOR").is_none_or(|v| v != "0" && v != "false"))
99    } else {
100        None
101    };
102    let terminal = Terminal {
103        tty: stdout.is_terminal(),
104        color,
105    };
106    match run_with(
107        std::env::args_os().skip(1),
108        annotate,
109        terminal,
110        BufWriter::new(stdout),
111    ) {
112        Ok(code) => ExitCode::from(code),
113        Err(message) => {
114            eprintln!("jsonvet: {message}");
115            ExitCode::from(2)
116        }
117    }
118}
119
120/// The end of a run: the `line` of counts as text on a terminal (`text`), and as an annotation
121/// when `found` problems, then whatever the format closes with.
122fn print_summary(
123    printer: &mut Printer<impl Write>,
124    line: Option<&str>,
125    text: bool,
126    found: bool,
127) -> io::Result<()> {
128    if let Some(line) = line {
129        if text {
130            printer.text_line(line)?;
131        }
132        if found {
133            printer.summary_annotation(line)?;
134        }
135    }
136    printer.finish()
137}
138
139/// Where output goes, as far as text output's look depends on it. Public only for
140/// `benches/codspeed.rs`; not part of the crate's API.
141#[doc(hidden)]
142#[derive(Clone, Copy, Debug, Default)]
143pub struct Terminal {
144    /// Standard output is a terminal. A summary and code frames are for people: hooks and CI
145    /// read standard output from a pipe and see one line per problem.
146    pub tty: bool,
147    /// What the environment asks for when `--color` is `auto`: `Some(false)` for `NO_COLOR`,
148    /// `Some(true)` for `FORCE_COLOR`.
149    pub color: Option<bool>,
150}
151
152/// Whether `text` loads as the config file `path` (a `pyproject.toml` reads its `[tool.jsonvet]`
153/// table), with the error message if not. Public only for the fuzz target `config_file`; not
154/// part of the crate's API.
155#[doc(hidden)]
156pub fn check_config_text(path: &Path, text: &str) -> Result<(), String> {
157    settings::load_str(path, text, false, None).map(drop)
158}
159
160/// The command, with its inputs passed in: 0 when no file has an error, 1 when one does.
161/// On a terminal, text output ends with a count of the problems. A usage error comes back as
162/// `Err`, which `run` turns into exit status 2. Public only for `benches/codspeed.rs`, which
163/// measures `--schema` in-process; not part of the crate's API.
164#[doc(hidden)]
165#[allow(
166    clippy::too_many_lines,
167    reason = "the checking pipeline keeps its stages in execution order"
168)]
169pub fn run_with(
170    args: impl IntoIterator<Item = OsString>,
171    annotate: bool,
172    terminal: Terminal,
173    out: impl Write,
174) -> Result<u8, String> {
175    // Usage errors quote arguments and the schema file, which can hold a newline or an escape.
176    let args = match args::parse(args)
177        .map_err(|e| messages::fill("error.usage", &[("error", &output::printable(&e))]))?
178    {
179        Command::Help => return print(out, messages::text("help")),
180        Command::Version => return print(out, concat!("jsonvet ", env!("CARGO_PKG_VERSION"))),
181        Command::ListRules => return print(out, &rules::list()),
182        Command::Explain(text) => return print(out, &text),
183        Command::Lsp => return lsp::run(),
184        Command::Check(args) => *args,
185    };
186    messages::english_only(args.format != Format::Text);
187    let resolver = Arc::new(Mutex::new(Resolver::new(&args.config, args.settings)?));
188    let settings_for = |path: &Path| lock(&resolver).for_file(path);
189    let threads = std::thread::available_parallelism().map_or(1, std::num::NonZero::get);
190    let files = expand(args.files, &resolver, threads)?;
191    if let Some(check_only) = args.fmt {
192        return format::run(&files, check_only, args.stdin_filename, &resolver, out);
193    }
194    let schemas = Schemas::default();
195    let load = |path: &Path, quiet| {
196        schemas.get(path, quiet, |admit| {
197            Schema::load(path, &*settings_for(path)?, quiet, admit)
198        })
199    };
200    let root = |path: &Path| lock(&resolver).root(path);
201
202    // Standard input goes by its `--stdin-filename`, for settings and in the output.
203    let stdin_name = args.stdin_filename.unwrap_or_else(|| STDIN.into());
204    let shown = |file: &OsStr| -> OsString {
205        if file == STDIN {
206            stdin_name.clone()
207        } else {
208            file.to_owned()
209        }
210    };
211    // Every config file, and every schema one names, is read before any file is checked, so a
212    // bad one stops the run at once.
213    let files = jobs(files, &shown, &settings_for, args.schema.as_deref(), &load)?;
214    let mut catalog = args
215        .schema_catalog
216        .then(catalog::Catalog::new)
217        .transpose()?;
218    let look = look(args.format, args.color, terminal);
219    // Standard input fixed to standard output leaves only standard error for the problems.
220    let stdin_fix = args.fix && files.iter().any(|(_, file)| file == STDIN);
221    let mut out = out;
222    let sink = Diagnostics::new(stdin_fix, &mut out);
223    let mut printer = Printer::new(sink, args.format, annotate, look, stdin_fix);
224    let mut fixed = FixRun::default();
225    let mut counts = Counts::default();
226    // Files are read and checked on every core, a chunk at a time, then printed here in the
227    // order given. A chunk ends once its files hold CHUNK_BYTES, so memory stays near that. With
228    // a schema a file takes SCHEMA_BYTES times its size, so the chunk is cut from the sizes
229    // before any file is read: otherwise every core would start on a file at once. Without
230    // a schema the files are not sized first, which made a walk of 10,000 files 7 ms slower.
231    // A file whose own `$schema` may name a schema is known only once read: it counts the same,
232    // and the texts read past the end of the chunk are kept, in `ahead`, for the next one.
233    // NOTE: each chunk waits for its slowest file before the next starts; hand files out
234    // one at a time across chunks if one big file among many small ones holds a run up.
235    let mut rest = &files[..];
236    let shares = schema_shares(&files, threads);
237    let mut ahead = Vec::new();
238    let written = loop {
239        if rest.is_empty() {
240            for note in schemas.missing_notes() {
241                printer.note(note);
242            }
243            let line = (!files.is_empty()).then(|| counts.summary(files.len()));
244            let found = counts.errors + counts.warnings > 0;
245            break print_summary(&mut printer, line.as_deref(), terminal.tty, found);
246        }
247        let shares = shares.get(files.len() - rest.len()..);
248        let with_schema =
249            |job: &Job, text: &str| args.schema_catalog || with_schema(job, &shown(&job.1), text);
250        let mut texts = take_chunk(&mut ahead, rest, shares, threads, &with_schema);
251        let chunk;
252        (chunk, rest) = rest.split_at(texts.len());
253        let known = if args.fix {
254            fix_chunk(
255                chunk,
256                &mut texts,
257                &shown,
258                args.unsafe_fixes,
259                threads,
260                &mut fixed,
261            )
262        } else {
263            vec![None; texts.len()]
264        };
265        let mut selected = None;
266        if let Some(catalog) = &mut catalog {
267            selected = Some(catalog.select(chunk, &texts, &shown, &schemas)?);
268            for note in catalog.take_notes() {
269                printer.note(note);
270            }
271        }
272        let chunk = selected.as_deref().unwrap_or(chunk);
273        let jobs: Vec<_> = chunk.iter().zip(&texts).zip(&known).collect();
274        let parts = (threads / jobs.len()).max(1);
275        let checked = parallel(
276            &jobs,
277            threads,
278            || false,
279            |((((settings, schema), file), (text, with)), known)| {
280                let name = shown(file);
281                let name = name.to_string_lossy();
282                text.as_deref().map(|text| match schema {
283                    Some(schema) => check(&name, text, settings, Some(schema), parts, None),
284                    None if *with => {
285                        check_own_schema(&name, text, settings, &schemas, &root, &load, parts)
286                    }
287                    None => check(&name, text, settings, None, parts, known.as_deref()),
288                })
289            },
290        );
291        schemas.over().map_err(|e| output::printable(&e))?;
292        let outcome = print_checked(chunk, checked, &shown, &mut printer, &mut counts);
293        if outcome.is_err() {
294            break outcome;
295        }
296    };
297    // A JSON or SARIF report on standard error is all of it, so the count stays out.
298    let count = !stdin_fix || args.format == Format::Text;
299    let written = written.and_then(|()| fixed.finish(count, &mut out));
300    conclude(written, &counts, args.max_warnings)
301}
302
303/// A file's report and its schema errors.
304type Checked<'t> = (Report<'t>, Vec<(usize, String, bool)>);
305
306/// Print what checking the files of `chunk` found, in order: the problems of each, or why it
307/// could not be read.
308fn print_checked(
309    chunk: &[Job],
310    checked: Vec<Result<Checked<'_>, &String>>,
311    shown: &impl Fn(&OsStr) -> OsString,
312    printer: &mut Printer<impl Write>,
313    counts: &mut Counts,
314) -> io::Result<()> {
315    chunk
316        .iter()
317        .zip(checked)
318        .try_for_each(|((_, file), checked)| match checked {
319            Ok((report, schema_errors)) => {
320                print_file(&shown(file), &report, &schema_errors, printer, counts)
321            }
322            Err(message) => {
323                counts.errors += 1;
324                counts.files += 1;
325                printer.unreadable(&shown(file), message)
326            }
327        })
328}
329
330/// The exit code once the output is written: 1 for an error, or for more warnings than
331/// `max_warnings` allows.
332fn conclude(
333    written: io::Result<()>,
334    counts: &Counts,
335    max_warnings: Option<usize>,
336) -> Result<u8, String> {
337    // A reader that has gone, as with `| head`, wants no more output: the verdict so far stands.
338    match written {
339        Err(e) if e.kind() != io::ErrorKind::BrokenPipe => {
340            return Err(messages::fill("error.write", &[("error", &e)]));
341        }
342        _ => {}
343    }
344    let too_many_warnings = max_warnings.is_some_and(|max| counts.warnings > max);
345    Ok(u8::from(counts.errors > 0 || too_many_warnings))
346}
347
348/// The texts of the first of `files`, read on up to `threads` threads until their [`weight`]s
349/// reach [`CHUNK_BYTES`], and always the first, each with whether `with_schema` says it is
350/// checked against a schema.
351fn read_chunk<S: Sync>(
352    files: &[(S, OsString)],
353    threads: usize,
354    with_schema: &(impl Fn(&(S, OsString), &str) -> bool + Sync),
355) -> Vec<(Result<String, String>, bool)> {
356    let bytes = AtomicUsize::new(0);
357    let full = || bytes.load(Ordering::Relaxed) >= CHUNK_BYTES;
358    parallel(files, threads, full, |file| {
359        let text = if file.1 == STDIN {
360            read_stdin()
361        } else {
362            read(Path::new(&file.1))
363        };
364        let with = text.as_ref().is_ok_and(|text| with_schema(file, text));
365        bytes.fetch_add(weight(&text, with), Ordering::Relaxed);
366        (text, with)
367    })
368}
369
370/// The texts of the next chunk of `rest`, the files left, each with whether it is checked
371/// against a schema: up to and including the one whose [`weight`] fills [`CHUNK_BYTES`], from
372/// `ahead`, the texts of the first of them read already, or if none are, from those read now.
373/// `shares` are the files' [`schema_share`]s, if any.
374fn take_chunk(
375    ahead: &mut Vec<(Result<String, String>, bool)>,
376    rest: &[Job],
377    shares: Option<&[usize]>,
378    threads: usize,
379    with_schema: &(impl Fn(&Job, &str) -> bool + Sync),
380) -> Vec<(Result<String, String>, bool)> {
381    if ahead.is_empty() {
382        let offered = match shares {
383            Some(shares) if !shares.is_empty() => &rest[..chunk_len(shares)],
384            _ => rest,
385        };
386        *ahead = read_chunk(offered, threads, with_schema);
387    }
388    let weights: Vec<usize> = ahead
389        .iter()
390        .map(|(text, with)| weight(text, *with))
391        .collect();
392    ahead.drain(..chunk_len(&weights)).collect()
393}
394
395/// The memory a text read takes while it is checked, as far as a chunk counts it: its size, or
396/// with a schema, [`SCHEMA_BYTES`] times that, capped at [`CHUNK_BYTES`].
397fn weight(text: &Result<String, String>, with_schema: bool) -> usize {
398    let size = text.as_ref().map_or(0, String::len);
399    if with_schema {
400        size.saturating_mul(SCHEMA_BYTES).min(CHUNK_BYTES)
401    } else {
402        size
403    }
404}
405
406/// The memory a file takes while a schema checks it, as far as a chunk counts it:
407/// [`SCHEMA_BYTES`] times its size, capped at [`CHUNK_BYTES`]. Standard input's size is known
408/// only once it is read, so it takes a chunk's worth. A file that cannot be read takes nothing:
409/// `read` reports why.
410fn schema_share(file: &OsStr) -> usize {
411    if file == STDIN {
412        return CHUNK_BYTES;
413    }
414    let size = std::fs::metadata(file).map_or(0, |meta| meta.len());
415    usize::try_from(size)
416        .unwrap_or(usize::MAX)
417        .saturating_mul(SCHEMA_BYTES)
418        .min(CHUNK_BYTES)
419}
420
421/// Each file's [`schema_share`], or 0 for a file with no schema; nothing if no file has one.
422fn schema_shares(files: &[Job], threads: usize) -> Vec<usize> {
423    if files.iter().all(|((_, schema), _)| schema.is_none()) {
424        return Vec::new();
425    }
426    parallel(
427        files,
428        threads,
429        || false,
430        |((_, schema), file)| schema.as_ref().map_or(0, |_| schema_share(file)),
431    )
432}
433
434/// How many of the files with these shares the next chunk takes: up to and including the one
435/// that fills [`CHUNK_BYTES`], so always at least one.
436fn chunk_len(shares: &[usize]) -> usize {
437    let mut total = 0;
438    shares
439        .iter()
440        .position(|share| {
441            total += share;
442            total >= CHUNK_BYTES
443        })
444        .map_or(shares.len(), |last| last + 1)
445}
446
447/// The files to check: each argument as given, except that a directory stands for the files under
448/// it that jsonvet checks by default, in sorted order. The walk skips hidden files and what
449/// `.gitignore`, `.ignore` and the `exclude` setting of each folder's settings name, as ripgrep
450/// would.
451fn expand(
452    args: Vec<OsString>,
453    resolver: &Arc<Mutex<Resolver>>,
454    threads: usize,
455) -> Result<Vec<OsString>, String> {
456    let mut files = Vec::with_capacity(args.len());
457    // Each argument costs a call to the system, so thousands of them are asked about in parallel:
458    // whether it is a folder, and which file it is, for `unique`.
459    let found = parallel(
460        &args,
461        threads,
462        || false,
463        |arg| {
464            let meta = std::fs::metadata(arg);
465            let dir = arg != STDIN && meta.as_ref().is_ok_and(std::fs::Metadata::is_dir);
466            (dir, identity(arg, meta))
467        },
468    );
469    // The identity of each of `files`, or `None` for a file a walk found.
470    let mut ids = Vec::with_capacity(args.len());
471    // Folders that share no file need no check for repeats: the walk follows no symlinks.
472    let mut roots = Vec::new();
473    let mut distinct = true;
474    for (arg, (dir, id)) in args.into_iter().zip(found) {
475        if !dir {
476            distinct = false;
477            files.push(arg);
478            ids.push(Some(id));
479            continue;
480        }
481        let start = files.len();
482        // The walk reads the ignore files of the folder, and of every folder above it when it
483        // can find them, before the filter sees anything.
484        let above = std::fs::canonicalize(&arg).ok();
485        match &above {
486            Some(root) => roots.push(root.clone()),
487            None => distinct = false,
488        }
489        if let Some(problem) = std::iter::once(Path::new(&arg))
490            .chain(above.iter().flat_map(|dir| dir.ancestors()))
491            .find_map(unsafe_ignore_file)
492        {
493            return Err(problem);
494        }
495        let skipped = Arc::new(Mutex::new(None));
496        let skip = Arc::clone(&skipped);
497        // The filter must own what it reads.
498        let resolver = Arc::clone(resolver);
499        // A config file that cannot be read excludes nothing here: the run reports it when it
500        // looks up the settings of the files under it.
501        let walk = ignore::WalkBuilder::new(&arg)
502            .filter_entry(move |entry| {
503                // The walk reads a folder's ignore files once the filter lets the folder in.
504                if entry.file_type().is_some_and(|t| t.is_dir())
505                    && let Some(problem) = unsafe_ignore_file(entry.path())
506                {
507                    // A poisoned lock still holds the message.
508                    skip.lock()
509                        .unwrap_or_else(std::sync::PoisonError::into_inner)
510                        .get_or_insert(problem);
511                    return false;
512                }
513                let settings = lock(&resolver).for_file(entry.path());
514                !settings.is_ok_and(|settings| settings.excluded(entry.path()))
515            })
516            .build();
517        for entry in walk {
518            let entry = entry
519                .map_err(|e| output::printable(&messages::fill("error.walk", &[("error", &e)])))?;
520            if entry.file_type().is_some_and(|t| t.is_file())
521                && Dialect::for_path(entry.path()).is_some()
522            {
523                files.push(entry.into_path().into_os_string());
524                ids.push(None);
525            }
526        }
527        if let Some(problem) = skipped
528            .lock()
529            .unwrap_or_else(std::sync::PoisonError::into_inner)
530            .take()
531        {
532            return Err(problem);
533        }
534        files[start..].sort_unstable();
535    }
536    // Checking costs a call to the system for each file, 12% of a run over 10,000 small files.
537    let overlap = |i: usize, root: &PathBuf| {
538        roots[..i]
539            .iter()
540            .any(|r| r.starts_with(root) || root.starts_with(r))
541    };
542    if distinct && !roots.iter().enumerate().any(|(i, root)| overlap(i, root)) {
543        return Ok(files);
544    }
545    Ok(unique(files, ids, threads))
546}
547
548/// What names a file apart from its spelling.
549#[derive(Clone, PartialEq, Eq, Hash)]
550enum Identity {
551    #[cfg(unix)]
552    Inode(u64, u64),
553    Path(PathBuf),
554}
555
556/// The identity of `file`, from `meta`, what `std::fs::metadata` said about it.
557fn identity(file: &OsStr, meta: io::Result<std::fs::Metadata>) -> Identity {
558    #[cfg(unix)]
559    if let Ok(meta) = meta {
560        use std::os::unix::fs::MetadataExt;
561        return Identity::Inode(meta.dev(), meta.ino());
562    }
563    // A file that cannot be found stays as spelled, so it is reported once per spelling.
564    Identity::Path(std::fs::canonicalize(file).unwrap_or_else(|_| file.into()))
565}
566
567/// `files` without any that is the same file as an earlier one, so that each is checked once,
568/// under its first spelling. Standard input can be given only once, as the arguments check.
569/// `ids` holds each file's identity where it is known already.
570fn unique(files: Vec<OsString>, ids: Vec<Option<Identity>>, threads: usize) -> Vec<OsString> {
571    let ids: Vec<Identity> = if ids.iter().all(Option::is_some) {
572        ids.into_iter().flatten().collect()
573    } else {
574        let files: Vec<_> = files.iter().zip(ids).collect();
575        parallel(
576            &files,
577            threads,
578            || false,
579            |(file, id)| {
580                id.clone()
581                    .unwrap_or_else(|| identity(file, std::fs::metadata(file)))
582            },
583        )
584    };
585    let mut seen = std::collections::HashSet::with_capacity(files.len());
586    files
587        .into_iter()
588        .zip(ids)
589        .filter_map(|(file, id)| seen.insert(id).then_some(file))
590        .collect()
591}
592
593/// The resolver, even if a panic elsewhere left its lock poisoned: it holds only a cache.
594fn lock(resolver: &Mutex<Resolver>) -> std::sync::MutexGuard<'_, Resolver> {
595    resolver
596        .lock()
597        .unwrap_or_else(std::sync::PoisonError::into_inner)
598}
599
600/// Where the problems go: standard output, or standard error when standard output carries a
601/// fixed text.
602enum Diagnostics<'w, W: Write> {
603    Out(&'w mut W),
604    Stderr(io::Stderr),
605}
606
607impl<'w, W: Write> Diagnostics<'w, W> {
608    fn new(to_stderr: bool, out: &'w mut W) -> Self {
609        if to_stderr {
610            Diagnostics::Stderr(io::stderr())
611        } else {
612            Diagnostics::Out(out)
613        }
614    }
615}
616
617impl<W: Write> Write for Diagnostics<'_, W> {
618    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
619        match self {
620            Diagnostics::Out(out) => out.write(buf),
621            Diagnostics::Stderr(err) => err.write(buf),
622        }
623    }
624
625    fn flush(&mut self) -> io::Result<()> {
626        match self {
627            Diagnostics::Out(out) => out.flush(),
628            Diagnostics::Stderr(err) => err.flush(),
629        }
630    }
631}
632
633/// What `--fix` did over the whole run.
634#[derive(Default)]
635struct FixRun {
636    problems: usize,
637    files: usize,
638    /// Standard input as it is after the fixes, for standard output.
639    stdin: Option<String>,
640}
641
642impl FixRun {
643    /// Print the count to standard error when `count` is set, then standard input's fixed text to
644    /// `out`.
645    fn finish(&self, count: bool, out: &mut impl Write) -> io::Result<()> {
646        if count && let Some(summary) = self.summary() {
647            eprintln!("{summary}");
648        }
649        match &self.stdin {
650            Some(text) => out.write_all(text.as_bytes()).and_then(|()| out.flush()),
651            None => Ok(()),
652        }
653    }
654
655    /// Such as `fixed 3 problems in 2 files`, or `None` when nothing was fixed.
656    fn summary(&self) -> Option<String> {
657        (self.problems > 0).then(|| {
658            messages::fill(
659                "summary.fixed",
660                &[
661                    (
662                        "problems",
663                        &messages::count("summary.problems", self.problems),
664                    ),
665                    ("files", &messages::count("summary.files", self.files)),
666                ],
667            )
668        })
669    }
670}
671
672/// Fix the `texts` of `chunk` in place, and write each changed file, so the check that follows
673/// reports what is left. A file that cannot be written gets the reason in place of its text.
674fn fix_chunk(
675    chunk: &[Job],
676    texts: &mut [(Result<String, String>, bool)],
677    shown: &(impl Fn(&OsStr) -> OsString + Sync),
678    unsafe_fixes: bool,
679    threads: usize,
680    run: &mut FixRun,
681) -> Vec<Option<Vec<Problem>>> {
682    let jobs: Vec<_> = chunk.iter().zip(texts.iter()).collect();
683    let done = parallel(
684        &jobs,
685        threads,
686        || false,
687        |(((settings, _), file), (text, _))| {
688            let text = text.as_ref().ok()?;
689            let name = shown(file);
690            let config = settings.config_for(&name.to_string_lossy());
691            let fixed = match jsonvet::try_fix(text, &config, unsafe_fixes) {
692                Ok(fixed) => fixed,
693                Err(problems) => return Some(Err(problems)),
694            };
695            let saved = if file == STDIN {
696                Ok(())
697            } else {
698                write_atomically(Path::new(file), &fixed.text)
699            };
700            Some(Ok((fixed, saved)))
701        },
702    );
703    drop(jobs);
704    let mut known = Vec::with_capacity(done.len());
705    for (((_, file), entry), done) in chunk.iter().zip(texts.iter_mut()).zip(done) {
706        // The scan's problems stand for the check only while the text is unchanged.
707        known.push(match &done {
708            Some(Err(problems)) => problems.clone(),
709            _ => None,
710        });
711        let done = done.and_then(Result::ok);
712        match done {
713            Some((fixed, Ok(()))) => {
714                run.problems += fixed.fixes.len();
715                run.files += 1;
716                entry.0 = Ok(fixed.text);
717            }
718            Some((_, Err(error))) => {
719                entry.0 = Err(error);
720            }
721            None => {}
722        }
723        if file == STDIN
724            && let Ok(text) = &entry.0
725        {
726            run.stdin = Some(text.clone());
727        }
728    }
729    known
730}
731
732#[derive(Default)]
733struct Counts {
734    errors: usize,
735    warnings: usize,
736    /// Files with at least one problem, or that could not be read.
737    files: usize,
738}
739
740impl Counts {
741    /// Such as `3 errors and 1 warning in 2 of 5 files`.
742    fn summary(&self, checked: usize) -> String {
743        let files = messages::count("summary.files", checked);
744        if self.errors + self.warnings == 0 {
745            return messages::fill("summary.none", &[("files", &files)]);
746        }
747        let errors = || messages::count("summary.errors", self.errors);
748        let warnings = || messages::count("summary.warnings", self.warnings);
749        let problems = match (self.errors, self.warnings) {
750            (_, 0) => errors(),
751            (0, _) => warnings(),
752            _ => messages::fill(
753                "summary.both",
754                &[("errors", &errors()), ("warnings", &warnings())],
755            ),
756        };
757        messages::fill(
758            "summary.some",
759            &[
760                ("problems", &problems),
761                ("failed", &self.files),
762                ("files", &files),
763            ],
764        )
765    }
766}
767
768/// `f` on the items, on up to `threads` threads counting this one, each with a [`STACK`]-byte
769/// stack, until `stop` says so before an item is taken: the results for the items done, in
770/// order, always at least the first. A panic in `f` carries on here.
771fn parallel<T: Sync, U: Send>(
772    items: &[T],
773    threads: usize,
774    stop: impl Fn() -> bool + Sync,
775    f: impl Fn(&T) -> U + Sync,
776) -> Vec<U> {
777    let next = AtomicUsize::new(0);
778    // Each thread takes the next item in turn, so the items taken are always the first ones,
779    // and the results are tagged to put them back in order.
780    let work = || {
781        let mut done = Vec::new();
782        loop {
783            if stop() && next.load(Ordering::Relaxed) > 0 {
784                break done;
785            }
786            let i = next.fetch_add(1, Ordering::Relaxed);
787            let Some(item) = items.get(i) else { break done };
788            done.push((i, f(item)));
789        }
790    };
791    let mut done = std::thread::scope(|scope| {
792        // A thread that cannot start leaves its share to the others.
793        let helpers: Vec<_> = (1..threads.min(items.len()))
794            .filter_map(|_| {
795                std::thread::Builder::new()
796                    .stack_size(STACK)
797                    .spawn_scoped(scope, work)
798                    .ok()
799            })
800            .collect();
801        let mut done = work();
802        for helper in helpers {
803            done.extend(
804                helper
805                    .join()
806                    .unwrap_or_else(|panic| std::panic::resume_unwind(panic)),
807            );
808        }
809        done
810    });
811    done.sort_unstable_by_key(|&(i, _)| i);
812    done.into_iter().map(|(_, result)| result).collect()
813}
814
815/// The parts to check a text in `dialect` in, on up to `parts` threads: more than one only for
816/// JSON Lines of [`SPLIT_BYTES`] or more.
817fn split(text: &str, dialect: Dialect, parts: usize) -> Vec<std::ops::Range<usize>> {
818    let split = dialect == Dialect::JsonLines && parts > 1 && text.len() >= SPLIT_BYTES;
819    jsonvet::line_parts(text, if split { parts } else { 1 })
820}
821
822/// Check a file's text, named by its lossy UTF-8 form for glob matching, and against `schema`
823/// if there is one, on up to `parts` threads: the report and the schema errors.
824fn check<'t>(
825    name: &str,
826    text: &'t str,
827    settings: &Settings,
828    schema: Option<&Schema>,
829    parts: usize,
830    known: Option<&[Problem]>,
831) -> (Report<'t>, Vec<(usize, String, bool)>) {
832    let config = settings.config_for(name);
833    let ranges = split(text, config.dialect(), parts);
834    if ranges.len() > 1 {
835        // Each line is a document of its own, and the error cap counts per document, so each
836        // part's schema errors are the ones a single pass finds in its lines.
837        let (lines, errors): (Vec<_>, Vec<_>) = parallel(
838            &ranges,
839            parts,
840            || false,
841            |lines| match schema {
842                Some(schema) => schema.check_lines(text, lines.clone(), &config),
843                None => (
844                    jsonvet::parse_lines(text, lines.clone(), &config, None),
845                    Vec::new(),
846                ),
847            },
848        )
849        .into_iter()
850        .unzip();
851        let (report, syntax_error) = jsonvet::join_lines(text, lines);
852        // A file with a syntax error in any part gets no schema errors.
853        let errors = if syntax_error {
854            Vec::new()
855        } else {
856            errors.into_iter().flatten().collect()
857        };
858        return (report, errors);
859    }
860    match schema {
861        // A file with a syntax error gets no schema errors.
862        Some(schema) => schema.check(text, &config),
863        // The problems `--fix` found on its way, when it changed nothing.
864        None => match known {
865            Some(problems) => (
866                jsonvet::report(text, problems.to_vec(), &config),
867                Vec::new(),
868            ),
869            None => (jsonvet::check(text, &config), Vec::new()),
870        },
871    }
872}
873
874/// Check `text` as a run checks the file at `path`: with its settings and its schema, if any.
875/// `Err` is what ends such a run: a config file or schema that cannot be used.
876/// `on_schema` receives local schema dependencies, including missing ones and failed loads.
877fn check_file<'t>(
878    path: &Path,
879    text: &'t str,
880    resolver: Resolver,
881    on_schema: &impl Fn(&Path),
882) -> Result<Checked<'t>, String> {
883    let resolver = Mutex::new(resolver);
884    let settings_for = |path: &Path| lock(&resolver).for_file(path);
885    let schemas = Schemas::default();
886    let load = |path: &Path, quiet| {
887        on_schema(path);
888        schemas.get(path, quiet, |admit| {
889            Schema::load(path, &*settings_for(path)?, quiet, admit)
890        })
891    };
892    let root = |path: &Path| lock(&resolver).root(path);
893    let jobs = jobs(
894        vec![path.into()],
895        &OsStr::to_owned,
896        &settings_for,
897        None,
898        &load,
899    )?;
900    // One file in, one job out.
901    let Some(job @ ((settings, schema), _)) = jobs.first() else {
902        return Err(String::new());
903    };
904    let name = path.to_string_lossy();
905    let checked = match schema {
906        Some(schema) => check(&name, text, settings, Some(schema), 1, None),
907        None if with_schema(job, path.as_os_str(), text) => {
908            check_own_schema(&name, text, settings, &schemas, &root, &load, 1)
909        }
910        None => check(&name, text, settings, None, 1, None),
911    };
912    schemas.observe_missing(on_schema);
913    schemas.over()?;
914    Ok(checked)
915}
916
917/// How text output looks: code frames on a terminal and lines otherwise, so tools reading a pipe
918/// always get `file:line:col`, and in color as `--color` and the environment ask.
919fn look(format: Format, color: Option<bool>, terminal: Terminal) -> Look {
920    let text = format == Format::Text;
921    Look {
922        frames: text && terminal.tty,
923        color: text && color.or(terminal.color).unwrap_or(terminal.tty),
924    }
925}
926
927/// Print one file's problems and schema errors, and add them to `counts`, each before it is
928/// printed, so a failed write still leaves the verdict. The file count is added last: only the
929/// summary reads it, and a failed write ends the run before the summary.
930fn print_file(
931    file: &OsStr,
932    report: &Report<'_>,
933    schema_errors: &[(usize, String, bool)],
934    printer: &mut Printer<impl Write>,
935    counts: &mut Counts,
936) -> io::Result<()> {
937    let before = counts.errors + counts.warnings;
938    for problem in report.problems() {
939        match problem.severity() {
940            Severity::Error => counts.errors += 1,
941            Severity::Warning => counts.warnings += 1,
942            _ => {}
943        }
944        printer.problem(file, report, problem)?;
945    }
946    for (offset, message, summary) in schema_errors {
947        counts.errors += usize::from(!summary);
948        printer.schema_error(file, report, *offset, message, *summary)?;
949    }
950    counts.files += usize::from(counts.errors + counts.warnings > before);
951    Ok(())
952}
953
954/// `path` opened for reading, with its size, if it is a regular file or a symlink to one. The
955/// type is checked on the open file, not the path, so a path swapped for a FIFO or a device after
956/// a check is still refused, and on Unix opening a FIFO does not wait for a writer.
957fn open_regular(path: &Path) -> io::Result<Option<(std::fs::File, u64)>> {
958    // shortcut: elsewhere the path is checked before the open, which a swap can race; Windows
959    // has no FIFO to swap in at a file path.
960    #[cfg(not(unix))]
961    if !std::fs::metadata(path)?.is_file() {
962        return Ok(None);
963    }
964    let mut options = std::fs::OpenOptions::new();
965    options.read(true);
966    // A regular file reads the same without blocking.
967    #[cfg(unix)]
968    std::os::unix::fs::OpenOptionsExt::custom_flags(&mut options, libc::O_NONBLOCK);
969    let file = options.open(path)?;
970    let meta = file.metadata()?;
971    Ok(meta.is_file().then_some((file, meta.len())))
972}
973
974/// A file's text, or why it has none. Only regular files are read, so a FIFO cannot hang the
975/// run and a device such as `/dev/zero` cannot fill memory. A symlink to a regular file is
976/// followed, as `cat` would.
977fn read(path: &Path) -> Result<String, String> {
978    let read = open_regular(path).and_then(|opened| {
979        if let Some((file, len)) = opened {
980            // The size is known already: `fs::read`, and `File`'s own `read_to_end`, would ask
981            // the system for it again. `take` reads to the end without asking. A file too big for
982            // memory fails alone, as an error, rather than aborting the run, but only where an
983            // oversized reservation fails: Linux's default heuristic overcommit
984            // (`vm.overcommit_memory=0`). Under mode 1 or on macOS it succeeds and the read can
985            // fill memory. NOTE: no cap against available memory, as std has no portable query.
986            let mut bytes = Vec::new();
987            bytes.try_reserve_exact(usize::try_from(len).unwrap_or(usize::MAX))?;
988            file.take(u64::MAX)
989                .read_to_end(&mut bytes)
990                .map(|_| Some(bytes))
991        } else {
992            Ok(None)
993        }
994    });
995    match read {
996        Ok(Some(bytes)) => decode(bytes),
997        Ok(None) => Err(messages::text("read.not-regular").to_owned()),
998        Err(e) if e.kind() == io::ErrorKind::NotFound => {
999            Err(messages::text("read.missing").to_owned())
1000        }
1001        Err(e) => Err(messages::fill("read.error", &[("error", &e)])),
1002    }
1003}
1004
1005/// Standard input's text, or why it has none. Empty input is an empty file.
1006fn read_stdin() -> Result<String, String> {
1007    let mut bytes = Vec::new();
1008    match io::stdin().lock().read_to_end(&mut bytes) {
1009        Ok(_) => decode(bytes),
1010        Err(e) => Err(messages::fill("read.error", &[("error", &e)])),
1011    }
1012}
1013
1014/// A file's bytes as UTF-8 text. A UTF-16 or UTF-32 file says so, rather than that its first
1015/// byte is not UTF-8: by its byte order mark, or without one by where the zero bytes of its first
1016/// two characters fall, as RFC 4627 section 3 tells the encodings apart.
1017fn decode(bytes: Vec<u8>) -> Result<String, String> {
1018    // Without a mark, a text these patterns match would fail the check anyway: a zero byte is
1019    // U+0000, and outside a string no dialect allows it. JSON5 does allow it raw inside one, so
1020    // a first byte that opens a string is left alone.
1021    let opens_string = |b: u8| b == b'"' || b == b'\'';
1022    // UTF-32 LE first: its mark starts with UTF-16 LE's.
1023    let encoding = match *bytes.as_slice() {
1024        [0xFF, 0xFE, 0, 0, ..] => Some("UTF-32 (LE)"),
1025        [0, 0, 0xFE, 0xFF, ..] => Some("UTF-32 (BE)"),
1026        [0xFF, 0xFE, ..] => Some("UTF-16 (LE)"),
1027        [0xFE, 0xFF, ..] => Some("UTF-16 (BE)"),
1028        [0, 0, 0, a, ..] if a != 0 => Some("UTF-32 (BE)"),
1029        [0, a, 0, b, ..] if a != 0 && b != 0 => Some("UTF-16 (BE)"),
1030        [a, 0, 0, 0, ..] if a != 0 && !opens_string(a) => Some("UTF-32 (LE)"),
1031        [a, 0, b, 0, ..] if a != 0 && b != 0 && !opens_string(a) => Some("UTF-16 (LE)"),
1032        _ => None,
1033    };
1034    if let Some(encoding) = encoding {
1035        return Err(messages::fill("read.encoding", &[("encoding", &encoding)]));
1036    }
1037    String::from_utf8(bytes).map_err(|e| {
1038        let byte = e.utf8_error().valid_up_to();
1039        messages::fill("read.not-utf8", &[("byte", &byte)])
1040    })
1041}
1042
1043/// The most a config file or an ignore file may hold: real ones take a few kilobytes.
1044pub(crate) const MAX_CONFIG_BYTES: u64 = 16 << 20;
1045
1046/// The most a `--schema` file may hold: 40 times the largest schema on schemastore.org
1047/// (1.6 MB), and room for the 5,000,000 objects and arrays a schema may hold at 2 to 3 bytes each.
1048pub(crate) const MAX_SCHEMA_BYTES: u64 = 64 << 20;
1049
1050/// The text of a config or schema file of at most `max` bytes. As in [`read`], only a regular
1051/// file, or a symlink to one, is opened, so a FIFO cannot hang the run and `/dev/zero` cannot
1052/// fill memory.
1053pub(crate) fn read_small(path: &Path, max: u64) -> io::Result<String> {
1054    let Some((file, _)) = open_regular(path)? else {
1055        return Err(io::Error::other(messages::text("read.not-regular")));
1056    };
1057    let mut text = String::new();
1058    // The cap holds even for a file that grows after the check.
1059    file.take(max + 1).read_to_string(&mut text)?;
1060    if text.len() as u64 > max {
1061        let mb = max >> 20;
1062        return Err(io::Error::other(messages::fill(
1063            "read.too-large",
1064            &[("mb", &mb)],
1065        )));
1066    }
1067    Ok(text)
1068}
1069
1070/// Why a folder walk must not read the `.gitignore` or `.ignore` in `dir`, if it must not. The
1071/// `ignore` crate reads them to the end with no check of what they are, so a link to
1072/// `/dev/zero` takes memory until the run is killed and a FIFO hangs it. A file that cannot be
1073/// looked at is no problem: the crate skips it too.
1074fn unsafe_ignore_file(dir: &Path) -> Option<String> {
1075    [".gitignore", ".ignore"].into_iter().find_map(|name| {
1076        let path = dir.join(name);
1077        let meta = std::fs::metadata(&path).ok()?;
1078        let problem = if !meta.is_file() {
1079            messages::text("read.not-regular").to_owned()
1080        } else if meta.len() > MAX_CONFIG_BYTES {
1081            messages::fill("read.too-large", &[("mb", &(MAX_CONFIG_BYTES >> 20))])
1082        } else {
1083            return None;
1084        };
1085        let error = format!("{}: {problem}", path.display());
1086        Some(output::printable(&messages::fill(
1087            "error.walk",
1088            &[("error", &error)],
1089        )))
1090    })
1091}
1092
1093fn print(mut out: impl Write, text: &str) -> Result<u8, String> {
1094    match writeln!(out, "{text}").and_then(|()| out.flush()) {
1095        Err(e) if e.kind() != io::ErrorKind::BrokenPipe => {
1096            Err(messages::fill("error.write", &[("error", &e)]))
1097        }
1098        _ => Ok(0),
1099    }
1100}
1101
1102#[cfg(test)]
1103mod tests {
1104    use std::fmt::Write as _;
1105    use std::sync::atomic::{AtomicUsize, Ordering};
1106
1107    use jsonvet::Dialect;
1108
1109    use super::{
1110        CHUNK_BYTES, MAX_CONFIG_BYTES, SCHEMA_BYTES, SPLIT_BYTES, STDIN, Schema, Settings, check,
1111        chunk_len, parallel, read_small, schema_share, schema_shares, split, take_chunk,
1112        unsafe_ignore_file,
1113    };
1114    use crate::schema::select::Job;
1115    use std::sync::Arc;
1116
1117    /// The type is checked on the opened file, and opening a FIFO with no writer returns at
1118    /// once, so a path swapped for one between a check and the open cannot hang the run.
1119    #[cfg(unix)]
1120    #[test]
1121    fn opening_a_fifo_does_not_wait_for_a_writer() {
1122        let dir = std::env::temp_dir().join(format!("jsonvet-fifo-{}", std::process::id()));
1123        std::fs::create_dir_all(&dir).expect("a temporary directory");
1124        let fifo = dir.join("fifo.json");
1125        let made = std::process::Command::new("mkfifo")
1126            .arg(&fifo)
1127            .status()
1128            .expect("run mkfifo");
1129        assert!(made.success());
1130        let (send, receive) = std::sync::mpsc::channel();
1131        let path = fifo.clone();
1132        std::thread::spawn(move || {
1133            let opened = super::open_regular(&path).map(|o| o.is_none());
1134            let _ = send.send(opened.map_err(|e| e.to_string()));
1135        });
1136        let refused = receive.recv_timeout(std::time::Duration::from_secs(5));
1137        let _ = std::fs::remove_dir_all(&dir);
1138        assert_eq!(
1139            refused,
1140            Ok(Ok(true)),
1141            "the open waited, or did not refuse the FIFO"
1142        );
1143    }
1144
1145    /// A schema that takes any value, in a fresh folder named for `test`, which also holds an
1146    /// empty folder `x`.
1147    fn any_schema(test: &str) -> (std::path::PathBuf, Schema) {
1148        let dir = std::env::temp_dir().join(format!("jsonvet-{test}-{}", std::process::id()));
1149        std::fs::create_dir_all(dir.join("x")).expect("a temporary directory");
1150        std::fs::write(dir.join("s.json"), "{}").expect("written");
1151        let schema = Schema::load(&dir.join("s.json"), &Settings::default(), false, |_| Ok(()));
1152        (dir, schema.expect("a schema"))
1153    }
1154
1155    #[test]
1156    fn config_text_reports_a_bad_file() {
1157        let path = std::path::Path::new("/nonexistent/jsonvet.toml");
1158        assert_eq!(super::check_config_text(path, "rules = {}"), Ok(()));
1159        let message = super::check_config_text(path, "rules = [").expect_err("bad toml");
1160        assert!(
1161            message.starts_with("/nonexistent/jsonvet.toml:1:"),
1162            "{message}"
1163        );
1164    }
1165
1166    #[test]
1167    fn only_files_with_a_schema_take_a_share() {
1168        let (dir, schema) = any_schema("shares");
1169        let _ = std::fs::remove_dir_all(&dir);
1170        let schema = Arc::new(schema);
1171        let settings = Arc::new(Settings::default());
1172        let job =
1173            |schema: Option<&Arc<Schema>>| ((Arc::clone(&settings), schema.cloned()), STDIN.into());
1174        assert_eq!(
1175            schema_shares(&[job(None), job(None)], 2),
1176            Vec::<usize>::new()
1177        );
1178        assert_eq!(
1179            schema_shares(&[job(None), job(Some(&schema)), job(None)], 2),
1180            [0, CHUNK_BYTES, 0]
1181        );
1182    }
1183
1184    #[test]
1185    fn small_files_are_read_up_to_their_limit_and_no_further() {
1186        let dir = std::env::temp_dir().join(format!("jsonvet-small-{}", std::process::id()));
1187        std::fs::create_dir_all(&dir).expect("a temporary directory");
1188        let path = dir.join("f");
1189        std::fs::write(&path, "0123456789").expect("written");
1190        assert_eq!(read_small(&path, 10).expect("ten bytes"), "0123456789");
1191        assert!(read_small(&path, 9).is_err());
1192        // An ignore file is judged by its size alone; sparse files, so nothing is written.
1193        let sized = |len: u64| {
1194            std::fs::File::create(dir.join(".gitignore"))
1195                .and_then(|file| file.set_len(len))
1196                .expect("a large file");
1197        };
1198        sized(MAX_CONFIG_BYTES);
1199        assert_eq!(unsafe_ignore_file(&dir), None);
1200        sized(MAX_CONFIG_BYTES + 1);
1201        assert!(unsafe_ignore_file(&dir).is_some_and(|e| e.ends_with("larger than 16 MB")));
1202        let _ = std::fs::remove_dir_all(&dir);
1203    }
1204
1205    #[test]
1206    fn only_json_lines_of_the_split_size_is_split_one_part_per_thread() {
1207        let line = "[1]\n".repeat(SPLIT_BYTES / 4);
1208        assert_eq!(split(&line, Dialect::JsonLines, 8).len(), 8);
1209        assert_eq!(split(&line, Dialect::JsonLines, 1).len(), 1);
1210        assert_eq!(split(&line, Dialect::Json, 8).len(), 1);
1211        assert_eq!(split(&line[4..], Dialect::JsonLines, 8).len(), 1);
1212    }
1213
1214    #[test]
1215    fn a_large_json_lines_file_checked_in_parts_reports_what_one_pass_does() {
1216        // Problems in every part: duplicate keys, and strings the schema wants as integers. Every
1217        // thousandth line has more schema errors than a document reports, and two in each
1218        // thousand have values the schema cannot check.
1219        let pad = "x".repeat(500);
1220        let wide: Vec<String> = (0..150).map(|k| format!("\"k{k}\": \"s\"")).collect();
1221        let wide = format!("{{{}}}", wide.join(", "));
1222        let mut text = String::from("\u{FEFF}");
1223        for i in 0..4_000 {
1224            let _ = match i % 7 {
1225                _ if i % 1_000 == 999 => writeln!(text, "{wide}"),
1226                _ if i % 1_000 == 500 => writeln!(text, "{{\"a\": 1e400}}"),
1227                _ if i % 1_000 == 501 => writeln!(text, r#"{{"\ud800": 1, "\ud801": 2}}"#),
1228                0 => writeln!(text, "{{\"a\": {i}, \"a\": 1}}"),
1229                1 => write!(text, "{{\"a\": \"{i}\", \"p\": \"{pad}\"}}\r\n"),
1230                _ => writeln!(text, "{{\"a\": {i}, \"p\": \"{pad}\"}}"),
1231            };
1232        }
1233        assert!(text.len() > SPLIT_BYTES);
1234        let deep = format!("{}{}\n", "[".repeat(600), "]".repeat(600));
1235        let texts = [
1236            text.clone(),
1237            format!("{text}{{\"a\": 1\n{text}"),
1238            format!("{text}{deep}{text}"),
1239        ];
1240        let dir = std::env::temp_dir().join(format!("jsonvet-split-{}", std::process::id()));
1241        std::fs::create_dir_all(&dir).expect("a temporary directory");
1242        let path = dir.join("schema.json");
1243        let schema = concat!(
1244            r#"{"properties": {"a": {"type": "integer"}}, "#,
1245            r#""patternProperties": {"^k": {"type": "integer"}}}"#,
1246        );
1247        std::fs::write(&path, schema).expect("written");
1248        let settings = Settings::default();
1249        let schema = Schema::load(&path, &settings, false, |_| Ok(())).expect("a schema");
1250        let _ = std::fs::remove_dir_all(&dir);
1251        for text in &texts {
1252            for schema in [None, Some(&schema)] {
1253                let (whole, whole_errors) = check("x.jsonl", text, &settings, schema, 1, None);
1254                let (split, split_errors) = check("x.jsonl", text, &settings, schema, 8, None);
1255                assert_ne!(whole.problems(), &[]);
1256                assert_eq!(split.problems(), whole.problems());
1257                assert_eq!(split_errors, whole_errors);
1258            }
1259        }
1260        let (_, errors) = check("x.jsonl", &texts[0], &settings, Some(&schema), 8, None);
1261        // 570 strings, 100 and a summary for each wide line, and one for each line not checked.
1262        assert_eq!(errors.len(), 570 + 4 * 101 + 8);
1263        let unchecked = errors.iter().filter(|(_, m, _)| m.contains("not checked"));
1264        assert_eq!(unchecked.count(), 8);
1265    }
1266
1267    #[test]
1268    fn parallel_keeps_the_order_and_stops_after_a_prefix() {
1269        let items: Vec<usize> = (0..1000).collect();
1270        for threads in [1, 3, 16] {
1271            let squares = parallel(&items, threads, || false, |i| i * i);
1272            assert_eq!(squares, items.iter().map(|i| i * i).collect::<Vec<_>>());
1273            // Stopped from the start, it still does the first item, and a thread that looks
1274            // before another has taken it may take one more.
1275            let stopped = parallel(&items, threads, || true, |&i| i);
1276            assert!((1..=threads).contains(&stopped.len()), "{stopped:?}");
1277            assert_eq!(stopped, (0..stopped.len()).collect::<Vec<_>>());
1278            let done = AtomicUsize::new(0);
1279            let some = parallel(
1280                &items,
1281                threads,
1282                || done.load(Ordering::Relaxed) >= 100,
1283                |&i| {
1284                    done.fetch_add(1, Ordering::Relaxed);
1285                    i
1286                },
1287            );
1288            // At least 100, plus at most one per thread already under way.
1289            assert!((100..100 + threads).contains(&some.len()), "{}", some.len());
1290            assert_eq!(some, (0..some.len()).collect::<Vec<_>>());
1291        }
1292        assert_eq!(
1293            parallel(&[] as &[u8], 4, || false, |&b| b),
1294            Vec::<u8>::new()
1295        );
1296    }
1297
1298    #[test]
1299    fn a_chunk_ends_with_the_file_that_fills_it() {
1300        assert_eq!(chunk_len(&[]), 0);
1301        assert_eq!(chunk_len(&[1, 2, 3]), 3);
1302        assert_eq!(chunk_len(&[CHUNK_BYTES, 1]), 1);
1303        assert_eq!(chunk_len(&[CHUNK_BYTES - 1, 1, 1]), 2);
1304        assert_eq!(chunk_len(&[CHUNK_BYTES - 2, 1, 1]), 3);
1305        assert_eq!(chunk_len(&[0, CHUNK_BYTES - 1, 0, 2, 5]), 4);
1306    }
1307
1308    #[test]
1309    fn take_chunk_respects_shares() {
1310        let dir = std::env::temp_dir().join(format!("jsonvet-chunk-{}", std::process::id()));
1311        std::fs::create_dir_all(&dir).expect("dir");
1312        let f1 = dir.join("1.json");
1313        let f2 = dir.join("2.json");
1314        std::fs::write(&f1, "{}").expect("write");
1315        std::fs::write(&f2, "{}").expect("write");
1316        let settings = Arc::new(Settings::default());
1317        let job1: Job = ((settings.clone(), None), f1.into_os_string());
1318        let job2: Job = ((settings.clone(), None), f2.into_os_string());
1319        let rest = [job1, job2];
1320        let mut ahead = Vec::new();
1321        let shares = [CHUNK_BYTES, CHUNK_BYTES];
1322        let taken = take_chunk(&mut ahead, &rest, Some(&shares), 1, &|_, _| false);
1323        assert_eq!(taken.len(), 1);
1324        assert_eq!(ahead, []);
1325        let _ = std::fs::remove_dir_all(&dir);
1326    }
1327
1328    #[test]
1329    fn a_file_counts_its_size_twenty_times_with_a_schema() {
1330        let dir = std::env::temp_dir().join(format!("jsonvet-share-{}", std::process::id()));
1331        std::fs::create_dir_all(&dir).expect("a temp dir");
1332        let path = dir.join("a.json");
1333        std::fs::write(&path, " ".repeat(1000)).expect("write");
1334        let path = path.as_os_str();
1335        assert_eq!(schema_share(path), 1000 * SCHEMA_BYTES);
1336        std::fs::write(path, " ".repeat(CHUNK_BYTES / SCHEMA_BYTES + 1)).expect("write");
1337        assert_eq!(schema_share(path), CHUNK_BYTES, "capped");
1338        assert_eq!(schema_share(dir.join("gone.json").as_os_str()), 0);
1339        assert_eq!(schema_share(std::ffi::OsStr::new("-")), CHUNK_BYTES);
1340        let _ = std::fs::remove_dir_all(&dir);
1341    }
1342
1343    #[test]
1344    fn frames_show_on_a_terminal_and_color_follows_the_option_then_the_environment() {
1345        use super::{Format, Look, Terminal, look};
1346        let terminal = |tty, color| Terminal { tty, color };
1347        // Off a terminal, `color_forced_off_a_terminal_keeps_one_line_per_problem` in
1348        // tests/cli.rs covers each case end to end.
1349        let frames = |color| Look {
1350            frames: true,
1351            color,
1352        };
1353        let lines = Look {
1354            frames: false,
1355            color: false,
1356        };
1357        let cases = [
1358            (Format::Text, None, terminal(true, None), frames(true)),
1359            (
1360                Format::Text,
1361                None,
1362                terminal(true, Some(false)),
1363                frames(false),
1364            ),
1365            (
1366                Format::Text,
1367                Some(false),
1368                terminal(true, Some(true)),
1369                frames(false),
1370            ),
1371            (Format::Sarif, Some(true), terminal(true, None), lines),
1372            (Format::Json, Some(true), terminal(true, None), lines),
1373        ];
1374        for (format, color, terminal, want) in cases {
1375            assert_eq!(
1376                look(format, color, terminal),
1377                want,
1378                "{format:?} {color:?} {terminal:?}"
1379            );
1380        }
1381    }
1382
1383    #[test]
1384    fn the_summary_counts_problems_and_files() {
1385        let dir = std::env::temp_dir().join(format!("jsonvet-summary-{}", std::process::id()));
1386        std::fs::create_dir_all(&dir).expect("a temp dir");
1387        let dup = dir.join("dup.json");
1388        let ok = dir.join("ok.json");
1389        std::fs::write(&dup, "{\"a\": 1, \"a\": 2}").expect("write");
1390        std::fs::write(&ok, "{}").expect("write");
1391        let run = |summary: bool, files: &[&std::path::Path]| {
1392            let mut out = Vec::new();
1393            let args = ["--no-config".into()]
1394                .into_iter()
1395                .chain(files.iter().map(|f| f.as_os_str().to_owned()));
1396            let terminal = super::Terminal {
1397                tty: summary,
1398                color: Some(false),
1399            };
1400            let code = super::run_with(args, false, terminal, &mut out).expect("a check");
1401            (code, String::from_utf8(out).expect("UTF-8"))
1402        };
1403        let (code, out) = run(true, &[&dup, &ok]);
1404        assert_eq!(code, 1);
1405        assert!(out.ends_with("\n1 error in 1 of 2 files\n"), "{out}");
1406        // A file that cannot be read counts as a file with an error.
1407        let gone = dir.join("gone.json");
1408        // The clean file last, so counting it instead of a file with problems shows.
1409        let dup2 = dir.join("dup2.json");
1410        std::fs::write(&dup2, "{\"a\": 1, \"a\": 2}").expect("write");
1411        let (_, out) = run(true, &[&dup, &gone, &dup2, &ok]);
1412        assert!(out.ends_with("\n3 errors in 3 of 4 files\n"), "{out}");
1413        assert_eq!(run(true, &[&ok]).1, "no problems in 1 file\n");
1414        // No summary off a terminal, nor for no files.
1415        assert!(!run(false, &[&dup]).1.contains(" in 1 of 1 file"));
1416        assert_eq!(run(true, &[]).1, "");
1417        // SARIF and JSON stay one document on a terminal too.
1418        for format in ["sarif", "json"] {
1419            let args = ["--no-config", "--format", format]
1420                .map(std::ffi::OsString::from)
1421                .into_iter()
1422                .chain([dup.clone().into_os_string()]);
1423            let mut out = Vec::new();
1424            let terminal = super::Terminal {
1425                tty: true,
1426                color: None,
1427            };
1428            super::run_with(args, false, terminal, &mut out).expect("a check");
1429            let out = String::from_utf8(out).expect("UTF-8");
1430            assert!(
1431                serde_json::from_str::<serde_json::Value>(&out).is_ok(),
1432                "{out}"
1433            );
1434        }
1435        let _ = std::fs::remove_dir_all(&dir);
1436    }
1437
1438    /// Text output takes a translation; SARIF and JSON stay English with one in place.
1439    #[test]
1440    fn a_translation_reaches_text_output_only() {
1441        let dir = std::env::temp_dir().join(format!("jsonvet-locale-{}", std::process::id()));
1442        std::fs::create_dir_all(&dir).expect("a temp dir");
1443        let dup = dir.join("dup.json");
1444        std::fs::write(&dup, "{\"a\": 1, \"a\": 2}").expect("write");
1445        crate::messages::use_translation(
1446            "\"duplicate-key\" = \"XX {path}\"\n\"related.first-at\" = \"XX {line}:{column}\"\n",
1447        );
1448        let run = |format: &str| {
1449            let args = ["--no-config", "--format", format]
1450                .map(std::ffi::OsString::from)
1451                .into_iter()
1452                .chain([dup.clone().into_os_string()]);
1453            let mut out = Vec::new();
1454            super::run_with(args, false, super::Terminal::default(), &mut out).expect("a check");
1455            String::from_utf8(out).expect("UTF-8")
1456        };
1457        let text = run("text");
1458        assert!(text.contains("XX $.a (XX 1:2)"), "{text}");
1459        for format in ["json", "sarif"] {
1460            let out = run(format);
1461            assert!(out.contains("duplicate key $.a (first at 1:2)"), "{out}");
1462            assert!(!out.contains("XX"), "{out}");
1463        }
1464        let _ = std::fs::remove_dir_all(&dir);
1465    }
1466
1467    /// A JSON Lines file and a `--schema` error get the same code frames on a terminal as any
1468    /// other problem, at the line they are on.
1469    #[test]
1470    fn json_lines_and_schema_errors_get_code_frames_on_a_terminal() {
1471        let dir = std::env::temp_dir().join(format!("jsonvet-frames-{}", std::process::id()));
1472        std::fs::create_dir_all(&dir).expect("a temp dir");
1473        let lines = dir.join("l.jsonl");
1474        let schema = dir.join("s.json");
1475        let data = dir.join("d.json");
1476        std::fs::write(&lines, "{\"a\": 1}\n{\"a\": True}\n").expect("write");
1477        std::fs::write(&schema, r#"{"properties": {"a": {"type": "string"}}}"#).expect("write");
1478        std::fs::write(&data, "{\n  \"a\": 1\n}\n").expect("write");
1479        let run = |args: &[&std::ffi::OsStr]| {
1480            let args = ["--no-config".into()]
1481                .into_iter()
1482                .chain(args.iter().map(|a| (*a).to_owned()));
1483            let terminal = super::Terminal {
1484                tty: true,
1485                color: Some(false),
1486            };
1487            let mut out = Vec::new();
1488            let code = super::run_with(args, false, terminal, &mut out).expect("a check");
1489            let out = String::from_utf8(out).expect("UTF-8");
1490            (
1491                code,
1492                out.replace(
1493                    &format!("{}{}", dir.display(), std::path::MAIN_SEPARATOR),
1494                    "",
1495                ),
1496            )
1497        };
1498        assert_eq!(
1499            run(&[lines.as_os_str()]),
1500            (
1501                1,
1502                "error[syntax]: expected a value, found `True`; did you mean `true`?\n \
1503                 --> l.jsonl:2:7\n  |\n2 | {\"a\": True}\n  |       ^^^^\n  |\n  = help: https://github.com/simonvanlierde/jsonvet/blob/main/docs/usage.md#syntax\n\n\
1504                 1 error in 1 of 1 file\n"
1505                    .to_owned()
1506            )
1507        );
1508        assert_eq!(
1509            run(&["--schema".as_ref(), schema.as_os_str(), data.as_os_str()]),
1510            (
1511                1,
1512                "error[schema]: 1 is not of type \"string\" (at $.a)\n \
1513                 --> d.json:2:8\n  |\n2 |   \"a\": 1\n  |        ^\n  |\n  = help: https://github.com/simonvanlierde/jsonvet/blob/main/docs/usage.md#schema\n\n\
1514                 1 error in 1 of 1 file\n"
1515                    .to_owned()
1516            )
1517        );
1518        let _ = std::fs::remove_dir_all(&dir);
1519    }
1520
1521    #[test]
1522    fn the_summary_names_errors_and_warnings_together() {
1523        let counts = |errors, warnings| super::Counts {
1524            errors,
1525            warnings,
1526            files: 1,
1527        };
1528        assert_eq!(
1529            counts(1, 1).summary(2),
1530            "1 error and 1 warning in 1 of 2 files"
1531        );
1532        assert_eq!(counts(0, 2).summary(1), "2 warnings in 1 of 1 file");
1533    }
1534
1535    #[test]
1536    fn a_line_separator_ends_a_frame_line_only_in_a_json5_file() {
1537        let dir = std::env::temp_dir().join(format!("jsonvet-u2028-{}", std::process::id()));
1538        std::fs::create_dir_all(&dir).expect("a temp dir");
1539        let frame = |name: &str| {
1540            let file = dir.join(name);
1541            std::fs::write(&file, "{\"a\": \"x\u{2028}y\", \"a\": 2}\n").expect("write");
1542            let mut out = Vec::new();
1543            let args = ["--no-config".into(), file.into_os_string()];
1544            let terminal = super::Terminal {
1545                tty: true,
1546                color: Some(false),
1547            };
1548            super::run_with(args, false, terminal, &mut out).expect("a check");
1549            String::from_utf8(out).expect("UTF-8")
1550        };
1551        let five = frame("a.json5");
1552        // Both the duplicate and the first copy, whose line ends at the separator.
1553        assert!(five.contains("\n2 | y\", \"a\": 2}\n"), "{five}");
1554        assert!(five.contains("\n1 | {\"a\": \"x\n"), "{five}");
1555        let other = frame("a.jsonc");
1556        // The frame shows U+2028 as a replacement character, on the one line.
1557        assert!(
1558            other
1559                .lines()
1560                .any(|l| l.starts_with("1 | {\"a\": \"x\u{FFFD}y\", \"a\": 2}")),
1561            "{other}"
1562        );
1563        let _ = std::fs::remove_dir_all(&dir);
1564    }
1565
1566    #[test]
1567    fn a_panic_on_the_big_stack_carries_on_in_the_caller() {
1568        static RUNS: AtomicUsize = AtomicUsize::new(0);
1569        let panic = std::panic::catch_unwind(|| {
1570            super::on_big_stack(|| -> u8 {
1571                RUNS.fetch_add(1, Ordering::Relaxed);
1572                panic!("on the big stack")
1573            })
1574        })
1575        .expect_err("the panic carries on");
1576        assert_eq!(panic.downcast_ref::<&str>(), Some(&"on the big stack"));
1577        assert_eq!(RUNS.load(Ordering::Relaxed), 1);
1578    }
1579}