Skip to main content

jsonvet_cli/
lib.rs

1//! The jsonvet command line, shared by the `jsonvet` and `check-json5` binaries.
2
3use std::ffi::{OsStr, OsString};
4use std::io::{self, BufWriter, IsTerminal, Read, Write};
5use std::path::Path;
6use std::process::ExitCode;
7
8use std::sync::atomic::{AtomicUsize, Ordering};
9use std::sync::{Arc, Mutex};
10
11use jsonvet::{Dialect, Report, Severity};
12
13mod args;
14mod glob;
15mod json;
16mod output;
17mod rules;
18mod sarif;
19mod schema;
20mod settings;
21
22use args::{Command, STDIN};
23use output::Printer;
24use schema::Schema;
25use settings::{Resolver, Settings};
26
27/// How deep `--schema` reads a file. Past it, the file has a syntax error. jsonschema 0.58 takes
28/// memory quadratic in depth for an error deep under a recursive `$ref`: 1.3 GB for a 50 KB file
29/// nested 10,000 levels.
30const SCHEMA_MAX_DEPTH: usize = 512;
31
32/// The stack every run uses, for `--schema`: jsonschema validates recursively, and building,
33/// validating and dropping a value [`SCHEMA_MAX_DEPTH`] levels deep against a recursive schema
34/// took up to 2.2 MB in a debug build, more than Windows' 1 MB main thread. Only the pages used
35/// are committed, so a plain check costs no more memory.
36const STACK: usize = 64 << 20;
37
38/// How much memory the files read at once may take before they are checked and printed.
39const CHUNK_BYTES: usize = 64 << 20;
40
41/// How many bytes of memory a byte of file text takes with `--schema`, which holds the file's
42/// events and value: 14 to 27 (docs/design.md, "Memory").
43const SCHEMA_BYTES: usize = 20;
44
45/// From this size, a JSON Lines file is split at newlines and its parts checked on the cores
46/// its chunk leaves free.
47const SPLIT_BYTES: usize = 1 << 20;
48
49/// Run the command with the process's arguments and environment, and return its exit code.
50pub fn run() -> ExitCode {
51    on_big_stack(run_here)
52}
53
54/// Run `f` on a thread with a [`STACK`]-byte stack, or here if no thread can start. A panic in
55/// `f` carries on here.
56fn on_big_stack<T: Send + 'static>(f: fn() -> T) -> T {
57    let thread = std::thread::Builder::new().name("jsonvet".into());
58    match thread.stack_size(STACK).spawn(f) {
59        Ok(thread) => thread
60            .join()
61            .unwrap_or_else(|panic| std::panic::resume_unwind(panic)),
62        Err(_) => f(),
63    }
64}
65
66fn run_here() -> ExitCode {
67    let annotate = std::env::var_os("GITHUB_ACTIONS").is_some_and(|v| v == "true");
68    let stdout = io::stdout().lock();
69    // A summary is for people: hooks and CI read stdout from a pipe and see the same output.
70    let summary = stdout.is_terminal();
71    match run_with(
72        std::env::args_os().skip(1),
73        annotate,
74        summary,
75        BufWriter::new(stdout),
76    ) {
77        Ok(code) => ExitCode::from(code),
78        Err(message) => {
79            eprintln!("jsonvet: {message}");
80            ExitCode::from(2)
81        }
82    }
83}
84
85/// The command, with its inputs passed in: 0 when no file has an error, 1 when one does.
86/// With `summary`, text output ends with a count of the problems. A usage error comes back as
87/// `Err`, which `run` turns into exit status 2.
88fn run_with(
89    args: impl IntoIterator<Item = OsString>,
90    annotate: bool,
91    summary: bool,
92    out: impl Write,
93) -> Result<u8, String> {
94    // Usage errors quote arguments and the schema file, which can hold a newline or an escape.
95    let args = match args::parse(args).map_err(|e| {
96        format!(
97            "{}\n\nRun `jsonvet --help` for usage.",
98            output::printable(&e)
99        )
100    })? {
101        Command::Help => return print(out, args::HELP),
102        Command::Version => return print(out, concat!("jsonvet ", env!("CARGO_PKG_VERSION"))),
103        Command::ListRules => return print(out, &rules::list()),
104        Command::Explain(text) => return print(out, &text),
105        Command::Check(args) => args,
106    };
107    let resolver = Arc::new(Mutex::new(Resolver::new(&args.config, args.settings)?));
108    let settings_for = |path: &Path| lock(&resolver).for_file(path);
109    let threads = std::thread::available_parallelism().map_or(1, std::num::NonZero::get);
110    let files = expand(args.files, &resolver, threads)?;
111    let schema = args
112        .schema
113        .as_deref()
114        .map(|path| Schema::load(path, &*settings_for(path)?))
115        .transpose()
116        .map_err(|e| output::printable(&e))?;
117
118    // Standard input goes by its `--stdin-filename`, for settings and in the output.
119    let stdin_name = args.stdin_filename.unwrap_or_else(|| STDIN.into());
120    let shown = |file: &OsStr| -> OsString {
121        if file == STDIN {
122            stdin_name.clone()
123        } else {
124            file.to_owned()
125        }
126    };
127    // Every config file is read before any file is checked, so a bad one stops the run at once.
128    let files = files
129        .into_iter()
130        .map(|file| Ok((settings_for(Path::new(&shown(&file)))?, file)))
131        .collect::<Result<Vec<_>, String>>()?;
132    let mut printer = Printer::new(out, args.format, annotate);
133    let mut counts = Counts::default();
134    // Files are read and checked on every core, a chunk at a time, then printed here in the
135    // order given. A chunk ends once its files hold CHUNK_BYTES, so memory stays near that. With
136    // `--schema` a file takes 20 times its size, so the chunk is cut from the sizes before any
137    // file is read: else every core takes a file at once. Without, sizing every file first made
138    // a walk of 10,000 files 7 ms slower.
139    // shortcut: each chunk waits for its slowest file before the next starts; hand files out
140    // one at a time across chunks if one big file among many small ones holds a run up.
141    let mut rest = &files[..];
142    let shares = match schema {
143        Some(_) => parallel(&files, threads, || false, |(_, file)| schema_share(file)),
144        None => Vec::new(),
145    };
146    let written = loop {
147        if rest.is_empty() {
148            let line = (summary && !files.is_empty()).then(|| counts.summary(files.len()));
149            break line
150                .map_or(Ok(()), |line| printer.text_line(&line))
151                .and_then(|()| printer.finish());
152        }
153        let offered = match shares.get(files.len() - rest.len()..) {
154            Some(shares) if !shares.is_empty() => &rest[..chunk_len(shares)],
155            _ => rest,
156        };
157        let texts = read_chunk(offered, threads);
158        let chunk;
159        (chunk, rest) = rest.split_at(texts.len());
160        let jobs: Vec<_> = chunk.iter().zip(&texts).collect();
161        let parts = (threads / jobs.len()).max(1);
162        let checked = parallel(
163            &jobs,
164            threads,
165            || false,
166            |((settings, file), text)| {
167                let name = shown(file);
168                let name = name.to_string_lossy();
169                text.as_deref()
170                    .map(|text| check(&name, text, settings, schema.as_ref(), parts))
171            },
172        );
173        let outcome =
174            chunk
175                .iter()
176                .zip(checked)
177                .try_for_each(|((_, file), checked)| match checked {
178                    Ok((report, schema_errors)) => print_file(
179                        &shown(file),
180                        &report,
181                        &schema_errors,
182                        &mut printer,
183                        &mut counts,
184                    ),
185                    Err(message) => {
186                        counts.errors += 1;
187                        counts.files += 1;
188                        printer.unreadable(&shown(file), message)
189                    }
190                });
191        if outcome.is_err() {
192            break outcome;
193        }
194    };
195    // A reader that has gone, as with `| head`, wants no more output: the verdict so far stands.
196    match written {
197        Err(e) if e.kind() != io::ErrorKind::BrokenPipe => {
198            return Err(format!("cannot write output: {e}"));
199        }
200        _ => {}
201    }
202    let too_many_warnings = args.max_warnings.is_some_and(|max| counts.warnings > max);
203    Ok(u8::from(counts.errors > 0 || too_many_warnings))
204}
205
206/// The texts of the first of `files`, read on up to `threads` threads until they hold
207/// [`CHUNK_BYTES`], and always the first.
208fn read_chunk<S: Sync>(files: &[(S, OsString)], threads: usize) -> Vec<Result<String, String>> {
209    let bytes = AtomicUsize::new(0);
210    let full = || bytes.load(Ordering::Relaxed) >= CHUNK_BYTES;
211    parallel(files, threads, full, |(_, file)| {
212        let text = if file == STDIN {
213            read_stdin()
214        } else {
215            read(Path::new(file))
216        };
217        bytes.fetch_add(text.as_ref().map_or(0, String::len), Ordering::Relaxed);
218        text
219    })
220}
221
222/// The memory a file takes while `--schema` checks it, as far as a chunk counts it:
223/// [`SCHEMA_BYTES`] times its size, capped at [`CHUNK_BYTES`]. Standard input's size is known
224/// only once it is read, so it takes a chunk's worth. A file that cannot be read takes nothing:
225/// `read` reports why.
226fn schema_share(file: &OsStr) -> usize {
227    if file == STDIN {
228        return CHUNK_BYTES;
229    }
230    let size = std::fs::metadata(file).map_or(0, |meta| meta.len());
231    usize::try_from(size)
232        .unwrap_or(usize::MAX)
233        .saturating_mul(SCHEMA_BYTES)
234        .min(CHUNK_BYTES)
235}
236
237/// How many of the files with these shares the next chunk takes: up to and including the one
238/// that fills [`CHUNK_BYTES`], so always at least one.
239fn chunk_len(shares: &[usize]) -> usize {
240    let mut total = 0;
241    shares
242        .iter()
243        .position(|share| {
244            total += share;
245            total >= CHUNK_BYTES
246        })
247        .map_or(shares.len(), |last| last + 1)
248}
249
250/// The files to check: each argument as given, except that a directory stands for the files under
251/// it that jsonvet checks by default, in sorted order. The walk skips hidden files and what
252/// `.gitignore`, `.ignore` and the `exclude` setting of each folder's settings name, as ripgrep
253/// would.
254fn expand(
255    args: Vec<OsString>,
256    resolver: &Arc<Mutex<Resolver>>,
257    threads: usize,
258) -> Result<Vec<OsString>, String> {
259    let mut files = Vec::with_capacity(args.len());
260    // Each argument costs a call to the system, so thousands of them are asked about in parallel.
261    let dirs = parallel(
262        &args,
263        threads,
264        || false,
265        |arg| arg != STDIN && Path::new(arg).is_dir(),
266    );
267    for (arg, dir) in args.into_iter().zip(dirs) {
268        if !dir {
269            files.push(arg);
270            continue;
271        }
272        let start = files.len();
273        // The walk reads the ignore files of the folder, and of every folder above it when it
274        // can find them, before the filter sees anything.
275        let above = std::fs::canonicalize(&arg).ok();
276        if let Some(problem) = std::iter::once(Path::new(&arg))
277            .chain(above.iter().flat_map(|dir| dir.ancestors()))
278            .find_map(unsafe_ignore_file)
279        {
280            return Err(problem);
281        }
282        let skipped = Arc::new(Mutex::new(None));
283        let skip = Arc::clone(&skipped);
284        // The filter must own what it reads.
285        let resolver = Arc::clone(resolver);
286        // A config file that cannot be read excludes nothing here: the run reports it when it
287        // looks up the settings of the files under it.
288        let walk = ignore::WalkBuilder::new(&arg)
289            .filter_entry(move |entry| {
290                // The walk reads a folder's ignore files once the filter lets the folder in.
291                if entry.file_type().is_some_and(|t| t.is_dir())
292                    && let Some(problem) = unsafe_ignore_file(entry.path())
293                {
294                    // A poisoned lock still holds the message.
295                    skip.lock()
296                        .unwrap_or_else(std::sync::PoisonError::into_inner)
297                        .get_or_insert(problem);
298                    return false;
299                }
300                let settings = lock(&resolver).for_file(entry.path());
301                !settings.is_ok_and(|settings| settings.excluded(entry.path()))
302            })
303            .build();
304        for entry in walk {
305            let entry = entry.map_err(|e| output::printable(&format!("cannot walk: {e}")))?;
306            if entry.file_type().is_some_and(|t| t.is_file())
307                && Dialect::for_path(entry.path()).is_some()
308            {
309                files.push(entry.into_path().into_os_string());
310            }
311        }
312        if let Some(problem) = skipped
313            .lock()
314            .unwrap_or_else(std::sync::PoisonError::into_inner)
315            .take()
316        {
317            return Err(problem);
318        }
319        files[start..].sort_unstable();
320    }
321    Ok(files)
322}
323
324/// The resolver, even if a panic elsewhere left its lock poisoned: it holds only a cache.
325fn lock(resolver: &Mutex<Resolver>) -> std::sync::MutexGuard<'_, Resolver> {
326    resolver
327        .lock()
328        .unwrap_or_else(std::sync::PoisonError::into_inner)
329}
330
331#[derive(Default)]
332struct Counts {
333    errors: usize,
334    warnings: usize,
335    /// Files with at least one problem, or that could not be read.
336    files: usize,
337}
338
339impl Counts {
340    /// Such as `3 errors and 1 warning in 2 of 5 files`.
341    fn summary(&self, checked: usize) -> String {
342        let n = |count: usize, what: &str| {
343            format!("{count} {what}{}", if count == 1 { "" } else { "s" })
344        };
345        if self.errors + self.warnings == 0 {
346            return format!("no problems in {}", n(checked, "file"));
347        }
348        let problems = match (self.errors, self.warnings) {
349            (e, 0) => n(e, "error"),
350            (0, w) => n(w, "warning"),
351            (e, w) => format!("{} and {}", n(e, "error"), n(w, "warning")),
352        };
353        format!("{problems} in {} of {}", self.files, n(checked, "file"))
354    }
355}
356
357/// `f` on the items, on up to `threads` threads counting this one, each with a [`STACK`]-byte
358/// stack, until `stop` says so before an item is taken: the results for the items done, in
359/// order, always at least the first. A panic in `f` carries on here.
360fn parallel<T: Sync, U: Send>(
361    items: &[T],
362    threads: usize,
363    stop: impl Fn() -> bool + Sync,
364    f: impl Fn(&T) -> U + Sync,
365) -> Vec<U> {
366    let next = AtomicUsize::new(0);
367    // Each thread takes the next item in turn, so the items taken are always the first ones,
368    // and the results are tagged to put them back in order.
369    let work = || {
370        let mut done = Vec::new();
371        loop {
372            if stop() && next.load(Ordering::Relaxed) > 0 {
373                break done;
374            }
375            let i = next.fetch_add(1, Ordering::Relaxed);
376            let Some(item) = items.get(i) else { break done };
377            done.push((i, f(item)));
378        }
379    };
380    let mut done = std::thread::scope(|scope| {
381        // A thread that cannot start leaves its share to the others.
382        let helpers: Vec<_> = (1..threads.min(items.len()))
383            .filter_map(|_| {
384                std::thread::Builder::new()
385                    .stack_size(STACK)
386                    .spawn_scoped(scope, work)
387                    .ok()
388            })
389            .collect();
390        let mut done = work();
391        for helper in helpers {
392            done.extend(
393                helper
394                    .join()
395                    .unwrap_or_else(|panic| std::panic::resume_unwind(panic)),
396            );
397        }
398        done
399    });
400    done.sort_unstable_by_key(|&(i, _)| i);
401    done.into_iter().map(|(_, result)| result).collect()
402}
403
404/// The parts to check a text in `dialect` in, on up to `parts` threads: more than one only for
405/// JSON Lines of [`SPLIT_BYTES`] or more.
406fn split(text: &str, dialect: Dialect, parts: usize) -> Vec<std::ops::Range<usize>> {
407    let split = dialect == Dialect::JsonLines && parts > 1 && text.len() >= SPLIT_BYTES;
408    jsonvet::line_parts(text, if split { parts } else { 1 })
409}
410
411/// Check a file's text, named by its lossy UTF-8 form for glob matching, and against `schema`
412/// if there is one, on up to `parts` threads: the report and the schema errors.
413fn check<'t>(
414    name: &str,
415    text: &'t str,
416    settings: &Settings,
417    schema: Option<&Schema>,
418    parts: usize,
419) -> (Report<'t>, Vec<(usize, String, bool)>) {
420    let config = settings.config_for(name);
421    let ranges = split(text, config.dialect(), parts);
422    if ranges.len() > 1 {
423        let config = match schema {
424            Some(_) => config.max_depth(SCHEMA_MAX_DEPTH),
425            None => config,
426        };
427        let lines = parallel(
428            &ranges,
429            parts,
430            || false,
431            |lines| jsonvet::parse_lines(text, lines.clone(), &config, schema.is_some()),
432        );
433        let (report, events) = jsonvet::join_lines(text, lines);
434        let errors = schema
435            .zip(events)
436            .map_or_else(Vec::new, |(schema, events)| schema.errors(&events));
437        return (report, errors);
438    }
439    match schema {
440        Some(schema) => {
441            let (report, events) = jsonvet::parse(text, &config.max_depth(SCHEMA_MAX_DEPTH));
442            // `events` is `None` after a syntax error, so such a file gets no schema errors.
443            let errors = events.map_or_else(Vec::new, |events| schema.errors(&events));
444            (report, errors)
445        }
446        None => (jsonvet::check(text, &config), Vec::new()),
447    }
448}
449
450/// Print one file's problems and schema errors, and add them to `counts`, each before it is
451/// printed, so a failed write still leaves the verdict. The file count is added last: only the
452/// summary reads it, and a failed write ends the run before the summary.
453fn print_file(
454    file: &OsStr,
455    report: &Report<'_>,
456    schema_errors: &[(usize, String, bool)],
457    printer: &mut Printer<impl Write>,
458    counts: &mut Counts,
459) -> io::Result<()> {
460    let before = counts.errors + counts.warnings;
461    for problem in report.problems() {
462        match problem.severity() {
463            Severity::Error => counts.errors += 1,
464            Severity::Warning => counts.warnings += 1,
465            _ => {}
466        }
467        printer.problem(file, report, problem)?;
468    }
469    for (offset, message, summary) in schema_errors {
470        counts.errors += usize::from(!summary);
471        printer.schema_error(file, report, *offset, message, *summary)?;
472    }
473    counts.files += usize::from(counts.errors + counts.warnings > before);
474    Ok(())
475}
476
477/// A file's text, or why it has none. Only regular files are read, so a FIFO cannot hang the
478/// run and a device such as `/dev/zero` cannot fill memory. A symlink to a regular file is
479/// followed, as `cat` would.
480fn read(path: &Path) -> Result<String, String> {
481    let read = std::fs::metadata(path).and_then(|meta| {
482        if meta.is_file() {
483            std::fs::read(path).map(Some)
484        } else {
485            Ok(None)
486        }
487    });
488    match read {
489        Ok(Some(bytes)) => decode(bytes),
490        Ok(None) => Err("not a regular file".to_owned()),
491        Err(e) if e.kind() == io::ErrorKind::NotFound => Err("no such file".to_owned()),
492        Err(e) => Err(format!("cannot read: {e}")),
493    }
494}
495
496/// Standard input's text, or why it has none. Empty input is an empty file.
497fn read_stdin() -> Result<String, String> {
498    let mut bytes = Vec::new();
499    match io::stdin().lock().read_to_end(&mut bytes) {
500        Ok(_) => decode(bytes),
501        Err(e) => Err(format!("cannot read: {e}")),
502    }
503}
504
505/// A file's bytes as UTF-8 text. A UTF-16 or UTF-32 file says so, rather than that its first
506/// byte is not UTF-8: by its byte order mark, or without one by where the zero bytes of its first
507/// two characters fall, as RFC 4627 section 3 tells the encodings apart.
508fn decode(bytes: Vec<u8>) -> Result<String, String> {
509    // Without a mark, a match always failed as UTF-8 before: a zero byte is U+0000, and outside
510    // a string no dialect allows it. JSON5 does allow it raw inside one, so a first byte that
511    // opens a string is left alone.
512    let opens_string = |b: u8| b == b'"' || b == b'\'';
513    // UTF-32 LE first: its mark starts with UTF-16 LE's.
514    let encoding = match *bytes.as_slice() {
515        [0xFF, 0xFE, 0, 0, ..] => Some("UTF-32 (LE)"),
516        [0, 0, 0xFE, 0xFF, ..] => Some("UTF-32 (BE)"),
517        [0xFF, 0xFE, ..] => Some("UTF-16 (LE)"),
518        [0xFE, 0xFF, ..] => Some("UTF-16 (BE)"),
519        [0, 0, 0, a, ..] if a != 0 => Some("UTF-32 (BE)"),
520        [0, a, 0, b, ..] if a != 0 && b != 0 => Some("UTF-16 (BE)"),
521        [a, 0, 0, 0, ..] if a != 0 && !opens_string(a) => Some("UTF-32 (LE)"),
522        [a, 0, b, 0, ..] if a != 0 && b != 0 && !opens_string(a) => Some("UTF-16 (LE)"),
523        _ => None,
524    };
525    if let Some(encoding) = encoding {
526        return Err(format!("file is {encoding}; jsonvet reads UTF-8"));
527    }
528    String::from_utf8(bytes)
529        .map_err(|e| format!("not valid UTF-8 (byte {})", e.utf8_error().valid_up_to()))
530}
531
532/// The most a config file or an ignore file may hold: real ones take a few kilobytes.
533pub(crate) const MAX_CONFIG_BYTES: u64 = 16 << 20;
534
535/// The most a `--schema` file may hold: 40 times the largest schema on schemastore.org
536/// (1.6 MB), and room for the 5,000,000 objects and arrays a schema may hold at 2 to 3 bytes each.
537pub(crate) const MAX_SCHEMA_BYTES: u64 = 64 << 20;
538
539/// The text of a config or schema file of at most `max` bytes. As in [`read`], only a regular
540/// file, or a symlink to one, is opened, so a FIFO cannot hang the run and `/dev/zero` cannot
541/// fill memory.
542pub(crate) fn read_small(path: &Path, max: u64) -> io::Result<String> {
543    if !std::fs::metadata(path)?.is_file() {
544        return Err(io::Error::other("not a regular file"));
545    }
546    let mut text = String::new();
547    // The cap holds even for a file that grows after the check.
548    std::fs::File::open(path)?
549        .take(max + 1)
550        .read_to_string(&mut text)?;
551    if text.len() as u64 > max {
552        return Err(io::Error::other(format!("larger than {} MB", max >> 20)));
553    }
554    Ok(text)
555}
556
557/// Why a folder walk must not read the `.gitignore` or `.ignore` in `dir`, if it must not. The
558/// `ignore` crate reads them to the end with no check of what they are, so a link to
559/// `/dev/zero` takes memory until the run is killed and a FIFO hangs it. A file that cannot be
560/// looked at is no problem: the crate skips it too.
561fn unsafe_ignore_file(dir: &Path) -> Option<String> {
562    [".gitignore", ".ignore"].into_iter().find_map(|name| {
563        let path = dir.join(name);
564        let meta = std::fs::metadata(&path).ok()?;
565        let problem = if !meta.is_file() {
566            "not a regular file".to_owned()
567        } else if meta.len() > MAX_CONFIG_BYTES {
568            format!("larger than {} MB", MAX_CONFIG_BYTES >> 20)
569        } else {
570            return None;
571        };
572        Some(output::printable(&format!(
573            "cannot walk: {}: {problem}",
574            path.display()
575        )))
576    })
577}
578
579fn print(mut out: impl Write, text: &str) -> Result<u8, String> {
580    match writeln!(out, "{text}").and_then(|()| out.flush()) {
581        Err(e) if e.kind() != io::ErrorKind::BrokenPipe => Err(format!("cannot write output: {e}")),
582        _ => Ok(0),
583    }
584}
585
586#[cfg(test)]
587mod tests {
588    use std::fmt::Write as _;
589    use std::sync::atomic::{AtomicUsize, Ordering};
590
591    use jsonvet::Dialect;
592
593    use super::{
594        CHUNK_BYTES, MAX_CONFIG_BYTES, SCHEMA_BYTES, SPLIT_BYTES, Schema, Settings, check,
595        chunk_len, parallel, read_small, schema_share, split, unsafe_ignore_file,
596    };
597
598    #[test]
599    fn small_files_are_read_up_to_their_limit_and_no_further() {
600        let dir = std::env::temp_dir().join(format!("jsonvet-small-{}", std::process::id()));
601        std::fs::create_dir_all(&dir).expect("a temporary directory");
602        let path = dir.join("f");
603        std::fs::write(&path, "0123456789").expect("written");
604        assert_eq!(read_small(&path, 10).expect("ten bytes"), "0123456789");
605        assert!(read_small(&path, 9).is_err());
606        // An ignore file is judged by its size alone; sparse files, so nothing is written.
607        let sized = |len: u64| {
608            std::fs::File::create(dir.join(".gitignore"))
609                .and_then(|file| file.set_len(len))
610                .expect("a large file");
611        };
612        sized(MAX_CONFIG_BYTES);
613        assert_eq!(unsafe_ignore_file(&dir), None);
614        sized(MAX_CONFIG_BYTES + 1);
615        assert!(unsafe_ignore_file(&dir).is_some_and(|e| e.ends_with("larger than 16 MB")));
616        let _ = std::fs::remove_dir_all(&dir);
617    }
618
619    #[test]
620    fn only_json_lines_of_the_split_size_is_split_one_part_per_thread() {
621        let line = "[1]\n".repeat(SPLIT_BYTES / 4);
622        assert_eq!(split(&line, Dialect::JsonLines, 8).len(), 8);
623        assert_eq!(split(&line, Dialect::JsonLines, 1).len(), 1);
624        assert_eq!(split(&line, Dialect::Json, 8).len(), 1);
625        assert_eq!(split(&line[4..], Dialect::JsonLines, 8).len(), 1);
626    }
627
628    #[test]
629    fn a_large_json_lines_file_checked_in_parts_reports_what_one_pass_does() {
630        // Problems in every part: duplicate keys, and strings the schema wants as integers.
631        let pad = "x".repeat(500);
632        let mut text = String::from("\u{FEFF}");
633        for i in 0..4_000 {
634            let _ = match i % 7 {
635                0 => writeln!(text, "{{\"a\": {i}, \"a\": 1}}"),
636                1 => write!(text, "{{\"a\": \"{i}\", \"p\": \"{pad}\"}}\r\n"),
637                _ => writeln!(text, "{{\"a\": {i}, \"p\": \"{pad}\"}}"),
638            };
639        }
640        assert!(text.len() > SPLIT_BYTES);
641        let deep = format!("{}{}\n", "[".repeat(600), "]".repeat(600));
642        let texts = [
643            text.clone(),
644            format!("{text}{{\"a\": 1\n{text}"),
645            format!("{text}{deep}{text}"),
646        ];
647        let dir = std::env::temp_dir().join(format!("jsonvet-split-{}", std::process::id()));
648        std::fs::create_dir_all(&dir).expect("a temporary directory");
649        let path = dir.join("schema.json");
650        std::fs::write(&path, r#"{"properties": {"a": {"type": "integer"}}}"#).expect("written");
651        let settings = Settings::default();
652        let schema = Schema::load(&path, &settings).expect("a schema");
653        let _ = std::fs::remove_dir_all(&dir);
654        for text in &texts {
655            for schema in [None, Some(&schema)] {
656                let (whole, whole_errors) = check("x.jsonl", text, &settings, schema, 1);
657                let (split, split_errors) = check("x.jsonl", text, &settings, schema, 8);
658                assert_ne!(whole.problems(), &[]);
659                assert_eq!(split.problems(), whole.problems());
660                assert_eq!(split_errors, whole_errors);
661            }
662        }
663        let (_, errors) = check("x.jsonl", &texts[0], &settings, Some(&schema), 8);
664        assert_eq!(errors.len(), 572);
665    }
666
667    #[test]
668    fn parallel_keeps_the_order_and_stops_after_a_prefix() {
669        let items: Vec<usize> = (0..1000).collect();
670        for threads in [1, 3, 16] {
671            let squares = parallel(&items, threads, || false, |i| i * i);
672            assert_eq!(squares, items.iter().map(|i| i * i).collect::<Vec<_>>());
673            // Stopped from the start, it still does the first item.
674            assert_eq!(parallel(&items, threads, || true, |&i| i), [0]);
675            let done = AtomicUsize::new(0);
676            let some = parallel(
677                &items,
678                threads,
679                || done.load(Ordering::Relaxed) >= 100,
680                |&i| {
681                    done.fetch_add(1, Ordering::Relaxed);
682                    i
683                },
684            );
685            // At least 100, plus at most one per thread already under way.
686            assert!((100..100 + threads).contains(&some.len()), "{}", some.len());
687            assert_eq!(some, (0..some.len()).collect::<Vec<_>>());
688        }
689        assert_eq!(
690            parallel(&[] as &[u8], 4, || false, |&b| b),
691            Vec::<u8>::new()
692        );
693    }
694
695    #[test]
696    fn a_chunk_ends_with_the_file_that_fills_it() {
697        assert_eq!(chunk_len(&[]), 0);
698        assert_eq!(chunk_len(&[1, 2, 3]), 3);
699        assert_eq!(chunk_len(&[CHUNK_BYTES, 1]), 1);
700        assert_eq!(chunk_len(&[CHUNK_BYTES - 1, 1, 1]), 2);
701        assert_eq!(chunk_len(&[CHUNK_BYTES - 2, 1, 1]), 3);
702        assert_eq!(chunk_len(&[0, CHUNK_BYTES - 1, 0, 2, 5]), 4);
703    }
704
705    #[test]
706    fn a_file_counts_its_size_twenty_times_with_a_schema() {
707        let dir = std::env::temp_dir().join(format!("jsonvet-share-{}", std::process::id()));
708        std::fs::create_dir_all(&dir).expect("a temp dir");
709        let path = dir.join("a.json");
710        std::fs::write(&path, " ".repeat(1000)).expect("write");
711        let path = path.as_os_str();
712        assert_eq!(schema_share(path), 1000 * SCHEMA_BYTES);
713        std::fs::write(path, " ".repeat(CHUNK_BYTES / SCHEMA_BYTES + 1)).expect("write");
714        assert_eq!(schema_share(path), CHUNK_BYTES, "capped");
715        assert_eq!(schema_share(dir.join("gone.json").as_os_str()), 0);
716        assert_eq!(schema_share(std::ffi::OsStr::new("-")), CHUNK_BYTES);
717        let _ = std::fs::remove_dir_all(&dir);
718    }
719
720    #[test]
721    fn the_summary_counts_problems_and_files() {
722        let dir = std::env::temp_dir().join(format!("jsonvet-summary-{}", std::process::id()));
723        std::fs::create_dir_all(&dir).expect("a temp dir");
724        let dup = dir.join("dup.json");
725        let ok = dir.join("ok.json");
726        std::fs::write(&dup, "{\"a\": 1, \"a\": 2}").expect("write");
727        std::fs::write(&ok, "{}").expect("write");
728        let run = |summary: bool, files: &[&std::path::Path]| {
729            let mut out = Vec::new();
730            let args = ["--no-config".into()]
731                .into_iter()
732                .chain(files.iter().map(|f| f.as_os_str().to_owned()));
733            let code = super::run_with(args, false, summary, &mut out).expect("a check");
734            (code, String::from_utf8(out).expect("UTF-8"))
735        };
736        let (code, out) = run(true, &[&dup, &ok]);
737        assert_eq!(code, 1);
738        assert!(out.ends_with("\n1 error in 1 of 2 files\n"), "{out}");
739        // A file that cannot be read counts as a file with an error.
740        let gone = dir.join("gone.json");
741        // The clean file last, so counting it instead of a file with problems shows.
742        let (_, out) = run(true, &[&dup, &gone, &dup, &ok]);
743        assert!(out.ends_with("\n3 errors in 3 of 4 files\n"), "{out}");
744        assert_eq!(run(true, &[&ok]).1, "no problems in 1 file\n");
745        // No summary off a terminal, nor for no files.
746        assert!(!run(false, &[&dup]).1.contains(" in 1 of 1 file"));
747        assert_eq!(run(true, &[]).1, "");
748        // SARIF and JSON stay one document on a terminal too.
749        for format in ["sarif", "json"] {
750            let args = ["--no-config", "--format", format]
751                .map(std::ffi::OsString::from)
752                .into_iter()
753                .chain([dup.clone().into_os_string()]);
754            let mut out = Vec::new();
755            super::run_with(args, false, true, &mut out).expect("a check");
756            let out = String::from_utf8(out).expect("UTF-8");
757            assert!(
758                serde_json::from_str::<serde_json::Value>(&out).is_ok(),
759                "{out}"
760            );
761        }
762        let _ = std::fs::remove_dir_all(&dir);
763    }
764
765    #[test]
766    fn a_panic_on_the_big_stack_carries_on_in_the_caller() {
767        static RUNS: AtomicUsize = AtomicUsize::new(0);
768        let panic = std::panic::catch_unwind(|| {
769            super::on_big_stack(|| -> u8 {
770                RUNS.fetch_add(1, Ordering::Relaxed);
771                panic!("on the big stack")
772            })
773        })
774        .expect_err("the panic carries on");
775        assert_eq!(panic.downcast_ref::<&str>(), Some(&"on the big stack"));
776        assert_eq!(RUNS.load(Ordering::Relaxed), 1);
777    }
778}